diff --git a/.asf.yaml b/.asf.yaml index 1a766f5330..ba8b7740a5 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -30,20 +30,95 @@ github: - rust enabled_merge_buttons: squash: true + squash_commit_message: PR_TITLE_AND_COMMIT_DETAILS merge: false rebase: false - protected_branches: - main: - required_status_checks: - # strict means "Require branches to be up to date before merging". - strict: true - + # Disabled: GitHub blocks ruleset bypass while classic protection exists. + # The "Default Branch Protection" ruleset replaces it. + protected_branches: ~ + rulesets: + - name: "Default Branch Protection" + type: branch + branches: + includes: + - "~DEFAULT_BRANCH" + excludes: [] + bypass_teams: + - root + restrict_deletion: true + restrict_force_push: true + required_linear_history: true required_pull_request_reviews: required_approving_review_count: 1 - dismiss_stale_reviews: true + dismiss_stale_reviews: false + - name: Merge Queue + target: branch + enforcement: active + bypass_actors: + # apache/root (ASF Infra). + - actor_id: 118420 + actor_type: Team + bypass_mode: always + # apache/iceberg-public. Iceberg committers can opt into this team via + # https://gitbox.apache.org/boxer/. + - actor_id: 20107235 + actor_type: Team + bypass_mode: pull_request + conditions: + ref_name: + include: + - "~DEFAULT_BRANCH" + exclude: [] + rules: + - type: required_status_checks + parameters: + # PR branches need not be up to date with main: Merge Queue + # already tests each entry against the latest main. + strict_required_status_checks_policy: false + # Check context names (job names, not workflow files) that Merge + # Queue waits for. Every producer workflow must run on both + # `pull_request` and `merge_group`, without path filters, so each + # context always reports. + required_status_checks: + # Repository policy and security checks. + - context: Analyze Actions + - context: Run zizmor 🌈 + - context: asf-allowlist-check - required_linear_history: true - del_branch_on_merge: true + # Rust workflow gates. + - context: ci-required + - context: check-public-api + + # Python bindings workflow gates. + - context: bindings-python-ci-required + - type: merge_queue + parameters: + check_response_timeout_minutes: 360 + grouping_strategy: ALLGREEN + max_entries_to_build: 3 + max_entries_to_merge: 3 + merge_method: SQUASH + min_entries_to_merge: 1 + min_entries_to_merge_wait_minutes: 0 + pull_requests: + # allow pull requests to merge automatically once all requirements are met + allow_auto_merge: true + # auto-delete head branches after being merged + del_branch_on_merge: true + # Limit open, non-draft PRs per user without write access. See CONTRIBUTING.md. + creation_cap: + enabled: true + max_open_pull_requests: 3 + # The cap is not applied to these contributors. ASF makes GitHub's bypass list + # match this list exactly, so a login added by hand in the repository settings + # is removed on the next sync. An unknown login fails the whole sync. + bypass_users: + - JanKaul + - c-thiel + - mbutrovich + - comphead + - dannycjones + - xanderbailey features: wiki: false issues: true @@ -52,12 +127,20 @@ github: collaborators: - JanKaul - c-thiel + - mbutrovich + - CTTY + - comphead ghp_branch: gh-pages ghp_path: / + copilot_code_review: + enabled: true + review_drafts: false + review_on_push: false notifications: commits: commits@iceberg.apache.org issues: issues@iceberg.apache.org pullrequests: issues@iceberg.apache.org + jobs: ci-jobs@iceberg.apache.org jira_options: link label link label discussions: issues@iceberg.apache.org diff --git a/.cargo/audit.toml b/.cargo/audit.toml index a46052f3b5..03605e556d 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -21,8 +21,38 @@ ignore = [ # Marvin Attack: potential key recovery through timing sidechannels # Issues: https://github.com/apache/iceberg-rust/issues/221 "RUSTSEC-2023-0071", - # `derivative` is unmaintained; consider using an alternative + # `paste` is unmaintained; consider using an alternative # # Introduced by hive_metastore, tracked at https://github.com/cloudwego/pilota/issues/293 - "RUSTSEC-2024-0388", + "RUSTSEC-2024-0436", + # `webpki-roots` < 1.0.10 bundles a distrusted emSign root. + # + # sqlx-core 0.8.6 pulls in webpki-roots 0.26.11, the final 0.26.x release. + # It only re-exports webpki-roots 1.x, which the lockfile pins to the fixed + # 1.0.10, but cargo-audit flags it by version number. Remove once sqlx moves + # to webpki-roots 1.x directly. + "RUSTSEC-2026-0381", + # Bundled SQLite can expose memory when opening an attacker-controlled + # database and WAL file. SQLite is enabled only by dev-dependencies here, + # and tests create their own databases. + # sqlx 0.8.6 and 0.9.0 exclude the patched libsqlite3-sys >= 0.39.0. + # Remove once sqlx supports it and the lockfile uses a patched version. + "RUSTSEC-2026-0459", + # `zstd` < 0.14.3 / `zstd-safe` < 8.2.0 memory-safety bugs. + # + # Fixed only in zstd 0.14, but parquet 59 and apache-avro 0.22 require + # zstd 0.13. Remove once on parquet >= 60 + # (https://github.com/apache/iceberg-rust/pull/3257), an apache-avro release + # after 0.22, and workspace `zstd` >= 0.14.3. + # + # None of the affected code is reachable here. To re-check, run + # `cargo tree --workspace --all-features --target all -e features -i zstd-safe` + # and confirm only `std` is enabled. + # + # Affects only zstd's `with_context` constructors, which nothing calls. + "RUSTSEC-2026-0391", + # Requires the `seekable` feature, which is off. + "RUSTSEC-2026-0398", + # Requires the `legacy` feature, which is off. + "RUSTSEC-2026-0403", ] diff --git a/.gitattributes b/.gitattributes index 908bda4b5b..0b27994b33 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,12 @@ +# Paths left out of the source archive built by dev/release/create_rc.sh. +# Files needed to build, test, and verify from source (dev/, .licenserc.yaml, +# lint configs) must stay in. +# +# .github must stay in too: `uses: $/...` action references are resolved from +# the repository archive, which honors export-ignore. website export-ignore +.asf.yaml export-ignore +.devcontainer export-ignore +.gitattributes export-ignore +.gitignore export-ignore +.idea export-ignore diff --git a/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml b/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml index 0759a65723..9f938b11a1 100644 --- a/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml @@ -22,19 +22,15 @@ description: Problems, bugs and issues with Apache Iceberg Rust labels: bug body: - - type: dropdown + - type: input attributes: label: Apache Iceberg Rust version - description: What Apache Iceberg Rust version are you using? - multiple: false - options: - - 0.6.0 (latest version) - - 0.5.1 - - 0.4.0 - - 0.3.0 - - 0.2.0 + description: > + What Apache Iceberg Rust version are you using? + placeholder: > + For example: v0.10.0 validations: - required: false + required: true - type: textarea attributes: label: Describe the bug diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 440a098c81..543bb521a6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,4 +18,10 @@ Provide a summary of the modifications in this PR. List the main changes such as Specify what test covers (unit test, integration test, etc.). If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? ---> \ No newline at end of file +--> + +## AI Disclosure + + diff --git a/.github/actions/generate-python-license/action.yml b/.github/actions/generate-python-license/action.yml new file mode 100644 index 0000000000..ac79233ac7 --- /dev/null +++ b/.github/actions/generate-python-license/action.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Generate wheel third-party licenses +description: 'Regenerates bindings/python/LICENSE and NOTICE with the root files plus the licenses and notices of the Rust dependencies bundled in the wheels. The sdist keeps the root files.' +runs: + using: "composite" + steps: + - name: Install cargo-about + # Keep version in sync with the install-cargo-about target in bindings/python/Makefile. + uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 + with: + tool: cargo-about@0.9.2 + + - name: Generate LICENSE and NOTICE + shell: bash + working-directory: bindings/python + run: | + # --fail misses warnings and license template fallbacks. + cargo about generate -c about.toml --locked --fail --manifest-path Cargo.toml -o cargo-about-output.txt about.hbs 2>&1 | tee "$RUNNER_TEMP/cargo-about.log" + if [[ -s "$RUNNER_TEMP/cargo-about.log" ]] || grep -n '' cargo-about-output.txt; then + echo "::error::bindings/python/about.toml needs an update, see bindings/python/LICENSING.md" + exit 1 + fi + # Both are symlinks to the root files, remove them so the redirects don't write through. + rm LICENSE NOTICE + cat ../../LICENSE cargo-about-output.txt third-party-licenses.txt > LICENSE + { cat ../../NOTICE; echo; cat third-party-notices.txt; } > NOTICE diff --git a/crates/integration_tests/testdata/spark/entrypoint.sh b/.github/actions/get-msrv/action.yml old mode 100755 new mode 100644 similarity index 63% rename from crates/integration_tests/testdata/spark/entrypoint.sh rename to .github/actions/get-msrv/action.yml index 51ec329f03..f0b03e2777 --- a/crates/integration_tests/testdata/spark/entrypoint.sh +++ b/.github/actions/get-msrv/action.yml @@ -1,5 +1,3 @@ -#!/bin/bash -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,18 +14,22 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# - -set -e - -start-master.sh -p 7077 -start-worker.sh spark://spark-iceberg:7077 -echo "Starting provision" -python3 ./provision.py +name: Get MSRV +description: Get the Minimum Supported Rust Version from Cargo.toml -echo "Finished provisioning" -touch /tmp/ready +outputs: + msrv: + description: The MSRV extracted from Cargo.toml + value: ${{ steps.get-msrv.outputs.msrv }} -echo "Print logs" -tail -f $SPARK_HOME/logs/* +runs: + using: composite + steps: + - name: Get MSRV + id: get-msrv + shell: bash + run: | + msrv=$(grep '^rust-version' Cargo.toml | sed 's/.*"\(.*\)".*/\1/') + echo "msrv=$msrv" >> $GITHUB_OUTPUT + echo "MSRV is $msrv" diff --git a/.github/actions/overwrite-package-version/action.yml b/.github/actions/overwrite-package-version/action.yml index 8a2739456e..9131171d9d 100644 --- a/.github/actions/overwrite-package-version/action.yml +++ b/.github/actions/overwrite-package-version/action.yml @@ -16,46 +16,26 @@ # under the License. name: 'Update Package Version' -description: 'Updates pyproject.toml version with a provided timestamp' +description: 'Sets a PEP 440 version in pyproject.toml for Python builds' inputs: - timestamp: - description: 'Timestamp to override to the package version' + version: + description: 'PEP 440 version to set in pyproject.toml' required: true runs: using: "composite" steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install toml - run: pip install toml + - name: Install toml-cli + # Avoid `--locked` as its lock file is not recently updated, + # and its dependencies at the time rely on unstable nightly features since removed. + run: cargo install toml-cli shell: bash - - name: Get and update version + - name: Set pyproject version shell: bash env: - TIMESTAMP: ${{ inputs.timestamp }} + VERSION: ${{ inputs.version }} run: | - # Read the current version from the Rust crate's Cargo.toml - CURRENT_VERSION=$(python -c "import toml; print(toml.load('bindings/python/Cargo.toml')['package']['version'])") - export NEW_VERSION="${CURRENT_VERSION}.dev${TIMESTAMP}" - echo "Current version: ${CURRENT_VERSION}" - echo "New dev version: ${NEW_VERSION}" - - # Update bindings/python/pyproject.toml: - # - Set project.version = NEW_VERSION - # - Ensure 'version' is not listed in project.dynamic - python -c " - import toml - import os - config = toml.load('bindings/python/pyproject.toml') - new_version = os.environ['NEW_VERSION'] - config['project']['version'] = new_version - if 'dynamic' in config['project']: - config['project']['dynamic'] = [v for v in config['project']['dynamic'] if v != 'version'] - with open('bindings/python/pyproject.toml', 'w') as f: - toml.dump(config, f) - print(f'Updated version to: {new_version}') - " \ No newline at end of file + echo "Setting pyproject version to: ${VERSION}" + PYPROJECT="bindings/python/pyproject.toml" + toml set "$PYPROJECT" project.version "${VERSION}" > tmp.toml && mv tmp.toml "$PYPROJECT" + sed 's/dynamic = \["version"\]/dynamic = []/' "$PYPROJECT" > tmp.toml && mv tmp.toml "$PYPROJECT" diff --git a/.github/actions/setup-builder/action.yml b/.github/actions/setup-builder/action.yml index 532174590f..99c6d2b61c 100644 --- a/.github/actions/setup-builder/action.yml +++ b/.github/actions/setup-builder/action.yml @@ -26,8 +26,8 @@ runs: using: "composite" steps: - name: Setup specified Rust toolchain - shell: bash if: ${{ inputs.rust-version != '' }} + shell: bash env: RUST_VERSION: ${{ inputs.rust-version }} run: | @@ -42,6 +42,8 @@ runs: echo "Installing toolchain according to rust-toolchain.toml" rustup show rustup component add rustfmt clippy + # Unused, and its version varies by runner image, which splits rust-cache keys. + rustup toolchain uninstall stable - name: Fixup git permissions # https://github.com/actions/checkout/issues/766 shell: bash diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000000..c00ef406b2 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,33 @@ +# Instructions + +## Code review + +You are a pragmatic senior Rust developer. +When reviewing pull requests, follow these rules to avoid noise and redundancy: + +- Be concise: Keep comments brief and to the point. Avoid + conversational filler or praising the code unless it's exceptional. +- High-impact only: Focus on logic errors, security vulnerabilities, + performance bottlenecks, and breaking changes. +- Skip the Obvious: Do not describe what the code is doing. Assume the + reader understands the code. +- Ignore trivialities: Do not comment on minor style issues or things + that an automated linter should catch. +- Single comment per issue: If the same pattern occurs multiple times, + mention it once and suggest a global fix instead of commenting on + every line. +- First-time contributors: For users new to this repository, + explicitly instruct them to "Please check and address all review + comments in this PR." + +Also flag these, once per PR with the locations: + +- Comment churn: Code comments or docs that restate the code, narrate + history, or justify review decisions. +- Unrelated edits: Renames, reformatting, or reordering of code the + change does not otherwise need. +- Duplication: New helpers, types, or tests that duplicate existing ones. +- API growth: New public API the change does not need. Check new entries + in `public-api.txt`. +- PR description: Descriptions that restate the diff or pad the + template. Mention it in the review summary. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a94ae0151b..f4af99ef5c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,15 +21,22 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" - day: "sunday" + interval: "monthly" + cooldown: + default-days: 7 + groups: + # The codeql-action init/autobuild/analyze steps must all run the + # same version - split PRs cause a version mismatch that fails the + # Analyze jobs. Group them so a single PR bumps all of them together. + codeql-action: + patterns: + - "github/codeql-action*" # Maintain dependencies for iceberg - package-ecosystem: "cargo" directory: "/" schedule: - interval: "weekly" - day: "sunday" + interval: "monthly" open-pull-requests-limit: 50 versioning-strategy: lockfile-only ignore: @@ -42,3 +49,13 @@ updates: patterns: - "arrow*" - "parquet" + cooldown: + default-days: 7 + + # Maintain dependencies for python bindings + - package-ecosystem: "uv" + directory: "/bindings/python" + schedule: + interval: "monthly" + cooldown: + default-days: 7 diff --git a/crates/iceberg/testdata/file_io_gcs/docker-compose.yaml b/.github/labeler.yml similarity index 81% rename from crates/iceberg/testdata/file_io_gcs/docker-compose.yaml rename to .github/labeler.yml index 6935a08644..0cf48f105a 100644 --- a/crates/iceberg/testdata/file_io_gcs/docker-compose.yaml +++ b/.github/labeler.yml @@ -15,9 +15,7 @@ # specific language governing permissions and limitations # under the License. -services: - gcs-server: - image: fsouza/fake-gcs-server@sha256:36b0116fae5236e8def76ccb07761a9ca323e476f366a5f4bf449cac19deaf2d - expose: - - 4443 - command: --scheme http +# Issue and pull request labeler configuration: https://github.com/github/issue-labeler +# Labels issues and PRs whose title or body matches the regex. +datafusion: + - '/datafusion/i' diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml new file mode 100644 index 0000000000..84b490ed3c --- /dev/null +++ b/.github/workflows/asf-allowlist-check.yml @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Verifies all GitHub Actions refs are on the ASF allowlist. +# Actions not on the allowlist silently fail with "Startup failure" — no logs, +# no notifications, and PRs may appear green because no checks ran. +# See https://github.com/apache/infrastructure-actions/issues/574 +name: "ASF Allowlist Check" + +concurrency: + group: ${{github.workflow}}-${{github.ref}}-${{github.event_name}} + cancel-in-progress: true + +on: + pull_request: + merge_group: + +permissions: + contents: read + +jobs: + asf-allowlist-check: + runs-on: ubuntu-slim + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The allowlist checker recognizes ./, but not $/, as a local action prefix. + # Normalize its disposable checkout while preserving $/ in source workflows. + - name: Normalize self-repository references for the allowlist checker + run: | + find .github -type f -name '*.yml' -exec sed -i 's#uses: \$/#uses: ./#g' {} + + - uses: apache/infrastructure-actions/allowlist-check@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # allowlist-check/v1.0.0 diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 47b0c47874..66a8b6758b 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -18,34 +18,41 @@ name: Security audit concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + group: ${{github.workflow}}-${{github.ref}}-${{github.event_name}} cancel-in-progress: true on: push: + branches: + - main paths: - "**/Cargo.toml" - "**/Cargo.lock" pull_request: - paths: - - "**/Cargo.toml" - - "**/Cargo.lock" - - ".github/workflows/audit.yml" + merge_group: schedule: - cron: '0 0 * * *' +permissions: + # All other permissions are set to none + contents: read + checks: write + issues: write + jobs: security_audit: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + if: github.repository == 'apache/iceberg-rust' steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: rust-version: stable - - uses: rustsec/audit-check@v2.0.0 + - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} - ignore: RUSTSEC-2024-0436 diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 5c904b319d..7252a0704f 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -21,11 +21,10 @@ on: push: branches: - main - pull_request: paths: - '**' # Include all files and directories in the repository by default. - '!.github/ISSUE_TEMPLATE/**' # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation. - - '!scripts/**' + - '!dev/release/**' - '!website/**' - '!.asf.yml' - '!.gitattributes' @@ -35,62 +34,105 @@ on: - '!LICENSE' - '!NOTICE' - '!README.md' + pull_request: + merge_group: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true -jobs: - check-rust: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - name: Check format - working-directory: "bindings/python" - run: cargo fmt --all -- --check - - name: Check clippy - working-directory: "bindings/python" - run: cargo clippy --all-targets --all-features -- -D warnings +permissions: + contents: read +jobs: check-python: - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.9.3" + enable-cache: true - name: Install tools run: | - pip install ruff + uv tool install ruff@0.15.22 - name: Check format working-directory: "bindings/python" run: | - ruff format . --diff + uvx ruff format . --diff - name: Check style working-directory: "bindings/python" run: | - ruff check . + uvx ruff check . test: runs-on: ${{ matrix.os }} strategy: + max-parallel: 15 matrix: os: - - ubuntu-latest + - ubuntu-24.04 - macos-latest - windows-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: 3.9 - - uses: PyO3/maturin-action@v1 + python-version: 3.12 + - name: Setup Rust toolchain + uses: $/.github/actions/setup-builder + # Linux maturin builds may run inside manylinux Docker, so host Rust caches only help Windows/macOS. + - name: Cache Rust artifacts + if: runner.os != 'Linux' + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: bindings-python + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + - uses: $/.github/actions/generate-python-license + - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: working-directory: "bindings/python" command: build - args: --out dist --sdist + args: --out dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.9.3" + enable-cache: true + - name: Sync dependencies + working-directory: "bindings/python" + shell: bash + run: | + uv sync --group dev --no-install-project + - name: Install built wheel + working-directory: "bindings/python" + shell: bash + # --no-index keeps resolution inside dist/: --find-links alone only adds a + # source, so a newer PyPI release would win and be tested instead. + run: | + uv pip install --no-build --reinstall --no-index --find-links dist/ pyiceberg-core - name: Run tests working-directory: "bindings/python" shell: bash + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_DATASET: ${{ secrets.HF_DATASET }} + run: | + make test + + bindings-python-ci-required: + if: ${{ always() }} + needs: [check-python, test] + runs-on: ubuntu-slim + steps: + - name: Verify Bindings Python CI jobs succeeded + env: + RESULTS: ${{ join(needs.*.result, ' ') }} run: | - set -e - pip install hatch==1.12.0 - hatch run dev:pip install dist/pyiceberg_core-*.whl --force-reinstall - hatch run dev:test + read -ra results <<< "$RESULTS" + for result in "${results[@]}"; do + test "$result" = "success" + done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa3f54d317..5a852a070d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,52 +21,55 @@ on: push: branches: - main - pull_request: paths: - - '**' # Include all files and directories in the repository by default. - - '!.github/ISSUE_TEMPLATE/**' # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation. - - '!scripts/**' + # Artifacts that must be included in releases, such as LICENSE and NOTICE, are deliberately not excluded so we can verify they will be bundled. + # Include all files and directories, then filter out files and directories that don't impact code, tests, or release artifacts. + - '**' + - '!.github/ISSUE_TEMPLATE/**' + - '!dev/release/**' - '!website/**' - '!.asf.yml' - '!.gitattributes' - '!.gitignore' - '!CONTRIBUTING.md' - '!CHANGELOG.md' - - '!LICENSE' - - '!NOTICE' - '!README.md' + pull_request: + merge_group: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true -env: - rust_msrv: "1.87" +permissions: + contents: read jobs: - check: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: - - ubuntu-latest - - macos-latest + lint: + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder + + - name: Install lint tools + # Keep versions in sync with the local install targets in Makefile. + uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 + with: + tool: taplo@0.9.3,cargo-machete@0.7.0,cargo-deny@0.19.9 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 - name: Check License Header - uses: apache/skywalking-eyes/header@v0.7.0 + uses: apache/skywalking-eyes/header@a196742f472feaffafea537ce5a2a4c3c53a8de4 # v0.9.0 - name: Check toml format - run: make check-toml - - - name: Install protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + run: taplo fmt --check - name: Cargo format run: make check-fmt @@ -74,107 +77,231 @@ jobs: - name: Check diff run: git diff --exit-code + - name: Check Cargo.lock is up-to-date + run: cargo update --workspace --locked + + - name: Check crates package LICENSE and NOTICE + run: make check-license-notice + + - name: Check dependency licenses + run: python3 ./dev/release/licenses.py dependencies check + + # Trusted publishing cannot create a crate (https://crates.io/docs/trusted-publishing), + # so a new publishable crate must be reserved on crates.io before it is merged. + # See "Adding a new crate" in website/src/release.md. + - name: Check publishable crates exist on crates.io + run: | + pkgs=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.publish != []) | .name') + # A failed `cargo metadata` would otherwise leave the loop empty and the step green. + [ -n "$pkgs" ] || { echo "::error::No publishable crates found in the workspace, expected at least one"; exit 1; } + for pkg in $pkgs; do + # --registry is required, otherwise cargo resolves the local workspace member. + if ! cargo info --registry crates-io "$pkg" >/dev/null; then + echo "::error::$pkg is not on crates.io. Reserve it before merging; see 'Adding a new crate' in website/src/release.md" + exit 1 + fi + done + + - name: Cargo Machete + run: cargo machete + + - name: Check typos + uses: crate-ci/typos@7f9d0819ca1c008ade73f4b41fa9d258d48f605b # v1.51.1 + + clippy: + needs: lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Rust toolchain + uses: $/.github/actions/setup-builder + - name: Cargo clippy run: make check-clippy - - name: Cargo Machete - run: make cargo-machete + - name: Cargo doc + run: make check-doc build: + needs: lint runs-on: ${{ matrix.os }} + env: + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" strategy: + max-parallel: 15 matrix: os: - - ubuntu-latest + - ubuntu-24.04 - macos-latest - windows-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Cache Rust artifacts - uses: Swatinem/rust-cache@v2 - - - name: Install protoc - uses: arduino/setup-protoc@v3 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Build run: make build + # Checks each crate individually to catch missing feature declarations that + # are otherwise masked by Cargo's feature unification across the workspace. + check_standalone: + needs: lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Rust toolchain + uses: $/.github/actions/setup-builder + + - name: Cache Rust artifacts + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Check each crate standalone + run: | + for pkg in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[].name'); do + echo "Checking $pkg..." + cargo check -p "$pkg" --all-targets || exit 1 + done + + # Dry-runs the publish without compiling, which catches dependency cycles between + # workspace crates, versions that do not resolve on crates.io, and missing metadata. + # publish.yml runs the full dry-run with compilation on release candidate tags. + - name: Check each crate can be published + run: cargo publish --workspace --all-features --dry-run --no-verify + + # Verifies the core iceberg crate compiles without default features, ensuring + # optional functionality is properly gated behind feature flags. build_with_no_default_features: + needs: lint runs-on: ${{ matrix.os }} + env: + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" strategy: + max-parallel: 15 matrix: os: - - ubuntu-latest + - ubuntu-24.04 - macos-latest - windows-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Cache Rust artifacts - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Build run: cargo build -p iceberg --no-default-features - unit: - runs-on: ubuntu-latest + tests: + needs: lint + runs-on: ubuntu-24.04 + env: + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + strategy: + max-parallel: 15 + matrix: + test-suite: + - { name: "default", args: "--all-targets --all-features --workspace" } + - { name: "doc", args: "--doc --all-features --workspace" } + name: Tests (${{ matrix.test-suite.name }}) steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - remove-dotnet: "true" - remove-android: "true" - remove-haskell: "true" - remove-codeql: "true" - remove-docker-images: "true" - root-reserve-mb: 10240 - temp-reserve-mb: 10240 - - - uses: actions/checkout@v5 + persist-credentials: false - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - - name: Install protoc - uses: arduino/setup-protoc@v3 + - name: Cache Rust artifacts + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + key: ${{ matrix.test-suite.name }} + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - - name: Cache Rust artifacts - uses: Swatinem/rust-cache@v2 + - name: Install cargo-nextest + if: matrix.test-suite.name == 'default' + uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 + with: + tool: cargo-nextest - - name: Test - run: cargo test --no-fail-fast --all-targets --all-features --workspace + - name: Start Docker containers + if: matrix.test-suite.name == 'default' + run: make docker-up - - name: Smol Test - run: cargo test --no-fail-fast --all-targets --no-default-features --features "smol" --features "storage-all" --workspace + - name: Run tests + shell: bash + env: + # HuggingFace Hub integration tests self-skip when these are unset + # (e.g. PRs from forks); see crates/storage/opendal/tests/file_io_hf_test.rs + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_BUCKET: ${{ secrets.HF_BUCKET }} + HF_DATASET: ${{ secrets.HF_DATASET }} + run: | + if [ "${{ matrix.test-suite.name }}" = "default" ]; then + cargo nextest run ${{ matrix.test-suite.args }} + else + cargo test --no-fail-fast ${{ matrix.test-suite.args }} + fi - - name: Doc Test - run: cargo test --no-fail-fast --doc --all-features --workspace + - name: Stop Docker containers + if: always() && matrix.test-suite.name == 'default' + run: make docker-down msrv: name: Verify MSRV - runs-on: ubuntu-latest + needs: lint + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v5 - - name: Install protoc - uses: arduino/setup-protoc@v3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + persist-credentials: false + - name: Get MSRV + id: get-msrv + uses: $/.github/actions/get-msrv - name: Setup MSRV Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: - rust-version: ${{ env.rust_msrv }} + rust-version: ${{ steps.get-msrv.outputs.msrv }} - name: Setup Nightly Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder - name: Check MSRV run: make check-msrv + + ci-required: + if: ${{ always() }} + needs: [lint, clippy, build, check_standalone, build_with_no_default_features, tests, msrv] + runs-on: ubuntu-slim + steps: + - name: Verify CI jobs succeeded + env: + RESULTS: ${{ join(needs.*.result, ' ') }} + run: | + read -ra results <<< "$RESULTS" + for result in "${results[@]}"; do + test "$result" = "success" + done diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000000..c61a6d6a81 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "CodeQL" + +concurrency: + group: ${{github.workflow}}-${{github.ref}}-${{github.event_name}} + cancel-in-progress: true + +on: + push: + branches: [ "main" ] + pull_request: + merge_group: + schedule: + - cron: '16 4 * * 1' + +permissions: + contents: read + +jobs: + analyze: + name: Analyze Actions + runs-on: ubuntu-slim + permissions: + contents: read + security-events: write + packages: read + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + languages: actions + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + category: "/language:actions" diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml new file mode 100644 index 0000000000..016800e0dc --- /dev/null +++ b/.github/workflows/labeler.yml @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Issue and Pull Request Labeler" +on: + issues: + types: [opened, edited] + pull_request_target: # zizmor: ignore[dangerous-triggers] + types: [opened, edited] + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + triage: + if: github.repository_owner == 'apache' + runs-on: ubuntu-slim + steps: + - uses: github/issue-labeler@ccca1e16fcb66762200e616fc29b57b940c1cb67 # v3.5 + with: + configuration-path: .github/labeler.yml # label -> regex rules + enable-versioned-regex: 0 # required; 0 = no versioned configs + include-title: 1 # match title + include-body: 1 # match body (default) diff --git a/.github/workflows/ci_typos.yml b/.github/workflows/public-api.yml similarity index 60% rename from .github/workflows/ci_typos.yml rename to .github/workflows/public-api.yml index 184b16ac29..872faf80e8 100644 --- a/.github/workflows/ci_typos.yml +++ b/.github/workflows/public-api.yml @@ -15,31 +15,39 @@ # specific language governing permissions and limitations # under the License. -name: Typos Check +name: Public API on: push: branches: - main + paths: + - 'crates/**' pull_request: - branches: - - main + merge_group: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true -env: - RUST_BACKTRACE: 1 +permissions: + contents: read jobs: - typos-check: - name: typos check - runs-on: ubuntu-latest - timeout-minutes: 10 - env: - FORCE_COLOR: 1 + check-public-api: + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v5 - - name: Check typos - uses: crate-ci/typos@v1.36.3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Rust toolchain + uses: $/.github/actions/setup-builder + + - name: Cache Rust artifacts + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Check public API + run: make check-public-api diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 71d35001da..aec8c92b75 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,36 +26,44 @@ on: - "v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+" workflow_dispatch: -env: - rust_msrv: "1.87" +permissions: + contents: read jobs: publish: - runs-on: ubuntu-latest - strategy: - # Publish package one by one instead of flooding the registry - max-parallel: 1 - matrix: - # Order here is sensitive, as it will be used to determine the order of publishing - package: - - "crates/iceberg" - - "crates/catalog/glue" - - "crates/catalog/hms" - - "crates/catalog/rest" - - "crates/catalog/sql" - - "crates/integrations/datafusion" + runs-on: ubuntu-24.04 + environment: publish + permissions: + contents: read + id-token: write # Required for crates.io trusted publishing steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Get MSRV + id: get-msrv + uses: $/.github/actions/get-msrv - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: - rust-version: ${{ env.rust_msrv }} + rust-version: ${{ steps.get-msrv.outputs.msrv }} + + # Packages and builds every crate as it would be published, without uploading, + # so publish failures are caught during the release candidate process. + - name: Dry-run publish for release candidates + if: ${{ startsWith(github.ref, 'refs/tags/') && contains(github.ref, '-') }} + run: cargo publish --workspace --all-features --dry-run + shell: bash + + - uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6 + id: auth - - name: Publish ${{ matrix.package }} - working-directory: ${{ matrix.package }} + - name: Publish crates # Only publish if it's a tag and the tag is not a pre-release if: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') }} - run: cargo publish --all-features + run: cargo publish --workspace --all-features + shell: bash env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index 828e4e16b3..42c7f20974 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -18,43 +18,35 @@ name: Publish Python 🐍 distribution 📦 to PyPI on: - workflow_run: - workflows: ["Publish"] # Trigger this workflow after the "publish.yml" workflow completes - types: - - completed workflow_dispatch: - -env: - rust_msrv: "1.87" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }}-${{ github.event_name }} - cancel-in-progress: true + inputs: + release_tag: + description: 'Release tag (e.g., v0.4.0 or v0.4.0-rc.1)' + required: true + type: string permissions: contents: read jobs: - check-cargo-publish: - runs-on: ubuntu-latest - # Only run if the triggering workflow succeeded OR if manually triggered - if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} - steps: - - run: echo 'The Publish workflow passed or was manually triggered' - validate-release-tag: - runs-on: ubuntu-latest - needs: [check-cargo-publish] + runs-on: ubuntu-24.04 outputs: cargo-version: ${{ steps.validate.outputs.cargo-version }} is-rc: ${{ steps.validate.outputs.is-rc }} steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: ${{ github.event_name == 'workflow_dispatch' }} + with: + persist-credentials: false + - name: Validate release tag format id: validate - # Note, `workflow_run.head_branch` does not contain `refs/tags/` prefix, just the tag name, i.e. `v0.4.0` or `v0.4.0-rc.1` # Valid formats: v.. OR v..-rc. + shell: bash + env: + RELEASE_TAG: ${{ inputs.release_tag }} run: | - RELEASE_TAG="${{ github.event.workflow_run.head_branch }}" echo "Validating release tag: $RELEASE_TAG" if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then echo "❌ Invalid release tag format: $RELEASE_TAG" @@ -63,10 +55,39 @@ jobs: exit 1 fi echo "✅ Release tag format is valid: $RELEASE_TAG" - + + # Verify that the workflow is being run on the same version as the release tag + DESIRED_REF="refs/tags/${RELEASE_TAG}" + if [[ "${GITHUB_REF}" != "${DESIRED_REF}" ]]; then + echo "❌ Error: expected workflow execution on ref ${DESIRED_REF}, got ref: ${GITHUB_REF}" + echo "Workflow must be dispatched against the tag ref for the desired release." + echo "Dispatching workflow from main or another branch is not supported." + exit 1 + fi + # Strip 'v' prefix for cargo version CARGO_VERSION="${RELEASE_TAG#v}" echo "Cargo version (without v prefix): $CARGO_VERSION" + + # For manual triggers, validate that the tag matches the version in Cargo.toml + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + # Extract base version (without -rc.X suffix) for comparison with Cargo.toml + BASE_VERSION="${CARGO_VERSION%-rc.*}" + echo "Base version (for Cargo.toml comparison): $BASE_VERSION" + + # Read workspace version via cargo metadata + CARGO_TOML_VERSION=$(cargo metadata --format-version 1 --no-deps -q | jq -r '.packages[0].version') + echo "Workspace version: $CARGO_TOML_VERSION" + + if [ "$BASE_VERSION" != "$CARGO_TOML_VERSION" ]; then + echo "❌ Version mismatch!" + echo " Release tag base version: $BASE_VERSION" + echo " Cargo.toml (workspace) version: $CARGO_TOML_VERSION" + echo "Please ensure the release tag matches the version in Cargo.toml" + exit 1 + fi + echo "✅ Version matches workspace Cargo.toml" + fi # Check if this is a release candidate if [[ "$RELEASE_TAG" =~ -rc\.[0-9]+$ ]]; then @@ -82,32 +103,35 @@ jobs: echo "is-rc=$IS_RC" >> $GITHUB_OUTPUT sdist: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [validate-release-tag] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - name: Install toml-cli - if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} - run: cargo install toml-cli - - - name: Set cargo version for RC + - name: Compute RC version if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} - working-directory: "bindings/python" + id: version + shell: bash + env: + CARGO_VERSION: ${{ needs.validate-release-tag.outputs.cargo-version }} run: | - echo "Setting cargo version to: ${{ needs.validate-release-tag.outputs.cargo-version }}" - toml set Cargo.toml package.version ${{ needs.validate-release-tag.outputs.cargo-version }} > Cargo.toml.tmp - # doing this explicitly to avoid issue in Windows where `mv` does not overwrite existing file - rm Cargo.toml - mv Cargo.toml.tmp Cargo.toml + # Convert semver RC (0.9.0-rc.1) to PEP 440 (0.9.0rc1) + echo "pep440=$(echo "$CARGO_VERSION" | sed 's/-rc\.\([0-9]*\)/rc\1/')" >> $GITHUB_OUTPUT + + - uses: $/.github/actions/overwrite-package-version + if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} + with: + version: ${{ steps.version.outputs.pep440 }} - - uses: PyO3/maturin-action@v1 + - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: working-directory: "bindings/python" command: sdist args: -o dist - name: Upload sdist - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wheels-sdist path: bindings/python/dist @@ -116,50 +140,74 @@ jobs: runs-on: "${{ matrix.os }}" needs: [validate-release-tag] strategy: + max-parallel: 15 matrix: + # Keep `targets` in bindings/python/about.toml in sync with these. include: - { os: windows-latest } + - { os: windows-11-arm } - { os: macos-latest, target: "universal2-apple-darwin" } - - { os: ubuntu-latest, target: "x86_64" } + - { os: ubuntu-24.04, target: "x86_64" } - { - os: ubuntu-latest, + os: ubuntu-24.04, target: "aarch64", manylinux: "manylinux_2_28", } - - { os: ubuntu-latest, target: "armv7l" } + - { os: ubuntu-24.04, target: "armv7l" } steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - name: Install toml-cli - if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} - run: cargo install toml-cli - - - name: Set cargo version for RC + - name: Compute RC version if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} - working-directory: "bindings/python" + id: version + shell: bash + env: + CARGO_VERSION: ${{ needs.validate-release-tag.outputs.cargo-version }} run: | - echo "Setting cargo version to: ${{ needs.validate-release-tag.outputs.cargo-version }}" - toml set Cargo.toml package.version ${{ needs.validate-release-tag.outputs.cargo-version }} > Cargo.toml.tmp - # doing this explicitly to avoid issue in Windows where `mv` does not overwrite existing file - rm Cargo.toml - mv Cargo.toml.tmp Cargo.toml + # Convert semver RC (0.9.0-rc.1) to PEP 440 (0.9.0rc1) + echo "pep440=$(echo "$CARGO_VERSION" | sed 's/-rc\.\([0-9]*\)/rc\1/')" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v6 + - uses: $/.github/actions/overwrite-package-version + if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} with: - python-version: 3.9 + version: ${{ steps.version.outputs.pep440 }} + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.12 + - name: Get MSRV + id: get-msrv + uses: $/.github/actions/get-msrv + - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: - rust-version: ${{ env.rust_msrv }} - - uses: PyO3/maturin-action@v1 + rust-version: ${{ steps.get-msrv.outputs.msrv }} + - uses: $/.github/actions/generate-python-license + - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: target: ${{ matrix.target }} manylinux: ${{ matrix.manylinux || 'auto' }} working-directory: "bindings/python" command: build - args: --release -o dist + args: --release -o dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.9.3" + enable-cache: true + # Verify the wheel is abi3-compatible and installable without building from source. + # Skipped for cross-compiled targets since they share the same maturin config; + # if abi3 is broken, the native target build will catch it. + - name: Verify wheel installs with no build fallback + if: matrix.target != 'aarch64' && matrix.target != 'armv7l' + working-directory: "bindings/python" + shell: bash + run: | + uv pip install --no-build --reinstall --system --no-index --find-links dist/ pyiceberg-core - name: Upload wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wheels-${{ matrix.os }}-${{ matrix.target }} path: bindings/python/dist @@ -167,7 +215,7 @@ jobs: pypi-publish: name: Publish Python 🐍 distribution 📦 to Pypi needs: [sdist, wheels] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 environment: name: pypi @@ -178,14 +226,14 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v5 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: wheels-* merge-multiple: true path: bindings/python/dist - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 with: skip-existing: true packages-dir: bindings/python/dist + verbose: true diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml index 5e6fc3ce38..e6d34aa60d 100644 --- a/.github/workflows/release_python_nightly.yml +++ b/.github/workflows/release_python_nightly.yml @@ -17,20 +17,22 @@ name: "Nightly PyPI Build" +concurrency: + group: ${{github.workflow}}-${{github.ref}}-${{github.event_name}} + cancel-in-progress: true + on: schedule: - cron: "0 0 * * *" # Runs at midnight UTC every day workflow_dispatch: # Allows manual triggering -env: - rust_msrv: "1.87" - permissions: contents: read jobs: set-version: - runs-on: ubuntu-latest + if: github.repository == 'apache/iceberg-rust' || github.event_name == 'workflow_dispatch' # Run on schedule for apache repo, or on manual dispatch from any repo + runs-on: ubuntu-slim outputs: timestamp: ${{ steps.set-ts.outputs.TIMESTAMP }} steps: @@ -40,76 +42,120 @@ jobs: sdist: needs: set-version - if: github.repository == 'apache/iceberg-rust' # Only run for apache repo - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v5 - - - uses: ./.github/actions/overwrite-package-version # Overwrite package version with timestamp + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - timestamp: ${{ needs.set-version.outputs.TIMESTAMP }} + persist-credentials: false + + - name: Compute dev version + id: version + shell: bash + env: + TIMESTAMP: ${{ needs.set-version.outputs.timestamp }} + run: | + CURRENT=$(cargo metadata --format-version 1 --no-deps -q | jq -r '.packages[0].version') + echo "pep440=${CURRENT}.dev${TIMESTAMP}" >> $GITHUB_OUTPUT + + - uses: $/.github/actions/overwrite-package-version + with: + version: ${{ steps.version.outputs.pep440 }} - - uses: PyO3/maturin-action@v1 + - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: working-directory: "bindings/python" command: sdist args: -o dist - name: Upload sdist - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wheels-sdist path: bindings/python/dist wheels: needs: set-version - if: github.repository == 'apache/iceberg-rust' # Only run for apache repo runs-on: "${{ matrix.os }}" strategy: + max-parallel: 15 matrix: + # Keep `targets` in bindings/python/about.toml in sync with these. include: - { os: windows-latest } + - { os: windows-11-arm } - { os: macos-latest, target: "universal2-apple-darwin" } - - { os: ubuntu-latest, target: "x86_64" } + - { os: ubuntu-24.04, target: "x86_64" } - { - os: ubuntu-latest, + os: ubuntu-24.04, target: "aarch64", manylinux: "manylinux_2_28", } - - { os: ubuntu-latest, target: "armv7l" } + - { os: ubuntu-24.04, target: "armv7l" } steps: - - uses: actions/checkout@v5 - - - uses: ./.github/actions/overwrite-package-version # Overwrite package version with timestamp + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - timestamp: ${{ needs.set-version.outputs.TIMESTAMP }} + persist-credentials: false + + - name: Compute dev version + id: version + shell: bash + env: + TIMESTAMP: ${{ needs.set-version.outputs.timestamp }} + run: | + CURRENT=$(cargo metadata --format-version 1 --no-deps -q | jq -r '.packages[0].version') + echo "pep440=${CURRENT}.dev${TIMESTAMP}" >> $GITHUB_OUTPUT + + - uses: $/.github/actions/overwrite-package-version + with: + version: ${{ steps.version.outputs.pep440 }} - - uses: actions/setup-python@v6 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: 3.9 + python-version: 3.12 + + - name: Get MSRV + id: get-msrv + uses: $/.github/actions/get-msrv - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder + uses: $/.github/actions/setup-builder with: - rust-version: ${{ env.rust_msrv }} + rust-version: ${{ steps.get-msrv.outputs.msrv }} + + - uses: $/.github/actions/generate-python-license - - uses: PyO3/maturin-action@v1 + - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0 with: target: ${{ matrix.target }} manylinux: ${{ matrix.manylinux || 'auto' }} working-directory: "bindings/python" command: build - args: --release -o dist + args: --release -o dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one + + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.9.3" + enable-cache: true + # Verify the wheel is abi3-compatible and installable without building from source. + # Skipped for cross-compiled targets since they share the same maturin config; + # if abi3 is broken, the native target build will catch it. + - name: Verify wheel installs with no build fallback + if: matrix.target != 'aarch64' && matrix.target != 'armv7l' + working-directory: "bindings/python" + shell: bash + run: | + uv pip install --no-build --reinstall --system --no-index --find-links dist/ pyiceberg-core - name: Upload wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wheels-${{ matrix.os }}-${{ matrix.target }} path: bindings/python/dist testpypi-publish: + if: github.repository == 'apache/iceberg-rust' # Only run for apache repo needs: [sdist, wheels] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 environment: name: testpypi @@ -120,7 +166,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v5 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: wheels-* merge-multiple: true @@ -128,8 +174,36 @@ jobs: - name: List downloaded artifacts run: ls -R bindings/python/dist - name: Publish to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 + id: publish-testpypi + continue-on-error: true + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 with: repository-url: https://test.pypi.org/legacy/ skip-existing: true packages-dir: bindings/python/dist + verbose: true + - name: Display error message on publish failure + if: steps.publish-testpypi.outcome == 'failure' + run: | + echo "::error::Failed to publish to TestPyPI" + echo "" + echo "⚠️ TestPyPI Publish Failed" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "" + echo "This may be due to TestPyPI storage limits." + echo "See: https://docs.pypi.org/project-management/storage-limits" + echo "" + echo "To resolve this issue, use the pypi-cleanup utility to clean up old TestPyPI artifacts:" + echo "https://pypi.org/project/pypi-cleanup/" + echo "" + echo " uvx pypi-cleanup --package pyiceberg-core --host https://test.pypi.org/ \\" + echo " --verbose -d 10 --do-it --username " + echo "" + echo "Requirements:" + echo " • Must be a maintainer for pyiceberg-core on TestPyPI" + echo " (https://test.pypi.org/project/pyiceberg-core)" + echo " • Requires TestPyPI password and 2FA" + echo " • ⚠️ ONLY do this for TestPyPI, NOT for production PyPI!" + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 1 diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 4b79c33274..baa082c8df 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -17,7 +17,7 @@ # under the License. # -name: "Close Stale Issues" +name: "Close Stale Issues and PRs" on: schedule: - cron: '0 0 * * *' @@ -25,20 +25,20 @@ on: permissions: # All other permissions are set to none issues: write + pull-requests: write jobs: stale: if: github.repository_owner == 'apache' - runs-on: ubuntu-22.04 + runs-on: ubuntu-slim steps: - - uses: actions/stale@v10.0.0 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: + # stale issues stale-issue-label: 'stale' - exempt-issue-labels: 'not-stale' + exempt-issue-labels: 'not-stale,security' days-before-issue-stale: 180 days-before-issue-close: 14 - # Only close stale issues, leave PRs alone - days-before-pr-stale: -1 stale-issue-message: > This issue has been automatically marked as stale because it has been open for 180 days with no activity. It will be closed in next 14 days if no further activity occurs. To @@ -47,3 +47,12 @@ jobs: close-issue-message: > This issue has been closed because it has not received any activity in the last 14 days since being marked as 'stale' + # stale PRs + stale-pr-label: 'stale' + exempt-pr-labels: 'not-stale,security' + stale-pr-message: 'This pull request has been marked as stale due to 30 days of inactivity. It will be closed in 1 week if no further activity occurs. If you think that’s incorrect or this pull request requires a review, please simply write any comment. If closed, you can revive the PR at any time and @mention a reviewer or discuss it on the dev@iceberg.apache.org list. Thank you for your contributions.' + close-pr-message: 'This pull request has been closed due to lack of activity. This is not a judgement on the merit of the PR in any way. It is just a way of keeping the PR queue manageable. If you think that is incorrect, or the pull request requires review, you can revive the PR at any time.' + days-before-pr-stale: 30 + days-before-pr-close: 7 + ascending: true + operations-per-run: 200 diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index c5925da6ae..bdd9e953ec 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -30,24 +30,24 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true +permissions: + contents: read + jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup mdBook - uses: peaceiris/actions-mdbook@v2 + uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0 with: mdbook-version: "0.4.36" - - name: Install protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Build working-directory: website run: mdbook build @@ -61,7 +61,7 @@ jobs: cp -r target/doc ./website/book/api - name: Deploy to gh-pages - uses: peaceiris/actions-gh-pages@v4.0.0 + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 if: github.event_name == 'push' && github.ref_name == 'main' with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 0000000000..71a8cee3a1 --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: GitHub Actions Security Analysis with zizmor 🌈 + +concurrency: + group: ${{github.workflow}}-${{github.ref}}-${{github.event_name}} + cancel-in-progress: true + +on: + pull_request: + merge_group: + +permissions: {} + +jobs: + zizmor: + name: Run zizmor 🌈 + runs-on: ubuntu-24.04 + permissions: {} + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Run zizmor 🌈 + uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 + with: + advanced-security: false diff --git a/.licenserc.yaml b/.licenserc.yaml index 01625e32cb..a3b586dff3 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -21,16 +21,27 @@ header: copyright-owner: Apache Software Foundation paths-ignore: - - 'LICENSE' - - 'NOTICE' - - '.gitattributes' - - '**/*.json' + - "LICENSE" + - "NOTICE" + - "bindings/python/third-party-licenses.txt" + - "bindings/python/third-party-notices.txt" + - ".gitattributes" + - "**/*.json" # Generated content by mdbook - - 'website/book' - # Generated content by scripts - - '**/DEPENDENCIES.*.tsv' + - "website/book" + # Generated content by cargo-public-api + - "**/public-api.txt" + # Generated content by trybuild + - "**/*.stderr" # Release distributions - - 'dist/*' - - 'Cargo.lock' - - '.github/PULL_REQUEST_TEMPLATE.md' + - "dist/*" + - "target" + - "Cargo.lock" + - "bindings/python/uv.lock" + # GitHub configuration + - ".github/PULL_REQUEST_TEMPLATE.md" + - ".github/copilot-instructions.md" + # Python environment + - "bindings/python/.venv/" + - "bindings/python/.pytest_cache/" comment: on-failure diff --git a/.typos.toml b/.typos.toml index 9363f17c9a..cde21f7091 100644 --- a/.typos.toml +++ b/.typos.toml @@ -15,5 +15,13 @@ # specific language governing permissions and limitations # under the License. +[type.rust] +extend-ignore-identifiers-re = ["^bimap$"] + +[default.extend-words] +AGS = "AGS" +ags = "ags" +ser = "ser" + [files] -extend-exclude = ["**/testdata", "CHANGELOG.md"] +extend-exclude = ["**/testdata", "CHANGELOG.md", "**/public-api.txt"] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..f33279679c --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,85 @@ + + +# Apache Iceberg Rust — Agent Instructions + +This file provides repository-specific guidance for automated agents working +in this repository. + +## Comments, docs, and diff scope + +Default to no comment. Leave existing comments as they are unless you change +the code they describe or they are wrong. + +- Comment only what the code cannot say: invariants, non-obvious + constraints, links to external behavior. Do not restate the code. +- Do not describe how the code changed or explain review history + ("previously", "instead of", "unlike X"). Put that in the commit message + or PR description. +- Public items get a one-line doc summary of what they do for the caller, + not how. Expand only for contract the caller must know, such as the + errors it returns (`# Errors`) or when it panics (`# Panics`). +- Test names say what they check. Do not narrate asserts. +- Cover the edge cases and error paths the change adds, not only the + happy path. +- Build the expected value and assert equality with it, rather than + checking fields one at a time. +- Do not add tests that duplicate existing coverage. +- Do not rename, reformat, refactor, or reorder code unrelated to the + change. A bug fix does not carry an API redesign. +- Reuse existing helpers, types, and test fixtures (such as + `TableTestFixture`) before adding new ones. Use arrow-rs kernels + instead of reimplementing them. +- Use the narrowest visibility that works. Make items `pub` only when they + are part of the API the change intends to expose. Run + `make generate-public-api` to update each crate's `public-api.txt`, and + check its diff for API growth. + +## Public API + +Justify each new public item: check that it needs to be public and that it +fits the existing abstractions. Consider how it would be changed or removed +later, and prefer additive changes to breaking ones. Derives such as +`#[derive(Clone)]` count as public API. A type that needs table-scoped context +to construct gets an accessor on `Table` instead of a public `new()`, so new +parameters do not break callers. Prefer expandable types, such as structs and enums, for configuration +over adding parameters to a function, so options can grow without breaking +callers. + +Iceberg Java has a +[deprecation policy](https://github.com/apache/iceberg/blob/9203eaff9eb97fef9d1f5a3ef4460e8453adaed9/site/docs/contribute.md?plain=1#L142-L170) +that iceberg-rust has not adopted. Treat it as a reference for how long a +deprecation lasts, not as a rule. In a review, name every new or changed public +item and consider whether the change needs it. + +## Pull requests + +- Follow the PR template. Keep each section short: the problem, the + approach, and anything a reviewer would not guess from the diff. In + AI Disclosure, name the areas of uncertainty or assumptions a reviewer + should focus on. +- Do not restate the diff: no file-by-file walkthroughs, per-function + summaries, or "changes made" checklists. + +## Security Model + +When assessing potential vulnerabilities or calibrating automated security +findings, use [`SECURITY-THREAT-MODEL.md`](SECURITY-THREAT-MODEL.md) as the +authoritative detailed description of this repository's security boundaries, +trust assumptions, and non-boundaries. diff --git a/CHANGELOG.md b/CHANGELOG.md index f66f64f478..7ab34c96d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,583 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Added + +* Support timestamp-based time travel through `TableScanBuilder::as_of_time` and `snapshot_id_as_of_time`. + +### Breaking Changes + +* `TableScan::to_arrow` now requires `ArrowReaderOptions`; pass `Default::default()` for default reader settings. Move reader-setting calls from `TableScanBuilder` and `ArrowReaderBuilder` to `ArrowReaderOptions::builder()`, then pass the options to `to_arrow(options)` or `ArrowReaderBuilder::with_options(options)`. Configure Parquet I/O through the nested `ParquetReadOptions`. Scan `with_concurrency_limit()` now bounds manifest planning only; set `with_data_file_concurrency_limit` on the reader-options builder to bound data-file and delete-file reads. Unset reader concurrency uses execution-host parallelism. See [#3514](https://github.com/apache/iceberg-rust/pull/3514). + +* `FileWrite::close`, `EncryptedOutputFile::write`, `ManifestListWriter::close`, and `PuffinWriter::close` now return `Result` instead of `Result<()>`. The returned size is the on-disk size, including encryption overhead. Custom `FileWrite` implementations must return the number of bytes stored. +* `EncryptedOutputFile::key_metadata()` is replaced by `key_metadata_with_saved_file_metadata(&FileMetadata)`. Pass the metadata returned by `write()` or the writer's `close()` to include the stored length before encoding key metadata. +* `EncryptedInputFile::metadata()` is now synchronous and derives the plaintext size from key metadata without a storage stat. Remove `.await` from calls to this method. +* AGS1 readers now require `StandardKeyMetadata::file_length` and reject missing or invalid lengths without falling back to a storage stat. AGS1-encrypted manifests, manifest lists, and Puffin files written by earlier development builds without this field must be rewritten using a build that can still read them before upgrading. This matches the Java client's read contract. +* `iceberg_catalog_loader::BoxedCatalogBuilder` has a new required method, `with_runtime`. Types implementing `CatalogBuilder` get it through the blanket impl; types implementing `BoxedCatalogBuilder` directly (for example, a wrapper around another `Box`) must implement it, typically by forwarding the runtime to the builder they wrap. +* `Transaction::commit` now returns `ErrorKind::PreconditionFailed` for read-only tables (`Table::readonly()`, e.g. from `StaticTable::into_table` or `TableBuilder::readonly(true)`), including empty transactions, which previously returned the table unchanged. Java treats an empty transaction on a static table as a no-op. +* `ArrowReader` reads nested struct fields as null for Parquet files that have neither field ids nor a name mapping, as iceberg-java does. Earlier versions returned their stored values by position. Tables migrated without field ids need a name mapping (`schema.name-mapping.default`) that covers nested fields to read them. + +## [v0.10.1] - 2026-07-28 + +* chore: Update crossbeam-epoch to 0.9.20, pin CI ruff version to 0.15.22 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2911 + +## [v0.10.0] - 2026-07-07 + +### Breaking Changes + +* **API Changes:** + * feat!(catalog): adding support for purge_table by @CTTY in https://github.com/apache/iceberg-rust/pull/2232 + * feat!: Enhance compression codec enum. by @emkornfield in https://github.com/apache/iceberg-rust/pull/2288 + * feat!(runtime): Support custom Runtime in Catalog by @CTTY in https://github.com/apache/iceberg-rust/pull/2308 + * refactor!(writer): borrow TableMetadata in DefaultLocationGenerator::new by @smaheshwar-pltr in https://github.com/apache/iceberg-rust/pull/2604 + +* **Compatibility:** + * chore: Bump MSRV to 1.94 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2652 + +## All Changes +* feat: Honor compression settings for metadata.json on write by @emkornfield in https://github.com/apache/iceberg-rust/pull/1876 +* chore(deps): Bump quinn-proto from 0.11.13 to 0.11.14 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2228 +* Add catalog test suite to unify catalog's behavior. by @blackmwk in https://github.com/apache/iceberg-rust/pull/2131 +* chore(deps): Bump datafusion from 52.2.0 to 52.3.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2235 +* chore(deps): Bump serde_with from 3.17.0 to 3.18.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2233 +* chore(deps): Bump tempfile from 3.26.0 to 3.27.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2234 +* chore(deps): Bump lz4_flex from 0.12.0 to 0.12.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2239 +* chore(deps): Bump lz4_flex from 0.12.0 to 0.12.1 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2238 +* feat(io): Add delete_stream to Storage trait by @CTTY in https://github.com/apache/iceberg-rust/pull/2216 +* feat(storage): implement opendal resolving storage by @CTTY in https://github.com/apache/iceberg-rust/pull/2231 +* doc: Update README.md to include more components by @CTTY in https://github.com/apache/iceberg-rust/pull/2248 +* fix(python): use resolving storage for python binding by @CTTY in https://github.com/apache/iceberg-rust/pull/2246 +* chore: Change publish parallism back to 1 by @CTTY in https://github.com/apache/iceberg-rust/pull/2254 +* Infra: Remove GitHub Actions from Dependabot configuration by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2267 +* ci: pin third-party actions to Apache-approved SHAs by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2266 +* chore(deps): fix audit check and disable inherited aws sdk default features by @maybe-vibe in https://github.com/apache/iceberg-rust/pull/2274 +* chore(deps): Bump minijinja from 2.17.1 to 2.18.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2273 +* chore(deps): Bump datafusion from 52.3.0 to 52.4.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2271 +* chore(deps): Bump datafusion-cli from 52.2.0 to 52.3.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2272 +* chore(deps): Bump datafusion-sqllogictest from 52.2.0 to 52.3.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2270 +* fix: explicitly set Python interpreter in maturin build to prevent using wrong version by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2277 +* chore(deps): Bump rustls-webpki from 0.103.7 to 0.103.10 in bindings/python by @blackmwk in https://github.com/apache/iceberg-rust/pull/2278 +* chore(deps): Bump bytes from 1.11.0 to 1.11.1 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2281 +* chore(deps): Bump time from 0.3.44 to 0.3.47 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2282 +* docs: Add DataFusion Comet to Users section of README by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2283 +* feat(encryption) [1/N] Support encryption: Add crypto for AES-GCM by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2026 +* feat!(catalog): adding support for purge_table by @CTTY in https://github.com/apache/iceberg-rust/pull/2232 +* deps: upgrade DataFusion to 53.0, Arrow to 58 by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2206 +* chore(deps): Bump uuid from 1.22.0 to 1.23.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2291 +* ci: improve github workflows by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2289 +* Make `convert_filters_to_predicate` public by @LLDay in https://github.com/apache/iceberg-rust/pull/2118 +* ci: fix zizmor security findings by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2290 +* fix(spec): clean up -1 snapshot ID sentinel usage and add deserialization test by @geserdugarov in https://github.com/apache/iceberg-rust/pull/2294 +* feat!: Enhance compression codec enum. by @emkornfield in https://github.com/apache/iceberg-rust/pull/2288 +* ci: allows nightly to run on workflow dispatch by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2304 +* ci: ensure use bash shell for env by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2305 +* feat(iceberg): Add snapshot utils to scan ancestors by @CTTY in https://github.com/apache/iceberg-rust/pull/2285 +* Fix zizmor workflow by @blackmwk in https://github.com/apache/iceberg-rust/pull/2324 +* chore(deps): Bump crate-ci/typos from 1.44.0 to 1.45.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2318 +* chore(deps): Bump taiki-e/install-action from 2.70.0 to 2.73.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2319 +* chore(deps): Bump aws-sdk-s3tables from 1.53.0 to 1.54.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2320 +* Fix ci workflow failure by @blackmwk in https://github.com/apache/iceberg-rust/pull/2325 +* chore(deps): Bump tokio from 1.50.0 to 1.51.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2321 +* chore(deps): Bump minijinja from 2.18.0 to 2.19.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2322 +* fix(s3tables): use 's3' as the default scheme by @rchowell in https://github.com/apache/iceberg-rust/pull/2313 +* fix(storage/s3): default to virtual-host-style addressing by @plusplusjiajia in https://github.com/apache/iceberg-rust/pull/2330 +* Fix RUSTSEC-2026-0097 by @blackmwk in https://github.com/apache/iceberg-rust/pull/2331 +* chore: update comment tag to match the exact version to fix zizmor by @CTTY in https://github.com/apache/iceberg-rust/pull/2333 +* ci: fix zizmor workflow by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2334 +* chore(deps): Bump astral-sh/setup-uv from 7.3.1 to 8.0.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2314 +* tests(s3tables): add s3tables end-to-end test to verify table creation/loading plus reading/writing by @dannycjones in https://github.com/apache/iceberg-rust/pull/2315 +* fix: build_fallback_field_id_map produces incorrect column indices for schemas with nested types by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2307 +* fix: incorrect Parquet INT96 timestamp values from ArrowReader by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2301 +* feat(encryption) [2/N] Support encryption: Add streaming encryption/decryption by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2286 +* refactor(storage): remove the configured_scheme parameter from storage impls by @rchowell in https://github.com/apache/iceberg-rust/pull/2338 +* chore(deps): Bump PyO3/maturin-action from 1.50.1 to 1.51.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2346 +* chore(deps): Bump actions/upload-artifact from 7.0.0 to 7.0.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2345 +* support fixedbinary(n) by @jeff-sqds in https://github.com/apache/iceberg-rust/pull/2348 +* chore: bump datafusion to 53.1.0 by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2350 +* chore(deps): Bump rustls-webpki from 0.103.10 to 0.103.12 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2352 +* fix: NaN pushdown correctly pushes down NaNs correctness issue by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2351 +* fix: Bump rustls-webpki from 0.103.12 to 0.103.13 to fix RUSTSEC-2026-0104 by @CTTY in https://github.com/apache/iceberg-rust/pull/2356 +* feat(datafusion): show pushed-down limit in IcebergTableScan EXPLAIN output by @toutane in https://github.com/apache/iceberg-rust/pull/2360 +* Split arrow reader into smaller modules by @blackmwk in https://github.com/apache/iceberg-rust/pull/2358 +* fix: allow missing sequence-number in v2 snapshots for v1-upgraded tables by @jembishop in https://github.com/apache/iceberg-rust/pull/2127 +* chore(deps): Bump tokio from 1.51.1 to 1.52.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2374 +* chore(deps): Bump datafusion-cli from 53.0.0 to 53.1.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2373 +* chore(deps): Bump crate-ci/typos from 1.45.0 to 1.45.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2372 +* chore(deps): Bump astral-sh/setup-uv from 8.0.0 to 8.1.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2371 +* chore(deps): Bump taiki-e/install-action from 2.75.7 to 2.75.18 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2370 +* chore(deps): Bump datafusion-sqllogictest from 53.0.0 to 53.1.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2369 +* chore(deps): Bump zizmorcore/zizmor-action from 0.5.2 to 0.5.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2368 +* docs: bump version for python binding by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2377 +* feat(reader): Add read_with_metrics() for scan I/O metrics by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2349 +* feat(encryption) [3/N] Support encryption: KMS by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2339 +* chore: Improve release process by @CTTY in https://github.com/apache/iceberg-rust/pull/2381 +* docs(arrow): clarify ScanMetrics::bytes_read includes delete files by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2379 +* feat(encryption) [4/N] Support encryption: StandardKeyMetadata by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2340 +* ci(python): verify wheel abi3 compatibility with --no-build by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2388 +* chore(deps): Bump aws-sdk-glue from 1.142.0 to 1.142.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2399 +* chore(deps): Bump taiki-e/install-action from 2.75.18 to 2.75.22 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2400 +* fix(sqllogictest): add missing tokio `rt-multi-thread` feature by @toutane in https://github.com/apache/iceberg-rust/pull/2386 +* feat(datatypes): add optional and required constructors for MapType by @TwinklerG in https://github.com/apache/iceberg-rust/pull/2405 +* CI: Use specific patch versions in workflow action comments by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2412 +* docs: Correct docs for Int128 and UInt128 being big-endian not little-endian by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2397 +* fix(tests): remove unnecessary `.to_string()` calls for metadata_location by @TwinklerG in https://github.com/apache/iceberg-rust/pull/2407 +* fix(catalog/memory): relax strict ms inequality in test_update_table by @Kurtiscwright in https://github.com/apache/iceberg-rust/pull/2395 +* chore(deps): bump opendal to 0.56, migrate to reqsign-core 3.0 by @kszucs in https://github.com/apache/iceberg-rust/pull/2401 +* fix(transaction): fix previous snapshot summary fetching by @dentiny in https://github.com/apache/iceberg-rust/pull/2391 +* chore(deps): Bump reqwest from 0.12.28 to 0.13.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2429 +* chore(deps): Bump serde_with from 3.18.0 to 3.19.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2428 +* chore(deps): Bump github/codeql-action from 4.35.2 to 4.35.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2426 +* chore(deps): Bump taiki-e/install-action from 2.75.22 to 2.75.30 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2425 +* chore(deps): Bump crate-ci/typos from 1.45.1 to 1.46.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2424 +* chore(deps): Bump the arrow-parquet group with 6 updates by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2427 +* feat(schema): Add `update_schema` action to enable table schema updates by @tomighita in https://github.com/apache/iceberg-rust/pull/2120 +* Fix RUSTSEC-2026-0122 by @blackmwk in https://github.com/apache/iceberg-rust/pull/2437 +* Fix pyiceberg-core dev dependency resolution by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2442 +* chore(deps): Bump urllib3 from 2.6.3 to 2.7.0 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2434 +* feat(encryption) [5/N] Support encryption: Encryption Manager by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2383 +* chore(deps-dev): Bump pytest from 8.4.2 to 9.0.3 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2326 +* chore: add `uv` to dependabot for /bindings/python by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2444 +* chore: update deps for /bindings/python by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2445 +* chore(deps): Bump reqwest from 0.12.28 to 0.13.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2450 +* chore(deps): Bump rand from 0.9.4 to 0.10.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2449 +* chore(deps): Bump taiki-e/install-action from 2.75.30 to 2.77.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2446 +* chore(deps): Bump the arrow-parquet group with 2 updates by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2448 +* ci: add check_standalone job to verify each crate compiles independently by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2389 +* fix(puffin): add missing magic number validation for puffins by @dentiny in https://github.com/apache/iceberg-rust/pull/2416 +* feat!(runtime): Support custom Runtime in Catalog by @CTTY in https://github.com/apache/iceberg-rust/pull/2308 +* chore(deps): Bump taiki-e/install-action from 2.77.1 to 2.77.4 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2461 +* chore(deps): Bump reqwest from 0.12.28 to 0.13.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2460 +* chore(deps): Bump crate-ci/typos from 1.46.0 to 1.46.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2459 +* chore(deps): Bump github/codeql-action from 4.35.3 to 4.35.4 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2457 +* chore(deps-dev): Bump maturin from 1.13.1 to 1.13.2 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2458 +* feat(encryption) [6/N] support `ArrowReaderOptions` in `get_metadata` by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2402 +* feat(partition): adjust transform boundary for timestamp types by @dentiny in https://github.com/apache/iceberg-rust/pull/2422 +* chore(deps): Bump idna from 3.13 to 3.15 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2470 +* feat: HuggingFace Hub storage backend and CDC table properties by @kszucs in https://github.com/apache/iceberg-rust/pull/2375 +* fix(spec): support JSON binary literals by @fallintoplace in https://github.com/apache/iceberg-rust/pull/2476 +* chore: Add a lock check to CI by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2484 +* chore(deps): Bump serde_with from 3.19.0 to 3.20.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2499 +* chore(deps): Bump crate-ci/typos from 1.46.1 to 1.46.2 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2491 +* chore(deps): Bump taiki-e/install-action from 2.77.4 to 2.79.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2492 +* chore(deps): Bump peaceiris/actions-gh-pages from 4.0.0 to 4.1.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2493 +* chore(deps): Bump zizmorcore/zizmor-action from 0.5.3 to 0.5.6 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2494 +* chore(deps-dev): Bump maturin from 1.13.2 to 1.13.3 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2495 +* chore(deps): Bump github/codeql-action from 4.35.4 to 4.35.5 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2497 +* docs: add pyiceberg-core release validation by @officialasishkumar in https://github.com/apache/iceberg-rust/pull/2502 +* chore(deps): Bump reqwest from 0.12.28 to 0.13.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2498 +* ci(toml): check toml fmt by @TwinklerG in https://github.com/apache/iceberg-rust/pull/2504 +* Add Hugging Face storage option to storage/opendal/README by @lhoestq in https://github.com/apache/iceberg-rust/pull/2487 +* chore(deps): Bump the arrow-parquet group across 1 directory with 6 updates by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2496 +* Use cargo publish to publish crates by @blackmwk in https://github.com/apache/iceberg-rust/pull/2489 +* feat(arrow): pass `&FieldRef` instead of `&Field` in `ArrowSchemaVisitor` to achieve cheap clone by @TwinklerG in https://github.com/apache/iceberg-rust/pull/2471 +* Add Spice.ai to Users by @lukekim in https://github.com/apache/iceberg-rust/pull/2516 +* chore(spec): remove stale #[allow(dead_code)] from snapshot_summary by @SreeramGarlapati in https://github.com/apache/iceberg-rust/pull/2514 +* fix(spec): don't panic on overwrite-truncate when previous totals exceed i32::MAX or are non-numeric by @SreeramGarlapati in https://github.com/apache/iceberg-rust/pull/2511 +* Infra: add bindings/python into root workspace by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2506 +* feat(iceberg): log warning when available_parallelism falls back to default by @huan233usc in https://github.com/apache/iceberg-rust/pull/2520 +* chore(deps): Bump urllib3 from 2.6.3 to 2.7.0 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2523 +* docs: add Iceberg security model by @sungwy in https://github.com/apache/iceberg-rust/pull/2524 +* Add some scripts to help release by @blackmwk in https://github.com/apache/iceberg-rust/pull/2522 +* feat(ci): add cargo-public-api checks for public API stability by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2525 +* Ignore RUSTSEC-2024-0014 by @blackmwk in https://github.com/apache/iceberg-rust/pull/2531 +* chore(deps): Bump minijinja from 2.19.0 to 2.20.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2555 +* chore(deps): Bump reqwest from 0.12.28 to 0.13.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2554 +* chore(deps): Bump github/codeql-action from 4.35.5 to 4.36.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2553 +* chore(deps): Bump dashmap from 6.1.0 to 6.2.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2552 +* chore(deps): Bump the arrow-parquet group with 2 updates by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2551 +* chore(deps): Bump actions/stale from 10.2.0 to 10.3.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2549 +* chore(deps): Bump crate-ci/typos from 1.46.2 to 1.46.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2548 +* chore(deps): Bump taiki-e/install-action from 2.77.4 to 2.79.7 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2547 +* chore(ci): omit blanket/auto-trait/auto-derived impls from public API by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2542 +* fix(transaction): preserve delete-only manifests in FastAppend by @viirya in https://github.com/apache/iceberg-rust/pull/2545 +* Increase the throughput of the `validate_duplicate_files` by @tomighita in https://github.com/apache/iceberg-rust/pull/2296 +* chore(refactor): add manifest list reader by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2541 +* fix(ci): correct checkout version comment for zizmor by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2578 +* Split manifest_list into smaller modules by @blackmwk in https://github.com/apache/iceberg-rust/pull/2577 +* chore(refactor) add file scan task builder by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2582 +* feat(avro): enable snappy codec support by @romusz in https://github.com/apache/iceberg-rust/pull/2573 +* chore(refactor) refactor ManifestWriterBuilder to take Box and location by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2568 +* chore(refactor) refactor manifest list writer to take Box to prep for encryption by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2564 +* chore(deps): bump opendal from 0.56 to 0.57 by @kszucs in https://github.com/apache/iceberg-rust/pull/2588 +* ci: consolidate HF/CDC integration tests into existing CI jobs by @kszucs in https://github.com/apache/iceberg-rust/pull/2587 +* chore(deps): Bump taiki-e/install-action from 2.79.7 to 2.81.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2599 +* chore(deps): Bump crate-ci/typos from 1.46.3 to 1.47.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2598 +* chore(deps-dev): Bump huggingface-hub from 1.15.0 to 1.17.0 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2597 +* chore(deps): Bump reqwest from 0.12.28 to 0.13.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2600 +* fix(catalog-sql): forward catalog props to FileIO by @kszucs in https://github.com/apache/iceberg-rust/pull/2528 +* fix(storage-opendal): add TimeoutLayer inside RetryLayer to bound hangs by @AlJohri in https://github.com/apache/iceberg-rust/pull/2455 +* fix(catalog): Only list iceberg tables for Glue by @CTTY in https://github.com/apache/iceberg-rust/pull/2570 +* fix(spec): Fix incorrect totals when previous is omitted or invalid by @dannycjones in https://github.com/apache/iceberg-rust/pull/2589 +* fix(s3tables): add TableNotFound error handling by @dannycjones in https://github.com/apache/iceberg-rust/pull/2595 +* fix(reader): `filter_row_groups_by_byte_range` duplicates rows for sub-row-group file splits by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2615 +* fix: use name-mapped field IDs for projection and predicate pushdown by @viirya in https://github.com/apache/iceberg-rust/pull/2612 +* refactor!(writer): borrow TableMetadata in DefaultLocationGenerator::new by @smaheshwar-pltr in https://github.com/apache/iceberg-rust/pull/2604 +* feat(encryption) [7/N] Read encrypted manifest list by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2453 +* cleanup: remove stale #[allow(dead_code)] from expr visitors and spec by @huan233usc in https://github.com/apache/iceberg-rust/pull/2616 +* fix(catalog/s3tables): stop dropping endpoint_url for empty properties by @Kurtiscwright in https://github.com/apache/iceberg-rust/pull/2623 +* chore(ci) cargo lock taplo cli by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2627 +* ci(python): avoid duplicate bindings build by @abnobdoss in https://github.com/apache/iceberg-rust/pull/2636 +* chore(deps): Bump crate-ci/typos from 1.47.0 to 1.47.2 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2646 +* feat(encryption): [10/N] Block encrypted writes until we support it in full. by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2626 +* chore(ci): ignore pyo3 RUSTSEC-2026-0176 and -0177 in audit by @NoahKusaba in https://github.com/apache/iceberg-rust/pull/2637 +* chore(deps): Bump reqwest from 0.12.28 to 0.13.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2645 +* chore(deps): Bump github/codeql-action from 4.36.0 to 4.36.2 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2644 +* chore(deps-dev): Bump huggingface-hub from 1.17.0 to 1.18.0 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2643 +* chore(deps): Bump serde_with from 3.20.0 to 3.21.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2642 +* chore(deps): Bump actions/checkout from 6.0.2 to 6.0.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2641 +* chore(deps): Bump astral-sh/setup-uv from 8.1.0 to 8.2.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2639 +* chore(deps): Bump taiki-e/install-action from 2.81.1 to 2.81.7 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2640 +* fix: add space in decimal type serialization for cross-engine compat by @SreeramGarlapati in https://github.com/apache/iceberg-rust/pull/2538 +* fix: support decimal datum conversion by precision by @abnobdoss in https://github.com/apache/iceberg-rust/pull/2508 +* chore(ci): migrate to trusted publishing by @dannycjones in https://github.com/apache/iceberg-rust/pull/2593 +* chore: Bump MSRV to 1.94 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2652 +* feat(datafusion): push down isnan over NaN-preserving numeric expressions by @huan233usc in https://github.com/apache/iceberg-rust/pull/2592 +* ci: Update version tag in publish.yml by @CTTY in https://github.com/apache/iceberg-rust/pull/2662 +* feat(transaction): add ExpireSnapshotsAction by @dhruvarya-db in https://github.com/apache/iceberg-rust/pull/2591 +* Fix: replace --profile py-release with --release in Python release workflows by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2638 +* feat(transaction): default expiry to history.expire.* table properties by @dhruvarya-db in https://github.com/apache/iceberg-rust/pull/2664 +* ci: save Rust caches only from main by @abnobdoss in https://github.com/apache/iceberg-rust/pull/2669 +* Extract name mapping from table properties by @prakharjain09 in https://github.com/apache/iceberg-rust/pull/2519 +* chore(dev): bump iceberg-rest-fixture to 1.10.1 by @huan233usc in https://github.com/apache/iceberg-rust/pull/2673 +* chore: Move dependency list generation from a release task to a PR author responsibility by @dannycjones in https://github.com/apache/iceberg-rust/pull/2676 +* ci: disable debuginfo and incremental in Rust CI by @abnobdoss in https://github.com/apache/iceberg-rust/pull/2672 +* chore(deps): Bump rust-lang/crates-io-auth-action from 1.0.3 to 1.0.4 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2688 +* CI: Check ASF action allowlist on every PR by @ebyhr in https://github.com/apache/iceberg-rust/pull/2694 +* fix(rest): skip serializing unset optional fields in CreateTableRequest by @rahulsmahadev in https://github.com/apache/iceberg-rust/pull/2610 +* chore: Update release runbook by @dannycjones in https://github.com/apache/iceberg-rust/pull/2674 +* feat(transaction): remove statistics file metadata for expired snapshots by @dhruvarya-db in https://github.com/apache/iceberg-rust/pull/2667 +* fix: fix RUSTSEC-2026-0185 by @CTTY in https://github.com/apache/iceberg-rust/pull/2703 +* fix(reader): graceful handling of missing column index by @sandugood in https://github.com/apache/iceberg-rust/pull/2693 +* fix: avoid lost-wakeup hang in DeleteFileIndex::get_deletes_for_data_file by @dhruvarya-db in https://github.com/apache/iceberg-rust/pull/2696 +* chore: Move dependency list generation back to release manager task by @dannycjones in https://github.com/apache/iceberg-rust/pull/2706 +* fix(transaction): don't let user properties override computed summary metrics by @viirya in https://github.com/apache/iceberg-rust/pull/2726 +* chore(deps): Bump rust-lang/crates-io-auth-action from 1.0.4 to 1.0.5 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2728 +* chore(deps-dev): Bump huggingface-hub from 1.18.0 to 1.20.1 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2730 +* chore(deps-dev): Bump maturin from 1.13.3 to 1.14.1 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2731 +* chore(deps-dev): Bump pytest from 9.0.3 to 9.1.1 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2732 +* chore(deps): Bump actions/checkout from 6.0.3 to 7.0.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2733 +* chore(deps): Bump bytes from 1.11.1 to 1.12.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2734 +* chore(deps): Bump taiki-e/install-action from 2.81.7 to 2.82.2 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2735 +* chore(deps): Bump minijinja from 2.20.0 to 2.21.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2736 +* chore: Update all cargo install references to use --locked by @dannycjones in https://github.com/apache/iceberg-rust/pull/2739 +* fix: Update Python binding test fixtures to close catalogs by @dannycjones in https://github.com/apache/iceberg-rust/pull/2721 +* chore: Update changelog and dependency lists for 0.10.0 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2745 +* chore(ci): Remove '--locked' flag on toml-cli install by @dannycjones in https://github.com/apache/iceberg-rust/pull/2750 +* ci: cache Python bindings Rust artifacts by @abnobdoss in https://github.com/apache/iceberg-rust/pull/2679 +* chore: Update changelog for 0.10.0 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2751 +* fix(datafusion): return single row with count 0 for empty inserts by @u70b3 in https://github.com/apache/iceberg-rust/pull/2712 +* fix(datafusion): reject non-append insert operations by @u70b3 in https://github.com/apache/iceberg-rust/pull/2714 +* feat(writer): honor write.parquet.* via ParquetWriterBuilder::from_table_properties by @kszucs in https://github.com/apache/iceberg-rust/pull/2561 +* refactor(encryption) remove key_metadata from the constructor of an unencrypted manifest writer by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2666 +* chore(deps): Bump reqwest from 0.12.28 to 0.13.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2690 +* chore(deps): Bump zeroize from 1.8.2 to 1.9.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2691 +* refactor(encryption) use `StandardKeyMetadata` from `EncryptedOutputFile` in `ManifestWriterBuilder` by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2628 +* Resolve RUSTSEC-2026-0194 RUSTSEC-2026-0195 by @blackmwk in https://github.com/apache/iceberg-rust/pull/2761 +* Restrict trigger push branch for GitHub Workflow by @apupier in https://github.com/apache/iceberg-rust/pull/2757 +* chore: update verify_rc script to verify tag against tarball by @CTTY in https://github.com/apache/iceberg-rust/pull/2760 +* Resolve RUSTSEC-2026-0190 by @blackmwk in https://github.com/apache/iceberg-rust/pull/2763 +* chore(deps): Bump reqwest from 0.12.28 to 0.13.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2770 +* chore(deps): Bump enum-ordinalize from 4.3.2 to 4.4.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2769 +* chore(deps): Bump actions/setup-python from 6.2.0 to 6.3.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2768 +* chore(deps): Bump taiki-e/install-action from 2.82.2 to 2.82.5 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2767 +* chore(deps-dev): Bump huggingface-hub from 1.20.1 to 1.21.0 in /bindings/python by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2766 +* ci: split lint checks and slim clippy by @abnobdoss in https://github.com/apache/iceberg-rust/pull/2771 +* Fix zizmor pin by @blackmwk in https://github.com/apache/iceberg-rust/pull/2762 +* feat(encryption) [8/N] Read encrypted manifest file by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2586 +* Reject negative positional delete positions by @fallintoplace in https://github.com/apache/iceberg-rust/pull/2631 +* chore: Update changelog for 0.10.0 by @dannycjones in https://github.com/apache/iceberg-rust/pull/2782 +* Include license and notice files in iceberg-storage-opendal crate by @ankane in https://github.com/apache/iceberg-rust/pull/2823 +* fix(license): Add missing LICENSE and NOTICE symlinks for iceberg-cache-moka and iceberg-catalog-loader by @dannycjones in https://github.com/apache/iceberg-rust/pull/2828 + +## [v0.9.0] - 2026-03-10 + +### Breaking Changes + +* **API Changes:** + * refactor!(storage): Move OpenDal Storage to a new crate iceberg-storage-opendal (#2207) by @CTTY in https://github.com/apache/iceberg-rust/pull/2207 + * feat!(storage): Integration Storage trait with FileIO (#2116) by @CTTY in https://github.com/apache/iceberg-rust/pull/2116 + * feat!(io): Implement Storage for OpenDal (#2080) by @CTTY in https://github.com/apache/iceberg-rust/pull/2080 + +* **Compatibility:** + * chore: Bumping MSRV to 1.92.0 (#2224) by @CTTY in https://github.com/apache/iceberg-rust/pull/2224 + * chore: Update MSRV to 1.91.0 (#2204) by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2204 + +* **Dependency Updates:** + * chore(deps): upgrade to DataFusion 52 (#1997) by @ethan-tyler in https://github.com/apache/iceberg-rust/pull/1997 + * feat(spec): replace rust_decimal with fastnum for 38-digit precision (#2063) by @xbattlax in https://github.com/apache/iceberg-rust/pull/2063 + +### All Changes + +* feat(reader): support timestamp type in create_column (#2180) by @chenzl25 in https://github.com/apache/iceberg-rust/pull/2180 +* chore: Bumping MSRV to 1.92.0 (#2224) by @CTTY in https://github.com/apache/iceberg-rust/pull/2224 +* perf(reader): Avoid second `create_parquet_record_batch_stream_builder()` call for migrated tables (#2176) by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2176 +* doc: Update IO feature flag and examples (#2214) by @CTTY in https://github.com/apache/iceberg-rust/pull/2214 +* chore(deps): Bump uuid from 1.21.0 to 1.22.0 (#2221) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2221 +* chore(deps): Bump minijinja from 2.16.0 to 2.17.1 (#2222) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2222 +* chore: update to DataFusion 52.2 (#2199) by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2199 +* perf(reader): Implement AsyncFileReader get_byte_ranges and coalesce close ranges (#2181) by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2181 +* refactor!(storage): Move OpenDal Storage to a new crate iceberg-storage-opendal (#2207) by @CTTY in https://github.com/apache/iceberg-rust/pull/2207 +* chore: Update MSRV to 1.91.0 (#2204) by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2204 +* feat!(storage): Integration Storage trait with FileIO (#2116) by @CTTY in https://github.com/apache/iceberg-rust/pull/2116 +* chore(deps): Bump minijinja from 2.14.0 to 2.16.0 (#2195) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2195 +* chore(deps): Bump the arrow-parquet group with 7 updates (#2192) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2192 +* chore(deps): Bump aws-sdk-glue from 1.133.0 to 1.134.0 (#2196) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2196 +* chore(deps): Bump actions/upload-artifact from 6 to 7 (#2189) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2189 +* chore(deps): Bump serde_with from 3.16.1 to 3.17.0 (#2194) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2194 +* chore(deps): Bump uuid from 1.19.0 to 1.21.0 (#2193) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2193 +* chore(deps): Bump actions/download-artifact from 7 to 8 (#2191) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2191 +* chore(deps): Bump crate-ci/typos from 1.43.5 to 1.44.0 (#2190) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2190 +* chore(deps): Bump fs-err from 3.2.2 to 3.3.0 (#2197) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2197 +* chore(deps): Bump tempfile from 3.24.0 to 3.25.0 (#2198) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2198 +* chore: use ubuntu-slim for lightweight jobs (#2187) by @zhjwpku in https://github.com/apache/iceberg-rust/pull/2187 +* chore(deps): upgrade to DataFusion 52 (#1997) by @ethan-tyler in https://github.com/apache/iceberg-rust/pull/1997 +* feat(datafusion): support isnan predicate pushdown to Iceberg (#2142) by @charlesdong1991 in https://github.com/apache/iceberg-rust/pull/2142 +* perf(reader): Pass data file size and delete file size to reader to avoid `stat()` calls (#2175) by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2175 +* infra: improve python binding setup (#2178) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2178 +* perf(reader): Add Parquet metadata size hint option to ArrowReaderBuilder (#2173) by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2173 +* ci: Update stale.yml GitHub workflow to handle stale PRs (#2171) by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2171 +* chore(deps): Bump urllib3 from 2.5.0 to 2.6.3 in /bindings/python (#2170) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2170 +* chore(deps): Bump crate-ci/typos from 1.43.4 to 1.43.5 (#2160) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2160 +* chore(deps): Bump actions/checkout from 4 to 6 (#2161) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2161 +* Potential fix for code scanning alert no. 1: Cache Poisoning via low-privileged code injection (#2164) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2164 +* chore(ci): add explicit least-privilege workflow permissions (#2163) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2163 +* chore(deps): Bump actions/stale from 10.1.1 to 10.2.0 (#2159) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2159 +* chore(deps): Bump minijinja from 2.15.1 to 2.16.0 (#2162) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2162 +* feat(playground): add memory catalog support (#2121) by @Shekharrajak in https://github.com/apache/iceberg-rust/pull/2121 +* CI(Python): dont use --sdist in maturin build for CI (#2166) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2166 +* CI: Add CodeQL workflow for GitHub Actions security scanning (#2151) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2151 +* chore(deps): Bump the arrow-parquet group with 2 updates (#2139) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2139 +* infra: set github actions max-parallel to 15 (#2150) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2150 +* chore(deps): Bump uuid from 1.20.0 to 1.21.0 (#2141) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2141 +* chore(deps): Bump tempfile from 3.23.0 to 3.25.0 (#2140) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2140 +* chore(deps): Bump crate-ci/typos from 1.43.3 to 1.43.4 (#2138) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2138 +* fix(rest): Filter sensitive headers from error logs (#2117) (#2130) by @cmackenzie1 in https://github.com/apache/iceberg-rust/pull/2117 +* fix: Interpret s3tables warehouse as table_location not metadata loca… (#2115) by @emkornfield in https://github.com/apache/iceberg-rust/pull/2115 +* refactor(storage): Reorganize storage code into a new module (#2109) by @CTTY in https://github.com/apache/iceberg-rust/pull/2109 +* Use uv instead of pip for python packages (#2129) by @blackmwk in https://github.com/apache/iceberg-rust/pull/2129 +* chore(deps): Bump crate-ci/typos from 1.42.3 to 1.43.3 (#2124) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2124 +* chore(deps): Bump the arrow-parquet group with 6 updates (#2125) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2125 +* chore(deps): Bump fs-err from 3.2.0 to 3.3.0 (#2126) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2126 +* chore(deps): Bump time from 0.3.44 to 0.3.47 in /bindings/python (#2112) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2112 +* chore(deps): Bump time from 0.3.44 to 0.3.47 (#2113) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2113 +* fix(python): Bump Python version upper bound (#2110) by @geruh in https://github.com/apache/iceberg-rust/pull/2110 +* feat(io): Implement native MemoryStorage (#2097) by @CTTY in https://github.com/apache/iceberg-rust/pull/2097 +* feat(io): Implement native LocalFsStorage (#2094) by @CTTY in https://github.com/apache/iceberg-rust/pull/2094 +* chore(deps): Bump bytes from 1.10.1 to 1.11.1 in /bindings/python (#2101) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2101 +* chore(deps): Bump bytes from 1.11.0 to 1.11.1 (#2102) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2102 +* feat!(io): Implement Storage for OpenDal (#2080) by @CTTY in https://github.com/apache/iceberg-rust/pull/2080 +* chore(deps): Bump crate-ci/typos from 1.42.1 to 1.42.3 (#2098) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2098 +* chore(deps): Bump minijinja from 2.13.0 to 2.15.1 (#2099) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2099 +* chore(python): Use PyPI release of pyiceberg instead of git dependency (#2095) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2095 +* feat(datafusion): Add Timestamp scalar value conversion for predicate pushdown (#2069) by @viirya in https://github.com/apache/iceberg-rust/pull/2069 +* fix: SqlCatalogBuilder does not save supplied name in SqlCatalogConfig before initializing SqlCatalog (#2079) by @brodiealexander in https://github.com/apache/iceberg-rust/pull/2079 +* feat(datafusion): Add Boolean predicate pushdown support (#2082) by @viirya in https://github.com/apache/iceberg-rust/pull/2082 +* Fix moto server port conflict with macOS AirPlay Receiver (#2090) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2090 +* site: merge container docs and add a section about docker desktop (#2091) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2091 +* Add ETL project link to README (#2087) by @imor in https://github.com/apache/iceberg-rust/pull/2087 +* feat(spec): replace rust_decimal with fastnum for 38-digit precision (#2063) by @xbattlax in https://github.com/apache/iceberg-rust/pull/2063 +* Split sqllogictests into different schedule (#2085) by @liurenjie1024 in https://github.com/apache/iceberg-rust/pull/2085 +* feat: Factor out CompressionCodec to common location and add Gzip (#2081) by @emkornfield in https://github.com/apache/iceberg-rust/pull/2081 +* Use nextest to speedup tests (#2078) by @liurenjie1024 in https://github.com/apache/iceberg-rust/pull/2078 +* Let integration tests run in parallel (#2076) by @liurenjie1024 in https://github.com/apache/iceberg-rust/pull/2076 +* Cleanup integration tests (#2074) by @liurenjie1024 in https://github.com/apache/iceberg-rust/pull/2074 +* feat(io): Add specific storage configs (#2072) by @CTTY in https://github.com/apache/iceberg-rust/pull/2072 +* Simplify test setup by starting container only once. (#2071) by @liurenjie1024 in https://github.com/apache/iceberg-rust/pull/2071 +* feat(io): Add Storage, StorageFactory, StorageConfig (#2061) by @CTTY in https://github.com/apache/iceberg-rust/pull/2061 +* feat(datafusion): Add LIKE predicate pushdown for StartsWith patterns (#2014) by @viirya in https://github.com/apache/iceberg-rust/pull/2014 +* chore(deps): Bump crate-ci/typos from 1.42.0 to 1.42.1 (#2066) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2066 +* chore(deps): Bump uuid from 1.19.0 to 1.20.0 (#2067) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2067 +* feat(io): Rename Storage to OpenDalStorage (#2059) by @CTTY in https://github.com/apache/iceberg-rust/pull/2059 +* feat(datafusion): Add support for DROP TABLE (#2033) by @CTTY in https://github.com/apache/iceberg-rust/pull/2033 +* feat(datafusion): Add Binary scalar value conversion for predicate pushdown (#2048) by @viirya in https://github.com/apache/iceberg-rust/pull/2048 +* perf(reader): Fast path ArrowReader::read when concurrency is 1 to avoid waker churn and add determinism to FileScanTask processing (#2020) by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2020 +* infra: fix manual trigger for the "release python" github workflow (#2049) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2049 +* fix: reuse partition field IDs for equivalent fields in AddSpec (#2011) by @aditya-subrahmanyan in https://github.com/apache/iceberg-rust/pull/2011 +* infra: dynamically set msrv in github workflows (#2040) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2040 +* chore(deps): Bump the arrow-parquet group with 2 updates (#2046) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2046 +* chore(deps): Bump rust_decimal from 1.39.0 to 1.40.0 (#2047) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2047 +* infra: auto update docker image (#2044) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2044 +* infra: use spark base image for docker (#2043) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2043 +* infra: upgrade spark to 3.5.8 (#2045) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2045 +* site: release static version from downloads page (#2039) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2039 +* fix: allow v2 to v3 table upgrades with existing snapshots (#2010) by @aditya-subrahmanyan in https://github.com/apache/iceberg-rust/pull/2010 +* rfc: Making Storage a Trait (#1885) by @CTTY in https://github.com/apache/iceberg-rust/pull/1885 +* infra: Restrict audit workflow to main apache/iceberg-rust repository (#1974) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1974 +* chore: Update copyright year in NOTICE files (#2024) by @AndreaBozzo in https://github.com/apache/iceberg-rust/pull/2024 +* feat(datafusion): Support `CREATE TABLE` for DataFusion (#1972) by @CTTY in https://github.com/apache/iceberg-rust/pull/1972 +* feat(datafusion): Add schema validation for partition projection (#2008) by @viirya in https://github.com/apache/iceberg-rust/pull/2008 +* add `make site` for local website preview (#2017) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2017 +* docs: split up the make command in release doc (#2016) by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/2016 +* docs: Minor update to README.md (#2018) by @aditya-subrahmanyan in https://github.com/apache/iceberg-rust/pull/2018 +* chore(deps): Bump the arrow-parquet group with 6 updates (#2013) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2013 +* chore(deps): Bump crate-ci/typos from 1.41.0 to 1.42.0 (#2012) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2012 +* feat(spec): add table_properties() method to TableMetadata (#2002) by @xbattlax in https://github.com/apache/iceberg-rust/pull/2002 +* feat(datafusion): Add LIMIT pushdown support (#2006) by @viirya in https://github.com/apache/iceberg-rust/pull/2006 +* fix: Reserved sort order ID cannot contain any fields (#1978) by @aditya-subrahmanyan in https://github.com/apache/iceberg-rust/pull/1978 +* fix: Use correct byte representation for decimal hashing (#1998) by @aditya-subrahmanyan in https://github.com/apache/iceberg-rust/pull/1998 +* chore(deps): Bump crate-ci/typos from 1.40.0 to 1.41.0 (#1982) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1982 +* feat(datafusion): Apply SortExec when writing in clustered mode (#2005) by @CTTY in https://github.com/apache/iceberg-rust/pull/2005 + +## [v0.8.0] - 2026-01-06 + +### Breaking Changes + +* **API Changes:** + * refactor: Remove redundant parameters from SnapshotProducer validation methods by @Li0k in https://github.com/apache/iceberg-rust/pull/1853 + * chore: Remove deprecated `remove_all` in FileIO by @jonathanc-n in https://github.com/apache/iceberg-rust/pull/1863 + * refactor: Drop smol runtime support by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1900 + +* **Compatibility:** + * chore: bump MSRV to 1.88, fix warnings and clippy errors by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1902 + +* **Dependency Updates:** + * Upgrade opendal to v0.55 by @dentiny in https://github.com/apache/iceberg-rust/pull/1895 + * deps: bump DataFusion to 51, Arrow to 57, pyo to 0.26 by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1899 + +* **Other:** + * Remove wildcard pattern in exhaustive enums by @lgingerich in https://github.com/apache/iceberg-rust/pull/1925 + +### All Changes + +* chore(deps): Bump tempfile from 3.22.0 to 3.23.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1717 +* chore(deps): Bump rand from 0.8.5 to 0.9.2 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1716 +* chore(deps): Bump crate-ci/typos from 1.36.2 to 1.36.3 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1715 +* refactor: Improve REST catalog's authenticate method by @imor in https://github.com/apache/iceberg-rust/pull/1712 +* chore(deps): Bump serde_with from 3.14.0 to 3.14.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1727 +* refactor(writer): Refactor writers for the future partitioning writers by @CTTY in https://github.com/apache/iceberg-rust/pull/1657 +* Set lock on version of Pydantic by @Fokko in https://github.com/apache/iceberg-rust/pull/1737 +* chore(deps): Bump crate-ci/typos from 1.36.3 to 1.37.2 by @Standing-Man in https://github.com/apache/iceberg-rust/pull/1734 +* feat: support more partition transformations for PartitionSpec::partition_to_path by @mnpw in https://github.com/apache/iceberg-rust/pull/1730 +* chore: Update website for 0.7.0 by @CTTY in https://github.com/apache/iceberg-rust/pull/1738 +* feat(sql-catalog): implement register table for sql catalog by @Standing-Man in https://github.com/apache/iceberg-rust/pull/1724 +* fix: ensure CoalescePartitionsExec is enabled for IcebergCommitExec by @sgrebnov in https://github.com/apache/iceberg-rust/pull/1723 +* chore(deps): Bump regex from 1.11.2 to 1.12.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1741 +* chore(deps): Bump crate-ci/typos from 1.37.2 to 1.38.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1740 +* Improve `IcebergCommitExec` to correctly populate properties/schema by @sgrebnov in https://github.com/apache/iceberg-rust/pull/1721 +* feat(spec): add `table_properties.rs` to spec by @kaushiksrini in https://github.com/apache/iceberg-rust/pull/1733 +* chore(deps): Bump actions/stale from 10.0.0 to 10.1.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1726 +* docs: remove -src suffix from artifact name by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1743 +* feat(reader): Make ArrowReaderBuilder::new public by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1748 +* feat(writer): Add clustered and fanout writer by @CTTY in https://github.com/apache/iceberg-rust/pull/1735 +* feat(catalog): impl builder for SqlCatalog by @335g in https://github.com/apache/iceberg-rust/pull/1666 +* fix: fix read parquert file when schema change by @chenzl25 in https://github.com/apache/iceberg-rust/pull/1750 +* docs: Fix broken orbstack and podman links in CONTRIBUTING.md by @petern48 in https://github.com/apache/iceberg-rust/pull/1757 +* chore(deps): Bump tokio from 1.47.1 to 1.48.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1763 +* chore(deps): Bump backon from 1.5.2 to 1.6.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1762 +* fix: global eq delete matching should apply to only strictly older files, and fix partition scoped matching to consider spec id by @amogh-jahagirdar in https://github.com/apache/iceberg-rust/pull/1758 +* chore(deps): Bump apache/skywalking-eyes from 0.7.0 to 0.8.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1760 +* chore(deps): Bump rust_decimal from 1.38.0 to 1.39.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1761 +* feat(datafusion): implement the project node to add the partition columns by @fvaleye in https://github.com/apache/iceberg-rust/pull/1602 +* fix: snapshot was producing empty summary by @imor in https://github.com/apache/iceberg-rust/pull/1767 +* docs: Add examples for PartitioningWriter by @CTTY in https://github.com/apache/iceberg-rust/pull/1754 +* feat(sqllogictest): Add support for iceberg datafusion sqllogictest integration by @lliangyu-lin in https://github.com/apache/iceberg-rust/pull/1764 +* fix(build): Pin home version after merging #1764 by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1783 +* minor: Update Cargo.lock to add home by @CTTY in https://github.com/apache/iceberg-rust/pull/1785 +* chore(deps): Bump aws-sdk-s3tables from 1.40.0 to 1.41.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1790 +* chore(deps): Bump rand from 0.8.5 to 0.9.2 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1789 +* chore(deps): Bump actions/download-artifact from 5 to 6 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1788 +* chore(deps): Bump actions/upload-artifact from 4 to 5 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1787 +* fix(reader): filter row groups when FileScanTask contains byte ranges by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1779 +* refactor(arrow,datafusion): Reuse PartitionValueCalculator in RecordBatchPartitionSplitter by @CTTY in https://github.com/apache/iceberg-rust/pull/1781 +* feat: Update Datafusion to v49 by @DerGut in https://github.com/apache/iceberg-rust/pull/1704 +* deps: unpin pydantic by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1793 +* feat(reader): Add Date32 support to RecordBatchTransformer create_column by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1792 +* feat(catalog): Implement update_table for S3TablesCatalog by @CTTY in https://github.com/apache/iceberg-rust/pull/1594 +* feat: Update Datafusion to v50 by @DerGut in https://github.com/apache/iceberg-rust/pull/1728 +* ci: Migrate to uv for python by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1796 +* ci: Relax msrv check thanks to rust 2024 by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1795 +* ci: Don't dismiss stale review to make contribution easier by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1799 +* add Makefile to bindings/python by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1800 +* chore: inline format args by @colinmarc in https://github.com/apache/iceberg-rust/pull/1805 +* refactor: Migrate from tera to minijinja by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1798 +* fix(reader): fix position delete bugs with row group skipping by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1806 +* feat(datafusion): implement the partitioning node for DataFusion to define the partitioning by @fvaleye in https://github.com/apache/iceberg-rust/pull/1620 +* feat(reader): Date32 from days since epoch for Literal:try_from_json by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1803 +* chore(deps): Bump aws-sdk-glue from 1.125.0 to 1.126.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1812 +* chore(deps): Bump astral-sh/setup-uv from 6 to 7 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1811 +* chore(deps): Bump crate-ci/typos from 1.38.1 to 1.39.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1810 +* feat(reader): position-based column projection for Parquet files without field IDs (migrated tables) by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1777 +* fix(reader): Equality delete files with partial schemas (containing only equality columns) by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1782 +* infra: use apache/hive:4.0.0 as hive Dockerfile base image by @geruh in https://github.com/apache/iceberg-rust/pull/1823 +* fix: StructType fails to deserialize JSON with type field by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1822 +* feat: Support for V3 Metadata by @c-thiel in https://github.com/apache/iceberg-rust/pull/1682 +* fix(reader): Support both position and equality delete files on the same FileScanTask by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1778 +* feat(datafusion): Add TaskWriter for DataFusion by @CTTY in https://github.com/apache/iceberg-rust/pull/1769 +* fix: support reading compressed metadata by @colinmarc in https://github.com/apache/iceberg-rust/pull/1802 +* Support deserializing bytes by @Fokko in https://github.com/apache/iceberg-rust/pull/1820 +* fix: Bump CI Spark version to 3.5.7 by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1832 +* infra: use python 3.12 for release by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1836 +* pyiceberg-core: create smaller artifacts by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1841 +* infra: add collaborators to .asf.yaml by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1842 +* pyiceberg-core: use pyo3 abi3-py310 by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1843 +* ci: parallelize unit test with matrix by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1833 +* pyiceberg-core: create even smaller artifacts by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1844 +* chore: Split values.rs into separate files by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1840 +* feat(datafusion): Support `INSERT INTO` partitioned tables by @CTTY in https://github.com/apache/iceberg-rust/pull/1827 +* docs: Add Wrappers project to README by @burmecia in https://github.com/apache/iceberg-rust/pull/1852 +* feat(reader): Add PartitionSpec support to FileScanTask and RecordBatchTransformer by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1821 +* feat(reader): null struct default values in create_column by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1847 +* refactor: Remove redundant parameters from SnapshotProducer validation methods by @Li0k in https://github.com/apache/iceberg-rust/pull/1853 +* infra: add verbose=true to pypa/gh-action-pypi-publish by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1846 +* use RecordBatchTransformerBuilder instead of RecordBatchTransformer by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1857 +* chore(deps): Bump bytes from 1.10.1 to 1.11.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1861 +* chore(deps): Bump serde_with from 3.15.1 to 3.16.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1859 +* chore(deps): Bump fs-err from 3.1.3 to 3.2.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1860 +* chore(deps): Bump crate-ci/typos from 1.39.0 to 1.39.2 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1858 +* chore: Remove deprecated `remove_all` in FileIO by @jonathanc-n in https://github.com/apache/iceberg-rust/pull/1863 +* infra: notify on github workflow failure by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1870 +* feat(reader): Add binary support to `get_arrow_datum` for equality deletes with binary type by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1848 +* Raise concurrency errors properly for glue tables by @jembishop in https://github.com/apache/iceberg-rust/pull/1875 +* infra: add instructions for cleaning up testpypi artifacts by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1855 +* chore(deps): Bump actions/checkout from 5 to 6 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1883 +* Update apache-avro to v0.21.0 by @N-Boutaib in https://github.com/apache/iceberg-rust/pull/1881 +* docs: Clarify functionality of `SnapshotProduceOperation` by @jonathanc-n in https://github.com/apache/iceberg-rust/pull/1874 +* feat(datafusion): Split IcebergTableProvider into static and non-static table provider by @CTTY in https://github.com/apache/iceberg-rust/pull/1879 +* infra: use new `del_branch_on_merge` in .asf.yaml by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1888 +* Upgrade opendal to v0.55 by @dentiny in https://github.com/apache/iceberg-rust/pull/1895 +* chore(deps): Bump http from 1.3.1 to 1.4.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1892 +* chore(deps): Bump crate-ci/typos from 1.39.2 to 1.40.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1891 +* feat(datafusion): Add `sort_by_partition` to sort the input partitioned data by @CTTY in https://github.com/apache/iceberg-rust/pull/1618 +* rfc: Modularize `iceberg` Implementations by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1854 +* refactor(writer): Make writer builders non-consuming in build by @leonzchang in https://github.com/apache/iceberg-rust/pull/1889 +* fix: Keep snapshot log on replace by @c-thiel in https://github.com/apache/iceberg-rust/pull/1896 +* chore(deps): Bump actions/stale from 10.1.0 to 10.1.1 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1908 +* feat(datafusion): Add sqllogictest for DataFusion INSERT INTO by @CTTY in https://github.com/apache/iceberg-rust/pull/1887 +* refactor: Drop smol runtime support by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1900 +* chore(deps): Bump minijinja from 2.12.0 to 2.13.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1909 +* chore(deps): Bump uuid from 1.18.1 to 1.19.0 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1910 +* feat(core): Add support for `_file` column by @gbrgr in https://github.com/apache/iceberg-rust/pull/1824 +* feat: Make `rest` types public, add documentation by @c-thiel in https://github.com/apache/iceberg-rust/pull/1901 +* chore: bump MSRV to 1.88, fix warnings and clippy errors by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1902 +* ci: Make s3tables ready for publish by @Xuanwo in https://github.com/apache/iceberg-rust/pull/1916 +* deps: bump DataFusion to 51, Arrow to 57, pyo to 0.26 by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1899 +* fix: Serialize `split_offsets` as null when empty by @AndreaBozzo in https://github.com/apache/iceberg-rust/pull/1906 +* feat(catalog): Implement update_table for SqlCatalog by @lgingerich in https://github.com/apache/iceberg-rust/pull/1911 +* fix: Respect precision and scale for Decimal128 in value.rs by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1921 +* fix: restore no-op logic in constants_map for NULL identity-partitioned columns by @mbutrovich in https://github.com/apache/iceberg-rust/pull/1922 +* fix: stack overflow when loading large equality deletes by @dojiong in https://github.com/apache/iceberg-rust/pull/1915 +* chore(deps): Bump actions/upload-artifact from 5 to 6 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1931 +* chore(deps): Bump actions/download-artifact from 6 to 7 by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/1932 +* Remove wildcard pattern in exhaustive enums by @lgingerich in https://github.com/apache/iceberg-rust/pull/1925 +* fix: prioritize delete manifests to prevent scan deadlock by @dojiong in https://github.com/apache/iceberg-rust/pull/1937 +* feat: Include statistics for Reserved Fields by @Fokko in https://github.com/apache/iceberg-rust/pull/1849 +* fix(website): Update expected messages by @CTTY in https://github.com/apache/iceberg-rust/pull/1942 +* feat: Implement shared delete file loading and caching for ArrowReader by @dojiong in https://github.com/apache/iceberg-rust/pull/1941 +* infra: license header check ignore target/ dir by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1954 +* infra: release script, validate proper ICEBERG_VERSION variable by @kevinjqliu in https://github.com/apache/iceberg-rust/pull/1956 +* refactor(arrow): Rename parameter in delete_filter for clarity by @robertmu in https://github.com/apache/iceberg-rust/pull/1955 +* feat(sqllogictest): use serde derived structs for schedule parsing by @AndreaBozzo in https://github.com/apache/iceberg-rust/pull/1953 +* fix: follow IEEE 754 totalOrder for `float` and `double` by @Standing-Man in https://github.com/apache/iceberg-rust/pull/1959 +* fix: return proper error rather than persisting error message on snapshot by @Standing-Man in https://github.com/apache/iceberg-rust/pull/1960 +* feat(arrow): Convert Arrow schema to Iceberg schema with auto assigned field ids by @CTTY in https://github.com/apache/iceberg-rust/pull/1928 +* fix: MemoryCatalog to return absolute NamespaceIdents by @eickler in https://github.com/apache/iceberg-rust/pull/1970 +* fix(spec): Include delete file content to V3 manifest by @CTTY in https://github.com/apache/iceberg-rust/pull/1979 +* fix: fix typo check error by @Standing-Man in https://github.com/apache/iceberg-rust/pull/1989 +* Fix ci audit failure by @liurenjie1024 in https://github.com/apache/iceberg-rust/pull/1988 +* feat: make FanoutWriter writer configurable by @Standing-Man in https://github.com/apache/iceberg-rust/pull/1962 + ## [v0.7.0] - 2025-09-23 ### Breaking Changes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4b175b055..d0d571ea88 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,12 @@ First, thank you for contributing to Iceberg Rust! The goal of this document is Once your changes are ready you must submit your branch as a [pull request](https://github.com/apache/iceberg-rust/pulls). +#### Open Pull Request Limit + +With coding agents, the number of open PRs now far exceeds our review capacity. To help reviewers focus on fewer PRs, users without write access are limited to **3 open, non-draft PRs** at a time. If you've reached the limit, wait for existing PRs to be merged, close some of them, or convert some to drafts until you have fewer than three open PRs before opening another. + +Contributors who have shown sustained engagement with the project can be exempted from the limit. The exemptions are listed under `bypass_users` in [`.asf.yaml`](https://github.com/apache/iceberg-rust/blob/main/.asf.yaml). Exempt contributors should open lower-priority or blocked PRs as drafts so reviewers can see where to focus. + #### Title The pull request title must follow the format outlined in the [conventional commits spec](https://www.conventionalcommits.org). [Conventional commits](https://www.conventionalcommits.org) is a standardized format for commit messages. iceberg-rust only requires this format for commits on the `main` branch. And because iceberg-rust squashes commits before merging branches, this means that only the pull request title must conform to this format. @@ -62,6 +68,19 @@ All pull requests are squash merged. We generally discourage large pull requests When a pull request is under review, please avoid using force push as it makes it difficult for reviewer to track changes. If you need to keep the branch up to date with the main branch, consider using `git merge` instead. +#### Merging an urgent PR + +Committers can merge an urgent, approved pull request without waiting for the merge queue or required checks (for example, a security fix while CI is stuck). Use this only when necessary; otherwise, let the merge queue run. + +1. Join the [`apache/iceberg-public`](https://github.com/orgs/apache/teams/iceberg-public) team via [Boxer](https://gitbox.apache.org/boxer/) (one-time; membership syncs in about 15 minutes). +2. Get the pull request approved by a committer. +3. If the pull request is in the merge queue, click **Remove from queue**. +4. Merge it directly: + + ```sh + gh pr merge --squash --admin + ``` + ### CI Currently, iceberg-rust uses GitHub Actions to run tests. The workflows are defined in `.github/workflows`. @@ -101,10 +120,7 @@ cargo 1.69.0 (6e9a83356 2023-04-12) #### Install Docker or Podman -Currently, iceberg-rust uses Docker to set up environment for integration tests. Native Docker has some limitations, please check (https://github.com/apache/iceberg-rust/pull/748). Please use Orbstack or Podman. - -For MacOS users, you can install [OrbStack as a docker alternative](reference/orbstack.md). -For Podman users, refer to [Using Podman instead of Docker](reference/podman.md) +Currently, iceberg-rust uses Docker to set up environment for integration tests. See [Container Runtimes](website/src/reference/container-runtimes.md) for setup instructions. ## Build @@ -121,6 +137,14 @@ tested in CI and developers have reproducible builds. In `Cargo.toml`, we specify the minimum version required to use iceberg-rust. This allows users to choose their dependency versions without always upgrading to the latest. +Dependency licenses must comply with the [ASF 3rd Party License Policy](https://www.apache.org/legal/resolved.html). +CI checks them against `deny.toml` with `cargo deny`; run `make check-dependency-licenses` to check locally. If a +license is rejected, find its category in the policy: + +- Category A: add the license to `allow`. +- Category B: add a per-crate entry to `exceptions`. +- Category X: not allowed by ASF policy; do not take the dependency. + ## Code of Conduct We expect all community members to follow our [Code of Conduct](https://www.apache.org/foundation/policies/conduct.html). diff --git a/Cargo.lock b/Cargo.lock index aa30a87e89..667120ff16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -18,10 +9,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] -name = "adler32" -version = "1.2.0" +name = "aead" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] [[package]] name = "aes" @@ -29,20 +24,34 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", + "cfg-if 1.0.4", + "cipher 0.4.4", + "cpufeatures 0.2.17", ] [[package]] -name = "ahash" -version = "0.7.8" +name = "aes" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" dependencies = [ - "getrandom 0.2.16", - "once_cell", - "version_check", + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.0", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes 0.8.4", + "cipher 0.4.4", + "ctr 0.9.2", + "ghash", + "subtle", ] [[package]] @@ -51,9 +60,9 @@ version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "const-random", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "serde", "version_check", @@ -62,24 +71,30 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" -version = "2.0.4" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" [[package]] name = "alloc-stdlib" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" dependencies = [ "alloc-no-stdlib", ] @@ -101,9 +116,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.20" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -116,97 +131,88 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.10" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anyhow" -version = "1.0.99" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "apache-avro" -version = "0.17.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" dependencies = [ "bigdecimal", - "bzip2 0.4.4", + "bon", "crc32fast", - "digest", - "libflate", + "digest 0.11.3", "log", - "num-bigint", + "miniz_oxide 0.9.1", + "num-bigint 0.4.8", + "ouroboros", "quad-rand", - "rand 0.8.5", + "rand 0.10.2", "regex-lite", "serde", "serde_bytes", "serde_json", "snap", - "strum 0.26.3", - "strum_macros 0.26.4", - "thiserror 1.0.69", - "typed-builder 0.19.1", + "strum 0.28.0", + "thiserror 2.0.18", "uuid", - "xz2", - "zstd", + "zstd 0.13.3", ] [[package]] -name = "apache-avro" -version = "0.20.0" +name = "approx" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" dependencies = [ - "bigdecimal", - "bon", - "digest", - "log", - "miniz_oxide", - "num-bigint", - "quad-rand", - "rand 0.9.2", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "strum 0.27.2", - "strum_macros 0.27.2", - "thiserror 2.0.16", - "uuid", - "zstd", + "num-traits", +] + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", ] [[package]] @@ -223,15 +229,15 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f15b4c6b148206ff3a2b35002e08929c2462467b62b9c02036d9c34f9ef994" +checksum = "d965156f2220e94dcf676e04322b7b0b41c97e85cb3e1c365b8e56d0f70be999" dependencies = [ "arrow-arith", "arrow-array", @@ -242,6 +248,7 @@ dependencies = [ "arrow-ipc", "arrow-json", "arrow-ord", + "arrow-pyarrow", "arrow-row", "arrow-schema", "arrow-select", @@ -250,72 +257,109 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" +checksum = "ca16fdd1daeb1eccefb42df52ec65448ef33eec72488598675d46dfa9ce58923" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "num", + "num-traits", ] [[package]] name = "arrow-array" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" +checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "chrono-tz 0.10.4", + "chrono-tz", "half", - "hashbrown 0.15.5", - "num", + "hashbrown 0.17.1", + "libc", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-avro" +version = "60.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfd40a71a38e4366b28742cc46912b9289d89e0c858895c8276ac0fc83dacdc6" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "bytes", + "crc", + "flate2", + "indexmap 2.14.0", + "rand 0.10.2", + "serde", + "serde_json", + "snap", + "strum_macros 0.28.0", + "uuid", + "zstd 0.14.3", ] [[package]] name = "arrow-buffer" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169b1d5d6cb390dd92ce582b06b23815c7953e9dfaaea75556e89d890d19993d" +checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade" dependencies = [ "bytes", "half", - "num", + "num-bigint 0.5.1", + "num-traits", ] [[package]] name = "arrow-cast" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" +checksum = "baa83a0752814a1922d5ee75032524ca9bbfb7f60ee35a67ed4ab8b275260aaf" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", + "arrow-ord", "arrow-schema", "arrow-select", - "atoi", - "base64 0.22.1", + "atoi 3.1.1", + "base64 0.23.0", "chrono", - "comfy-table", "half", "lexical-core", - "num", + "num-traits", "ryu", ] +[[package]] +name = "arrow-cmp" +version = "60.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", +] + [[package]] name = "arrow-csv" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "012c9fef3f4a11573b2c74aec53712ff9fdae4a95f4ce452d1bbf088ee00f06b" +checksum = "e4572af05fadde76a063139805c089d0915b7eb64ed5bef14456c6e458a80b28" dependencies = [ "arrow-array", "arrow-cast", @@ -328,70 +372,87 @@ dependencies = [ [[package]] name = "arrow-data" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" +checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num", + "num-integer", + "num-traits", ] [[package]] name = "arrow-ipc" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ea5967e8b2af39aff5d9de2197df16e305f47f404781d3230b2dc672da5d92" +checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", + "arrow-select", "flatbuffers", - "lz4_flex", ] [[package]] name = "arrow-json" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5709d974c4ea5be96d900c01576c7c0b99705f4a3eec343648cb1ca863988a9c" +checksum = "d843cca4b5361d7667564083b151a30d816d2b7e7ce2baa53948e2188f087f06" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half", - "indexmap 2.11.0", + "indexmap 2.14.0", + "itoa", "lexical-core", "memchr", - "num", - "serde", + "num-traits", + "ryu", + "serde_core", "serde_json", "simdutf8", ] [[package]] name = "arrow-ord" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" +checksum = "4c58f2ae6dd1480fc413f7320a547e39dcc499c23d4f36ca19572b685811e6e4" dependencies = [ "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "arrow-select", ] +[[package]] +name = "arrow-pyarrow" +version = "60.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b6dfd82e81f080a076cb8ae5d53ea55b18db98edd4abb3960a6ca9d5d3c8184" +dependencies = [ + "arrow-array", + "arrow-data", + "arrow-schema", + "pyo3", +] + [[package]] name = "arrow-row" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bf7393166beaf79b4bed9bfdf19e97472af32ce5b6b48169d321518a08cae2" +checksum = "68f9db8345b0638416d3b56d117d262d13070fdef788c6d75b53a180215c477e" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,33 +463,35 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" +checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3" dependencies = [ - "serde", + "bitflags 2.13.0", + "serde_core", "serde_json", ] [[package]] name = "arrow-select" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" +checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", - "num", + "num-traits", ] [[package]] name = "arrow-string" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" +checksum = "ab1be9306b9a413b5b004e2a5a19caac728870d76e8891b351f040d554c93862" dependencies = [ "arrow-array", "arrow-buffer", @@ -436,7 +499,7 @@ dependencies = [ "arrow-schema", "arrow-select", "memchr", - "num", + "num-traits", "regex", "regex-syntax", ] @@ -469,118 +532,17 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-channel" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-compression" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" -dependencies = [ - "bzip2 0.5.2", - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", - "xz2", - "zstd", - "zstd-safe", -] - -[[package]] -name = "async-executor" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497c00e0fd83a72a79a39fcbd8e3e2f055d6f6c7e025f3b3d91f4f8e76527fb8" -dependencies = [ - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "pin-project-lite", - "slab", -] - -[[package]] -name = "async-fs" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f7e37c0ed80b2a977691c47dae8625cfb21e205827106c64f7c588766b2e50" -dependencies = [ - "async-lock", - "blocking", - "futures-lite", -] - -[[package]] -name = "async-io" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19634d6336019ef220f09fd31168ce5c184b295cbf80345437cc36094ef223ca" -dependencies = [ - "async-lock", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite", - "parking", - "polling", - "rustix 1.0.8", - "slab", - "windows-sys 0.60.2", -] - [[package]] name = "async-lock" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ "event-listener", "event-listener-strategy", "pin-project-lite", ] -[[package]] -name = "async-net" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b948000fad4873c1c9339d60f2623323a0cfd3816e5181033c6a5cb68b2accf7" -dependencies = [ - "async-io", - "blocking", - "futures-lite", -] - -[[package]] -name = "async-process" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65daa13722ad51e6ab1a1b9c01299142bc75135b337923cfa10e79bbbd669f00" -dependencies = [ - "async-channel", - "async-io", - "async-lock", - "async-signal", - "async-task", - "blocking", - "cfg-if", - "event-listener", - "futures-lite", - "rustix 1.0.8", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -589,33 +551,9 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", -] - -[[package]] -name = "async-signal" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f567af260ef69e1d52c2b560ce0ea230763e6fbb9214a85d768760a920e3e3c1" -dependencies = [ - "async-io", - "async-lock", - "atomic-waker", - "cfg-if", - "futures-core", - "futures-io", - "rustix 1.0.8", - "signal-hook-registry", - "slab", - "windows-sys 0.60.2", + "syn", ] -[[package]] -name = "async-task" -version = "4.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" - [[package]] name = "async-trait" version = "0.1.89" @@ -624,7 +562,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -636,6 +574,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d12c37257553c19e9aee6cba77814fdc9e07844abfc8b3a2c377cd920ee5a993" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -644,15 +591,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.6" +version = "1.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bc1b40fb26027769f16960d2f4a6bc20c4bb755d403e552c8c1a73af433c246" +checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" dependencies = [ "aws-credential-types", "aws-runtime", @@ -660,17 +607,18 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.63.6", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.1.0", "aws-smithy-types", "aws-types", "bytes", "fastrand", "hex", - "http 1.3.1", - "ring", + "http 1.5.0", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -680,9 +628,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.6" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d025db5d9f52cbc413b167136afb3d8aeea708c0d8884783cf6253be5e22f6f2" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -692,9 +640,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.13.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c953fe1ba023e6b7730c0d4b031d06f267f23a46167dcbd40316644b10a17ba" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -702,35 +650,36 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.30.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbfd150b5dbdb988bcc8fb1fe787eb6b7ee6180ca24da683b61ea5405f3d43ff" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ - "bindgen", "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] name = "aws-runtime" -version = "1.5.10" +version = "1.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c034a1bc1d70e16e7f4e4caf7e9f7693e4c9c24cd91cf17c2a0b21abaebc7c8b" +checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.63.6", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", + "bytes-utils", "fastrand", - "http 0.2.12", - "http-body 0.4.6", + "http 1.5.0", + "http-body 1.0.1", "percent-encoding", "pin-project-lite", "tracing", @@ -739,15 +688,17 @@ dependencies = [ [[package]] name = "aws-sdk-glue" -version = "1.119.0" +version = "1.153.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9170d379508a356718ca6c27ffa5c00b5077ecdf7ab5cb27a70834a58e00dd13" +checksum = "48b33661f9c0b02b6c23d42646f43ea9eab6db37fdcbd80f85fabe72a70b9d8e" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.63.6", "aws-smithy-json", + "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -755,21 +706,24 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-s3tables" -version = "1.37.0" +version = "1.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5b1975c398d8778a07ddd3de2bff50f03455907d4524c5e74eab05954ac95c" +checksum = "ebf4f0e9ec9c0cd8f978b0672086924c1b6c4c19776773bf1a6f0020d24c954d" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.63.6", "aws-smithy-json", + "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -777,21 +731,24 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sso" -version = "1.83.0" +version = "1.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cd43af212d2a1c4dedff6f044d7e1961e5d9e7cfe773d70f31d9842413886" +checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.63.6", "aws-smithy-json", + "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -799,21 +756,24 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.84.0" +version = "1.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20ec4a95bd48e0db7a424356a161f8d87bd6a4f0af37204775f0da03d9e39fc3" +checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.63.6", "aws-smithy-json", + "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -821,21 +781,24 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.85.0" +version = "1.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "410309ad0df4606bc721aff0d89c3407682845453247213a0ccc5ff8801ee107" +checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.63.6", "aws-smithy-json", + "aws-smithy-observability 0.2.6", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -844,37 +807,38 @@ dependencies = [ "aws-types", "fastrand", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.3.4" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084c34162187d39e3740cb635acd73c4e3a551a36146ad6fe8883c929c9f876c" +checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" dependencies = [ "aws-credential-types", - "aws-smithy-http", + "aws-smithy-http 0.63.6", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", - "http 1.3.1", + "http 1.5.0", "percent-encoding", - "sha2", + "sha2 0.11.0", "time", "tracing", ] [[package]] name = "aws-smithy-async" -version = "1.2.5" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" dependencies = [ "futures-util", "pin-project-lite", @@ -883,18 +847,40 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.3" +version = "0.63.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c4dacf2d38996cf729f55e7a762b30918229917eca115de45dfa8dfb97796c9" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", + "futures-util", + "http 1.5.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.5.0", + "http-body 1.0.1", + "http-body-util", "percent-encoding", "pin-project-lite", "pin-utils", @@ -903,57 +889,62 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147e8eea63a40315d704b97bf9bc9b8c1402ae94f89d5ad6f7550d963309da1b" +checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.3.27", - "h2 0.4.12", - "http 0.2.12", - "http 1.3.1", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper 1.7.0", - "hyper-rustls 0.24.2", - "hyper-rustls 0.27.7", + "h2", + "http 1.5.0", + "hyper", + "hyper-rustls", "hyper-util", "pin-project-lite", - "rustls 0.21.12", - "rustls 0.23.31", - "rustls-native-certs 0.8.1", + "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tower", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.61.5" +version = "0.62.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa31b350998e703e9826b2104dd6f63be0508666e1aba88137af060e8944047" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.1.0", "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.3" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.7" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" dependencies = [ "aws-smithy-types", "urlencoding", @@ -961,22 +952,24 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.9.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3946acbe1ead1301ba6862e712c7903ca9bb230bdf1fbd1b5ac54158ef2ab1f" +checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" dependencies = [ "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.64.0", "aws-smithy-http-client", - "aws-smithy-observability", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "bytes", "fastrand", "http 0.2.12", - "http 1.3.1", + "http 1.5.0", "http-body 0.4.6", "http-body 1.0.1", + "http-body-util", "pin-project-lite", "pin-utils", "tokio", @@ -985,15 +978,16 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07f5e0fc8a6b3f2303f331b94504bbf754d85488f402d6f1dd7a6080f99afe56" +checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.3.1", + "http 1.5.0", "pin-project-lite", "tokio", "tracing", @@ -1001,17 +995,50 @@ dependencies = [ ] [[package]] -name = "aws-smithy-types" -version = "1.3.2" +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ - "base64-simd", - "bytes", - "bytes-utils", - "futures-core", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + +[[package]] +name = "aws-smithy-types" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", "http 0.2.12", - "http 1.3.1", + "http 1.5.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -1028,22 +1055,23 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.10" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.8" +version = "1.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b069d19bf01e46298eaedd7c6f283fe565a59263e53eebec945f3e6398f42390" +checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", + "aws-smithy-schema 0.1.0", "aws-smithy-types", "rustc_version", "tracing", @@ -1051,41 +1079,26 @@ dependencies = [ [[package]] name = "backon" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand", "gloo-timers", "tokio", ] -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64" -version = "0.22.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" [[package]] name = "base64-simd" @@ -1099,19 +1112,19 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.8.0" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bigdecimal" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.8", "num-integer", "num-traits", "serde", @@ -1123,70 +1136,33 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.106", - "which", -] - [[package]] name = "bitflags" -version = "2.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" -dependencies = [ - "serde", -] - -[[package]] -name = "bitvec" -version = "1.0.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "blake2" -version = "0.10.6" +name = "bitflags" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" dependencies = [ - "digest", + "serde_core", ] [[package]] name = "blake3" -version = "1.8.2" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if", + "cfg-if 1.0.4", "constant_time_eq", + "cpufeatures 0.3.0", ] [[package]] @@ -1198,6 +1174,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -1208,23 +1193,29 @@ dependencies = [ ] [[package]] -name = "blocking" -version = "1.6.2" +name = "block-padding" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" dependencies = [ - "async-channel", - "async-task", - "futures-io", - "futures-lite", - "piper", + "hybrid-array", +] + +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", ] [[package]] name = "bon" -version = "3.7.2" +version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2529c31017402be841eb45892278a6c21a000c0a17643af326c73a73f83f0fb" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" dependencies = [ "bon-macros", "rustversion", @@ -1232,47 +1223,24 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.7.2" +version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82020dadcb845a345591863adb65d74fa8dc5c18a0b6d408470e13b7adc7005" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", "quote", "rustversion", - "syn 2.0.106", -] - -[[package]] -name = "borsh" -version = "1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" -dependencies = [ - "borsh-derive", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd1d3c0c2f5833f22386f252fe8ed005c7f59fdcddeef025c01b4c3b9fd9ac3" -dependencies = [ - "once_cell", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "brotli" -version = "8.0.2" +version = "9.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1281,58 +1249,60 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.0" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +checksum = "34c6783f42c21c952ad7112b354f0ba2f10760e9decea5b1b60523d6e270faaf" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bstr" -version = "1.12.0" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79" dependencies = [ "memchr", - "serde", + "regex-automata", + "serde_core", ] [[package]] name = "bumpalo" -version = "3.19.0" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] -name = "bytecheck" -version = "0.6.12" +name = "bytemuck" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" dependencies = [ - "bytecheck_derive", - "ptr_meta 0.1.4", - "simdutf8", + "bytemuck_derive", ] [[package]] -name = "bytecheck_derive" -version = "0.6.12" +name = "bytemuck_derive" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] -[[package]] -name = "bytemuck" -version = "1.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" - [[package]] name = "byteorder" version = "1.5.0" @@ -1341,9 +1311,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" dependencies = [ "serde", ] @@ -1359,48 +1329,28 @@ dependencies = [ ] [[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" +name = "cbc" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cc", - "pkg-config", + "cipher 0.4.4", ] [[package]] name = "cbc" -version = "0.1.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" dependencies = [ - "cipher", + "cipher 0.5.2", ] [[package]] name = "cc" -version = "1.2.36" +version = "1.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" dependencies = [ "find-msvc-tools", "jobserver", @@ -1409,19 +1359,16 @@ dependencies = [ ] [[package]] -name = "cexpr" -version = "0.6.0" +name = "cfg-if" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "cfg-if" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -1429,29 +1376,29 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.0", -] - -[[package]] -name = "chrono-tz" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" -dependencies = [ - "chrono", - "chrono-tz-build", - "phf 0.11.3", + "windows-link", ] [[package]] @@ -1461,18 +1408,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "phf 0.12.1", -] - -[[package]] -name = "chrono-tz-build" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" -dependencies = [ - "parse-zoneinfo", - "phf 0.11.3", - "phf_codegen", + "phf", ] [[package]] @@ -1481,26 +1417,26 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", - "inout", + "crypto-common 0.1.7", + "inout 0.1.4", ] [[package]] -name = "clang-sys" -version = "1.8.1" +name = "cipher" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ - "glob", - "libc", - "libloading", + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.2", ] [[package]] name = "clap" -version = "4.5.47" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1508,9 +1444,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.47" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -1520,92 +1456,73 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.47" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "clap_lex" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" - -[[package]] -name = "clipboard-win" -version = "5.4.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" -dependencies = [ - "error-code", -] +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "cmake" -version = "0.1.54" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] [[package]] -name = "colorchoice" -version = "1.0.4" +name = "cmov" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] -name = "colored" -version = "3.0.0" +name = "colorchoice" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" -dependencies = [ - "windows-sys 0.59.0", -] +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] -name = "comfy-table" -version = "7.2.0" +name = "colored" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8e18d0dca9578507f13f9803add0df13362b02c501c1c17734f0dbb52eaf0b" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "unicode-segmentation", - "unicode-width 0.2.1", + "windows-sys 0.61.2", ] [[package]] -name = "concurrent-queue" -version = "2.5.0" +name = "combine" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ - "crossbeam-utils", + "bytes", + "memchr", ] [[package]] -name = "console" -version = "0.15.11" +name = "const-oid" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" -dependencies = [ - "encode_unicode", - "libc", - "once_cell", - "unicode-width 0.2.1", - "windows-sys 0.59.0", -] +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const-oid" -version = "0.9.6" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" [[package]] name = "const-random" @@ -1622,16 +1539,31 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "tiny-keccak", ] +[[package]] +name = "const-str" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" + +[[package]] +name = "const_panic" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" +dependencies = [ + "typewit", +] + [[package]] name = "constant_time_eq" -version = "0.3.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "core-foundation" @@ -1660,14 +1592,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "core2" -version = "0.4.0" +name = "countio" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +checksum = "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" dependencies = [ - "memchr", + "futures-io", ] +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1677,28 +1615,38 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" dependencies = [ "crc-catalog", ] [[package]] name = "crc-catalog" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -1707,51 +1655,41 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ - "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -1761,43 +1699,96 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" dependencies = [ "csv-core", "itoa", "ryu", - "serde", + "serde_core", ] [[package]] name = "csv-core" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" dependencies = [ "memchr", ] [[package]] name = "ctor" -version = "0.2.9" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" +checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" dependencies = [ - "quote", - "syn 2.0.106", + "ctor-proc-macro", + "dtor", +] + +[[package]] +name = "ctor" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb22e947478ccf9dc44d8922042c677a63fbb88f2cb468521d1145816e5087cb" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + +[[package]] +name = "ctor-proc-macro" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher 0.4.4", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher 0.5.2", +] + +[[package]] +name = "ctutils" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" +dependencies = [ + "cmov", ] [[package]] @@ -1812,12 +1803,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -1831,21 +1822,20 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.106", + "syn", ] [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.106", + "syn", ] [[package]] @@ -1856,33 +1846,27 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.21.3", + "darling_core 0.23.0", "quote", - "syn 2.0.106", + "syn", ] -[[package]] -name = "dary_heap" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" - [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "crossbeam-utils", "hashbrown 0.14.5", "lock_api", @@ -1891,715 +1875,34 @@ dependencies = [ ] [[package]] -name = "datafusion" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a11e19a7ccc5bb979c95c1dceef663eab39c9061b3bbf8d1937faf0f03bf41f" -dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", - "async-trait", - "bytes", - "bzip2 0.5.2", - "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", - "flate2", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "parquet", - "rand 0.9.2", - "regex", - "sqlparser", - "tempfile", - "tokio", - "url", - "uuid", - "xz2", - "zstd", -] - -[[package]] -name = "datafusion-catalog" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94985e67cab97b1099db2a7af11f31a45008b282aba921c1e1d35327c212ec18" -dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "tokio", -] - -[[package]] -name = "datafusion-catalog-listing" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e002df133bdb7b0b9b429d89a69aa77b35caeadee4498b2ce1c7c23a99516988" -dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "log", - "object_store", - "tokio", -] - -[[package]] -name = "datafusion-cli" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85591b54815b0c9d3fbf3b0425b75e9fc49ef73e6886d6c19d622986e2551b53" -dependencies = [ - "arrow", - "async-trait", - "aws-config", - "aws-credential-types", - "clap", - "datafusion", - "dirs", - "env_logger", - "futures", - "mimalloc", - "object_store", - "parking_lot", - "parquet", - "regex", - "rustyline", - "tokio", - "url", -] - -[[package]] -name = "datafusion-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13242fc58fd753787b0a538e5ae77d356cb9d0656fa85a591a33c5f106267f6" -dependencies = [ - "ahash 0.8.12", - "apache-avro 0.17.0", - "arrow", - "arrow-ipc", - "base64 0.22.1", - "half", - "hashbrown 0.14.5", - "indexmap 2.11.0", - "libc", - "log", - "object_store", - "parquet", - "paste", - "recursive", - "sqlparser", - "tokio", - "web-time", -] - -[[package]] -name = "datafusion-common-runtime" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2239f964e95c3a5d6b4a8cde07e646de8995c1396a7fd62c6e784f5341db499" -dependencies = [ - "futures", - "log", - "tokio", -] - -[[package]] -name = "datafusion-datasource" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf792579bc8bf07d1b2f68c2d5382f8a63679cce8fbebfd4ba95742b6e08864" -dependencies = [ - "arrow", - "async-compression", - "async-trait", - "bytes", - "bzip2 0.5.2", - "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "flate2", - "futures", - "glob", - "itertools 0.14.0", - "log", - "object_store", - "parquet", - "rand 0.9.2", - "tempfile", - "tokio", - "tokio-util", - "url", - "xz2", - "zstd", -] - -[[package]] -name = "datafusion-datasource-avro" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de733d231abb0fba663ff60fd37bf7171fa8b2e46e8a99e41362001821d116e" -dependencies = [ - "apache-avro 0.17.0", - "arrow", - "async-trait", - "bytes", - "chrono", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "num-traits", - "object_store", - "tokio", -] - -[[package]] -name = "datafusion-datasource-csv" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc114f9a1415174f3e8d2719c371fc72092ef2195a7955404cfe6b2ba29a706" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "regex", - "tokio", -] - -[[package]] -name = "datafusion-datasource-json" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88dd5e215c420a52362b9988ecd4cefd71081b730663d4f7d886f706111fc75" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "serde_json", - "tokio", -] - -[[package]] -name = "datafusion-datasource-parquet" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33692acdd1fbe75280d14f4676fe43f39e9cb36296df56575aa2cac9a819e4cf" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "parquet", - "rand 0.9.2", - "tokio", -] - -[[package]] -name = "datafusion-doc" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e7b648387b0c1937b83cb328533c06c923799e73a9e3750b762667f32662c0" - -[[package]] -name = "datafusion-execution" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9609d83d52ff8315283c6dad3b97566e877d8f366fab4c3297742f33dcd636c7" -dependencies = [ - "arrow", - "dashmap", - "datafusion-common", - "datafusion-expr", - "futures", - "log", - "object_store", - "parking_lot", - "rand 0.9.2", - "tempfile", - "url", -] - -[[package]] -name = "datafusion-expr" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75230cd67f650ef0399eb00f54d4a073698f2c0262948298e5299fc7324da63" -dependencies = [ - "arrow", - "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", - "indexmap 2.11.0", - "paste", - "recursive", - "serde_json", - "sqlparser", -] - -[[package]] -name = "datafusion-expr-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fafb3a045ed6c49cfca0cd090f62cf871ca6326cc3355cb0aaf1260fa760b6" -dependencies = [ - "arrow", - "datafusion-common", - "indexmap 2.11.0", - "itertools 0.14.0", - "paste", -] - -[[package]] -name = "datafusion-functions" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9a9cf655265861a20453b1e58357147eab59bdc90ce7f2f68f1f35104d3bb" -dependencies = [ - "arrow", - "arrow-buffer", - "base64 0.22.1", - "blake2", - "blake3", - "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", - "hex", - "itertools 0.14.0", - "log", - "md-5", - "rand 0.9.2", - "regex", - "sha2", - "unicode-segmentation", - "uuid", -] - -[[package]] -name = "datafusion-functions-aggregate" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f07e49733d847be0a05235e17b884d326a2fd402c97a89fe8bcf0bfba310005" -dependencies = [ - "ahash 0.8.12", - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "half", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-aggregate-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4512607e10d72b0b0a1dc08f42cb5bd5284cb8348b7fea49dc83409493e32b1b" -dependencies = [ - "ahash 0.8.12", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-functions-nested" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab331806e34f5545e5f03396e4d5068077395b1665795d8f88c14ec4f1e0b7a" -dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-macros", - "datafusion-physical-expr-common", - "itertools 0.14.0", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-table" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ac2c0be983a06950ef077e34e0174aa0cb9e346f3aeae459823158037ade37" -dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", - "parking_lot", - "paste", -] - -[[package]] -name = "datafusion-functions-window" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f3d92731de384c90906941d36dcadf6a86d4128409a9c5cd916662baed5f53" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-window-common" -version = "48.0.1" +name = "defmt" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c679f8bf0971704ec8fd4249fcbb2eb49d6a12cc3e7a840ac047b4928d3541b5" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "bitflags 1.3.2", + "defmt-macros", ] [[package]] -name = "datafusion-macros" -version = "48.0.1" +name = "defmt-macros" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2821de7cb0362d12e75a5196b636a59ea3584ec1e1cc7dc6f5e34b9e8389d251" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" dependencies = [ - "datafusion-expr", + "defmt-parser", + "proc-macro2", "quote", - "syn 2.0.106", -] - -[[package]] -name = "datafusion-optimizer" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1594c7a97219ede334f25347ad8d57056621e7f4f35a0693c8da876e10dd6a53" -dependencies = [ - "arrow", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "indexmap 2.11.0", - "itertools 0.14.0", - "log", - "recursive", - "regex", - "regex-syntax", -] - -[[package]] -name = "datafusion-physical-expr" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6da0f2412088d23f6b01929dedd687b5aee63b19b674eb73d00c3eb3c883b7" -dependencies = [ - "ahash 0.8.12", - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", - "half", - "hashbrown 0.14.5", - "indexmap 2.11.0", - "itertools 0.14.0", - "log", - "paste", - "petgraph 0.8.2", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb0dbd9213078a593c3fe28783beaa625a4e6c6a6c797856ee2ba234311fb96" -dependencies = [ - "ahash 0.8.12", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "hashbrown 0.14.5", - "itertools 0.14.0", -] - -[[package]] -name = "datafusion-physical-optimizer" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d140854b2db3ef8ac611caad12bfb2e1e1de827077429322a6188f18fc0026a" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "itertools 0.14.0", - "log", - "recursive", -] - -[[package]] -name = "datafusion-physical-plan" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46cbdf21a01206be76d467f325273b22c559c744a012ead5018dfe79597de08" -dependencies = [ - "ahash 0.8.12", - "arrow", - "arrow-ord", - "arrow-schema", - "async-trait", - "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "futures", - "half", - "hashbrown 0.14.5", - "indexmap 2.11.0", - "itertools 0.14.0", - "log", - "parking_lot", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "datafusion-session" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a72733766ddb5b41534910926e8da5836622316f6283307fd9fb7e19811a59c" -dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-sql", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "tokio", + "syn", ] [[package]] -name = "datafusion-spark" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f469b1301a5a2dec8b944eacb7b1194ed8c0f8203ce460b7451bb6fa2e89060" -dependencies = [ - "arrow", - "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-macros", - "log", -] - -[[package]] -name = "datafusion-sql" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5162338cdec9cc7ea13a0e6015c361acad5ec1d88d83f7c86301f789473971f" -dependencies = [ - "arrow", - "bigdecimal", - "datafusion-common", - "datafusion-expr", - "indexmap 2.11.0", - "log", - "recursive", - "regex", - "sqlparser", -] - -[[package]] -name = "datafusion-sqllogictest" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a05ab096d3dd4c234f4ecf3c9c08b71396ac1810d4d57d3a13049bbf8a1b30" -dependencies = [ - "arrow", - "async-trait", - "bigdecimal", - "clap", - "datafusion", - "datafusion-spark", - "datafusion-substrait", - "futures", - "half", - "indicatif", - "itertools 0.14.0", - "log", - "object_store", - "rust_decimal", - "sqllogictest", - "sqlparser", - "tempfile", - "thiserror 2.0.16", - "tokio", -] - -[[package]] -name = "datafusion-substrait" -version = "48.0.1" +name = "defmt-parser" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6029c08002772fb2c38a191d21a8b03af78750f42d9b896222f6b0e16c84cf" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "async-recursion", - "async-trait", - "chrono", - "datafusion", - "itertools 0.14.0", - "object_store", - "pbjson-types", - "prost", - "substrait", - "tokio", - "url", + "thiserror 2.0.18", ] [[package]] @@ -2608,19 +1911,18 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] [[package]] name = "deranged" -version = "0.5.3" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d630bccd429a5bb5a64b5e94f693bfc48c9f8566418fda4c494cc94f911f87cc" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", - "serde", + "serde_core", ] [[package]] @@ -2641,7 +1943,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -2651,14 +1953,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.106", + "syn", ] [[package]] -name = "deunicode" -version = "1.6.2" +name = "des" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" +checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" +dependencies = [ + "cipher 0.5.2", +] [[package]] name = "diff" @@ -2672,12 +1977,24 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "6.0.0" @@ -2696,25 +2013,25 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.0", + "windows-sys 0.61.2", ] [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "dissimilar" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8975ffdaa0ef3661bfe02dbdcc06c9f829dfafe6a3c474de366a8d5e44276921" +checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" [[package]] name = "dlv-list" @@ -2725,6 +2042,18 @@ dependencies = [ "const-random", ] +[[package]] +name = "dns-lookup" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d" +dependencies = [ + "cfg-if 1.0.4", + "libc", + "socket2 0.6.4", + "windows-sys 0.60.2", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -2738,139 +2067,85 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" [[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" - -[[package]] -name = "educe" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" -dependencies = [ - "enum-ordinalize", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "either" -version = "1.15.0" +name = "dtor" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" dependencies = [ - "serde", + "dtor-proc-macro", ] [[package]] -name = "encode_unicode" -version = "1.0.0" +name = "dtor-proc-macro" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" [[package]] -name = "endian-type" -version = "0.1.2" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] -name = "enum-ordinalize" -version = "4.3.0" +name = "dyn-clone" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea0dcfa4e54eeb516fe454635a95753ddd39acda650ce703031c6973e315dd5" -dependencies = [ - "enum-ordinalize-derive", -] +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] -name = "enum-ordinalize-derive" -version = "4.3.1" +name = "either" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "serde", ] [[package]] -name = "env_filter" -version = "0.1.3" +name = "equivalent" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" -dependencies = [ - "log", - "regex", -] +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] -name = "env_logger" -version = "0.11.8" +name = "erased-serde" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", + "serde", + "serde_core", + "typeid", ] -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - [[package]] name = "errno" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] -[[package]] -name = "error-code" -version = "3.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" - -[[package]] -name = "escape8259" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" - [[package]] name = "etcetera" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "home", "windows-sys 0.48.0", ] [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -2896,71 +2171,60 @@ dependencies = [ ] [[package]] -name = "fallible-iterator" -version = "0.2.0" +name = "fastnum" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "faststr" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baec6a0289d7f1fe5665586ef7340af82e3037207bef60f5785e57569776f0c8" -dependencies = [ - "bytes", - "rkyv 0.8.11", - "serde", - "simdutf8", -] +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] -name = "fd-lock" -version = "4.0.4" +name = "faststr" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" dependencies = [ - "cfg-if", - "rustix 1.0.8", - "windows-sys 0.59.0", + "bytes", + "rkyv", + "serde", + "simdutf8", ] [[package]] name = "find-msvc-tools" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" - -[[package]] -name = "fixedbitset" -version = "0.5.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flatbuffers" -version = "25.2.10" +version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags", + "bitflags 2.13.0", "rustc_version", ] [[package]] name = "flate2" -version = "1.1.2" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "libz-rs-sys", - "miniz_oxide", + "miniz_oxide 0.8.9", + "zlib-rs", ] [[package]] @@ -2971,7 +2235,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin", + "spin 0.9.9", ] [[package]] @@ -2997,17 +2261,11 @@ dependencies = [ [[package]] name = "fragile" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" - -[[package]] -name = "fs-err" -version = "3.1.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d7be93788013f265201256d58f04936a8079ad5dc898743aa20525f503b683" +checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9" dependencies = [ - "autocfg", + "futures-core", ] [[package]] @@ -3016,17 +2274,11 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - [[package]] name = "futures" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -3039,9 +2291,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -3049,15 +2301,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -3077,51 +2329,44 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-lite" -version = "2.6.1" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -3131,22 +2376,44 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] [[package]] -name = "generator" -version = "0.8.7" +name = "g2gen" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" +checksum = "c5a7e0eb46f83a20260b850117d204366674e85d3a908d90865c78df9a6b1dfc" dependencies = [ - "cc", - "cfg-if", - "libc", - "log", - "rustversion", - "windows", + "g2poly", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "g2p" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "539e2644c030d3bf4cd208cb842d2ce2f80e82e6e8472390bcef83ceba0d80ad" +dependencies = [ + "g2gen", + "g2poly", +] + +[[package]] +name = "g2poly" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312d2295c7302019c395cfb90dacd00a82a2eabd700429bba9c7a3f38dbbe11b" + +[[package]] +name = "gearhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" +dependencies = [ + "cfg-if 0.1.10", ] [[package]] @@ -3159,13 +2426,33 @@ dependencies = [ "version_check", ] +[[package]] +name = "geo-traits" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" +dependencies = [ + "geo-types", +] + +[[package]] +name = "geo-types" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5" +dependencies = [ + "approx", + "num-traits", + "serde", +] + [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -3174,54 +2461,66 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ - "cfg-if", - "js-sys", + "cfg-if 1.0.4", "libc", - "r-efi", - "wasi 0.14.4+wasi-0.2.4", - "wasm-bindgen", + "r-efi 5.3.0", + "wasip2", ] [[package]] -name = "gimli" -version = "0.31.1" +name = "getrandom" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if 1.0.4", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] [[package]] -name = "glob" -version = "0.3.3" +name = "ghash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] [[package]] -name = "globset" -version = "0.4.16" +name = "git-version" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" dependencies = [ - "aho-corasick", - "bstr", - "log", - "regex-automata", - "regex-syntax", + "git-version-macro", ] [[package]] -name = "globwalk" -version = "0.9.1" +name = "git-version-macro" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ - "bitflags", - "ignore", - "walkdir", + "proc-macro2", + "quote", + "syn", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "gloo-timers" version = "0.3.0" @@ -3236,36 +2535,17 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.11.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "h2" -version = "0.4.12" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.3.1", - "indexmap 2.11.0", + "http 1.5.0", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -3274,13 +2554,15 @@ dependencies = [ [[package]] name = "half" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "cfg-if", + "bytemuck", + "cfg-if 1.0.4", "crunchy", "num-traits", + "zerocopy", ] [[package]] @@ -3288,19 +2570,12 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.8", -] [[package]] name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash 0.8.12", - "allocator-api2", -] [[package]] name = "hashbrown" @@ -3313,6 +2588,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.10.0" @@ -3322,17 +2603,64 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hdfs-native" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd181084003308224efddf737417832186839ce2882d2468ddad0114cfb3551" +dependencies = [ + "aes 0.9.3", + "base64 0.22.1", + "bitflags 2.13.0", + "bumpalo", + "bytes", + "cbc 0.2.1", + "chrono", + "cipher 0.5.2", + "crc", + "ctr 0.10.1", + "des", + "dns-lookup", + "futures", + "g2p", + "hex", + "hmac 0.13.0", + "libc", + "libloading", + "log", + "md-5 0.11.0", + "num-traits", + "once_cell", + "prost", + "prost-types", + "rand 0.10.2", + "regex", + "roxmltree", + "socket2 0.6.4", + "thiserror 2.0.18", + "tokio", + "url", + "uuid", + "whoami 2.1.2", +] + +[[package]] +name = "heapify" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" + [[package]] name = "heck" -version = "0.5.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] -name = "hermit-abi" -version = "0.5.2" +name = "heck" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hex" @@ -3340,6 +2668,28 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-xet" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" +dependencies = [ + "async-trait", + "bytes", + "http 1.5.0", + "more-asserts", + "serde", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "uuid", + "xet-client", + "xet-core-structures", + "xet-data", + "xet-runtime", +] + [[package]] name = "hive_metastore" version = "0.2.0" @@ -3358,7 +2708,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -3367,16 +2717,25 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3392,12 +2751,11 @@ dependencies = [ [[package]] name = "http" -version = "1.3.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", - "fnv", "itoa", ] @@ -3419,7 +2777,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.3.1", + "http 1.5.0", ] [[package]] @@ -3430,7 +2788,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.3.1", + "http 1.5.0", "http-body 1.0.1", "pin-project-lite", ] @@ -3447,131 +2805,89 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "humansize" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" -dependencies = [ - "libm", -] - [[package]] name = "humantime" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] -name = "hyper" -version = "0.14.32" +name = "hybrid-array" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", + "typenum", ] [[package]] name = "hyper" -version = "1.7.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", "futures-channel", - "futures-core", - "h2 0.4.12", - "http 1.3.1", - "http-body 1.0.1", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "pin-utils", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "rustls-native-certs 0.6.3", + "futures-core", + "h2", + "http 1.5.0", + "http-body 1.0.1", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", "tokio", - "tokio-rustls 0.24.1", + "want", ] [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.3.1", - "hyper 1.7.0", + "http 1.5.0", + "hyper", "hyper-util", - "rustls 0.23.31", - "rustls-native-certs 0.8.1", - "rustls-pki-types", + "rustls", + "rustls-native-certs", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tower-service", - "webpki-roots 1.0.2", ] [[package]] name = "hyper-util" -version = "0.1.16" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", - "futures-core", "futures-util", - "http 1.3.1", + "http 1.5.0", "http-body 1.0.1", - "hyper 1.7.0", + "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2 0.6.4", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] name = "iana-time-zone" -version = "0.1.63" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3593,13 +2909,15 @@ dependencies = [ [[package]] name = "iceberg" -version = "0.7.0" +version = "0.10.1" dependencies = [ + "aes-gcm", "anyhow", - "apache-avro 0.20.0", + "apache-avro", "array-init", "arrow-arith", "arrow-array", + "arrow-avro", "arrow-buffer", "arrow-cast", "arrow-ord", @@ -3613,49 +2931,54 @@ dependencies = [ "bimap", "bytes", "chrono", - "ctor", + "crc32fast", "derive_builder", "expect-test", + "fastnum", + "flate2", "fnv", + "form_urlencoded", "futures", + "iceberg-property-macro", "iceberg_test_utils", "itertools 0.13.0", + "minijinja", "mockall", "moka", "murmur3", - "num-bigint", "once_cell", - "opendal", "ordered-float 4.6.0", "parquet", + "parquet-geospatial", "pretty_assertions", - "rand 0.8.5", + "rand 0.9.5", "regex", - "reqsign", "reqwest", "roaring", - "rust_decimal", + "rstest", "serde", + "serde_arrow", "serde_bytes", "serde_derive", "serde_json", "serde_repr", "serde_with", - "smol", "strum 0.27.2", "tempfile", - "tera", - "thrift", "tokio", - "typed-builder 0.20.1", + "tracing", + "tracing-subscriber", + "typed-builder", + "typetag", "url", "uuid", - "zstd", + "zeroize", + "zstd 0.13.3", ] [[package]] name = "iceberg-cache-moka" -version = "0.7.0" +version = "0.10.1" dependencies = [ "iceberg", "moka", @@ -3663,38 +2986,36 @@ dependencies = [ [[package]] name = "iceberg-catalog-glue" -version = "0.7.0" +version = "0.10.1" dependencies = [ "anyhow", "async-trait", "aws-config", "aws-sdk-glue", - "ctor", "iceberg", + "iceberg-property-macro", + "iceberg-storage-opendal", "iceberg_test_utils", - "port_scanner", "serde_json", "tokio", - "tracing", ] [[package]] name = "iceberg-catalog-hms" -version = "0.7.0" +version = "0.10.1" dependencies = [ "anyhow", "async-trait", "chrono", - "ctor", "faststr", "hive_metastore", "iceberg", + "iceberg-storage-opendal", "iceberg_test_utils", "linkedbytes", "metainfo", "motore-macros", "pilota", - "port_scanner", "serde_json", "tokio", "tracing", @@ -3704,7 +3025,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-loader" -version = "0.7.0" +version = "0.10.1" dependencies = [ "async-trait", "iceberg", @@ -3712,98 +3033,100 @@ dependencies = [ "iceberg-catalog-hms", "iceberg-catalog-rest", "iceberg-catalog-s3tables", + "iceberg-catalog-sql", + "iceberg-storage-opendal", + "iceberg_test_utils", + "reqwest", + "rstest", + "sqlx", + "tempfile", "tokio", ] [[package]] name = "iceberg-catalog-rest" -version = "0.7.0" +version = "0.10.1" dependencies = [ "async-trait", + "aws-credential-types", + "aws-sigv4", + "base64 0.22.1", + "bytes", "chrono", - "ctor", - "http 1.3.1", + "futures", + "http 1.5.0", "iceberg", "iceberg_test_utils", "itertools 0.13.0", "mockito", - "port_scanner", + "moka", "reqwest", "serde", "serde_derive", "serde_json", + "sha2 0.10.9", "tokio", "tracing", - "typed-builder 0.20.1", + "typed-builder", "uuid", ] [[package]] name = "iceberg-catalog-s3tables" -version = "0.7.0" +version = "0.10.1" dependencies = [ + "arrow-array", + "arrow-schema", "async-trait", "aws-config", "aws-sdk-s3tables", + "futures", "iceberg", + "iceberg-storage-opendal", "iceberg_test_utils", "itertools 0.13.0", + "parquet", "tokio", + "uuid", ] [[package]] name = "iceberg-catalog-sql" -version = "0.7.0" +version = "0.10.1" dependencies = [ "async-trait", "iceberg", - "iceberg_test_utils", + "iceberg-property-macro", "itertools 0.13.0", "regex", "sqlx", + "strum 0.27.2", "tempfile", "tokio", - "typed-builder 0.20.1", -] - -[[package]] -name = "iceberg-datafusion" -version = "0.7.0" -dependencies = [ - "anyhow", - "async-trait", - "datafusion", - "expect-test", - "futures", - "iceberg", - "parquet", - "tempfile", - "tokio", - "uuid", + "tracing", ] [[package]] name = "iceberg-examples" -version = "0.7.0" +version = "0.10.1" dependencies = [ "futures", "iceberg", "iceberg-catalog-rest", + "iceberg-storage-opendal", "tokio", ] [[package]] name = "iceberg-integration-tests" -version = "0.7.0" +version = "0.10.1" dependencies = [ "arrow-array", "arrow-schema", - "ctor", - "datafusion", "futures", "iceberg", "iceberg-catalog-rest", - "iceberg-datafusion", + "iceberg-storage-opendal", "iceberg_test_utils", "ordered-float 2.10.1", "parquet", @@ -3812,59 +3135,61 @@ dependencies = [ ] [[package]] -name = "iceberg-playground" -version = "0.7.0" +name = "iceberg-property-macro" +version = "0.10.1" dependencies = [ - "anyhow", - "clap", - "datafusion", - "datafusion-cli", - "dirs", - "fs-err", "iceberg", - "iceberg-catalog-rest", - "iceberg-datafusion", - "mimalloc", - "stacker", - "tokio", - "toml", - "tracing", - "tracing-subscriber", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "trybuild", ] [[package]] -name = "iceberg-sqllogictest" -version = "0.7.0" +name = "iceberg-storage-opendal" +version = "0.10.1" dependencies = [ "anyhow", "async-trait", - "datafusion", - "datafusion-sqllogictest", - "enum-ordinalize", - "indicatif", + "bytes", + "cfg-if 1.0.4", + "futures", + "iceberg", + "iceberg-property-macro", + "iceberg_test_utils", + "opendal", + "reqsign-aws-v4", + "reqsign-core", + "reqwest", "serde", - "sqllogictest", + "serde_json", + "tempfile", "tokio", - "toml", "tracing", + "typetag", + "url", ] [[package]] name = "iceberg_test_utils" -version = "0.7.0" +version = "0.10.1" dependencies = [ - "tracing", + "iceberg", + "tokio", "tracing-subscriber", ] [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -3872,9 +3197,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -3885,11 +3210,10 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -3900,42 +3224,38 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", - "stable_deref_trait", - "tinystr", "writeable", "yoke", "zerofrom", @@ -3962,30 +3282,14 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", ] -[[package]] -name = "ignore" -version = "0.4.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" -dependencies = [ - "crossbeam-deque", - "globset", - "log", - "memchr", - "regex-automata", - "same-file", - "walkdir", - "winapi-util", -] - [[package]] name = "indexmap" version = "1.9.3" @@ -3999,26 +3303,14 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.17.1", "serde", -] - -[[package]] -name = "indicatif" -version = "0.17.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" -dependencies = [ - "console", - "number_prefix", - "portable-atomic", - "unicode-width 0.2.1", - "web-time", + "serde_core", ] [[package]] @@ -4027,67 +3319,50 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "block-padding", + "block-padding 0.3.3", "generic-array", ] [[package]] -name = "integer-encoding" -version = "3.0.4" +name = "inout" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "block-padding 0.4.2", + "hybrid-array", +] [[package]] name = "integer-encoding" -version = "4.0.2" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d762194228a2f1c11063e46e32e5acb96e66e906382b9eb5441f2e0504bbd5a" +checksum = "14c00403deb17c3221a1fe4fb571b9ed0370b3dcd116553c77fa294a3d918699" dependencies = [ "async-trait", "tokio", ] [[package]] -name = "io-uring" -version = "0.7.10" +name = "inventory" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ - "bitflags", - "cfg-if", - "libc", + "rustversion", ] [[package]] name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.8" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" -dependencies = [ - "memchr", - "serde", -] +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - -[[package]] -name = "itertools" -version = "0.12.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -4109,89 +3384,155 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.15" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +checksum = "ccfe6121cbe750cf81efa362d85c0bde7ea298ec43092d3a193baca59cdbd634" dependencies = [ + "defmt", "jiff-static", + "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", - "serde", + "serde_core", + "wasm-bindgen", + "windows-link", ] [[package]] name = "jiff-static" -version = "0.2.15" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e165e897f662d428f3cd3828a919dbe067c2d42bb1031eede74ef9d27ecdedd2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6142247df1a93c2b3587402a19710be3e6e942f1581a1702e76408f2c21d6590" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if 1.0.4", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "rustc_version", + "simd_cesu8", + "syn", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", ] [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.4.3", "libc", ] [[package]] name = "js-sys" -version = "0.3.78" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0b063578492ceec17683ef2f8c5e89121fbd0b172cbc280635ab7567db2738" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ - "once_cell", + "cfg-if 1.0.4", + "futures-util", "wasm-bindgen", ] [[package]] -name = "jsonwebtoken" -version = "9.3.1" +name = "konst" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +checksum = "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" dependencies = [ - "base64 0.22.1", - "js-sys", - "pem", - "ring", - "serde", - "serde_json", - "simple_asn1", + "const_panic", + "konst_proc_macros", + "typewit", ] +[[package]] +name = "konst_proc_macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" + [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.9", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "lexical-core" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -4202,120 +3543,80 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ "lexical-parse-integer", "lexical-util", - "static_assertions", ] [[package]] name = "lexical-parse-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "lexical-util" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" -dependencies = [ - "static_assertions", -] +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "lexical-write-float" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" -dependencies = [ - "lexical-util", - "lexical-write-integer", - "static_assertions", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ "lexical-util", - "static_assertions", + "lexical-write-integer", ] [[package]] -name = "libc" -version = "0.2.175" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" - -[[package]] -name = "libflate" -version = "2.1.0" +name = "lexical-write-integer" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" dependencies = [ - "adler32", - "core2", - "crc32fast", - "dary_heap", - "libflate_lz77", + "lexical-util", ] [[package]] -name = "libflate_lz77" -version = "2.1.0" +name = "libc" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" -dependencies = [ - "core2", - "hashbrown 0.14.5", - "rle-decode-fast", -] +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" -version = "0.8.8" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" dependencies = [ - "cfg-if", - "windows-targets 0.48.5", + "cfg-if 1.0.4", + "windows-link", ] [[package]] name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - -[[package]] -name = "libmimalloc-sys" -version = "0.1.44" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" -dependencies = [ - "cc", - "libc", -] +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.9" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ - "bitflags", + "bitflags 2.13.0", "libc", - "redox_syscall", + "plain", + "redox_syscall 0.9.0", ] [[package]] @@ -4330,25 +3631,10 @@ dependencies = [ ] [[package]] -name = "libtest-mimic" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5297962ef19edda4ce33aaa484386e0a5b3d7f2f4e037cbeee00503ef6b29d33" -dependencies = [ - "anstream", - "anstyle", - "clap", - "escape8259", -] - -[[package]] -name = "libz-rs-sys" -version = "0.5.2" +name = "link-section" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" -dependencies = [ - "zlib-rs", -] +checksum = "e333fe507b738576d6da5bb3f1a7d7a1c80307ed9ef31624c057d844c19c93e9" [[package]] name = "linked-hash-map" @@ -4368,51 +3654,37 @@ dependencies = [ ] [[package]] -name = "linux-raw-sys" -version = "0.4.15" +name = "linktime-proc-macro" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +checksum = "8c7b0a3383c2a1002d11349c92c85a666a5fb679e96c79d782cf0dbe557fd6ee" [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" - -[[package]] -name = "loom" -version = "0.7.2" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator", - "scoped-tls", - "tracing", - "tracing-subscriber", -] +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru-slab" @@ -4422,22 +3694,35 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.11.5" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ "twox-hash", ] [[package]] -name = "lzma-sys" -version = "0.1.20" +name = "lz4_flex" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ - "cc", - "libc", - "pkg-config", + "twox-hash", +] + +[[package]] +name = "marrow" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "789b43b5511b6037f7230829f87eac922fcfa13568362a5aa2eccb1bc0b07cb0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "bytemuck", + "half", + "serde", ] [[package]] @@ -4455,15 +3740,40 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if", - "digest", + "cfg-if 1.0.4", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if 1.0.4", + "digest 0.11.3", +] + +[[package]] +name = "mea" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2640d335e7273dacdcf51044026139b2e269c3bb0dfc3f8cb3496b85e3f6a42c" +dependencies = [ + "slab", ] [[package]] name = "memchr" -version = "2.7.5" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "memo-map" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" [[package]] name = "memoffset" @@ -4480,46 +3790,51 @@ version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdc67a1d6ef0340a7f5152b9fd34ad7477b4d518920f7557267f1fc6e5a62641" dependencies = [ - "ahash 0.8.12", + "ahash", "faststr", "paste", - "rustc-hash 2.1.1", + "rustc-hash", "tokio", ] [[package]] -name = "mimalloc" -version = "0.1.48" +name = "minijinja" +version = "2.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" +checksum = "86886cf6dbf4e614b19c9a1eec9775f021869d7eadde0fc73921a81b90c9b4c9" dependencies = [ - "libmimalloc-sys", + "memo-map", + "serde", ] [[package]] -name = "minimal-lexical" -version = "0.2.1" +name = "miniz_oxide" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", ] [[package]] name = "mio" -version = "1.0.4" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4528,7 +3843,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "downcast", "fragile", "mockall_derive", @@ -4542,29 +3857,30 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "mockito" -version = "1.7.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7760e0e418d9b7e5777c0374009ca4c93861b9066f18cb334a20ce50ab63aa48" +checksum = "90820618712cab19cfc46b274c6c22546a82affcb3c3bdf0f29e3db8e1bb92c0" dependencies = [ "assert-json-diff", "bytes", "colored", - "futures-util", - "http 1.3.1", + "futures-core", + "http 1.5.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper", "hyper-util", "log", - "rand 0.9.2", + "pin-project-lite", + "rand 0.9.5", "regex", "serde_json", "serde_urlencoded", @@ -4574,26 +3890,30 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.10" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ "async-lock", "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", + "equivalent", "event-listener", "futures-util", - "loom", "parking_lot", "portable-atomic", - "rustc_version", "smallvec", "tagptr", - "thiserror 1.0.69", "uuid", ] +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + [[package]] name = "motore" version = "0.4.1" @@ -4614,33 +3934,27 @@ checksum = "b40e46c845ac234bcba19db7ab252bc2778cbadd516a466d2f12b1580852d136" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - [[package]] name = "munge" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7feb0b48aa0a25f9fe0899482c6e1379ee7a11b24a53073eacdecb9adb6dc60" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" dependencies = [ "munge_macro", ] [[package]] name = "munge_macro" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2e3795a5d2da581a8b252fec6022eee01aea10161a4d1bf237d4cbe47f7e988" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -4655,97 +3969,70 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" -[[package]] -name = "nibble_vec" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" -dependencies = [ - "smallvec", -] - [[package]] name = "nix" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags", - "cfg-if", + "bitflags 2.13.0", + "cfg-if 1.0.4", "cfg_aliases", "libc", "memoffset", ] [[package]] -name = "nix" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", -] - -[[package]] -name = "nom" -version = "7.1.3" +name = "ntapi" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" dependencies = [ - "memchr", - "minimal-lexical", + "winapi", ] [[package]] name = "nu-ansi-term" -version = "0.50.1" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] -name = "num" -version = "0.4.3" +name = "num-bigint" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ - "num-bigint", - "num-complex", "num-integer", - "num-iter", - "num-rational", "num-traits", + "serde", ] [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] name = "num-bigint-dig" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ - "byteorder", "lazy_static", "libm", "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "zeroize", ] @@ -4761,9 +4048,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -4776,165 +4063,342 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" dependencies = [ - "autocfg", "num-integer", "num-traits", ] [[package]] -name = "num-rational" -version = "0.4.2" +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.0", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "opendal" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" +dependencies = [ + "ctor 1.0.8", + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-azdls", + "opendal-service-fs", + "opendal-service-gcs", + "opendal-service-hdfs-native", + "opendal-service-hf", + "opendal-service-oss", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" +dependencies = [ + "anyhow", + "base64 0.23.0", + "bytes", + "futures", + "http 1.5.0", + "jiff", + "log", + "md-5 0.11.0", + "mea", + "percent-encoding", + "quick-xml", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +dependencies = [ + "bytes", + "futures", + "http 1.5.0", + "http-body 1.0.1", + "opendal-core", + "reqwest", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" dependencies = [ - "num-bigint", - "num-integer", - "num-traits", + "futures", + "http 1.5.0", + "mea", + "opendal-core", ] [[package]] -name = "num-traits" -version = "0.2.19" +name = "opendal-layer-logging" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" dependencies = [ - "autocfg", - "libm", + "log", + "opendal-core", ] [[package]] -name = "num_cpus" -version = "1.17.0" +name = "opendal-layer-retry" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" dependencies = [ - "hermit-abi", - "libc", + "backon", + "log", + "opendal-core", ] [[package]] -name = "num_enum" -version = "0.7.4" +name = "opendal-layer-timeout" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a973b4e44ce6cad84ce69d797acf9a044532e4184c4f267913d1b546a0727b7a" +checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" dependencies = [ - "num_enum_derive", - "rustversion", + "opendal-core", + "tokio", ] [[package]] -name = "num_enum_derive" -version = "0.7.4" +name = "opendal-service-azdls" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d" +checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.106", + "base64 0.23.0", + "bytes", + "http 1.5.0", + "log", + "mea", + "opendal-core", + "opendal-service-azure-common", + "quick-xml", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "serde_json", ] [[package]] -name = "number_prefix" -version = "0.4.0" +name = "opendal-service-azure-common" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" +dependencies = [ + "http 1.5.0", + "opendal-core", +] [[package]] -name = "object" -version = "0.36.7" +name = "opendal-service-fs" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "826c4e17a30643b888fe983897f9a4b23b07066e1d069727a923cc8fb419a702" dependencies = [ - "memchr", + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", ] [[package]] -name = "object_store" -version = "0.12.3" +name = "opendal-service-gcs" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc4f07659e11cd45a341cd24d71e683e3be65d9ff1f8150061678fe60437496" +checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" dependencies = [ "async-trait", - "base64 0.22.1", "bytes", - "chrono", - "form_urlencoded", - "futures", - "http 1.3.1", - "http-body-util", - "humantime", - "hyper 1.7.0", - "itertools 0.14.0", - "md-5", - "parking_lot", + "http 1.5.0", + "log", + "opendal-core", "percent-encoding", - "quick-xml 0.38.3", - "rand 0.9.2", - "reqwest", - "ring", - "rustls-pemfile 2.2.0", + "quick-xml", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", "serde", "serde_json", - "serde_urlencoded", - "thiserror 2.0.16", "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", ] [[package]] -name = "once_cell" -version = "1.21.3" +name = "opendal-service-hdfs-native" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "bdeb0c6fd99ffe3acf9c58f9664842ea549e4fd984c8202a5e656d18de46dae4" +dependencies = [ + "bytes", + "futures", + "hdfs-native", + "log", + "opendal-core", + "serde", +] [[package]] -name = "once_cell_polyfill" -version = "1.70.1" +name = "opendal-service-hf" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" +dependencies = [ + "bytes", + "hf-xet", + "http 1.5.0", + "log", + "opendal-core", + "percent-encoding", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-oss" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" +dependencies = [ + "bytes", + "http 1.5.0", + "log", + "opendal-core", + "quick-xml", + "reqsign-aliyun-oss", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", +] [[package]] -name = "opendal" -version = "0.54.0" +name = "opendal-service-s3" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb9838d0575c6dbaf3fcec7255af8d5771996d4af900bbb6fa9a314dec00a1a" +checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" dependencies = [ - "anyhow", - "backon", - "base64 0.22.1", + "base64 0.23.0", "bytes", - "chrono", - "crc32c", - "futures", - "getrandom 0.2.16", - "http 1.3.1", - "http-body 1.0.1", + "crc-fast", + "http 1.5.0", "log", - "md-5", - "percent-encoding", - "quick-xml 0.37.5", - "reqsign", - "reqwest", + "md-5 0.11.0", + "opendal-core", + "quick-xml", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", "serde", - "serde_json", - "tokio", - "uuid", + "url", ] [[package]] name = "openssl-probe" -version = "0.1.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "option-ext" @@ -4958,7 +4422,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", "serde", ] @@ -4973,16 +4437,43 @@ dependencies = [ ] [[package]] -name = "outref" -version = "0.5.2" +name = "os_str_bytes" +version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +dependencies = [ + "memchr", +] [[package]] -name = "owo-colors" -version = "4.2.2" +name = "ouroboros" +version = "0.18.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48dd4f4a2c8405440fd0462561f0e5806bd0f77e86f51c761481bdd4018b545e" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn", +] + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "parking" @@ -4992,9 +4483,9 @@ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", "parking_lot_core", @@ -5002,60 +4493,63 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "parquet" -version = "55.2.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" +checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4" dependencies = [ - "ahash 0.8.12", + "ahash", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.22.1", + "base64 0.23.0", "brotli", "bytes", "chrono", "flate2", "futures", "half", - "hashbrown 0.15.5", - "lz4_flex", - "num", - "num-bigint", - "object_store", - "paste", + "hashbrown 0.17.1", + "lz4_flex 0.14.0", + "num-bigint 0.5.1", + "num-integer", + "num-traits", + "parquet-geospatial", + "ring", "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", - "zstd", + "zstd 0.14.3", ] [[package]] -name = "parse-zoneinfo" -version = "0.3.1" +name = "parquet-geospatial" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +checksum = "eef66af500dd4e6f546d88ed582277559f4dba847ad0f1388c118ea27908fddb" dependencies = [ - "regex", + "arrow-schema", + "geo-traits", + "serde", + "serde_json", + "wkb", ] [[package]] @@ -5064,61 +4558,24 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pbjson" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" -dependencies = [ - "base64 0.21.7", - "serde", -] - -[[package]] -name = "pbjson-build" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" -dependencies = [ - "heck", - "itertools 0.13.0", - "prost", - "prost-types", -] - -[[package]] -name = "pbjson-types" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54e5e7bfb1652f95bc361d76f3c780d8e526b134b85417e774166ee941f0887" -dependencies = [ - "bytes", - "chrono", - "pbjson", - "pbjson-build", - "prost", - "prost-build", - "serde", -] - [[package]] name = "pbkdf2" version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", - "hmac", + "digest 0.10.7", + "hmac 0.12.1", ] [[package]] name = "pem" -version = "3.0.5" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ "base64 0.22.1", - "serde", + "serde_core", ] [[package]] @@ -5136,117 +4593,13 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "pest" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" -dependencies = [ - "memchr", - "thiserror 2.0.16", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "pest_meta" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5" -dependencies = [ - "pest", - "sha2", -] - -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap 2.11.0", -] - -[[package]] -name = "petgraph" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap 2.11.0", - "serde", -] - -[[package]] -name = "phf" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" -dependencies = [ - "phf_shared 0.11.3", -] - [[package]] name = "phf" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared 0.12.1", -] - -[[package]] -name = "phf_codegen" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" -dependencies = [ - "phf_generator", - "phf_shared 0.11.3", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared 0.11.3", - "rand 0.8.5", -] - -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher", + "phf_shared", ] [[package]] @@ -5264,12 +4617,12 @@ version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "520e32b295867834617b8a352b6fd4837e578f328f4e48813db4904ebc0d70b2" dependencies = [ - "ahash 0.8.12", + "ahash", "anyhow", "async-recursion", "bytes", "faststr", - "integer-encoding 4.0.2", + "integer-encoding", "lazy_static", "linkedbytes", "ordered-float 4.6.0", @@ -5282,29 +4635,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -5312,17 +4665,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" -dependencies = [ - "atomic-waker", - "fastrand", - "futures-io", -] - [[package]] name = "pkcs1" version = "0.7.5" @@ -5340,12 +4682,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ - "aes", - "cbc", + "aes 0.8.4", + "cbc 0.1.2", "der", "pbkdf2", "scrypt", - "sha2", + "sha2 0.10.9", "spki", ] @@ -5363,79 +4705,48 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "polling" -version = "3.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5bd19146350fe804f7cb2669c851c03d69da628803dab0d98018142aaa5d829" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi", - "pin-project-lite", - "rustix 1.0.8", - "windows-sys 0.60.2", -] - -[[package]] -name = "port_scanner" -version = "0.1.5" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325a6d2ac5dee293c3b2612d4993b98aec1dff096b0a2dae70ed7d95784a05da" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] -name = "portable-atomic" -version = "1.11.1" +name = "plain" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] -name = "portable-atomic-util" -version = "0.2.4" +name = "polyval" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "portable-atomic", + "cfg-if 1.0.4", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", ] [[package]] -name = "postgres-protocol" -version = "0.6.8" +name = "portable-atomic" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" -dependencies = [ - "base64 0.22.1", - "byteorder", - "bytes", - "fallible-iterator", - "hmac", - "md-5", - "memchr", - "rand 0.9.2", - "sha2", - "stringprep", -] +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] -name = "postgres-types" -version = "0.2.9" +name = "portable-atomic-util" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ - "bytes", - "fallible-iterator", - "postgres-protocol", + "portable-atomic", ] [[package]] name = "potential_utf" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -5457,9 +4768,9 @@ dependencies = [ [[package]] name = "predicates" -version = "3.1.3" +version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" dependencies = [ "anstyle", "predicates-core", @@ -5467,15 +4778,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" [[package]] name = "predicates-tree" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" dependencies = [ "predicates-core", "termtree", @@ -5498,126 +4809,156 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.106", + "syn", ] [[package]] name = "proc-macro-crate" -version = "3.3.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ "toml_edit", ] [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] -name = "prost" -version = "0.13.5" +name = "proc-macro2-diagnostics" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ - "bytes", - "prost-derive", + "proc-macro2", + "quote", + "syn", + "version_check", + "yansi", ] [[package]] -name = "prost-build" -version = "0.13.5" +name = "prost" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ - "heck", - "itertools 0.14.0", - "log", - "multimap", - "once_cell", - "petgraph 0.7.1", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 2.0.106", - "tempfile", + "bytes", + "prost-derive", ] [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "prost-types" -version = "0.13.5" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] [[package]] -name = "psm" -version = "0.1.26" +name = "ptr_meta" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e944464ec8536cd1beb0bbfd96987eb5e3b72f2ecdafdc5c769a37f1fa2ae1f" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" dependencies = [ - "cc", + "ptr_meta_derive", ] [[package]] -name = "ptr_meta" -version = "0.1.4" +name = "ptr_meta_derive" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ - "ptr_meta_derive 0.1.4", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "ptr_meta" -version = "0.3.0" +name = "pyiceberg_core_rust" +version = "0.10.1" +dependencies = [ + "arrow", + "iceberg", + "pyo3", +] + +[[package]] +name = "pyo3" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "700d18fa267b73b9b521fd7e13580e2f446916f176cacee1ab63fcc8191f1655" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90" +checksum = "7b3fc0c4d08f6bb10e71fe39dfb9e2f59c6eb6854e22ec8092f50c69a4499adb" dependencies = [ - "ptr_meta_derive 0.3.0", + "target-lexicon", ] [[package]] -name = "ptr_meta_derive" -version = "0.1.4" +name = "pyo3-ffi" +version = "0.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc0b8e19df29aad7086cf977bb0c2a2f143e30567eb113e9cf72b62ca698330" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +checksum = "6100e8a4b5eba53afaa5ed078364851a0b2499c553a44c31026b929049b49dc6" dependencies = [ "proc-macro2", + "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn", ] [[package]] -name = "ptr_meta_derive" -version = "0.3.0" +name = "pyo3-macros-backend" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1" +checksum = "6143877a16e82b5a727b7127ff4cd86858a24a28d745d72f43e6f227c7b1bdb3" dependencies = [ + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -5628,19 +4969,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" -version = "0.37.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quick-xml" -version = "0.38.3" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -5648,19 +4979,19 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", - "rustls 0.23.31", - "socket2 0.6.0", - "thiserror 2.0.16", + "rustc-hash", + "rustls", + "socket2 0.6.4", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -5668,20 +4999,22 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ + "aws-lc-rs", "bytes", - "getrandom 0.3.3", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.2", + "rand 0.10.2", + "rand_pcg", "ring", - "rustc-hash 2.1.1", - "rustls 0.23.31", + "rustc-hash", + "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.16", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -5689,23 +5022,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.0", + "socket2 0.6.4", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -5717,35 +5050,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - -[[package]] -name = "radix_trie" -version = "0.2.1" +name = "r-efi" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" -dependencies = [ - "endian-type", - "nibble_vec", -] +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rancor" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947" +checksum = "daff8b7b3ccf5f7ba270b3e7a0a4d4c701c5797e38dec27c7e2c3dbb830fed1c" dependencies = [ - "ptr_meta 0.3.0", + "ptr_meta", ] [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -5755,12 +5078,23 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -5780,7 +5114,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -5789,46 +5123,59 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "serde", ] [[package]] name = "rand_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", ] [[package]] -name = "recursive" -version = "0.1.1" +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "recursive-proc-macro-impl", - "stacker", + "rand_core 0.10.1", ] [[package]] -name = "recursive-proc-macro-impl" -version = "0.1.1" +name = "redb" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" dependencies = [ - "quote", - "syn 2.0.106", + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.0", ] [[package]] name = "redox_syscall" -version = "0.5.17" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" dependencies = [ - "bitflags", + "bitflags 2.13.0", ] [[package]] @@ -5837,36 +5184,36 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "libredox", - "thiserror 2.0.16", + "thiserror 2.0.18", ] [[package]] name = "ref-cast" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "regex" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -5876,9 +5223,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -5887,104 +5234,185 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.6" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] -name = "regress" -version = "0.10.4" +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "rend" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "663ba70707f96e871406fe10d68128412e619b06d1d47cb91c3a4c6501176240" + +[[package]] +name = "reqsign-aliyun-oss" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145bb27393fe455dd64d6cbc8d059adfa392590a45eadf079c01b11857e7b010" +checksum = "a5e6d659fcdbca6fe2d7ef109c2e28499b7be80501f1bb86c10caf5ec8ac1219" dependencies = [ - "hashbrown 0.15.5", - "memchr", + "anyhow", + "form_urlencoded", + "http 1.5.0", + "log", + "percent-encoding", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", ] [[package]] -name = "rend" -version = "0.4.2" +name = "reqsign-aws-core" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c" dependencies = [ - "bytecheck", + "bytes", + "form_urlencoded", + "hex", + "http 1.5.0", + "log", + "percent-encoding", + "quick-xml", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", ] [[package]] -name = "rend" -version = "0.5.2" +name = "reqsign-aws-v4" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35e8a6bf28cd121053a66aa2e6a2e3eaffad4a60012179f0e864aa5ffeff215" +checksum = "4a0c499f4ed12d04c3d4c78fe4cb01aee22c9dae22848c14db2c6313d9df9f43" +dependencies = [ + "bytes", + "http 1.5.0", + "log", + "quick-xml", + "reqsign-aws-core", + "reqsign-core", + "serde", +] [[package]] -name = "reqsign" -version = "0.16.5" +name = "reqsign-azure-storage" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +checksum = "2824e7da3c2cc42ac3406c674eb57c89127fdcd97f3a73c608cfc680505ea134" dependencies = [ "anyhow", - "async-trait", - "base64 0.22.1", - "chrono", + "base64 0.23.0", + "bytes", "form_urlencoded", - "getrandom 0.2.16", + "http 1.5.0", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-core" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa" +dependencies = [ + "anyhow", + "base64 0.23.0", + "bytes", + "futures", "hex", - "hmac", - "home", - "http 1.3.1", - "jsonwebtoken", + "hmac 0.13.0", + "http 1.5.0", + "jiff", "log", - "once_cell", "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.5", - "reqwest", "rsa", - "rust-ini", "serde", "serde_json", - "sha1", - "sha2", + "sha1 0.11.0", + "sha2 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "663d9d55abd0df0830ef0ae43708297cc1371cf4e8ca91f3ac813c309cca8c98" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4080a227f82a09f68540ecd028622065d7ac4c0bcb8727a25bdcfc0526235792" +dependencies = [ + "form_urlencoded", + "http 1.5.0", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core", + "rsa", + "serde", + "serde_json", "tokio", ] [[package]] name = "reqwest" -version = "0.12.23" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.0", "bytes", "futures-core", "futures-util", - "h2 0.4.12", - "http 1.3.1", + "http 1.5.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", - "hyper-rustls 0.27.7", + "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.31", - "rustls-native-certs 0.8.1", + "rustls", "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -5994,7 +5422,20 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.2", +] + +[[package]] +name = "reqwest-middleware" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" +dependencies = [ + "anyhow", + "async-trait", + "http 1.5.0", + "reqwest", + "thiserror 2.0.18", + "tower-service", ] [[package]] @@ -6004,8 +5445,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if", - "getrandom 0.2.16", + "cfg-if 1.0.4", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -6013,145 +5454,119 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.7.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" -dependencies = [ - "bitvec", - "bytecheck", - "bytes", - "hashbrown 0.12.3", - "ptr_meta 0.1.4", - "rend 0.4.2", - "rkyv_derive 0.7.45", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv" -version = "0.8.11" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f5c3e5da784cd8c69d32cdc84673f3204536ca56e1fa01be31a74b92c932ac" +checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" dependencies = [ "bytes", - "hashbrown 0.15.5", - "indexmap 2.11.0", + "hashbrown 0.17.1", + "indexmap 2.14.0", "munge", - "ptr_meta 0.3.0", + "ptr_meta", "rancor", - "rend 0.5.2", - "rkyv_derive 0.8.11", + "rend", + "rkyv_derive", "tinyvec", "uuid", ] [[package]] name = "rkyv_derive" -version = "0.7.45" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] -name = "rkyv_derive" -version = "0.8.11" +name = "roaring" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4270433626cffc9c4c1d3707dd681f2a2718d3d7b09ad754bec137acecda8d22" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "bytemuck", + "byteorder", ] [[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - -[[package]] -name = "roaring" -version = "0.11.2" +name = "roxmltree" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f08d6a905edb32d74a5d5737a0c9d7e950c312f3c46cb0ca0a2ca09ea11878a0" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" dependencies = [ - "bytemuck", - "byteorder", + "memchr", ] [[package]] name = "rsa" -version = "0.9.8" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", "pkcs1", "pkcs8", "rand_core 0.6.4", - "sha2", + "sha2 0.10.9", "signature", "spki", - "subtle", - "zeroize", -] - -[[package]] -name = "rust-ini" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" -dependencies = [ - "cfg-if", - "ordered-multimap", + "subtle", + "zeroize", ] [[package]] -name = "rust_decimal" -version = "1.38.0" +name = "rstest" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8975fc98059f365204d635119cf9c5a60ae67b841ed49b5422a9a7e56cdfac0" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" dependencies = [ - "arrayvec", - "borsh", - "bytes", - "num-traits", - "postgres-types", - "rand 0.8.5", - "rkyv 0.7.45", - "serde", - "serde_json", + "futures-timer", + "futures-util", + "rstest_macros", ] [[package]] -name = "rustc-demangle" -version = "0.1.26" +name = "rstest_macros" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if 1.0.4", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] [[package]] -name = "rustc-hash" -version = "1.1.0" +name = "rust-ini" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if 1.0.4", + "ordered-multimap", +] [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" dependencies = [ - "rand 0.8.5", + "rand 0.9.5", ] [[package]] @@ -6165,124 +5580,86 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustix" -version = "1.0.8" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.0", "errno", "libc", - "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", + "linux-raw-sys", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.31" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.4", + "rustls-webpki", "subtle", "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework 2.11.1", -] - -[[package]] -name = "rustls-native-certs" -version = "0.8.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.4.0", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", + "security-framework", ] [[package]] name = "rustls-pki-types" -version = "1.12.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", ] [[package]] -name = "rustls-webpki" -version = "0.101.7" +name = "rustls-platform-verifier" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "ring", - "untrusted", + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", ] +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" -version = "0.103.4" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -6292,37 +5669,21 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] -name = "rustyline" -version = "16.0.0" +name = "ryu" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fd9ca5ebc709e8535e8ef7c658eb51457987e48c98ead2be482172accc408d" -dependencies = [ - "bitflags", - "cfg-if", - "clipboard-win", - "fd-lock", - "home", - "libc", - "log", - "memchr", - "nix 0.30.1", - "radix_trie", - "unicode-segmentation", - "unicode-width 0.2.1", - "utf8parse", - "windows-sys 0.59.0", -] +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] -name = "ryu" -version = "1.0.20" +name = "safe-transmute" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" [[package]] name = "salsa20" @@ -6330,7 +5691,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] @@ -6344,23 +5705,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "schemars" -version = "0.8.22" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ - "dyn-clone", - "schemars_derive", - "serde", - "serde_json", + "windows-sys 0.61.2", ] [[package]] @@ -6377,9 +5726,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.4" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -6387,24 +5736,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "schemars_derive" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 2.0.106", -] - -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" @@ -6419,45 +5750,16 @@ checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ "pbkdf2", "salsa20", - "sha2", -] - -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", + "sha2 0.10.9", ] [[package]] name = "security-framework" -version = "3.4.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b369d18893388b345804dc0007963c99b7d665ae71d275812d828c6f089640" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -6466,9 +5768,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -6476,12 +5778,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" -dependencies = [ - "serde", -] +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "seq-macro" @@ -6491,54 +5790,79 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_arrow" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eef6f03bec0418bbde0fa9c070a5fee5b793c9cc4a392613d37c1de7ee211db5" +dependencies = [ + "arrow-array", + "arrow-schema", + "bytemuck", + "chrono", + "half", + "marrow", + "serde", +] + [[package]] name = "serde_bytes" -version = "0.11.17" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" dependencies = [ "serde", + "serde_core", ] [[package]] -name = "serde_derive" -version = "1.0.219" +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "serde_derive", ] [[package]] -name = "serde_derive_internals" -version = "0.29.1" +name = "serde_derive" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "serde_json" -version = "1.0.143" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", - "ryu", "serde", + "serde_core", + "zmij", ] [[package]] @@ -6549,28 +5873,16 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_tokenstream" -version = "0.2.2" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64060d864397305347a78851c51588fd283767e7e7589829e8121d65512340f1" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ - "proc-macro2", - "quote", - "serde", - "syn 2.0.106", + "serde_core", ] [[package]] @@ -6587,19 +5899,20 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" dependencies = [ "base64 0.22.1", + "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.11.0", + "indexmap 2.14.0", + "jiff", "schemars 0.9.0", - "schemars 1.0.4", - "serde", - "serde_derive", + "schemars 1.2.1", + "serde_core", "serde_json", "serde_with_macros", "time", @@ -6607,38 +5920,36 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" +name = "sha1" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "indexmap 2.11.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", + "cfg-if 1.0.4", + "cpufeatures 0.2.17", + "digest 0.10.7", ] [[package]] name = "sha1" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ - "cfg-if", - "cpufeatures", - "digest", + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -6647,9 +5958,30 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if", - "cpufeatures", - "digest", + "cfg-if 1.0.4", + "cpufeatures 0.2.17", + "digest 0.10.7", + "sha2-asm", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sha2-asm" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" +dependencies = [ + "cc", ] [[package]] @@ -6661,18 +5993,30 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "bstr", + "dirs", + "os_str_bytes", +] + [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] @@ -6682,10 +6026,26 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -6698,66 +6058,27 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" -[[package]] -name = "simple_asn1" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror 2.0.16", - "time", -] - [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "slab" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" - -[[package]] -name = "slug" -version = "0.1.6" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882a80f72ee45de3cc9a5afeb2da0331d58df69e4e7d8eeb5d3c7784ae67e724" -dependencies = [ - "deunicode", - "wasm-bindgen", -] +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" dependencies = [ "serde", ] -[[package]] -name = "smol" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33bd3e260892199c3ccfc487c88b2da2265080acb316cd920da72fdfd7c599f" -dependencies = [ - "async-channel", - "async-executor", - "async-fs", - "async-io", - "async-lock", - "async-net", - "async-process", - "blocking", - "futures-lite", -] - [[package]] name = "snap" version = "1.1.1" @@ -6776,21 +6097,21 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "sonic-number" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a74044c092f4f43ca7a6cfd62854cf9fb5ac8502b131347c990bf22bef1dfe" +checksum = "3775c3390edf958191f1ab1e8c5c188907feebd0f3ce1604cb621f72961dbf32" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -6801,7 +6122,7 @@ checksum = "0275f9f2f07d47556fe60c2759da8bc4be6083b047b491b2d476aa0bfa558eb1" dependencies = [ "bumpalo", "bytes", - "cfg-if", + "cfg-if 1.0.4", "faststr", "itoa", "ref-cast", @@ -6810,27 +6131,33 @@ dependencies = [ "simdutf8", "sonic-number", "sonic-simd", - "thiserror 2.0.16", + "thiserror 2.0.18", ] [[package]] name = "sonic-simd" -version = "0.1.1" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b421f7b6aa4a5de8f685aaf398dfaa828346ee639d2b1c1061ab43d40baa6223" +checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" @@ -6841,53 +6168,6 @@ dependencies = [ "der", ] -[[package]] -name = "sqllogictest" -version = "0.28.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3566426f72a13e393aa34ca3d542c5b0eb86da4c0db137ee9b5cfccc6179e52d" -dependencies = [ - "async-trait", - "educe", - "fs-err", - "futures", - "glob", - "humantime", - "itertools 0.13.0", - "libtest-mimic", - "md-5", - "owo-colors", - "rand 0.8.5", - "regex", - "similar", - "subst", - "tempfile", - "thiserror 2.0.16", - "tracing", -] - -[[package]] -name = "sqlparser" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" -dependencies = [ - "log", - "recursive", - "sqlparser_derive", -] - -[[package]] -name = "sqlparser_derive" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - [[package]] name = "sqlx" version = "0.8.6" @@ -6919,17 +6199,17 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap 2.11.0", + "indexmap 2.14.0", "log", "memchr", "once_cell", "percent-encoding", - "rustls 0.23.31", + "rustls", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", - "thiserror 2.0.16", + "thiserror 2.0.18", "tokio", "tokio-stream", "tracing", @@ -6947,7 +6227,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.106", + "syn", ] [[package]] @@ -6958,17 +6238,17 @@ checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" dependencies = [ "dotenvy", "either", - "heck", + "heck 0.5.0", "hex", "once_cell", "proc-macro2", "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-sqlite", - "syn 2.0.106", + "syn", "tokio", "url", ] @@ -6979,13 +6259,13 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ - "atoi", + "atoi 2.0.0", "base64 0.22.1", - "bitflags", + "bitflags 2.13.0", "byteorder", "bytes", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -6995,23 +6275,23 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", - "rand 0.8.5", + "rand 0.8.6", "rsa", - "sha1", - "sha2", + "sha1 0.10.6", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.16", + "thiserror 2.0.18", "tracing", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -7020,9 +6300,9 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ - "atoi", + "atoi 2.0.0", "base64 0.22.1", - "bitflags", + "bitflags 2.13.0", "byteorder", "crc", "dotenvy", @@ -7032,23 +6312,23 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.16", + "thiserror 2.0.18", "tracing", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -7057,7 +6337,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ - "atoi", + "atoi 2.0.0", "flume", "futures-channel", "futures-core", @@ -7070,29 +6350,16 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.16", + "thiserror 2.0.18", "tracing", "url", ] [[package]] name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "stacker" -version = "0.1.21" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" -dependencies = [ - "cc", - "cfg-if", - "libc", - "psm", - "windows-sys 0.59.0", -] +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "static_assertions" @@ -7100,6 +6367,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -7117,12 +6394,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" - [[package]] name = "strum" version = "0.27.2" @@ -7133,16 +6404,12 @@ dependencies = [ ] [[package]] -name = "strum_macros" -version = "0.26.4" +name = "strum" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.106", + "strum_macros 0.28.0", ] [[package]] @@ -7151,45 +6418,22 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", -] - -[[package]] -name = "subst" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9a86e5144f63c2d18334698269a8bfae6eece345c70b64821ea5b35054ec99" -dependencies = [ - "memchr", - "unicode-width 0.1.14", + "syn", ] [[package]] -name = "substrait" -version = "0.56.0" +name = "strum_macros" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13de2e20128f2a018dab1cfa30be83ae069219a65968c6f89df66ad124de2397" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ - "heck", - "pbjson", - "pbjson-build", - "pbjson-types", - "prettyplease", - "prost", - "prost-build", - "prost-types", - "regress", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "serde_yaml", - "syn 2.0.106", - "typify", - "walkdir", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -7199,21 +6443,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] -name = "syn" -version = "1.0.109" +name = "symlink" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" [[package]] name = "syn" -version = "2.0.106" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", @@ -7237,7 +6476,42 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] @@ -7247,44 +6521,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] -name = "tap" +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "target-triple" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +checksum = "c3a6bfce3d99adfa72d24750a61f782f3036a81e7f86d8841ee1326deaebd171" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.3", + "getrandom 0.4.3", "once_cell", - "rustix 1.0.8", - "windows-sys 0.59.0", + "rustix", + "windows-sys 0.61.2", ] [[package]] -name = "tera" -version = "1.20.0" +name = "termcolor" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab9d851b45e865f178319da0abdbfe6acbc4328759ff18dafc3a41c16b4cd2ee" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ - "chrono", - "chrono-tz 0.9.0", - "globwalk", - "humansize", - "lazy_static", - "percent-encoding", - "pest", - "pest_derive", - "rand 0.8.5", - "regex", - "serde", - "serde_json", - "slug", - "unic-segment", + "winapi-util", ] [[package]] @@ -7304,11 +6571,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.16" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.16", + "thiserror-impl 2.0.18", ] [[package]] @@ -7319,18 +6586,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "thiserror-impl" -version = "2.0.16" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] @@ -7339,56 +6606,34 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if", -] - -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding 3.0.4", - "log", - "ordered-float 2.10.1", - "threadpool", + "cfg-if 1.0.4", ] [[package]] name = "time" -version = "0.3.43" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -7405,9 +6650,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -7415,9 +6660,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -7430,60 +6675,58 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.47.1" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", - "socket2 0.6.0", + "socket2 0.6.4", "tokio-macros", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] -name = "tokio-rustls" -version = "0.24.1" +name = "tokio-retry" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" dependencies = [ - "rustls 0.21.12", + "pin-project-lite", + "rand 0.10.2", "tokio", ] [[package]] name = "tokio-rustls" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.31", + "rustls", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -7492,9 +6735,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.16" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -7505,50 +6748,60 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.23" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" dependencies = [ - "serde", + "indexmap 2.14.0", + "serde_core", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_parser", + "toml_writer", + "winnow", ] [[package]] name = "toml_datetime" -version = "0.6.11" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ - "serde", + "serde_core", ] [[package]] name = "toml_edit" -version = "0.22.27" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ - "indexmap 2.11.0", - "serde", - "serde_spanned", + "indexmap 2.14.0", "toml_datetime", - "toml_write", + "toml_parser", "winnow", ] [[package]] -name = "toml_write" -version = "0.1.2" +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -7561,20 +6814,20 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.0", "bytes", "futures-util", - "http 1.3.1", + "http 1.5.0", "http-body 1.0.1", - "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", + "url", ] [[package]] @@ -7591,9 +6844,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -7601,22 +6854,35 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.18", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -7633,22 +6899,35 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" -version = "0.3.20" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", "once_cell", "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -7658,19 +6937,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] -name = "twox-hash" -version = "2.1.2" +name = "trybuild" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "1e605bf6b39357663d8ba4e984f8be8da8df6bb32e81031d6889024ea8fd68e4" +dependencies = [ + "glob", + "serde", + "serde_derive", + "serde_json", + "target-triple", + "termcolor", + "toml", +] [[package]] -name = "typed-builder" -version = "0.19.1" +name = "twox-hash" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" -dependencies = [ - "typed-builder-macro 0.19.1", -] +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typed-builder" @@ -7678,18 +6963,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" dependencies = [ - "typed-builder-macro 0.20.1", -] - -[[package]] -name = "typed-builder-macro" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "typed-builder-macro", ] [[package]] @@ -7700,117 +6974,50 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] -name = "typenum" -version = "1.18.0" +name = "typeid" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" [[package]] -name = "typify" -version = "0.4.3" +name = "typenum" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7144144e97e987c94758a3017c920a027feac0799df325d6df4fc8f08d02068e" -dependencies = [ - "typify-impl", - "typify-macro", -] +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] -name = "typify-impl" -version = "0.4.3" +name = "typetag" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062879d46aa4c9dfe0d33b035bbaf512da192131645d05deacb7033ec8581a09" +checksum = "c5a897b12c6c1151ad0b138b8db50252dc301f93bc3b027db05eec82aeed298c" dependencies = [ - "heck", - "log", - "proc-macro2", - "quote", - "regress", - "schemars 0.8.22", - "semver", + "erased-serde", + "inventory", + "once_cell", "serde", - "serde_json", - "syn 2.0.106", - "thiserror 2.0.16", - "unicode-ident", + "typetag-impl", ] [[package]] -name = "typify-macro" -version = "0.4.3" +name = "typetag-impl" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9708a3ceb6660ba3f8d2b8f0567e7d4b8b198e2b94d093b8a6077a751425de9e" +checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5" dependencies = [ "proc-macro2", "quote", - "schemars 0.8.22", - "semver", - "serde", - "serde_json", - "serde_tokenstream", - "syn 2.0.106", - "typify-impl", -] - -[[package]] -name = "ucd-trie" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" - -[[package]] -name = "unic-char-property" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" -dependencies = [ - "unic-char-range", -] - -[[package]] -name = "unic-char-range" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" - -[[package]] -name = "unic-common" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" - -[[package]] -name = "unic-segment" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" -dependencies = [ - "unic-ucd-segment", -] - -[[package]] -name = "unic-ucd-segment" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", + "syn", ] [[package]] -name = "unic-ucd-version" -version = "0.9.0" +name = "typewit" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" -dependencies = [ - "unic-common", -] +checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" [[package]] name = "unicode-bidi" @@ -7820,48 +7027,34 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-normalization" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" dependencies = [ "tinyvec", ] [[package]] name = "unicode-properties" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - -[[package]] -name = "unicode-width" -version = "0.2.1" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] -name = "unsafe-libyaml" -version = "0.2.11" +name = "universal-hash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] [[package]] name = "untrusted" @@ -7871,9 +7064,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -7901,13 +7094,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.18.1" +version = "1.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.4.3", "js-sys", - "serde", + "serde_core", "wasm-bindgen", ] @@ -7943,12 +7136,12 @@ dependencies = [ "metainfo", "motore", "mur3", - "nix 0.29.0", + "nix", "once_cell", "pin-project", - "rand 0.9.2", + "rand 0.9.5", "socket2 0.5.10", - "thiserror 2.0.16", + "thiserror 2.0.18", "tokio", "tokio-stream", "tower", @@ -7961,7 +7154,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a41a936651c3b8206339898466fd4f39596cfedf789e5ee6a8d02a45feb45f" dependencies = [ - "ahash 0.8.12", + "ahash", "anyhow", "bytes", "chrono", @@ -7976,10 +7169,10 @@ dependencies = [ "paste", "pilota", "pin-project", - "rustc-hash 2.1.1", + "rustc-hash", "scopeguard", "sonic-rs", - "thiserror 2.0.16", + "thiserror 2.0.18", "tokio", "tracing", "volo", @@ -8018,9 +7211,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" -version = "0.14.4+wasi-0.2.4" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a5f4a424faf49c3c2c344f166f0662341d470ea185e939657aaff130f0ec4a" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] @@ -8032,50 +7234,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] -name = "wasm-bindgen" -version = "0.2.101" +name = "wasite" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", + "wasi 0.14.7+wasi-0.2.4", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.101" +name = "wasm-bindgen" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.106", + "cfg-if 1.0.4", + "once_cell", + "rustversion", + "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.51" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca85039a9b469b38336411d6d6ced91f3fc87109a2a27b0c197663f5144dffe" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.101" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8083,31 +7277,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.101" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.106", - "wasm-bindgen-backend", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.101" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-streams" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ "futures-util", "js-sys", @@ -8118,9 +7312,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.78" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e4b637749ff0d92b8fad63aa1f7cff3cbe125fd49c175cd6345e7272638b12" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -8136,161 +7330,197 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.2", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.2" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] [[package]] -name = "which" -version = "4.4.2" +name = "whoami" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", + "libredox", + "wasite 0.1.0", ] [[package]] name = "whoami" -version = "1.6.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" dependencies = [ + "libc", "libredox", - "wasite", + "objc2-system-configuration", + "wasite 1.0.2", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.0", + "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows" -version = "0.61.3" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ "windows-collections", "windows-core", "windows-future", - "windows-link 0.1.3", "windows-numerics", ] [[package]] name = "windows-collections" -version = "0.2.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ "windows-core", ] [[package]] name = "windows-core" -version = "0.61.2" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.1.3", + "windows-link", "windows-result", "windows-strings", ] [[package]] name = "windows-future" -version = "0.2.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ "windows-core", - "windows-link 0.1.3", + "windows-link", "windows-threading", ] [[package]] name = "windows-implement" -version = "0.60.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "windows-link" -version = "0.1.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "windows-link" -version = "0.2.0" +name = "windows-numerics" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] [[package]] -name = "windows-numerics" -version = "0.2.0" +name = "windows-registry" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-core", - "windows-link 0.1.3", + "windows-link", + "windows-result", + "windows-strings", ] [[package]] name = "windows-result" -version = "0.3.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] name = "windows-strings" -version = "0.4.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] @@ -8311,31 +7541,22 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.3", + "windows-targets 0.53.5", ] [[package]] name = "windows-sys" -version = "0.61.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link 0.2.0", + "windows-link", ] [[package]] @@ -8371,28 +7592,28 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.3" +version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link 0.1.3", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] name = "windows-threading" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] @@ -8409,9 +7630,9 @@ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" @@ -8427,9 +7648,9 @@ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" @@ -8445,9 +7666,9 @@ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" [[package]] name = "windows_i686_gnullvm" @@ -8457,9 +7678,9 @@ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" @@ -8475,9 +7696,9 @@ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" @@ -8493,9 +7714,9 @@ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" @@ -8511,9 +7732,9 @@ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" @@ -8529,55 +7750,206 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.13" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] [[package]] name = "wit-bindgen" -version = "0.45.1" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wkb" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" +checksum = "a120b336c7ad17749026d50427c23d838ecb50cd64aaea6254b5030152f890a9" +dependencies = [ + "byteorder", + "geo-traits", + "num_enum", + "thiserror 1.0.69", +] [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] -name = "wyz" -version = "0.5.1" +name = "xattr" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ - "tap", + "libc", + "rustix", ] [[package]] -name = "xmlparser" -version = "0.13.6" +name = "xet-client" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "bytes", + "clap", + "crc32fast", + "futures", + "http 1.5.0", + "hyper", + "lazy_static", + "more-asserts", + "rand 0.10.2", + "redb", + "reqwest", + "reqwest-middleware", + "serde", + "serde_json", + "serde_repr", + "statrs", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-retry", + "tracing", + "tracing-subscriber", + "url", + "urlencoding", + "web-time", + "xet-core-structures", + "xet-runtime", +] [[package]] -name = "xz2" -version = "0.1.7" +name = "xet-core-structures" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" +dependencies = [ + "async-trait", + "base64 0.22.1", + "blake3", + "bytemuck", + "bytes", + "clap", + "countio", + "csv", + "futures", + "futures-util", + "getrandom 0.4.3", + "heapify", + "itertools 0.14.0", + "lazy_static", + "lz4_flex 0.13.1", + "more-asserts", + "rand 0.10.2", + "regex", + "safe-transmute", + "serde", + "static_assertions", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "uuid", + "web-time", + "xet-runtime", +] + +[[package]] +name = "xet-data" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "clap", + "gearhash", + "http 1.5.0", + "itertools 0.14.0", + "lazy_static", + "more-asserts", + "rand 0.10.2", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "url", + "uuid", + "walkdir", + "xet-client", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-runtime" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" dependencies = [ - "lzma-sys", + "anyhow", + "async-trait", + "bytes", + "chrono", + "colored", + "const-str", + "ctor 0.6.3", + "dirs", + "futures", + "git-version", + "humantime", + "konst", + "lazy_static", + "libc", + "more-asserts", + "oneshot", + "pin-project", + "rand 0.10.2", + "reqwest", + "serde", + "serde_json", + "shellexpand", + "sysinfo", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "tracing-appender", + "tracing-subscriber", + "whoami 2.1.2", + "winapi", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yansi" version = "1.0.1" @@ -8586,11 +7958,10 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -8598,68 +7969,68 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "75726053136156d419e285b9b7eddaaea9e3fea6ce32eed44a89901f0bd98de1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "4714fd92cf900833d49538023a9b3915155210801d1c1169eba513b2addefd71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -8668,9 +8039,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.4" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -8679,20 +8050,26 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn", ] [[package]] name = "zlib-rs" -version = "0.5.2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" +checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" @@ -8700,7 +8077,16 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe", + "zstd-safe 7.2.4", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe 8.2.0", ] [[package]] @@ -8712,11 +8098,20 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 999b911753..d40ed7290f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,14 +16,15 @@ # under the License. [workspace] -exclude = ["bindings/python"] members = [ + "bindings/python", "crates/catalog/*", "crates/examples", "crates/iceberg", "crates/integration_tests", "crates/integrations/*", - "crates/sqllogictest", + "crates/property-macro", + "crates/storage/*", "crates/test_utils", ] resolver = "2" @@ -31,98 +32,127 @@ resolver = "2" [workspace.package] edition = "2024" homepage = "https://rust.iceberg.apache.org/" -version = "0.7.0" +publish = false +version = "0.10.1" license = "Apache-2.0" repository = "https://github.com/apache/iceberg-rust" # Check the MSRV policy in README.md before changing this -rust-version = "1.87" +rust-version = "1.95" + +[workspace.lints.rust] +unused_qualifications = "deny" [workspace.dependencies] +aes-gcm = "0.10" anyhow = "1.0.72" -apache-avro = { version = "0.20", features = ["zstandard"] } +apache-avro = { version = "0.22", features = ["snappy", "zstandard"] } array-init = "2" -arrow-arith = { version = "55.1" } -arrow-array = { version = "55.1" } -arrow-buffer = { version = "55.1" } -arrow-cast = { version = "55.1" } -arrow-ord = { version = "55.1" } -arrow-schema = { version = "55.1" } -arrow-select = { version = "55.1" } -arrow-string = { version = "55.1" } +arrow = "60.0" +arrow-arith = "60.0" +arrow-array = "60.0" +arrow-avro = { version = "60.0", default-features = false, features = [ + "deflate", + "snappy", + "zstd", +] } +arrow-buffer = "60.0" +arrow-cast = "60.0" +arrow-ord = "60.0" +arrow-schema = "60.0" +arrow-select = "60.0" +arrow-string = "60.0" as-any = "0.3.2" -async-trait = "0.1.88" -aws-config = "1.8.1" -aws-sdk-glue = "1.39" -aws-sdk-s3tables = "1.28.0" +async-trait = "0.1.89" +aws-config = "1.8.7" +aws-credential-types = "1.2" +aws-sdk-glue = { version = "1.85", default-features = false, features = [ + "default-https-client", + "rt-tokio", +] } +aws-sdk-s3tables = { version = "1.28", default-features = false, features = [ + "default-https-client", + "rt-tokio", +] } +aws-sigv4 = { version = "1.4", default-features = false, features = [ + "sign-http", + "http1", +] } backon = "1.5.1" base64 = "0.22.1" bimap = "0.6" -bytes = "1.10" +bytes = "1.11" +cfg-if = "1" chrono = "0.4.41" -clap = { version = "4.5.41", features = ["derive", "cargo"] } -ctor = "0.2.8" -datafusion = "48.0.1" -datafusion-cli = "48.0.1" -datafusion-sqllogictest = "48.0.1" +crc32fast = "1" derive_builder = "0.20" -dirs = "6" -enum-ordinalize = "4.3.0" expect-test = "1" +fastnum = { version = "0.7", default-features = false, features = [ + "std", + "serde", +] } faststr = "0.2.31" +flate2 = "1.1.5" fnv = "1.0.7" -fs-err = "3.1.0" +form_urlencoded = "1.2.2" futures = "0.3" hive_metastore = "0.2.0" http = "1.2" -iceberg = { version = "0.7.0", path = "./crates/iceberg" } -iceberg-catalog-rest = { version = "0.7.0", path = "./crates/catalog/rest" } -iceberg-catalog-glue = { version = "0.7.0", path = "./crates/catalog/glue" } -iceberg-catalog-s3tables = { version = "0.7.0", path = "./crates/catalog/s3tables" } -iceberg-catalog-hms = { version = "0.7.0", path = "./crates/catalog/hms" } -iceberg-datafusion = { version = "0.7.0", path = "./crates/integrations/datafusion" } -indicatif = "0.17" +iceberg = { version = "0.10.0", path = "./crates/iceberg" } +iceberg-catalog-glue = { version = "0.10.0", path = "./crates/catalog/glue" } +iceberg-catalog-hms = { version = "0.10.0", path = "./crates/catalog/hms" } +iceberg-catalog-rest = { version = "0.10.0", path = "./crates/catalog/rest" } +iceberg-catalog-s3tables = { version = "0.10.0", path = "./crates/catalog/s3tables" } +iceberg-catalog-sql = { version = "0.10.0", path = "./crates/catalog/sql" } +iceberg-property-macro = { version = "0.10.0", path = "./crates/property-macro" } +iceberg-storage-opendal = { version = "0.10.0", path = "./crates/storage/opendal" } itertools = "0.13" linkedbytes = "0.1.8" metainfo = "0.7.14" -mimalloc = "0.1.46" +minijinja = "2.12.0" mockall = "0.13.1" mockito = "1" +moka = "0.12.10" motore-macros = "0.4.3" murmur3 = "0.5.2" -num-bigint = "0.4.6" once_cell = "1.20" -opendal = "0.54.0" +opendal = "0.58" ordered-float = "4" -parquet = "55.1" +parquet = "60.0" +parquet-geospatial = "60.0" pilota = "0.11.10" -port_scanner = "0.1.5" pretty_assertions = "1.4" -rand = "0.8.5" -regex = "1.10.5" -reqwest = { version = "0.12.12", default-features = false, features = ["json"] } +proc-macro2 = "1" +pyo3 = "0.29" +quote = "1" +rand = "0.9.3" +regex = "1.11.3" +reqwest = { version = "0.13.5", default-features = false, features = ["json"] } roaring = { version = "0.11" } -rust_decimal = "1.37.1" +rstest = "0.26" serde = { version = "1.0.219", features = ["rc"] } serde_bytes = "0.11.17" serde_derive = "1.0.219" serde_json = "1.0.142" serde_repr = "0.1.16" serde_with = "3.4" -smol = "2.0.2" -sqllogictest = "0.28.3" -stacker = "0.1.20" +sha2 = "0.10" +sqlx = { version = "0.8.1", default-features = false } strum = "0.27.2" +syn = "2" tempfile = "3.18" -tera = "1" -thrift = "0.17.0" -tokio = { version = "1.46.1", default-features = false } -toml = "0.8" -tracing = "0.1.40" -tracing-subscriber = "0.3.8" +tokio = { version = "1.47", default-features = false, features = [ + "sync", + "rt-multi-thread", +] } +tracing = "0.1.41" +tracing-subscriber = "0.3.20" +trybuild = "1" typed-builder = "0.20" -url = "2.5.4" +typetag = "0.2" +url = "2.5.7" uuid = { version = "1.18", features = ["v7"] } volo = "0.10.6" volo-thrift = "0.10.8" -zstd = "0.13.3" +zeroize = "1.7" +zstd = { version = "0.13.3", default-features = false } diff --git a/Makefile b/Makefile index 1b98fe8fa1..485d2866b0 100644 --- a/Makefile +++ b/Makefile @@ -21,36 +21,85 @@ build: cargo build --all-targets --all-features --workspace check-fmt: - cargo fmt --all -- --check + cargo fmt --all -- --check check-clippy: - cargo clippy --all-targets --all-features --workspace -- -D warnings + cargo clippy --all-targets --all-features --workspace -- -D warnings +check-doc: + RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --all-features --workspace + +# Keep version in sync with the CI lint install step in .github/workflows/ci.yml. install-cargo-machete: - cargo install cargo-machete@0.7.0 + cargo install --locked cargo-machete@0.7.0 cargo-machete: install-cargo-machete cargo machete +install-cargo-nextest: + cargo install --locked cargo-nextest + +nextest: install-cargo-nextest + cargo nextest run --all-targets --all-features --workspace + +# Keep version in sync with the CI lint install step in .github/workflows/ci.yml. install-taplo-cli: - cargo install taplo-cli@0.9.3 + cargo install --locked taplo-cli@0.9.3 fix-toml: install-taplo-cli taplo fmt check-toml: install-taplo-cli - taplo check + taplo fmt --check NIGHTLY_VERSION := $(shell awk -F'"' '/^channel/ {print $$2}' rust-toolchain.toml) MSRV_VERSION := $(shell awk -F'"' '/^rust-version/ {print $$2}' Cargo.toml) check-msrv: - @set -e; \ - trap 'git restore Cargo.lock' EXIT; \ - cargo +$(NIGHTLY_VERSION) generate-lockfile -Z direct-minimal-versions; \ - cargo +$(MSRV_VERSION) check --locked --workspace - -check: check-fmt check-clippy check-toml cargo-machete + cargo +$(MSRV_VERSION) check --workspace + +# Evaluated inside the recipes that need it rather than with $(shell) at parse +# time, so plain `make ` invocations don't pay a cargo metadata call. +PUBLIC_API_CRATES_CMD = cargo metadata --no-deps --format-version 1 | \ + jq -r '.packages[] | select(.publish == null) | "\(.name):\(.manifest_path)"' + +install-cargo-public-api: + cargo install --locked cargo-public-api@0.51.0 + +generate-public-api: install-cargo-public-api + @for entry in $$($(PUBLIC_API_CRATES_CMD)); do \ + crate=$${entry%%:*}; \ + manifest=$${entry##*:}; \ + crate_dir=$$(dirname "$$manifest"); \ + echo "Generating public API for $$crate..."; \ + cargo public-api -p "$$crate" --all-features -ss > "$$crate_dir/public-api.txt"; \ + done + +check-public-api: install-cargo-public-api + @fail=0; \ + for entry in $$($(PUBLIC_API_CRATES_CMD)); do \ + crate=$${entry%%:*}; \ + manifest=$${entry##*:}; \ + crate_dir=$$(dirname "$$manifest"); \ + echo "Checking public API for $$crate..."; \ + cargo public-api -p "$$crate" --all-features -ss | diff - "$$crate_dir/public-api.txt" || { \ + echo "ERROR: Public API for $$crate has changed. Run 'make generate-public-api' to update."; \ + fail=1; \ + }; \ + done; \ + if [ $$fail -ne 0 ]; then exit 1; fi + +check-license-notice: + python3 ./dev/release/licenses.py crates check + +# Keep version in sync with the CI lint install step in .github/workflows/ci.yml. +install-cargo-deny: + cargo install --locked cargo-deny@0.19.9 + +check-dependency-licenses: install-cargo-deny + python3 ./dev/release/licenses.py dependencies check + +check: check-fmt check-clippy check-doc check-toml cargo-machete check-license-notice check-dependency-licenses doc-test: cargo test --no-fail-fast --doc --all-features --workspace @@ -58,8 +107,25 @@ doc-test: unit-test: doc-test cargo test --no-fail-fast --lib --all-features --workspace -test: doc-test - cargo test --no-fail-fast --all-targets --all-features --workspace +test: docker-up + @trap '$(MAKE) docker-down' EXIT; \ + $(MAKE) nextest clean: cargo clean + +install-mdbook: + cargo install --locked mdbook@0.4.36 + +site: install-mdbook + cd website && mdbook serve + +# Docker targets for integration tests +docker-up: + docker compose -f dev/docker-compose.yaml up -d --build --wait --remove-orphans + +docker-down: + docker compose -f dev/docker-compose.yaml down -v --remove-orphans --timeout 0 + +docker-logs: + docker compose -f dev/docker-compose.yaml logs -f diff --git a/NOTICE b/NOTICE index 9340680cbd..73b77516fa 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Iceberg Rust -Copyright 2023-2024 The Apache Software Foundation +Copyright 2023-2026 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/README.md b/README.md index 88b8714cc5..fa974dc767 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,17 @@ Rust implementation of [Apache Iceberg™](https://iceberg.apache.org/). The Apache Iceberg Rust project is composed of the following components: -| Name | Release | Docs | -|--------------------------|-----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| -| [iceberg] | [![iceberg image]][iceberg link] | [![docs release]][iceberg release docs] [![docs dev]][iceberg dev docs] | -| [iceberg-datafusion] | [![iceberg-datafusion image]][iceberg-datafusion link] | [![docs release]][iceberg-datafusion release docs] [![docs dev]][iceberg-datafusion dev docs] | -| [iceberg-catalog-glue] | [![iceberg-catalog-glue image]][iceberg-catalog-glue link] | [![docs release]][iceberg-catalog-glue release docs] [![docs dev]][iceberg-catalog-glue dev docs] | -| [iceberg-catalog-hms] | [![iceberg-catalog-hms image]][iceberg-catalog-hms link] | [![docs release]][iceberg-catalog-hms release docs] [![docs dev]][iceberg-catalog-hms dev docs] | -| [iceberg-catalog-rest] | [![iceberg-catalog-rest image]][iceberg-catalog-rest link] | [![docs release]][iceberg-catalog-rest release docs] [![docs dev]][iceberg-catalog-rest dev docs] | +| Name | Release | Docs | +|-------------------------------|--------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| +| [iceberg] | [![iceberg image]][iceberg link] | [![docs release]][iceberg release docs] [![docs dev]][iceberg dev docs] | +| [iceberg-catalog-loader] | [![iceberg-catalog-loader image]][iceberg-catalog-loader link] | [![docs release]][iceberg-catalog-loader release docs] [![docs dev]][iceberg-catalog-loader dev docs] | +| [iceberg-catalog-glue] | [![iceberg-catalog-glue image]][iceberg-catalog-glue link] | [![docs release]][iceberg-catalog-glue release docs] [![docs dev]][iceberg-catalog-glue dev docs] | +| [iceberg-catalog-hms] | [![iceberg-catalog-hms image]][iceberg-catalog-hms link] | [![docs release]][iceberg-catalog-hms release docs] [![docs dev]][iceberg-catalog-hms dev docs] | +| [iceberg-catalog-rest] | [![iceberg-catalog-rest image]][iceberg-catalog-rest link] | [![docs release]][iceberg-catalog-rest release docs] [![docs dev]][iceberg-catalog-rest dev docs] | +| [iceberg-catalog-s3tables] | [![iceberg-catalog-s3tables image]][iceberg-catalog-s3tables link] | [![docs release]][iceberg-catalog-s3tables release docs] [![docs dev]][iceberg-catalog-s3tables dev docs] | +| [iceberg-catalog-sql] | [![iceberg-catalog-sql image]][iceberg-catalog-sql link] | [![docs release]][iceberg-catalog-sql release docs] [![docs dev]][iceberg-catalog-sql dev docs] | +| [iceberg-cache-moka] | [![iceberg-cache-moka image]][iceberg-cache-moka link] | [![docs release]][iceberg-cache-moka release docs] [![docs dev]][iceberg-cache-moka dev docs] | +| [iceberg-storage-opendal] | [![iceberg-storage-opendal image]][iceberg-storage-opendal link] | [![docs release]][iceberg-storage-opendal release docs] [![docs dev]][iceberg-storage-opendal dev docs] | [docs release]: https://img.shields.io/badge/docs-release-blue [docs dev]: https://img.shields.io/badge/docs-dev-blue @@ -43,12 +47,6 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg release docs]: https://docs.rs/iceberg [iceberg dev docs]: https://rust.iceberg.apache.org/api/iceberg/ -[iceberg-datafusion]: crates/integrations/datafusion/README.md -[iceberg-datafusion image]: https://img.shields.io/crates/v/iceberg-datafusion.svg -[iceberg-datafusion link]: https://crates.io/crates/iceberg-datafusion -[iceberg-datafusion dev docs]: https://rust.iceberg.apache.org/api/iceberg_datafusion/ -[iceberg-datafusion release docs]: https://docs.rs/iceberg-datafusion - [iceberg-catalog-glue]: crates/catalog/glue/README.md [iceberg-catalog-glue image]: https://img.shields.io/crates/v/iceberg-catalog-glue.svg [iceberg-catalog-glue link]: https://crates.io/crates/iceberg-catalog-glue @@ -61,27 +59,56 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg-catalog-hms release docs]: https://docs.rs/iceberg-catalog-hms [iceberg-catalog-hms dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_hms/ - [iceberg-catalog-rest]: crates/catalog/rest/README.md [iceberg-catalog-rest image]: https://img.shields.io/crates/v/iceberg-catalog-rest.svg [iceberg-catalog-rest link]: https://crates.io/crates/iceberg-catalog-rest [iceberg-catalog-rest release docs]: https://docs.rs/iceberg-catalog-rest [iceberg-catalog-rest dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_rest/ +[iceberg-catalog-sql]: crates/catalog/sql +[iceberg-catalog-sql image]: https://img.shields.io/crates/v/iceberg-catalog-sql.svg +[iceberg-catalog-sql link]: https://crates.io/crates/iceberg-catalog-sql +[iceberg-catalog-sql release docs]: https://docs.rs/iceberg-catalog-sql +[iceberg-catalog-sql dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_sql/ + +[iceberg-catalog-s3tables]: crates/catalog/s3tables/README.md +[iceberg-catalog-s3tables image]: https://img.shields.io/crates/v/iceberg-catalog-s3tables.svg +[iceberg-catalog-s3tables link]: https://crates.io/crates/iceberg-catalog-s3tables +[iceberg-catalog-s3tables release docs]: https://docs.rs/iceberg-catalog-s3tables +[iceberg-catalog-s3tables dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_s3tables/ + +[iceberg-storage-opendal]: crates/storage/opendal/README.md +[iceberg-storage-opendal image]: https://img.shields.io/crates/v/iceberg-storage-opendal.svg +[iceberg-storage-opendal link]: https://crates.io/crates/iceberg-storage-opendal +[iceberg-storage-opendal release docs]: https://docs.rs/iceberg-storage-opendal +[iceberg-storage-opendal dev docs]: https://rust.iceberg.apache.org/api/iceberg_storage_opendal/ + +[iceberg-catalog-loader]: crates/catalog/loader +[iceberg-catalog-loader image]: https://img.shields.io/crates/v/iceberg-catalog-loader.svg +[iceberg-catalog-loader link]: https://crates.io/crates/iceberg-catalog-loader +[iceberg-catalog-loader release docs]: https://docs.rs/iceberg-catalog-loader +[iceberg-catalog-loader dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_loader/ + +[iceberg-cache-moka]: crates/integrations/cache-moka +[iceberg-cache-moka image]: https://img.shields.io/crates/v/iceberg-cache-moka.svg +[iceberg-cache-moka link]: https://crates.io/crates/iceberg-cache-moka +[iceberg-cache-moka release docs]: https://docs.rs/iceberg-cache-moka +[iceberg-cache-moka dev docs]: https://rust.iceberg.apache.org/api/iceberg_cache_moka/ + ## Iceberg Rust Implementation Status The features that Iceberg Rust currently supports can be found [here](https://iceberg.apache.org/status/). ## Supported Rust Version -Iceberg Rust is built and tested with stable rust, and will keep a rolling MSRV(minimum supported rust version). +Iceberg Rust is built and tested with stable rust, and will keep a rolling MSRV (minimum supported rust version). At least three months from latest rust release is supported. MSRV is updated when we release iceberg-rust. Check the current MSRV on [crates.io](https://crates.io/crates/iceberg). ## Contribute -Apache Iceberg is an active open-source project, governed under the Apache Software Foundation (ASF). Iceberg-rust are always open to people who want to use or contribute to it. Here are some ways to get involved. +Apache Iceberg is an active open-source project, governed under the Apache Software Foundation (ASF). Apache Iceberg Rust is always open to people who want to use or contribute to it. Here are some ways to get involved. - Start with [Contributing Guide](CONTRIBUTING.md). - Submit [Issues](https://github.com/apache/iceberg-rust/issues/new) for bug report or feature requests. @@ -97,6 +124,10 @@ The Apache Iceberg community is built on the principles described in the [Apache - [Lakekeeper](https://github.com/lakekeeper/lakekeeper/): An Apache-licensed Iceberg REST Catalog with data access controls. - [Moonlink](https://github.com/Mooncake-Labs/moonlink): A Rust library that enables sub-second mirroring (CDC) of Postgres tables into Iceberg. - [RisingWave](https://github.com/risingwavelabs/risingwave): A Postgres-compatible SQL database designed for real-time event streaming data processing, analysis, and management. +- [Wrappers](https://github.com/supabase/wrappers): Postgres Foreign Data Wrapper development framework in Rust. +- [ETL](https://github.com/supabase/etl): Stream your Postgres data anywhere in real-time. +- [Apache DataFusion Comet](https://github.com/apache/datafusion-comet): High-performance accelerator for Apache Spark, built on top of the powerful Apache DataFusion query engine. +- [Spice.ai](https://github.com/spiceai/spiceai): An open-source compute and AI runtime for data-grounded agents. ## License diff --git a/SECURITY-THREAT-MODEL.md b/SECURITY-THREAT-MODEL.md new file mode 100644 index 0000000000..0ebfc1d3a2 --- /dev/null +++ b/SECURITY-THREAT-MODEL.md @@ -0,0 +1,227 @@ + + +# Apache Iceberg Rust Security Threat Model + +This document describes the detailed security threat model for Apache +Iceberg Rust. It is intended for maintainers and automated security triage. + +## Purpose + +Apache Iceberg Rust is primarily a client library and implementation of the +Iceberg table format and catalog interactions for Rust applications and +services. It is commonly embedded in larger systems that provide their own +authentication, authorization, and credential management. Because of that +deployment model, many bug classes that look security-relevant in the abstract +are not actually security vulnerabilities in Iceberg Rust itself. + +This model is intended to answer: + +- what Iceberg Rust generally treats as a security vulnerability +- what Iceberg Rust generally treats as correctness, hardening, or deployment + work +- which boundaries are primarily owned by Iceberg Rust versus the surrounding + catalog, application, or service +- which issue classes should be downgraded by default by scanners + +## Scope + +This model is scoped to the Apache Iceberg Rust repository itself: + +- table format and metadata handling +- catalog and REST catalog clients +- transport, credential, and configuration handling implemented in this repo +- bindings and helper tooling shipped in this repo + +It is not a general threat model for every Rust service that embeds Iceberg +Rust. + +In particular, it does not attempt to define the complete security model for: + +- applications or services that embed Iceberg Rust +- storage-level authorization enforced outside Iceberg Rust + +## Security Goals + +Iceberg Rust should: + +- avoid exposing secrets or delegated credentials to principals that were not + already trusted with them +- avoid creating new unauthorized capabilities in Iceberg Rust-owned + components +- avoid violating trust boundaries that Iceberg Rust itself owns, such as + leaking auth, transport, or credential-bearing state across catalog or + client boundaries in the same process + +Iceberg Rust does not aim to be the primary enforcement point for: + +- user-to-user authorization inside the embedding application +- storage-level authorization +- service-side credential scoping performed by an external catalog + +## Roles + +### Operator + +The operator configures the surrounding catalog, application, service, and +storage integration around Iceberg Rust. This role is trusted to choose +endpoints, warehouses, storage integrations, and credentials. + +### Catalog control plane + +The catalog control plane resolves tables and supplies metadata, locations, +configuration, and delegated credentials to Iceberg Rust. It may be +implemented by a REST catalog server or another catalog implementation. +Iceberg Rust assumes this control plane is trusted and outside its primary +security boundary. + +### REST catalog client + +The REST catalog client consumes catalog-provided metadata, configuration, and +credentials. Client-side bugs in routing, caching, or reuse may still be +security-relevant if they leak credential-bearing state across boundaries that +the Iceberg Rust client is expected to preserve. + +### Embedding application + +Applications and services embedding Iceberg Rust are responsible for their own +user-facing authorization boundaries unless Iceberg Rust explicitly documents +otherwise. + +### Table writer or maintainer + +This role may already have legitimate power to write or replace table +metadata, write or delete files, choose paths under an allowed warehouse or +table location, and invoke destructive maintenance operations. If a report +only shows a new way to achieve the same effect this role can already cause +legitimately, it is usually not a security issue in Iceberg Rust. + +## Trust Boundaries + +### Boundary 1: operator-trusted configuration + +The following are generally treated as trusted operator or deployment inputs: + +- catalog properties +- endpoint configuration +- warehouse and storage roots +- transport wiring and credential configuration + +If a report depends on the attacker controlling those values directly, it is +usually not a vulnerability in Iceberg Rust itself. + +### Boundary 2: catalog-supplied metadata + +Iceberg Rust often accepts metadata locations, table properties, namespace +properties, and related control-plane information from a catalog. By default, +Iceberg Rust treats those sources as trusted. + +This means a malicious catalog supplying incorrect or malicious metadata is +usually not an Iceberg Rust vulnerability by itself. + +### Boundary 3: REST catalog-supplied configuration and delegated storage access + +In REST deployments, Iceberg Rust may also accept service endpoints, +configuration, and delegated storage access from the REST catalog server. By +default, those are treated as trusted control-plane inputs unless Iceberg Rust +explicitly documents a stronger guarantee. + +This means a malicious REST catalog server sending dangerous endpoints is +usually not an Iceberg Rust vulnerability by itself. It also means many +credential-selection bugs are often correctness or specification issues rather +than security boundary failures. + +The major exception is secret exposure. If Iceberg Rust surfaces credentials +or secrets to a new audience that was not already trusted with them, that is +security-relevant. + +### Boundary 4: storage-level authorization + +Object store permissions are enforced by the storage provider and the +credentials the surrounding deployment chooses to hand to Iceberg Rust. +Iceberg Rust is not the root authority for bucket- or object-level +authorization. + +## In-Scope Security Vulnerabilities + +The following categories are generally security-relevant in Iceberg Rust when +the report is credible and reproducible. + +### 1. Secret or credential disclosure to a new audience + +Examples include: + +- catalog or storage credentials exposed through a user-visible surface +- one catalog's credentials or auth state leaking into another catalog or + client + +### 2. Iceberg Rust-owned trust-boundary violations + +Security issues exist when Iceberg Rust itself is expected to separate +catalogs, clients, or principals and fails to do so. + +Examples include: + +- process-global auth or transport state crossing catalog instances +- secret-bearing state from one principal reused for another principal within + an Iceberg Rust-owned boundary + +## Usually Out of Scope or Non-Security by Default + +These categories may still be real bugs worth fixing, but they are not usually +security vulnerabilities in Iceberg Rust itself. + +### 1. Correctness bugs + +Examples include incorrect metadata handling, ambiguous matching semantics, +and logic bugs that do not create a new trust-boundary violation. + +### 2. Parser hardening and malformed-input robustness + +Malformed-input crashes, runtime exceptions, and memory amplification are +usually treated as robustness or hardening work rather than security issues in +Iceberg Rust itself. + +### 3. Malicious catalog or external service scenarios + +Reports that require a malicious catalog or other external control-plane +service are usually outside Iceberg Rust's primary security boundary. + +### 4. Equivalent-harm reports + +If the actor already has a legitimate capability that can cause the same harm, +the new path is usually not a security issue. + +## Scanner Calibration Rules + +A scanner targeting Iceberg Rust should treat a finding as higher-confidence +only if it plausibly shows one of the following: + +- exposure of a secret or delegated credential to a new audience +- creation of a new unauthorized capability in an Iceberg Rust-owned component +- violation of an Iceberg Rust-owned trust boundary rather than a surrounding + catalog, application, service, or operator boundary + +A finding should be downgraded or rejected by default if it instead depends +primarily on: + +- malformed-input robustness or denial-of-service behavior +- a malicious catalog or external service +- a principal that already has equivalent power through legitimate write or + maintenance capabilities diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock deleted file mode 100644 index 1a619dca0f..0000000000 --- a/bindings/python/Cargo.lock +++ /dev/null @@ -1,5260 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "abi_stable" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445" -dependencies = [ - "abi_stable_derive", - "abi_stable_shared", - "const_panic", - "core_extensions", - "crossbeam-channel", - "generational-arena", - "libloading", - "lock_api", - "parking_lot", - "paste", - "repr_offset", - "rustc_version", - "serde", - "serde_derive", - "serde_json", -] - -[[package]] -name = "abi_stable_derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898" -dependencies = [ - "abi_stable_shared", - "as_derive_utils", - "core_extensions", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", - "typed-arena", -] - -[[package]] -name = "abi_stable_shared" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63" -dependencies = [ - "core_extensions", -] - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.16", - "once_cell", - "version_check", -] - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "const-random", - "getrandom 0.3.3", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anyhow" -version = "1.0.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" - -[[package]] -name = "apache-avro" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" -dependencies = [ - "bigdecimal", - "bon", - "digest", - "log", - "miniz_oxide", - "num-bigint", - "quad-rand", - "rand 0.9.2", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "strum", - "strum_macros", - "thiserror 2.0.16", - "uuid", -] - -[[package]] -name = "array-init" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" - -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - -[[package]] -name = "arrayvec" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" - -[[package]] -name = "arrow" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f15b4c6b148206ff3a2b35002e08929c2462467b62b9c02036d9c34f9ef994" -dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-pyarrow", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", -] - -[[package]] -name = "arrow-arith" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "num", -] - -[[package]] -name = "arrow-array" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" -dependencies = [ - "ahash 0.8.12", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "chrono", - "chrono-tz", - "half", - "hashbrown 0.15.5", - "num", -] - -[[package]] -name = "arrow-buffer" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169b1d5d6cb390dd92ce582b06b23815c7953e9dfaaea75556e89d890d19993d" -dependencies = [ - "bytes", - "half", - "num", -] - -[[package]] -name = "arrow-cast" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "atoi", - "base64", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num", - "ryu", -] - -[[package]] -name = "arrow-csv" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "012c9fef3f4a11573b2c74aec53712ff9fdae4a95f4ce452d1bbf088ee00f06b" -dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", - "chrono", - "csv", - "csv-core", - "regex", -] - -[[package]] -name = "arrow-data" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" -dependencies = [ - "arrow-buffer", - "arrow-schema", - "half", - "num", -] - -[[package]] -name = "arrow-ipc" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ea5967e8b2af39aff5d9de2197df16e305f47f404781d3230b2dc672da5d92" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "flatbuffers", - "lz4_flex", -] - -[[package]] -name = "arrow-json" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5709d974c4ea5be96d900c01576c7c0b99705f4a3eec343648cb1ca863988a9c" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", - "chrono", - "half", - "indexmap 2.11.0", - "lexical-core", - "memchr", - "num", - "serde", - "serde_json", - "simdutf8", -] - -[[package]] -name = "arrow-ord" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", -] - -[[package]] -name = "arrow-pyarrow" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e55ecf16b9b61d433f6e63c72fc6afcf2597d7db96583de88ebb887d1822268" -dependencies = [ - "arrow-array", - "arrow-data", - "arrow-schema", - "pyo3", -] - -[[package]] -name = "arrow-row" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bf7393166beaf79b4bed9bfdf19e97472af32ce5b6b48169d321518a08cae2" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "half", -] - -[[package]] -name = "arrow-schema" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" -dependencies = [ - "bitflags", - "serde", - "serde_json", -] - -[[package]] -name = "arrow-select" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" -dependencies = [ - "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "num", -] - -[[package]] -name = "arrow-string" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" -dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", - "memchr", - "num", - "regex", - "regex-syntax", -] - -[[package]] -name = "as-any" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" - -[[package]] -name = "as_derive_utils" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4" -dependencies = [ - "core_extensions", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "async-compression" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" -dependencies = [ - "bzip2", - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", - "xz2", - "zstd", - "zstd-safe", -] - -[[package]] -name = "async-ffi" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" -dependencies = [ - "abi_stable", -] - -[[package]] -name = "async-lock" -version = "3.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "backon" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" -dependencies = [ - "fastrand", - "gloo-timers", - "tokio", -] - -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bigdecimal" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" -dependencies = [ - "autocfg", - "libm", - "num-bigint", - "num-integer", - "num-traits", - "serde", -] - -[[package]] -name = "bimap" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" - -[[package]] -name = "bitflags" -version = "2.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" - -[[package]] -name = "bitvec" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" -dependencies = [ - "funty", - "radium", - "tap", - "wyz", -] - -[[package]] -name = "blake2" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" -dependencies = [ - "digest", -] - -[[package]] -name = "blake3" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bon" -version = "3.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2529c31017402be841eb45892278a6c21a000c0a17643af326c73a73f83f0fb" -dependencies = [ - "bon-macros", - "rustversion", -] - -[[package]] -name = "bon-macros" -version = "3.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82020dadcb845a345591863adb65d74fa8dc5c18a0b6d408470e13b7adc7005" -dependencies = [ - "darling 0.21.3", - "ident_case", - "prettyplease", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.106", -] - -[[package]] -name = "borsh" -version = "1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" -dependencies = [ - "borsh-derive", - "cfg_aliases", -] - -[[package]] -name = "borsh-derive" -version = "1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd1d3c0c2f5833f22386f252fe8ed005c7f59fdcddeef025c01b4c3b9fd9ac3" -dependencies = [ - "once_cell", - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "brotli" -version = "8.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "bumpalo" -version = "3.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" - -[[package]] -name = "bytecheck" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" -dependencies = [ - "bytecheck_derive", - "ptr_meta", - "simdutf8", -] - -[[package]] -name = "bytecheck_derive" -version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "bytemuck" -version = "1.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" - -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] -name = "cc" -version = "1.2.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" - -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - -[[package]] -name = "chrono" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link 0.2.0", -] - -[[package]] -name = "chrono-tz" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" -dependencies = [ - "chrono", - "phf", -] - -[[package]] -name = "comfy-table" -version = "7.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8e18d0dca9578507f13f9803add0df13362b02c501c1c17734f0dbb52eaf0b" -dependencies = [ - "unicode-segmentation", - "unicode-width", -] - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - -[[package]] -name = "const-random" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" -dependencies = [ - "const-random-macro", -] - -[[package]] -name = "const-random-macro" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" -dependencies = [ - "getrandom 0.2.16", - "once_cell", - "tiny-keccak", -] - -[[package]] -name = "const_panic" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb8a602185c3c95b52f86dc78e55a6df9a287a7a93ddbcf012509930880cf879" -dependencies = [ - "typewit", -] - -[[package]] -name = "constant_time_eq" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "core_extensions" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003" -dependencies = [ - "core_extensions_proc_macros", -] - -[[package]] -name = "core_extensions_proc_macros" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "csv" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" -dependencies = [ - "memchr", -] - -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", -] - -[[package]] -name = "darling" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" -dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.106", -] - -[[package]] -name = "darling_core" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.106", -] - -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core 0.20.11", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "darling_macro" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" -dependencies = [ - "darling_core 0.21.3", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "dashmap" -version = "6.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - -[[package]] -name = "datafusion" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a11e19a7ccc5bb979c95c1dceef663eab39c9061b3bbf8d1937faf0f03bf41f" -dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", - "async-trait", - "bytes", - "bzip2", - "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", - "flate2", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "parquet", - "rand 0.9.2", - "regex", - "sqlparser", - "tempfile", - "tokio", - "url", - "uuid", - "xz2", - "zstd", -] - -[[package]] -name = "datafusion-catalog" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94985e67cab97b1099db2a7af11f31a45008b282aba921c1e1d35327c212ec18" -dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "tokio", -] - -[[package]] -name = "datafusion-catalog-listing" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e002df133bdb7b0b9b429d89a69aa77b35caeadee4498b2ce1c7c23a99516988" -dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "log", - "object_store", - "tokio", -] - -[[package]] -name = "datafusion-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13242fc58fd753787b0a538e5ae77d356cb9d0656fa85a591a33c5f106267f6" -dependencies = [ - "ahash 0.8.12", - "arrow", - "arrow-ipc", - "base64", - "half", - "hashbrown 0.14.5", - "indexmap 2.11.0", - "libc", - "log", - "object_store", - "parquet", - "paste", - "recursive", - "sqlparser", - "tokio", - "web-time", -] - -[[package]] -name = "datafusion-common-runtime" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2239f964e95c3a5d6b4a8cde07e646de8995c1396a7fd62c6e784f5341db499" -dependencies = [ - "futures", - "log", - "tokio", -] - -[[package]] -name = "datafusion-datasource" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf792579bc8bf07d1b2f68c2d5382f8a63679cce8fbebfd4ba95742b6e08864" -dependencies = [ - "arrow", - "async-compression", - "async-trait", - "bytes", - "bzip2", - "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "flate2", - "futures", - "glob", - "itertools 0.14.0", - "log", - "object_store", - "parquet", - "rand 0.9.2", - "tempfile", - "tokio", - "tokio-util", - "url", - "xz2", - "zstd", -] - -[[package]] -name = "datafusion-datasource-csv" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc114f9a1415174f3e8d2719c371fc72092ef2195a7955404cfe6b2ba29a706" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "regex", - "tokio", -] - -[[package]] -name = "datafusion-datasource-json" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d88dd5e215c420a52362b9988ecd4cefd71081b730663d4f7d886f706111fc75" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "object_store", - "serde_json", - "tokio", -] - -[[package]] -name = "datafusion-datasource-parquet" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33692acdd1fbe75280d14f4676fe43f39e9cb36296df56575aa2cac9a819e4cf" -dependencies = [ - "arrow", - "async-trait", - "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "parquet", - "rand 0.9.2", - "tokio", -] - -[[package]] -name = "datafusion-doc" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e7b648387b0c1937b83cb328533c06c923799e73a9e3750b762667f32662c0" - -[[package]] -name = "datafusion-execution" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9609d83d52ff8315283c6dad3b97566e877d8f366fab4c3297742f33dcd636c7" -dependencies = [ - "arrow", - "dashmap", - "datafusion-common", - "datafusion-expr", - "futures", - "log", - "object_store", - "parking_lot", - "rand 0.9.2", - "tempfile", - "url", -] - -[[package]] -name = "datafusion-expr" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75230cd67f650ef0399eb00f54d4a073698f2c0262948298e5299fc7324da63" -dependencies = [ - "arrow", - "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", - "indexmap 2.11.0", - "paste", - "recursive", - "serde_json", - "sqlparser", -] - -[[package]] -name = "datafusion-expr-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fafb3a045ed6c49cfca0cd090f62cf871ca6326cc3355cb0aaf1260fa760b6" -dependencies = [ - "arrow", - "datafusion-common", - "indexmap 2.11.0", - "itertools 0.14.0", - "paste", -] - -[[package]] -name = "datafusion-ffi" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257de77f836e9bad3e0274d4ffc88555ebf559ae30d4e9c674b9809104c1cc3b" -dependencies = [ - "abi_stable", - "arrow", - "arrow-schema", - "async-ffi", - "async-trait", - "datafusion", - "datafusion-functions-aggregate-common", - "datafusion-proto", - "datafusion-proto-common", - "futures", - "log", - "prost", - "semver", - "tokio", -] - -[[package]] -name = "datafusion-functions" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf9a9cf655265861a20453b1e58357147eab59bdc90ce7f2f68f1f35104d3bb" -dependencies = [ - "arrow", - "arrow-buffer", - "base64", - "blake2", - "blake3", - "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", - "hex", - "itertools 0.14.0", - "log", - "md-5", - "rand 0.9.2", - "regex", - "sha2", - "unicode-segmentation", - "uuid", -] - -[[package]] -name = "datafusion-functions-aggregate" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f07e49733d847be0a05235e17b884d326a2fd402c97a89fe8bcf0bfba310005" -dependencies = [ - "ahash 0.8.12", - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "half", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-aggregate-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4512607e10d72b0b0a1dc08f42cb5bd5284cb8348b7fea49dc83409493e32b1b" -dependencies = [ - "ahash 0.8.12", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-functions-nested" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab331806e34f5545e5f03396e4d5068077395b1665795d8f88c14ec4f1e0b7a" -dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-macros", - "datafusion-physical-expr-common", - "itertools 0.14.0", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-table" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ac2c0be983a06950ef077e34e0174aa0cb9e346f3aeae459823158037ade37" -dependencies = [ - "arrow", - "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", - "parking_lot", - "paste", -] - -[[package]] -name = "datafusion-functions-window" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f3d92731de384c90906941d36dcadf6a86d4128409a9c5cd916662baed5f53" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "log", - "paste", -] - -[[package]] -name = "datafusion-functions-window-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c679f8bf0971704ec8fd4249fcbb2eb49d6a12cc3e7a840ac047b4928d3541b5" -dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", -] - -[[package]] -name = "datafusion-macros" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2821de7cb0362d12e75a5196b636a59ea3584ec1e1cc7dc6f5e34b9e8389d251" -dependencies = [ - "datafusion-expr", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "datafusion-optimizer" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1594c7a97219ede334f25347ad8d57056621e7f4f35a0693c8da876e10dd6a53" -dependencies = [ - "arrow", - "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr", - "indexmap 2.11.0", - "itertools 0.14.0", - "log", - "recursive", - "regex", - "regex-syntax", -] - -[[package]] -name = "datafusion-physical-expr" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6da0f2412088d23f6b01929dedd687b5aee63b19b674eb73d00c3eb3c883b7" -dependencies = [ - "ahash 0.8.12", - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", - "half", - "hashbrown 0.14.5", - "indexmap 2.11.0", - "itertools 0.14.0", - "log", - "paste", - "petgraph", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb0dbd9213078a593c3fe28783beaa625a4e6c6a6c797856ee2ba234311fb96" -dependencies = [ - "ahash 0.8.12", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "hashbrown 0.14.5", - "itertools 0.14.0", -] - -[[package]] -name = "datafusion-physical-optimizer" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d140854b2db3ef8ac611caad12bfb2e1e1de827077429322a6188f18fc0026a" -dependencies = [ - "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "itertools 0.14.0", - "log", - "recursive", -] - -[[package]] -name = "datafusion-physical-plan" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b46cbdf21a01206be76d467f325273b22c559c744a012ead5018dfe79597de08" -dependencies = [ - "ahash 0.8.12", - "arrow", - "arrow-ord", - "arrow-schema", - "async-trait", - "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "futures", - "half", - "hashbrown 0.14.5", - "indexmap 2.11.0", - "itertools 0.14.0", - "log", - "parking_lot", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "datafusion-proto" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3fc7a2744332c2ef8804274c21f9fa664b4ca5889169250a6fd6b649ee5d16c" -dependencies = [ - "arrow", - "chrono", - "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-proto-common", - "object_store", - "prost", -] - -[[package]] -name = "datafusion-proto-common" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "800add86852f12e3d249867425de2224c1e9fb7adc2930460548868781fbeded" -dependencies = [ - "arrow", - "datafusion-common", - "prost", -] - -[[package]] -name = "datafusion-session" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a72733766ddb5b41534910926e8da5836622316f6283307fd9fb7e19811a59c" -dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-sql", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "tokio", -] - -[[package]] -name = "datafusion-sql" -version = "48.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5162338cdec9cc7ea13a0e6015c361acad5ec1d88d83f7c86301f789473971f" -dependencies = [ - "arrow", - "bigdecimal", - "datafusion-common", - "datafusion-expr", - "indexmap 2.11.0", - "log", - "recursive", - "regex", - "sqlparser", -] - -[[package]] -name = "deranged" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d630bccd429a5bb5a64b5e94f693bfc48c9f8566418fda4c494cc94f911f87cc" -dependencies = [ - "powerfmt", - "serde", -] - -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.106", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "dissimilar" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8975ffdaa0ef3661bfe02dbdcc06c9f829dfafe6a3c474de366a8d5e44276921" - -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" -dependencies = [ - "libc", - "windows-sys 0.60.2", -] - -[[package]] -name = "event-listener" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", -] - -[[package]] -name = "expect-test" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63af43ff4431e848fb47472a920f14fa71c24de13255a5692e93d4e90302acb0" -dependencies = [ - "dissimilar", - "once_cell", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "find-msvc-tools" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" - -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] -name = "flatbuffers" -version = "25.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" -dependencies = [ - "bitflags", - "rustc_version", -] - -[[package]] -name = "flate2" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" -dependencies = [ - "crc32fast", - "libz-rs-sys", - "miniz_oxide", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generational-arena" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "generator" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" -dependencies = [ - "cc", - "cfg-if", - "libc", - "log", - "rustversion", - "windows", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi", - "wasi 0.14.4+wasi-0.2.4", - "wasm-bindgen", -] - -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - -[[package]] -name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" - -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "half" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" -dependencies = [ - "cfg-if", - "crunchy", - "num-traits", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.8", -] - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash 0.8.12", - "allocator-api2", -] - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "http" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "humantime" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" - -[[package]] -name = "hyper" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "pin-utils", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tower-service", - "webpki-roots", -] - -[[package]] -name = "hyper-util" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "iceberg" -version = "0.7.0" -dependencies = [ - "anyhow", - "apache-avro", - "array-init", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", - "as-any", - "async-trait", - "backon", - "base64", - "bimap", - "bytes", - "chrono", - "derive_builder", - "expect-test", - "fnv", - "futures", - "itertools 0.13.0", - "moka", - "murmur3", - "num-bigint", - "once_cell", - "opendal", - "ordered-float 4.6.0", - "parquet", - "rand 0.8.5", - "reqsign", - "reqwest", - "roaring", - "rust_decimal", - "serde", - "serde_bytes", - "serde_derive", - "serde_json", - "serde_repr", - "serde_with", - "strum", - "thrift", - "tokio", - "typed-builder", - "url", - "uuid", - "zstd", -] - -[[package]] -name = "iceberg-datafusion" -version = "0.7.0" -dependencies = [ - "anyhow", - "async-trait", - "datafusion", - "futures", - "iceberg", - "parquet", - "tokio", - "uuid", -] - -[[package]] -name = "icu_collections" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" -dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" - -[[package]] -name = "icu_properties" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "potential_utf", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" - -[[package]] -name = "icu_provider" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" -dependencies = [ - "displaydoc", - "icu_locale_core", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - -[[package]] -name = "indexmap" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" -dependencies = [ - "equivalent", - "hashbrown 0.15.5", - "serde", -] - -[[package]] -name = "indoc" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" - -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - -[[package]] -name = "io-uring" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" -dependencies = [ - "bitflags", - "cfg-if", - "libc", -] - -[[package]] -name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.3", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.78" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0b063578492ceec17683ef2f8c5e89121fbd0b172cbc280635ab7567db2738" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "lexical-core" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" -dependencies = [ - "lexical-parse-integer", - "lexical-util", - "static_assertions", -] - -[[package]] -name = "lexical-parse-integer" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" -dependencies = [ - "lexical-util", - "static_assertions", -] - -[[package]] -name = "lexical-util" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" -dependencies = [ - "static_assertions", -] - -[[package]] -name = "lexical-write-float" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" -dependencies = [ - "lexical-util", - "lexical-write-integer", - "static_assertions", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" -dependencies = [ - "lexical-util", - "static_assertions", -] - -[[package]] -name = "libc" -version = "0.2.175" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" - -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - -[[package]] -name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - -[[package]] -name = "libz-rs-sys" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" -dependencies = [ - "zlib-rs", -] - -[[package]] -name = "linux-raw-sys" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" - -[[package]] -name = "litemap" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" - -[[package]] -name = "lock_api" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" - -[[package]] -name = "loom" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator", - "scoped-tls", - "tracing", - "tracing-subscriber", -] - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "lz4_flex" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" -dependencies = [ - "twox-hash", -] - -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - -[[package]] -name = "memchr" -version = "2.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", -] - -[[package]] -name = "moka" -version = "0.12.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" -dependencies = [ - "async-lock", - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "event-listener", - "futures-util", - "loom", - "parking_lot", - "portable-atomic", - "rustc_version", - "smallvec", - "tagptr", - "thiserror 1.0.69", - "uuid", -] - -[[package]] -name = "murmur3" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" - -[[package]] -name = "nu-ansi-term" -version = "0.50.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" -dependencies = [ - "num-integer", - "num-traits", - "serde", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - -[[package]] -name = "object_store" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc4f07659e11cd45a341cd24d71e683e3be65d9ff1f8150061678fe60437496" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "futures", - "http", - "humantime", - "itertools 0.14.0", - "parking_lot", - "percent-encoding", - "thiserror 2.0.16", - "tokio", - "tracing", - "url", - "walkdir", - "wasm-bindgen-futures", - "web-time", -] - -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - -[[package]] -name = "opendal" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb9838d0575c6dbaf3fcec7255af8d5771996d4af900bbb6fa9a314dec00a1a" -dependencies = [ - "anyhow", - "backon", - "base64", - "bytes", - "chrono", - "crc32c", - "futures", - "getrandom 0.2.16", - "http", - "http-body", - "log", - "md-5", - "percent-encoding", - "quick-xml", - "reqsign", - "reqwest", - "serde", - "serde_json", - "tokio", - "uuid", -] - -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - -[[package]] -name = "ordered-float" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" -dependencies = [ - "num-traits", -] - -[[package]] -name = "ordered-multimap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" -dependencies = [ - "dlv-list", - "hashbrown 0.14.5", -] - -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - -[[package]] -name = "parking_lot" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.52.6", -] - -[[package]] -name = "parquet" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" -dependencies = [ - "ahash 0.8.12", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", - "base64", - "brotli", - "bytes", - "chrono", - "flate2", - "futures", - "half", - "hashbrown 0.15.5", - "lz4_flex", - "num", - "num-bigint", - "object_store", - "paste", - "seq-macro", - "simdutf8", - "snap", - "thrift", - "tokio", - "twox-hash", - "zstd", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "petgraph" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap 2.11.0", - "serde", -] - -[[package]] -name = "phf" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_shared" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "portable-atomic" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" - -[[package]] -name = "potential_utf" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" -dependencies = [ - "zerovec", -] - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.106", -] - -[[package]] -name = "proc-macro-crate" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" -dependencies = [ - "toml_edit", -] - -[[package]] -name = "proc-macro2" -version = "1.0.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools 0.14.0", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "psm" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e944464ec8536cd1beb0bbfd96987eb5e3b72f2ecdafdc5c769a37f1fa2ae1f" -dependencies = [ - "cc", -] - -[[package]] -name = "ptr_meta" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "pyiceberg_core_rust" -version = "0.7.0" -dependencies = [ - "arrow", - "datafusion-ffi", - "iceberg", - "iceberg-datafusion", - "pyo3", - "tokio", -] - -[[package]] -name = "pyo3" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" -dependencies = [ - "cfg-if", - "indoc", - "libc", - "memoffset", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", -] - -[[package]] -name = "pyo3-build-config" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" -dependencies = [ - "once_cell", - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" -dependencies = [ - "heck", - "proc-macro2", - "pyo3-build-config", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "quad-rand" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" - -[[package]] -name = "quick-xml" -version = "0.37.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quinn" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror 2.0.16", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" -dependencies = [ - "bytes", - "getrandom 0.3.3", - "lru-slab", - "rand 0.9.2", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror 2.0.16", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.60.2", -] - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.3", -] - -[[package]] -name = "recursive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" -dependencies = [ - "recursive-proc-macro-impl", - "stacker", -] - -[[package]] -name = "recursive-proc-macro-impl" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" -dependencies = [ - "quote", - "syn 2.0.106", -] - -[[package]] -name = "redox_syscall" -version = "0.5.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" -dependencies = [ - "bitflags", -] - -[[package]] -name = "ref-cast" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "regex" -version = "1.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-lite" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30" - -[[package]] -name = "regex-syntax" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" - -[[package]] -name = "rend" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" -dependencies = [ - "bytecheck", -] - -[[package]] -name = "repr_offset" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea" -dependencies = [ - "tstr", -] - -[[package]] -name = "reqsign" -version = "0.16.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" -dependencies = [ - "anyhow", - "async-trait", - "base64", - "chrono", - "form_urlencoded", - "getrandom 0.2.16", - "hex", - "hmac", - "home", - "http", - "log", - "percent-encoding", - "quick-xml", - "rand 0.8.5", - "reqwest", - "rust-ini", - "serde", - "serde_json", - "sha1", - "sha2", - "tokio", -] - -[[package]] -name = "reqwest" -version = "0.12.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" -dependencies = [ - "base64", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "webpki-roots", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.16", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rkyv" -version = "0.7.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" -dependencies = [ - "bitvec", - "bytecheck", - "bytes", - "hashbrown 0.12.3", - "ptr_meta", - "rend", - "rkyv_derive", - "seahash", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.7.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "roaring" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f08d6a905edb32d74a5d5737a0c9d7e950c312f3c46cb0ca0a2ca09ea11878a0" -dependencies = [ - "bytemuck", - "byteorder", -] - -[[package]] -name = "rust-ini" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - -[[package]] -name = "rust_decimal" -version = "1.37.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b203a6425500a03e0919c42d3c47caca51e79f1132046626d2c8871c5092035d" -dependencies = [ - "arrayvec", - "borsh", - "bytes", - "num-traits", - "rand 0.8.5", - "rkyv", - "serde", - "serde_json", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" - -[[package]] -name = "rustc-hash" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" - -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - -[[package]] -name = "rustix" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.60.2", -] - -[[package]] -name = "rustls" -version = "0.23.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" -dependencies = [ - "once_cell", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-pki-types" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" -dependencies = [ - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schemars" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" -dependencies = [ - "dyn-clone", - "ref-cast", - "serde", - "serde_json", -] - -[[package]] -name = "schemars" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" -dependencies = [ - "dyn-clone", - "ref-cast", - "serde", - "serde_json", -] - -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "seahash" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" - -[[package]] -name = "semver" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" - -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" - -[[package]] -name = "serde" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_bytes" -version = "0.11.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_derive" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "serde_json" -version = "1.0.143" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "serde_repr" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "3.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" -dependencies = [ - "base64", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.11.0", - "schemars 0.9.0", - "schemars 1.0.4", - "serde", - "serde_derive", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - -[[package]] -name = "siphasher" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" - -[[package]] -name = "slab" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "snap" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" - -[[package]] -name = "socket2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "sqlparser" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" -dependencies = [ - "log", - "recursive", - "sqlparser_derive", -] - -[[package]] -name = "sqlparser_derive" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "stacker" -version = "0.1.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" -dependencies = [ - "cc", - "cfg-if", - "libc", - "psm", - "windows-sys 0.59.0", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "strum" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - -[[package]] -name = "target-lexicon" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" - -[[package]] -name = "tempfile" -version = "3.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" -dependencies = [ - "fastrand", - "getrandom 0.3.3", - "once_cell", - "rustix", - "windows-sys 0.60.2", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" -dependencies = [ - "thiserror-impl 2.0.16", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "thread_local" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "log", - "ordered-float 2.10.1", - "threadpool", -] - -[[package]] -name = "time" -version = "0.3.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031" -dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" - -[[package]] -name = "time-macros" -version = "0.2.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - -[[package]] -name = "tinystr" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.47.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" -dependencies = [ - "backtrace", - "bytes", - "io-uring", - "libc", - "mio", - "pin-project-lite", - "slab", - "socket2", - "tokio-macros", - "windows-sys 0.59.0", -] - -[[package]] -name = "tokio-macros" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" - -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap 2.11.0", - "toml_datetime", - "winnow", -] - -[[package]] -name = "tower" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "iri-string", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "tracing-core" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "tstr" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7" -dependencies = [ - "tstr_proc_macros", -] - -[[package]] -name = "tstr_proc_macros" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" - -[[package]] -name = "twox-hash" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" - -[[package]] -name = "typed-arena" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" - -[[package]] -name = "typed-builder" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" -dependencies = [ - "typed-builder-macro", -] - -[[package]] -name = "typed-builder-macro" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "typenum" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" - -[[package]] -name = "typewit" -version = "1.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c98488b93df24b7c794d6a58c4198d7a2abde676324beaca84f7fb5b39c0811" - -[[package]] -name = "unicode-ident" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" - -[[package]] -name = "unicode-segmentation" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" - -[[package]] -name = "unicode-width" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" - -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "uuid" -version = "1.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" -dependencies = [ - "getrandom 0.3.3", - "js-sys", - "serde", - "wasm-bindgen", -] - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasi" -version = "0.14.4+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a5f4a424faf49c3c2c344f166f0662341d470ea185e939657aaff130f0ec4a" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.106", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.51" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca85039a9b469b38336411d6d6ced91f3fc87109a2a27b0c197663f5144dffe" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "wasm-streams" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.78" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e4b637749ff0d92b8fad63aa1f7cff3cbe125fd49c175cd6345e7272638b12" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys 0.61.0", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" -dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-link 0.1.3", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" -dependencies = [ - "windows-core", -] - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.1.3", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-future" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" -dependencies = [ - "windows-core", - "windows-link 0.1.3", - "windows-threading", -] - -[[package]] -name = "windows-implement" -version = "0.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "windows-interface" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - -[[package]] -name = "windows-link" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" - -[[package]] -name = "windows-numerics" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core", - "windows-link 0.1.3", -] - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link 0.1.3", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link 0.1.3", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.3", -] - -[[package]] -name = "windows-sys" -version = "0.61.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" -dependencies = [ - "windows-link 0.2.0", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" -dependencies = [ - "windows-link 0.1.3", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", -] - -[[package]] -name = "windows-threading" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" -dependencies = [ - "windows-link 0.1.3", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - -[[package]] -name = "winnow" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" -dependencies = [ - "memchr", -] - -[[package]] -name = "wit-bindgen" -version = "0.45.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" - -[[package]] -name = "writeable" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" - -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - -[[package]] -name = "yoke" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" - -[[package]] -name = "zerotrie" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - -[[package]] -name = "zlib-rs" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 879a1450b7..96e1f1ced8 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -16,24 +16,30 @@ # under the License. [package] -edition = "2024" -homepage = "https://rust.iceberg.apache.org" +edition = { workspace = true } +homepage = { workspace = true } name = "pyiceberg_core_rust" -rust-version = "1.87" -version = "0.7.0" # This crate is used to build python bindings, we don't want to publish it publish = false +rust-version = { workspace = true } +version = { workspace = true } -keywords = ["iceberg"] -license = "Apache-2.0" +categories = ["database"] +description = "Apache Iceberg Python Bindings" +keywords = ["iceberg", "python", "bindings"] +license = { workspace = true } +repository = { workspace = true } [lib] crate-type = ["cdylib"] [dependencies] -arrow = { version = "55.1", features = ["pyarrow", "chrono-tz"] } -iceberg = { path = "../../crates/iceberg" } -pyo3 = { version = "0.24.1", features = ["extension-module", "abi3-py39"] } -iceberg-datafusion = { path = "../../crates/integrations/datafusion" } -datafusion-ffi = { version = "48.0.1" } -tokio = { version = "1.46.1", default-features = false } +arrow = { workspace = true, features = ["chrono-tz", "pyarrow"] } +iceberg = { workspace = true } +pyo3 = { workspace = true, features = ["abi3-py310"] } + +[package.metadata.cargo-machete] +ignored = [] + +[lints] +workspace = true diff --git a/bindings/python/DEPENDENCIES.rust.tsv b/bindings/python/DEPENDENCIES.rust.tsv deleted file mode 100644 index 7565a13e1d..0000000000 --- a/bindings/python/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,445 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib -abi_stable@0.11.3 X X -abi_stable_derive@0.11.3 X X -abi_stable_shared@0.11.0 X X -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@55.2.0 X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-csv@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-json@55.2.0 X -arrow-ord@55.2.0 X -arrow-pyarrow@55.2.0 X -arrow-row@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -as_derive_utils@0.11.0 X X -async-compression@0.4.19 X X -async-ffi@0.5.0 X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -blake2@0.10.6 X X -blake3@1.8.2 X X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -bzip2@0.5.2 X X -bzip2-sys@0.1.13+1.0.8 X X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -chrono-tz@0.10.4 X X -comfy-table@7.2.0 X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -const_panic@0.2.14 X -constant_time_eq@0.3.1 X X X -core-foundation-sys@0.8.7 X X -core_extensions@1.5.4 X X -core_extensions_proc_macros@1.5.4 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -csv@1.3.1 X X -csv-core@0.1.12 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -dashmap@6.1.0 X -datafusion@48.0.1 X -datafusion-catalog@48.0.1 X -datafusion-catalog-listing@48.0.1 X -datafusion-common@48.0.1 X -datafusion-common-runtime@48.0.1 X -datafusion-datasource@48.0.1 X -datafusion-datasource-csv@48.0.1 X -datafusion-datasource-json@48.0.1 X -datafusion-datasource-parquet@48.0.1 X -datafusion-doc@48.0.1 X -datafusion-execution@48.0.1 X -datafusion-expr@48.0.1 X -datafusion-expr-common@48.0.1 X -datafusion-ffi@48.0.1 X -datafusion-functions@48.0.1 X -datafusion-functions-aggregate@48.0.1 X -datafusion-functions-aggregate-common@48.0.1 X -datafusion-functions-nested@48.0.1 X -datafusion-functions-table@48.0.1 X -datafusion-functions-window@48.0.1 X -datafusion-functions-window-common@48.0.1 X -datafusion-macros@48.0.1 X -datafusion-optimizer@48.0.1 X -datafusion-physical-expr@48.0.1 X -datafusion-physical-expr-common@48.0.1 X -datafusion-physical-optimizer@48.0.1 X -datafusion-physical-plan@48.0.1 X -datafusion-proto@48.0.1 X -datafusion-proto-common@48.0.1 X -datafusion-session@48.0.1 X -datafusion-sql@48.0.1 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -errno@0.3.13 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -fixedbitset@0.5.7 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generational-arena@0.2.9 X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -half@2.6.0 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -humantime@2.2.0 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-datafusion@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -indoc@2.0.6 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libloading@0.7.4 X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -linux-raw-sys@0.9.4 X X X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -lzma-sys@0.1.20 X X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -memoffset@0.9.1 X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -object_store@0.12.3 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -petgraph@0.8.2 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -prost@0.13.5 X -prost-derive@0.13.5 X -psm@0.1.26 X X -pyiceberg_core_rust@0.7.0 X -pyo3@0.24.2 X X -pyo3-build-config@0.24.2 X X -pyo3-ffi@0.24.2 X X -pyo3-macros@0.24.2 X X -pyo3-macros-backend@0.24.2 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -repr_offset@0.2.2 X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.37.2 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustix@1.0.8 X X X -rustls@0.23.31 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -same-file@1.0.6 X X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simdutf8@0.1.5 X X -siphasher@1.0.1 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.0 X X -sqlparser@0.55.0 X -sqlparser_derive@0.3.0 X -stable_deref_trait@1.2.0 X X -stacker@0.1.21 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@1.0.109 X X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -target-lexicon@0.13.2 X -tempfile@3.21.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -tstr@0.2.4 X -tstr_proc_macros@0.2.2 X -twox-hash@2.1.2 X -typed-arena@2.0.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -typewit@1.14.1 X -unicode-ident@1.0.18 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.2.1 X X -unindent@0.2.4 X X -untrusted@0.9.0 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -web-time@1.1.0 X X -webpki-roots@1.0.2 X -winapi@0.3.9 X X -winapi-i686-pc-windows-gnu@0.4.0 X X -winapi-util@0.1.11 X X -winapi-x86_64-pc-windows-gnu@0.4.0 X X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.0 X X -windows-targets@0.52.6 X X -windows-targets@0.53.3 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.0 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.0 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.0 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.0 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.0 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.0 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.0 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.0 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -xz2@0.1.7 X X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/bindings/python/LICENSING.md b/bindings/python/LICENSING.md new file mode 100644 index 0000000000..9eb48ecc08 --- /dev/null +++ b/bindings/python/LICENSING.md @@ -0,0 +1,73 @@ + + +# LICENSE and NOTICE for pyiceberg-core + +The `pyiceberg-core` wheels are a convenience artifact. We build them and upload them to PyPI as part of every release. + +The Rust crates that `pyiceberg-core` depends on are statically linked into each wheel. So the `LICENSE` and `NOTICE` inside a wheel must cover those crates as well as iceberg-rust itself. + +The sdist only contains iceberg-rust's own source, so it ships the root `LICENSE` and `NOTICE` unchanged. + +## How it works + +`LICENSE` and `NOTICE` are generated at build time, before `maturin` builds and packages each wheel. + +The [`generate-python-license`](../../.github/actions/generate-python-license/action.yml) action regenerates the wheel's `LICENSE` and `NOTICE` in `bindings/python/`: + +- `LICENSE`: the root `LICENSE`, then the [`cargo-about`](https://github.com/EmbarkStudios/cargo-about) output, then the manual entries in `third-party-licenses.txt`. +- `NOTICE`: the root `NOTICE`, then the manual entries in `third-party-notices.txt`. + +The action is used in these workflows: + +- [`release_python.yml`](../../.github/workflows/release_python.yml): the release wheels published to PyPI. +- [`release_python_nightly.yml`](../../.github/workflows/release_python_nightly.yml): the nightly wheels published to TestPyPI. +- [`bindings_python_ci.yml`](../../.github/workflows/bindings_python_ci.yml): pull requests. It only checks that generation doesn't fail. The generated files are not published. + +Note that in the repository, `bindings/python/LICENSE` and `NOTICE` are symlinks to the root files, so builds outside the release workflows ship the plain root files. + +## Relevant files + +- `about.hbs`: the `cargo-about` output template. Keep it a plain template. +- `about.toml`: configures `cargo-about`. + - `accepted`: the licenses we bundle, matching `deny.toml`. Add new ones at the end, as the order is the preference when a crate offers a choice. + - `targets`: the wheel targets. Keep in sync with the build matrices in `release_python.yml` and `release_python_nightly.yml`. + - Clarifications, for crates whose license files `cargo-about` can't read (such as `libm`). They pin the file by checksum (`shasum -a 256 `). See the [`cargo-about` docs](https://embarkstudios.github.io/cargo-about/cli/clarify.html). +- `third-party-licenses.txt`: otherwise, add the license text here. For example, `brotli-decompressor`'s MIT license, which the crate doesn't ship. It names the crate and version, so re-check it when the crate is upgraded. +- `third-party-notices.txt`: the `NOTICE` entries of dependencies that ship one (Arrow, Avro, moka). Find them from `bindings/python` (the repository's own crates appear too): + + ```shell + cargo about generate -c about.toml --locked --fail --manifest-path Cargo.toml --format json \ + | jq -r '.crates[].package.manifest_path' | xargs -n1 dirname \ + | xargs -I{} find {} -maxdepth 1 -iname 'NOTICE*' + ``` + +## Keeping entries up to date + +Pull request CI checks `about.toml`. It fails on a license that isn't in `accepted`, on a `cargo about` warning such as a stale clarification, and on `` in the output, which means no license file was found. To reproduce a failure, run this from `bindings/python`: + +```shell +make -s cargo-about | grep -n '' +``` + +CI doesn't check the other entries. Before each release, from `bindings/python`, check that: + +- `targets` in `about.toml` matches the wheel targets in `release_python.yml` and `release_python_nightly.yml`. +- Each crate in `third-party-licenses.txt` is in the `make -s cargo-about` output at the version the entry names. If the version changed, re-check the entry against the new version. If the crate is gone, remove the entry. +- `third-party-notices.txt` matches the `NOTICE` files of the dependencies, found with the command in [Relevant files](#relevant-files). diff --git a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml b/bindings/python/Makefile similarity index 58% rename from crates/iceberg/testdata/file_io_s3/docker-compose.yaml rename to bindings/python/Makefile index cbce31864e..256fedaf2d 100644 --- a/crates/iceberg/testdata/file_io_s3/docker-compose.yaml +++ b/bindings/python/Makefile @@ -15,16 +15,22 @@ # specific language governing permissions and limitations # under the License. -services: - minio: - image: minio/minio:RELEASE.2024-02-26T09-33-48Z - expose: - - 9000 - - 9001 - environment: - MINIO_ROOT_USER: 'admin' - MINIO_ROOT_PASSWORD: 'password' - MINIO_ADDRESS: ':9000' - MINIO_CONSOLE_ADDRESS: ':9001' - entrypoint: sh - command: -c 'mkdir -p /data/bucket1 && /usr/bin/minio server /data' +.PHONY: build install test install-cargo-about cargo-about + +build: + uv run maturin develop + +# uv sync may remove local editable pyiceberg-core, so install rebuilds it afterwards. +install: + uv sync --group dev --no-install-project + $(MAKE) build + +test: + uv run --no-sync pytest + +# Keep version in sync with .github/actions/generate-python-license/action.yml. +install-cargo-about: + cargo install --locked --features cli cargo-about@0.9.2 + +cargo-about: install-cargo-about + cargo about generate -c about.toml --locked --fail --manifest-path Cargo.toml about.hbs diff --git a/bindings/python/README.md b/bindings/python/README.md index 2dddc71ac1..f831c28e89 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -23,22 +23,30 @@ This project is used to build an Iceberg-rust powered core for [PyIceberg](https ## Setup -Install Hatch: +Install [uv](https://docs.astral.sh/uv/getting-started/installation/): ```shell -pip install hatch==1.12.0 +pip install uv==0.9.3 ``` -Hatch uses [uv](https://docs.astral.sh/uv/) as a backend by default, so [make sure that it is installed](https://docs.astral.sh/uv/getting-started/installation/) as well. +Set up the development environment: + +```shell +make install +``` ## Build ```shell -hatch run dev:develop +make build ``` ## Test ```shell -hatch run dev:test +make test ``` + +## LICENSE and NOTICE + +The wheels bundle the licenses and notices of their Rust dependencies. See [LICENSING.md](LICENSING.md) for how they are generated and how to keep them up to date. diff --git a/bindings/python/about.hbs b/bindings/python/about.hbs new file mode 100644 index 0000000000..c55b318a1e --- /dev/null +++ b/bindings/python/about.hbs @@ -0,0 +1,35 @@ +{{!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--}} + +-------------------------------------------------------------------------------- + +This package bundles Rust dependencies. Their licenses follow. +{{#each licenses}} + +-------------------------------------------------------------------------------- + +{{{name}}} + +Used by: +{{#each used_by}} + - {{crate.name}} {{crate.version}}{{#if crate.repository}} ({{crate.repository}}){{/if}} +{{/each}} + +{{{text}}} +{{/each}} diff --git a/bindings/python/about.toml b/bindings/python/about.toml new file mode 100644 index 0000000000..a804d6acb3 --- /dev/null +++ b/bindings/python/about.toml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Configuration for `cargo about`, which lists the license texts of the Rust +# dependencies compiled into pyiceberg-core. +# +# Keep `accepted` in line with deny.toml. Order is preference when a crate offers +# a choice, so add new licenses at the end. +# +# Accepted licenses must be ASF Category A or B: +# https://www.apache.org/legal/resolved.html +accepted = [ + "Apache-2.0", + "BSD-3-Clause", + "CC0-1.0", + "ISC", + "MIT", + # Not on that page; Category A per + # https://issues.apache.org/jira/browse/LEGAL-660 + "Unicode-3.0", + "Zlib", + "Apache-2.0 WITH LLVM-exception", + "BSD-2-Clause", + "BSL-1.0", + "CDLA-Permissive-2.0", +] +ignore-build-dependencies = true +ignore-dev-dependencies = true +# The wheel targets in release_python.yml and release_python_nightly.yml. Keep in sync. +targets = [ + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "armv7-unknown-linux-gnueabihf", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", +] + +# The published crate omits its LICENSE, so cargo-about fetches it from the +# upstream repository. +[alloc-stdlib.clarify] +license = "BSD-3-Clause" +[[alloc-stdlib.clarify.git]] +checksum = "c0c56f26d9c051cac4d200c34c84e7ae9aaa853e01a982a1df08b09931e518ae" +path = "LICENSE" + +# libm's license is in LICENSE.txt, which cargo-about doesn't match by name. +[libm.clarify] +license = "MIT" +[[libm.clarify.files]] +checksum = "3823dda7cf046602f4b4e77ec8e227863dc4736037cc85bb33d9f19febe16bb7" +path = "LICENSE.txt" diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 36b5834228..a60a6cc127 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -17,7 +17,7 @@ [build-system] build-backend = "maturin" -requires = ["maturin>=1.0,<2.0"] +requires = ["maturin>=1.9.4,<2.0"] [project] classifiers = [ @@ -25,33 +25,53 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Rust", ] -name = "pyiceberg-core" -readme = "project-description.md" -requires-python = "~=3.9" dynamic = ["version"] license = { file = "LICENSE" } +name = "pyiceberg-core" +readme = "project-description.md" +requires-python = ">=3.10,<4" [tool.maturin] -features = ["pyo3/extension-module"] +# sdist only: maturin adds these to wheels automatically (dist-info/licenses/). +include = [ + { path = "LICENSE", format = "sdist" }, + { path = "NOTICE", format = "sdist" }, +] module-name = "pyiceberg_core.pyiceberg_core_rust" +profile = "release" python-source = "python" -include = [ - { path = "LICENSE", format = ["sdist", "wheel"] }, - { path = "NOTICE", format = ["sdist", "wheel"] } +# Don't strip: depending on the binary's layout, a stripped macOS .so fails to import +# with "mis-aligned LINKEDIT string pool" (https://github.com/apache/iceberg-rust/issues/3646). +config = [ + "profile.release.lto=\"thin\"", + "profile.release.codegen-units=1", + "profile.release.opt-level=3", + "profile.release.strip=false", ] [tool.ruff.lint] ignore = ["F403", "F405"] -[tool.hatch.envs.dev] -dependencies = ["maturin>=1.0,<2.0", "pytest>=8.3.2", "datafusion==45.*", "pyiceberg[sql-sqlite,pyarrow]>=0.10.0", "fastavro>=1.11.1"] +[tool.pytest.ini_options] +filterwarnings = ["error"] -[tool.hatch.envs.dev.scripts] -build = "maturin build --out dist --sdist" -develop = "maturin develop" -test = "pytest" +[dependency-groups] +dev = [ + "maturin>=1.9.4,<2.0", + "pytest>=8.3.2", + # Set a PyIceberg lower bound; otherwise the resolver may select the + # unrelated 0.0.2 stub. + # Install PyArrow directly instead of using pyiceberg[pyarrow]: that extra + # depends on the published pyiceberg-core, which conflicts with this local build. + "pyiceberg>=0.11", + "pyarrow>=17", + "fastavro>=1.11.1", +] diff --git a/bindings/python/src/data_file.rs b/bindings/python/src/data_file.rs index 900d6c6014..7051e83280 100644 --- a/bindings/python/src/data_file.rs +++ b/bindings/python/src/data_file.rs @@ -23,6 +23,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyBytes; +use crate::error::to_py_err; + #[pyclass()] pub struct PyPrimitiveLiteral { inner: PrimitiveLiteral, @@ -120,20 +122,20 @@ impl PyDataFile { } #[getter] - fn upper_bounds(&self) -> HashMap> { + fn upper_bounds(&self) -> PyResult>> { self.inner .upper_bounds() .iter() - .map(|(k, v)| (*k, v.to_bytes().unwrap().to_vec())) + .map(|(k, v)| Ok((*k, v.to_bytes().map_err(to_py_err)?.to_vec()))) .collect() } #[getter] - fn lower_bounds(&self) -> HashMap> { + fn lower_bounds(&self) -> PyResult>> { self.inner .lower_bounds() .iter() - .map(|(k, v)| (*k, v.to_bytes().unwrap().to_vec())) + .map(|(k, v)| Ok((*k, v.to_bytes().map_err(to_py_err)?.to_vec()))) .collect() } @@ -143,7 +145,7 @@ impl PyDataFile { } #[getter] - fn split_offsets(&self) -> &[i64] { + fn split_offsets(&self) -> Option<&[i64]> { self.inner.split_offsets() } diff --git a/bindings/python/src/datafusion_table_provider.rs b/bindings/python/src/datafusion_table_provider.rs deleted file mode 100644 index b5e1bf952e..0000000000 --- a/bindings/python/src/datafusion_table_provider.rs +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::ffi::CString; -use std::sync::Arc; - -use datafusion_ffi::table_provider::FFI_TableProvider; -use iceberg::TableIdent; -use iceberg::io::FileIO; -use iceberg::table::StaticTable; -use iceberg_datafusion::table::IcebergTableProvider; -use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::*; -use pyo3::types::PyCapsule; - -use crate::runtime::runtime; - -#[pyclass(name = "IcebergDataFusionTable")] -pub struct PyIcebergDataFusionTable { - inner: Arc, -} - -#[pymethods] -impl PyIcebergDataFusionTable { - #[new] - fn new( - identifier: Vec, - metadata_location: String, - file_io_properties: Option>, - ) -> PyResult { - let runtime = runtime(); - - let provider = runtime.block_on(async { - let table_ident = TableIdent::from_strs(identifier) - .map_err(|e| PyRuntimeError::new_err(format!("Invalid table identifier: {e}")))?; - - let mut builder = FileIO::from_path(&metadata_location) - .map_err(|e| PyRuntimeError::new_err(format!("Failed to init FileIO: {e}")))?; - - if let Some(props) = file_io_properties { - builder = builder.with_props(props); - } - - let file_io = builder - .build() - .map_err(|e| PyRuntimeError::new_err(format!("Failed to build FileIO: {e}")))?; - - let static_table = - StaticTable::from_metadata_file(&metadata_location, table_ident, file_io) - .await - .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to load static table: {e}")) - })?; - - let table = static_table.into_table(); - - IcebergTableProvider::try_new_from_table(table) - .await - .map_err(|e| { - PyRuntimeError::new_err(format!("Failed to create table provider: {e}")) - }) - })?; - - Ok(Self { - inner: Arc::new(provider), - }) - } - - fn __datafusion_table_provider__<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { - let capsule_name = CString::new("datafusion_table_provider").unwrap(); - - let ffi_provider = FFI_TableProvider::new(self.inner.clone(), false, Some(runtime())); - - PyCapsule::new(py, ffi_provider, Some(capsule_name)) - } -} - -pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - let this = PyModule::new(py, "datafusion")?; - - this.add_class::()?; - - m.add_submodule(&this)?; - py.import("sys")? - .getattr("modules")? - .set_item("pyiceberg_core.datafusion", this)?; - - Ok(()) -} diff --git a/bindings/python/src/encryption.rs b/bindings/python/src/encryption.rs new file mode 100644 index 0000000000..c0b14875b5 --- /dev/null +++ b/bindings/python/src/encryption.rs @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg::encryption::StandardKeyMetadata; +use pyo3::prelude::*; +use pyo3::types::PyBytes; + +use crate::error::to_py_err; + +/// The encryption key, AAD prefix and file length needed to decrypt a single file. +#[pyclass( + frozen, + name = "StandardKeyMetadata", + module = "pyiceberg_core.encryption" +)] +pub struct PyStandardKeyMetadata { + // Python objects, so attribute access increfs rather than copying the key on every read. + encryption_key: Py, + aad_prefix: Option>, + file_length: Option, +} + +#[pymethods] +impl PyStandardKeyMetadata { + #[getter] + fn encryption_key(&self, py: Python<'_>) -> Py { + self.encryption_key.clone_ref(py) + } + + #[getter] + fn aad_prefix(&self, py: Python<'_>) -> Option> { + self.aad_prefix.as_ref().map(|prefix| prefix.clone_ref(py)) + } + + #[getter] + fn file_length(&self) -> Option { + self.file_length + } + + /// Redacts the key, so logging an instance cannot leak key material. + fn __repr__(&self, py: Python<'_>) -> PyResult { + let key_length = self.encryption_key.bind(py).as_bytes().len(); + let aad_prefix = match &self.aad_prefix { + Some(prefix) => prefix.bind(py).repr()?.to_string(), + None => "None".to_string(), + }; + let file_length = match self.file_length { + Some(file_length) => file_length.to_string(), + None => "None".to_string(), + }; + + Ok(format!( + "StandardKeyMetadata(encryption_key=, aad_prefix={aad_prefix}, file_length={file_length})" + )) + } +} + +/// Decode `StandardKeyMetadata` from its wire format. +#[pyfunction] +pub fn decode_standard_key_metadata( + py: Python<'_>, + data: &[u8], +) -> PyResult { + let metadata = StandardKeyMetadata::decode(data).map_err(to_py_err)?; + + Ok(PyStandardKeyMetadata { + encryption_key: PyBytes::new(py, metadata.encryption_key().as_bytes()).unbind(), + aad_prefix: metadata + .aad_prefix() + .map(|aad_prefix| PyBytes::new(py, aad_prefix).unbind()), + file_length: metadata.file_length(), + }) +} + +/// Encode `StandardKeyMetadata` to its wire format. +#[pyfunction] +#[pyo3(signature = (encryption_key, aad_prefix=None, file_length=None))] +pub fn encode_standard_key_metadata<'py>( + py: Python<'py>, + encryption_key: &[u8], + aad_prefix: Option<&[u8]>, + file_length: Option, +) -> PyResult> { + let mut metadata = StandardKeyMetadata::try_new(encryption_key).map_err(to_py_err)?; + + if let Some(aad_prefix) = aad_prefix { + metadata = metadata.with_aad_prefix(aad_prefix); + } + if let Some(file_length) = file_length { + metadata = metadata.with_file_length(file_length); + } + + Ok(PyBytes::new(py, &metadata.encode().map_err(to_py_err)?)) +} + +pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + let this = PyModule::new(py, "encryption")?; + + this.add_class::()?; + this.add_function(wrap_pyfunction!(decode_standard_key_metadata, &this)?)?; + this.add_function(wrap_pyfunction!(encode_standard_key_metadata, &this)?)?; + + m.add_submodule(&this)?; + py.import("sys")? + .getattr("modules")? + .set_item("pyiceberg_core.encryption", this) +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 131d864bdd..c50b1ce15a 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -18,16 +18,15 @@ use pyo3::prelude::*; mod data_file; -mod datafusion_table_provider; +mod encryption; mod error; mod manifest; -mod runtime; mod transform; #[pymodule] fn pyiceberg_core_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - datafusion_table_provider::register_module(py, m)?; transform::register_module(py, m)?; manifest::register_module(py, m)?; + encryption::register_module(py, m)?; Ok(()) } diff --git a/bindings/python/src/manifest.rs b/bindings/python/src/manifest.rs index ee756949c7..15b80c3e3e 100644 --- a/bindings/python/src/manifest.rs +++ b/bindings/python/src/manifest.rs @@ -23,6 +23,7 @@ use iceberg::spec::{ use pyo3::prelude::*; use crate::data_file::PyDataFile; +use crate::error::to_py_err; #[pyclass] pub struct PyManifest { @@ -52,7 +53,7 @@ pub struct PyFieldSummary { } #[pymethods] -impl crate::manifest::PyFieldSummary { +impl PyFieldSummary { #[getter] fn contains_null(&self) -> bool { self.inner.contains_null @@ -80,7 +81,7 @@ pub struct PyManifestFile { } #[pymethods] -impl crate::manifest::PyManifestFile { +impl PyManifestFile { #[getter] fn manifest_path(&self) -> &str { self.inner.manifest_path.as_str() @@ -145,14 +146,13 @@ impl crate::manifest::PyManifestFile { } #[getter] - fn partitions(&self) -> Vec { - self.inner - .partitions - .clone() - .unwrap() - .iter() - .map(|s| PyFieldSummary { inner: s.clone() }) - .collect() + fn partitions(&self) -> Option> { + self.inner.partitions.as_ref().map(|partitions| { + partitions + .iter() + .map(|s| PyFieldSummary { inner: s.clone() }) + .collect() + }) } #[getter] @@ -195,11 +195,10 @@ impl PyManifestEntry { } #[pyfunction] -pub fn read_manifest_entries(bs: &[u8]) -> PyManifest { - // TODO: Some error handling - PyManifest { - inner: Manifest::parse_avro(bs).unwrap(), - } +pub fn read_manifest_entries(bs: &[u8]) -> PyResult { + Ok(PyManifest { + inner: Manifest::parse_avro(bs).map_err(to_py_err)?, + }) } #[pyclass] @@ -208,7 +207,7 @@ pub struct PyManifestList { } #[pymethods] -impl crate::manifest::PyManifestList { +impl PyManifestList { fn entries(&self) -> Vec { self.inner .entries() @@ -221,10 +220,10 @@ impl crate::manifest::PyManifestList { } #[pyfunction] -pub fn read_manifest_list(bs: &[u8]) -> PyManifestList { - PyManifestList { - inner: ManifestList::parse_with_version(bs, FormatVersion::V2).unwrap(), - } +pub fn read_manifest_list(bs: &[u8]) -> PyResult { + Ok(PyManifestList { + inner: ManifestList::parse_with_version(bs, FormatVersion::V2).map_err(to_py_err)?, + }) } pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/bindings/python/src/transform.rs b/bindings/python/src/transform.rs index 24e9f061dd..b50a0fa84d 100644 --- a/bindings/python/src/transform.rs +++ b/bindings/python/src/transform.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{Array, ArrayData, make_array}; +use arrow::array::{ArrayData, make_array}; use arrow::pyarrow::{FromPyArrow, ToPyArrow}; use iceberg::spec::Transform; use iceberg::transform::create_transform_function; @@ -24,54 +24,54 @@ use pyo3::prelude::*; use crate::error::to_py_err; #[pyfunction] -pub fn identity(py: Python, array: PyObject) -> PyResult { +pub fn identity(py: Python, array: Py) -> PyResult> { apply(py, array, Transform::Identity) } #[pyfunction] -pub fn void(py: Python, array: PyObject) -> PyResult { +pub fn void(py: Python, array: Py) -> PyResult> { apply(py, array, Transform::Void) } #[pyfunction] -pub fn year(py: Python, array: PyObject) -> PyResult { +pub fn year(py: Python, array: Py) -> PyResult> { apply(py, array, Transform::Year) } #[pyfunction] -pub fn month(py: Python, array: PyObject) -> PyResult { +pub fn month(py: Python, array: Py) -> PyResult> { apply(py, array, Transform::Month) } #[pyfunction] -pub fn day(py: Python, array: PyObject) -> PyResult { +pub fn day(py: Python, array: Py) -> PyResult> { apply(py, array, Transform::Day) } #[pyfunction] -pub fn hour(py: Python, array: PyObject) -> PyResult { +pub fn hour(py: Python, array: Py) -> PyResult> { apply(py, array, Transform::Hour) } #[pyfunction] -pub fn bucket(py: Python, array: PyObject, num_buckets: u32) -> PyResult { +pub fn bucket(py: Python, array: Py, num_buckets: u32) -> PyResult> { apply(py, array, Transform::Bucket(num_buckets)) } #[pyfunction] -pub fn truncate(py: Python, array: PyObject, width: u32) -> PyResult { +pub fn truncate(py: Python, array: Py, width: u32) -> PyResult> { apply(py, array, Transform::Truncate(width)) } -fn apply(py: Python, array: PyObject, transform: Transform) -> PyResult { +fn apply(py: Python, array: Py, transform: Transform) -> PyResult> { // import let array = ArrayData::from_pyarrow_bound(array.bind(py))?; let array = make_array(array); let transform_function = create_transform_function(&transform).map_err(to_py_err)?; let array = transform_function.transform(array).map_err(to_py_err)?; // export - let array = array.into_data(); - array.to_pyarrow(py) + let array = array.to_data(); + Ok(array.to_pyarrow(py)?.unbind()) } pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py index 75ee3f36a3..150b6af581 100644 --- a/bindings/python/tests/conftest.py +++ b/bindings/python/tests/conftest.py @@ -16,28 +16,41 @@ # under the License. +from collections.abc import Generator from tempfile import TemporaryDirectory from typing import ( Any, - Dict, - Generator, ) import pytest from pydantic_core import to_json - from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema -from pyiceberg.transforms import IdentityTransform, DayTransform +from pyiceberg.transforms import DayTransform, IdentityTransform from pyiceberg.types import ( IntegerType, NestedField, TimestampType, ) +from pyiceberg.utils.config import Config + + +@pytest.fixture(scope="session", autouse=True) +def isolate_pyiceberg_config() -> Generator[None, None, None]: + monkeypatch = pytest.MonkeyPatch() + with TemporaryDirectory() as empty_home_dir: + monkeypatch.setenv("HOME", empty_home_dir) + monkeypatch.setenv("PYICEBERG_HOME", empty_home_dir) + + from pyiceberg import catalog + + monkeypatch.setattr(catalog, "_ENV_CONFIG", Config()) + yield + monkeypatch.undo() @pytest.fixture(scope="session") -def avro_schema_manifest_entry() -> Dict[str, Any]: +def avro_schema_manifest_entry() -> dict[str, Any]: return { "type": "record", "name": "manifest_entry", @@ -409,7 +422,7 @@ def test_partition_spec() -> Schema: @pytest.fixture(scope="session") def generated_manifest_entry_file( - avro_schema_manifest_entry: Dict[str, Any], + avro_schema_manifest_entry: dict[str, Any], test_schema: Schema, test_partition_spec: PartitionSpec, ) -> Generator[str, None, None]: diff --git a/bindings/python/tests/test_datafusion_table_provider.py b/bindings/python/tests/test_datafusion_table_provider.py deleted file mode 100644 index 915b787697..0000000000 --- a/bindings/python/tests/test_datafusion_table_provider.py +++ /dev/null @@ -1,160 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from datetime import date, datetime -import uuid -import pytest -from pyiceberg_core.datafusion import IcebergDataFusionTable -from datafusion import SessionContext -from pyiceberg.catalog import Catalog, load_catalog -import pyarrow as pa -from pathlib import Path -import datafusion - -assert ( - datafusion.__version__ >= "45" -) # iceberg table provider only works for datafusion >= 45 - - -@pytest.fixture(scope="session") -def warehouse(tmp_path_factory: pytest.TempPathFactory) -> Path: - return tmp_path_factory.mktemp("warehouse") - - -@pytest.fixture(scope="session") -def catalog(warehouse: Path) -> Catalog: - catalog = load_catalog( - "default", - **{ - "uri": f"sqlite:///{warehouse}/pyiceberg_catalog.db", - "warehouse": f"file://{warehouse}", - }, - ) - return catalog - - -@pytest.fixture(scope="session") -def arrow_table_with_null() -> "pa.Table": - """Pyarrow table with all kinds of columns.""" - import pyarrow as pa - - return pa.Table.from_pydict( - { - "bool": [False, None, True], - "string": ["a", None, "z"], - # Go over the 16 bytes to kick in truncation - "string_long": ["a" * 22, None, "z" * 22], - "int": [1, None, 9], - "long": [1, None, 9], - "float": [0.0, None, 0.9], - "double": [0.0, None, 0.9], - # 'time': [1_000_000, None, 3_000_000], # Example times: 1s, none, and 3s past midnight #Spark does not support time fields - "timestamp": [ - datetime(2023, 1, 1, 19, 25, 00), - None, - datetime(2023, 3, 1, 19, 25, 00), - ], - # "timestamptz": [ - # datetime(2023, 1, 1, 19, 25, 00, tzinfo=timezone.utc), - # None, - # datetime(2023, 3, 1, 19, 25, 00, tzinfo=timezone.utc), - # ], - "date": [date(2023, 1, 1), None, date(2023, 3, 1)], - # Not supported by Spark - # 'time': [time(1, 22, 0), None, time(19, 25, 0)], - # Not natively supported by Arrow - # 'uuid': [uuid.UUID('00000000-0000-0000-0000-000000000000').bytes, None, uuid.UUID('11111111-1111-1111-1111-111111111111').bytes], - "binary": [b"\01", None, b"\22"], - "fixed": [ - uuid.UUID("00000000-0000-0000-0000-000000000000").bytes, - None, - uuid.UUID("11111111-1111-1111-1111-111111111111").bytes, - ], - }, - ) - - -def test_register_iceberg_table_provider( - catalog: Catalog, arrow_table_with_null: pa.Table -) -> None: - catalog.create_namespace_if_not_exists("default") - iceberg_table = catalog.create_table_if_not_exists( - "default.dataset", - schema=arrow_table_with_null.schema, - ) - iceberg_table.append(arrow_table_with_null) - - iceberg_table_provider = IcebergDataFusionTable( - identifier=iceberg_table.name(), - metadata_location=iceberg_table.metadata_location, - file_io_properties=iceberg_table.io.properties, - ) - - ctx = SessionContext() - ctx.register_table_provider("test", iceberg_table_provider) - - datafusion_table = ctx.table("test") - assert datafusion_table is not None - - # check that the schema is the same - from pyiceberg.io.pyarrow import _pyarrow_schema_ensure_small_types - - assert _pyarrow_schema_ensure_small_types( - datafusion_table.schema() - ) == _pyarrow_schema_ensure_small_types(iceberg_table.schema().as_arrow()) - # large/small type mismatches, fixed in pyiceberg 0.10.0 - # assert datafusion_table.schema() == iceberg_table.schema().as_arrow() - - # check that the data is the same - assert ( - datafusion_table.to_arrow_table().to_pylist() - == iceberg_table.scan().to_arrow().to_pylist() - ) - # large/small type mismatches, fixed in pyiceberg 0.10.0 - # assert datafusion_table.to_arrow_table() == iceberg_table.scan().to_arrow() - - -def test_register_pyiceberg_table( - catalog: Catalog, arrow_table_with_null: pa.Table -) -> None: - from types import MethodType - - catalog.create_namespace_if_not_exists("default") - iceberg_table = catalog.create_table_if_not_exists( - "default.dataset", - schema=arrow_table_with_null.schema, - ) - iceberg_table.append(arrow_table_with_null) - - # monkey patch the __datafusion_table_provider__ method to the iceberg table - def __datafusion_table_provider__(self): - return IcebergDataFusionTable( - identifier=self.name(), - metadata_location=self.metadata_location, - file_io_properties=self.io.properties, - ).__datafusion_table_provider__() - - iceberg_table.__datafusion_table_provider__ = MethodType( - __datafusion_table_provider__, iceberg_table - ) - - ctx = SessionContext() - ctx.register_table_provider("test", iceberg_table) - - datafusion_table = ctx.table("test") - assert datafusion_table is not None diff --git a/bindings/python/tests/test_encryption.py b/bindings/python/tests/test_encryption.py new file mode 100644 index 0000000000..1e47156077 --- /dev/null +++ b/bindings/python/tests/test_encryption.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +from pyiceberg_core import encryption + +AES128_KEY = b"0123456789012345" +# A version byte, then the Avro-encoded encryption_key: zigzag length 16, then the key. +ENCODED_PREFIX = b"\x01\x20" + AES128_KEY + + +def fields(metadata): + return (metadata.encryption_key, metadata.aad_prefix, metadata.file_length) + + +# Pinned in both directions so the encoding stays compatible with the Java and Python +# implementations, including the null union tags written for absent optional fields. +@pytest.mark.parametrize( + "aad_prefix, file_length, encoded", + [ + (None, None, ENCODED_PREFIX + b"\x00\x00"), + (b"ad", None, ENCODED_PREFIX + b"\x02\x04ad\x00"), + (b"ad", 1024, ENCODED_PREFIX + b"\x02\x04ad\x02\x80\x10"), + (b"", None, ENCODED_PREFIX + b"\x02\x00\x00"), + ], +) +def test_wire_format(aad_prefix, file_length, encoded): + assert fields(encryption.decode_standard_key_metadata(encoded)) == ( + AES128_KEY, + aad_prefix, + file_length, + ) + assert ( + encryption.encode_standard_key_metadata(AES128_KEY, aad_prefix, file_length) + == encoded + ) + + +def test_repr_redacts_encryption_key(): + encoded = encryption.encode_standard_key_metadata(AES128_KEY, b"ad", 1024) + + rendered = repr(encryption.decode_standard_key_metadata(encoded)) + + assert AES128_KEY not in rendered.encode() + assert rendered == ( + "StandardKeyMetadata(encryption_key=, " + "aad_prefix=b'ad', file_length=1024)" + ) + + +@pytest.mark.parametrize("key_length", [16, 24, 32]) +def test_encode_accepts_aes_key_lengths(key_length): + encoded = encryption.encode_standard_key_metadata(bytes(key_length)) + metadata = encryption.decode_standard_key_metadata(encoded) + + assert fields(metadata) == (bytes(key_length), None, None) + + +@pytest.mark.parametrize("key_length", [0, 4, 15, 20, 33]) +def test_encode_rejects_invalid_key_length(key_length): + with pytest.raises(ValueError, match="key length"): + encryption.encode_standard_key_metadata(bytes(key_length)) + + +@pytest.mark.parametrize("key_length", [0, 4, 15, 20, 33]) +def test_decode_rejects_invalid_key_length(key_length): + # Key length is validated on decode too, not only on encode. The Avro zigzag length + # is 2 * key_length, a single byte for every length here. + data = b"\x01" + bytes([key_length * 2]) + bytes(key_length) + b"\x00\x00" + + with pytest.raises(ValueError, match="Invalid encryption key in key metadata"): + encryption.decode_standard_key_metadata(data) + + +@pytest.mark.parametrize("data", [b"\x02", b"\x02\x20" + AES128_KEY + b"\x00\x00"]) +def test_decode_rejects_unsupported_version(data): + with pytest.raises(ValueError, match=r"version: 2"): + encryption.decode_standard_key_metadata(data) + + +def test_decode_rejects_empty_buffer(): + with pytest.raises(ValueError, match="Empty key metadata"): + encryption.decode_standard_key_metadata(b"") + + +def test_decode_tolerates_trailing_bytes(): + # Deliberate, matching Java: a reader must tolerate fields a newer writer appended. + encoded = encryption.encode_standard_key_metadata(AES128_KEY, b"ad", 1024) + + metadata = encryption.decode_standard_key_metadata(encoded + b"\xde\xad\xbe\xef") + + assert fields(metadata) == (AES128_KEY, b"ad", 1024) + + +@pytest.mark.xfail( + reason="apache/avro-rs#664: a missing union tag decodes as null; fixed upstream in 0.23", + strict=True, +) +def test_decode_rejects_truncated_union_tags(): + # Version and key, with both union tags omitted, currently decodes as if the + # optional fields were absent rather than erroring. + with pytest.raises(ValueError): + encryption.decode_standard_key_metadata(ENCODED_PREFIX) diff --git a/bindings/python/tests/test_manifest.py b/bindings/python/tests/test_manifest.py index 701eac2509..147b81fccc 100644 --- a/bindings/python/tests/test_manifest.py +++ b/bindings/python/tests/test_manifest.py @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, List -import pytest - +from pathlib import Path +from typing import Any +import pytest from pyiceberg.io import FileIO from pyiceberg.io.pyarrow import PyArrowFileIO from pyiceberg.manifest import ( @@ -27,19 +27,41 @@ FileFormat, ManifestEntry, ManifestEntryStatus, + ManifestFile, _manifests, + write_manifest_list, ) @pytest.fixture(autouse=True) def clear_global_manifests_cache() -> None: # Clear the global cache before each test - _manifests.cache_clear() # type: ignore + cache_clear = getattr(_manifests, "cache_clear", None) + if callable(cache_clear): + # pyiceberg<=0.10: _manifests was lru_cache-decorated. + cache_clear() + return + + # pyiceberg>=0.11: manifests are cached in module-level LRUCache. + from pyiceberg import manifest as manifest_module + + manifest_cache = getattr(manifest_module, "_manifest_cache", None) + manifest_cache_lock = getattr(manifest_module, "_manifest_cache_lock", None) + + if manifest_cache is None: + return + + if manifest_cache_lock is None: + manifest_cache.clear() + return + + with manifest_cache_lock: + manifest_cache.clear() def fetch_manifest_entry( manifest_entry_file: str, io: FileIO, discard_deleted: bool = True -) -> List[ManifestEntry]: +) -> list[ManifestEntry]: from pyiceberg_core import manifest bs = io.new_input(manifest_entry_file).open().read() @@ -49,31 +71,43 @@ def fetch_manifest_entry( # but this is the easiest for now until we # have the write part in there as well def _convert_entry(entry: Any) -> ManifestEntry: - data_file = DataFile( - DataFileContent(entry.data_file.content), - entry.data_file.file_path, - FileFormat(entry.data_file.file_format), - [p.value() if p is not None else None for p in entry.data_file.partition], - entry.data_file.record_count, - entry.data_file.file_size_in_bytes, - entry.data_file.column_sizes, - entry.data_file.value_counts, - entry.data_file.null_value_counts, - entry.data_file.nan_value_counts, - entry.data_file.lower_bounds, - entry.data_file.upper_bounds, - entry.data_file.key_metadata, - entry.data_file.split_offsets, - entry.data_file.equality_ids, - entry.data_file.sort_order_id, + data_file_args = { + "content": DataFileContent(entry.data_file.content), + "file_path": entry.data_file.file_path, + "file_format": FileFormat(entry.data_file.file_format), + "partition": [ + p.value() if p is not None else None for p in entry.data_file.partition + ], + "record_count": entry.data_file.record_count, + "file_size_in_bytes": entry.data_file.file_size_in_bytes, + "column_sizes": entry.data_file.column_sizes, + "value_counts": entry.data_file.value_counts, + "null_value_counts": entry.data_file.null_value_counts, + "nan_value_counts": entry.data_file.nan_value_counts, + "lower_bounds": entry.data_file.lower_bounds, + "upper_bounds": entry.data_file.upper_bounds, + "key_metadata": entry.data_file.key_metadata, + "split_offsets": entry.data_file.split_offsets, + "equality_ids": entry.data_file.equality_ids, + "sort_order_id": entry.data_file.sort_order_id, + } + data_file = ( + DataFile.from_args(**data_file_args) + if hasattr(DataFile, "from_args") + else DataFile(**data_file_args) ) - return ManifestEntry( - ManifestEntryStatus(entry.status), - entry.snapshot_id, - entry.sequence_number, - entry.file_sequence_number, - data_file, + manifest_entry_args = { + "status": ManifestEntryStatus(entry.status), + "snapshot_id": entry.snapshot_id, + "sequence_number": entry.sequence_number, + "file_sequence_number": entry.file_sequence_number, + "data_file": data_file, + } + return ( + ManifestEntry.from_args(**manifest_entry_args) + if hasattr(ManifestEntry, "from_args") + else ManifestEntry(**manifest_entry_args) ) return [ @@ -140,3 +174,64 @@ def test_read_manifest_entry(generated_manifest_entry_file: str) -> None: assert data_file.split_offsets == [4] assert data_file.equality_ids is None assert data_file.sort_order_id == 0 + + +@pytest.mark.parametrize( + "payload", [b"", b"not-an-avro-file", b"Obj\x01truncated-after-the-avro-magic"] +) +def test_read_manifest_entries_raises_on_invalid_avro(payload: bytes) -> None: + from pyiceberg_core import manifest + + with pytest.raises(ValueError): + manifest.read_manifest_entries(payload) + + +@pytest.mark.parametrize( + "payload", [b"", b"not-an-avro-file", b"Obj\x01truncated-after-the-avro-magic"] +) +def test_read_manifest_list_raises_on_invalid_avro(payload: bytes) -> None: + from pyiceberg_core import manifest + + with pytest.raises(ValueError): + manifest.read_manifest_list(payload) + + +def test_manifest_file_without_partition_summaries(tmp_path: Path) -> None: + """Field summaries are optional, so a manifest list without them reads back as None.""" + from pyiceberg_core import manifest + + manifest_list_file = str(tmp_path / "snap.avro") + io = PyArrowFileIO() + + with write_manifest_list( + format_version=2, + output_file=io.new_output(manifest_list_file), + snapshot_id=25, + parent_snapshot_id=None, + sequence_number=1, + avro_compression="null", + ) as writer: + writer.add_manifests( + [ + ManifestFile.from_args( + manifest_path="s3://bucket/metadata/manifest.avro", + manifest_length=1024, + partition_spec_id=0, + added_snapshot_id=25, + sequence_number=1, + min_sequence_number=1, + added_files_count=1, + existing_files_count=0, + deleted_files_count=0, + added_rows_count=10, + existing_rows_count=0, + deleted_rows_count=0, + partitions=None, + ) + ] + ) + + bs = io.new_input(manifest_list_file).open().read() + manifest_file = manifest.read_manifest_list(bs).entries()[0] + + assert manifest_file.partitions is None diff --git a/bindings/python/tests/test_transform.py b/bindings/python/tests/test_transform.py index 6156f0d68a..4074dae965 100644 --- a/bindings/python/tests/test_transform.py +++ b/bindings/python/tests/test_transform.py @@ -92,9 +92,9 @@ def test_day_transform(): def test_hour_transform(): arr = pa.array( [ - datetime(1970, 1, 1, 19, 1, 23), - datetime(2000, 3, 1, 12, 1, 23), - datetime(22, 5, 1, 22, 1, 1), # Negative + datetime(1970, 1, 1, 19, 1, 23), # noqa: DTZ001 + datetime(2000, 3, 1, 12, 1, 23), # noqa: DTZ001 + datetime(22, 5, 1, 22, 1, 1), # noqa: DTZ001 # Negative ] ) result = transform.hour(arr) diff --git a/bindings/python/third-party-licenses.txt b/bindings/python/third-party-licenses.txt new file mode 100644 index 0000000000..b19fd73c5b --- /dev/null +++ b/bindings/python/third-party-licenses.txt @@ -0,0 +1,27 @@ +-------------------------------------------------------------------------------- + +MIT License + +Used by: + - brotli-decompressor 5.0.3 (https://github.com/dropbox/rust-brotli-decompressor) + (src/context.rs) + +Copyright 2013 Google Inc. All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/python/third-party-notices.txt b/bindings/python/third-party-notices.txt new file mode 100644 index 0000000000..74fef989ee --- /dev/null +++ b/bindings/python/third-party-notices.txt @@ -0,0 +1,30 @@ +This package bundles the following Rust dependencies that ship their own NOTICE. + +Apache Arrow (arrow, parquet and related crates) +Copyright 2016-2026 The Apache Software Foundation + +This product includes software from the chronoutil crate (MIT) + * Copyright (c) 2020-2022 Oliver Margetts + * https://github.com/olliemath/chronoutil + +This product includes software from the compact-thrift project (Apache 2.0) + * Copyright Jörn Horstmann + * https://github.com/jhorstmann/compact-thrift + +Apache Avro (apache-avro) +Copyright 2010-2025 The Apache Software Foundation + +The initial implementation of the Apache Avro Rust SDK has been +developed by Flavien Raynaud (https://github.com/flavray/avro-rs) +and donated to the Apache Avro project in 2020. + +Moka (moka) + +The majority of the Moka library is dual-licensed under the MIT and Apache 2.0 +licenses. However, the following files are licensed solely under the Apache +License 2.0: + +- src/common/frequency_sketch.rs +- src/common/timer_wheel.rs + +These files were ported from the Java Caffeine library and are not dual-licensed. diff --git a/bindings/python/uv.lock b/bindings/python/uv.lock new file mode 100644 index 0000000000..182447b20e --- /dev/null +++ b/bindings/python/uv.lock @@ -0,0 +1,1050 @@ +version = 1 +revision = 3 +requires-python = ">=3.10, <4" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version < '3.14'", +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "cachetools" +version = "6.2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/39/91/d9ae9a66b01102a18cd16db0cf4cd54187ffe10f0865cc80071a4104fbb3/cachetools-6.2.6.tar.gz", hash = "sha256:16c33e1f276b9a9c0b49ab5782d901e3ad3de0dd6da9bf9bcd29ac5672f2f9e6", size = 32363, upload-time = "2026-01-27T20:32:59.956Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/45/f458fa2c388e79dd9d8b9b0c99f1d31b568f27388f2fdba7bb66bbc0c6ed/cachetools-6.2.6-py3-none-any.whl", hash = "sha256:8c9717235b3c651603fff0076db52d6acbfd1b338b8ed50256092f7ce9c85bda", size = 11668, upload-time = "2026-01-27T20:32:58.527Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/08/0f303cb0b529e456bb116f2d50565a482694fbb94340bf56d44677e7ed03/charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d", size = 315182, upload-time = "2026-04-02T09:25:40.673Z" }, + { url = "https://files.pythonhosted.org/packages/24/47/b192933e94b546f1b1fe4df9cc1f84fcdbf2359f8d1081d46dd029b50207/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8", size = 209329, upload-time = "2026-04-02T09:25:42.354Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b4/01fa81c5ca6141024d89a8fc15968002b71da7f825dd14113207113fabbd/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790", size = 231230, upload-time = "2026-04-02T09:25:44.281Z" }, + { url = "https://files.pythonhosted.org/packages/20/f7/7b991776844dfa058017e600e6e55ff01984a063290ca5622c0b63162f68/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc", size = 225890, upload-time = "2026-04-02T09:25:45.475Z" }, + { url = "https://files.pythonhosted.org/packages/20/e7/bed0024a0f4ab0c8a9c64d4445f39b30c99bd1acd228291959e3de664247/charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393", size = 216930, upload-time = "2026-04-02T09:25:46.58Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ab/b18f0ab31cdd7b3ddb8bb76c4a414aeb8160c9810fdf1bc62f269a539d87/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153", size = 202109, upload-time = "2026-04-02T09:25:48.031Z" }, + { url = "https://files.pythonhosted.org/packages/82/e5/7e9440768a06dfb3075936490cb82dbf0ee20a133bf0dd8551fa096914ec/charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af", size = 214684, upload-time = "2026-04-02T09:25:49.245Z" }, + { url = "https://files.pythonhosted.org/packages/71/94/8c61d8da9f062fdf457c80acfa25060ec22bf1d34bbeaca4350f13bcfd07/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34", size = 212785, upload-time = "2026-04-02T09:25:50.671Z" }, + { url = "https://files.pythonhosted.org/packages/66/cd/6e9889c648e72c0ab2e5967528bb83508f354d706637bc7097190c874e13/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1", size = 203055, upload-time = "2026-04-02T09:25:51.802Z" }, + { url = "https://files.pythonhosted.org/packages/92/2e/7a951d6a08aefb7eb8e1b54cdfb580b1365afdd9dd484dc4bee9e5d8f258/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752", size = 232502, upload-time = "2026-04-02T09:25:53.388Z" }, + { url = "https://files.pythonhosted.org/packages/58/d5/abcf2d83bf8e0a1286df55cd0dc1d49af0da4282aa77e986df343e7de124/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53", size = 214295, upload-time = "2026-04-02T09:25:54.765Z" }, + { url = "https://files.pythonhosted.org/packages/47/3a/7d4cd7ed54be99973a0dc176032cba5cb1f258082c31fa6df35cff46acfc/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616", size = 227145, upload-time = "2026-04-02T09:25:55.904Z" }, + { url = "https://files.pythonhosted.org/packages/1d/98/3a45bf8247889cf28262ebd3d0872edff11565b2a1e3064ccb132db3fbb0/charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a", size = 218884, upload-time = "2026-04-02T09:25:57.074Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/2e8b7f8915ed5c9ef13aa828d82738e33888c485b65ebf744d615040c7ea/charset_normalizer-3.4.7-cp310-cp310-win32.whl", hash = "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374", size = 148343, upload-time = "2026-04-02T09:25:58.199Z" }, + { url = "https://files.pythonhosted.org/packages/35/1b/3b8c8c77184af465ee9ad88b5aea46ea6b2e1f7b9dc9502891e37af21e30/charset_normalizer-3.4.7-cp310-cp310-win_amd64.whl", hash = "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943", size = 159174, upload-time = "2026-04-02T09:25:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/feb40dca40dbb21e0a908801782d9288c64fc8d8e562c2098e9994c8c21b/charset_normalizer-3.4.7-cp310-cp310-win_arm64.whl", hash = "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008", size = 147805, upload-time = "2026-04-02T09:26:00.756Z" }, + { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, + { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, + { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742, upload-time = "2026-04-02T09:26:06.36Z" }, + { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061, upload-time = "2026-04-02T09:26:08.347Z" }, + { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239, upload-time = "2026-04-02T09:26:09.823Z" }, + { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173, upload-time = "2026-04-02T09:26:10.953Z" }, + { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841, upload-time = "2026-04-02T09:26:12.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304, upload-time = "2026-04-02T09:26:13.711Z" }, + { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455, upload-time = "2026-04-02T09:26:14.941Z" }, + { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036, upload-time = "2026-04-02T09:26:16.478Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739, upload-time = "2026-04-02T09:26:17.751Z" }, + { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277, upload-time = "2026-04-02T09:26:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819, upload-time = "2026-04-02T09:26:20.295Z" }, + { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281, upload-time = "2026-04-02T09:26:21.74Z" }, + { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843, upload-time = "2026-04-02T09:26:22.901Z" }, + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "fastavro" +version = "1.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/5b/ccb338db71f347e3bc031d268bf6dc41e5ead63b6997b8e72af92f05e18e/fastavro-1.12.2.tar.gz", hash = "sha256:3c79502d56cf6b76210032e1c53494ddfbc73c140bccf2ef4092b3f0825323ab", size = 1030127, upload-time = "2026-04-24T14:36:01.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/91/16c3508447e7cf9f413a6a01792a990ed94d17505fc80a7fb76027078aed/fastavro-1.12.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c7c6d26c731a0e1e8e7d4ae8f13ae524eb6ec0e90d99c8147a19fdbae14eb807", size = 976824, upload-time = "2026-04-24T14:36:04.233Z" }, + { url = "https://files.pythonhosted.org/packages/2d/3a/97534561a1b4615366345ac066ad1f54698a59aa510eece3153c3a603d29/fastavro-1.12.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7caeecf519eff50f007ca4bee16b6e0a8252e5fe682c94432192a20867239888", size = 3185186, upload-time = "2026-04-24T14:36:06.395Z" }, + { url = "https://files.pythonhosted.org/packages/ee/e4/26512b52f58305b9d2194169de2e82c16d5131f0a0b6359e50d34faf4021/fastavro-1.12.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:731aefe6c4bf2bafa0798ef83927676d06e44d1d18202cfb56d63b40422ab900", size = 3196799, upload-time = "2026-04-24T14:36:09.028Z" }, + { url = "https://files.pythonhosted.org/packages/58/69/22f3b29a4555eb805a26f209f12532df8aafa48685d1cd1879aa42758d04/fastavro-1.12.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f089f24225a28ddafa5cfad7c41cfa84db1a55f2d473370769a95c0e3bac60c9", size = 3112396, upload-time = "2026-04-24T14:36:11.401Z" }, + { url = "https://files.pythonhosted.org/packages/e9/2a/fc61ef522050e1079ccf1aee07192881f3b11129f5e2b76811fd4fc3bb2f/fastavro-1.12.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:653c4f90dd21d8a1e74309919e08934e420d9aef51d051d14bf5a1c0e8293c22", size = 3180452, upload-time = "2026-04-24T14:36:13.634Z" }, + { url = "https://files.pythonhosted.org/packages/a6/6a/43ce9d713e9f1122e19c80d94d0dc0a356b8562d33eea90081dac781dd97/fastavro-1.12.2-cp310-cp310-win_amd64.whl", hash = "sha256:030f17eb4c7978538a31b55dea451ceace851a88dc9816b1923f8fb8a260db4c", size = 445396, upload-time = "2026-04-24T14:36:15.243Z" }, + { url = "https://files.pythonhosted.org/packages/89/77/058f3c93348624cb695399b27f3f0c1c3d1190586065797e4a48f75d4147/fastavro-1.12.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d48cd7094598a7e9d4297e8bf4bbe0dc9dc2ba4367d83dbb603e3b3c6aa35566", size = 974559, upload-time = "2026-04-24T14:36:17.172Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ef/08bbfa643addd2b98a9ce536613e2098928aa5e3ca098fd5b74f3c03b96a/fastavro-1.12.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:070c6134604bd7b6fd44409406ac50445339682b2e872885db2e859f92d22e93", size = 3352777, upload-time = "2026-04-24T14:36:19.679Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ec/55c11108529bdb59e635899f737651f729485ea5af36e128fb6560969c3d/fastavro-1.12.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b73d50978d5e57416fa68461f9f3c8f39ea39e761cb1e12f919745adefe26a7", size = 3387036, upload-time = "2026-04-24T14:36:21.794Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b3/4459f7c61804e9b42b49f02fba8fbbb041af76c7cab43cee4018532ecd00/fastavro-1.12.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c57a9920400166398695d92580eca21fd7a79f3c67d691ac7e20a7d1b5300735", size = 3284780, upload-time = "2026-04-24T14:36:24.193Z" }, + { url = "https://files.pythonhosted.org/packages/5d/e3/d7f510b9b8c7b73409a6232a9a8d282faa8560f85d024d7212e4c5dff3df/fastavro-1.12.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:81f6108f3ac292fb6cd05758c9e531389d8fc5e94e8c949b9298f4fb0a239662", size = 3368557, upload-time = "2026-04-24T14:36:26.667Z" }, + { url = "https://files.pythonhosted.org/packages/cb/10/14fa0abf8e7da07258393ae2b783dd4bb60d1fb93ad790296d27561f33ce/fastavro-1.12.2-cp311-cp311-win_amd64.whl", hash = "sha256:eec44256856fd59d29d1f1d0950ace18a58e4228e7d49de5d5e1b1875b227dde", size = 446499, upload-time = "2026-04-24T14:36:28.547Z" }, + { url = "https://files.pythonhosted.org/packages/86/d2/c36f646296794c05d29a07bec84a6c56bfd285203e389a8954987ec1c515/fastavro-1.12.2-cp311-cp311-win_arm64.whl", hash = "sha256:ecd1b23ea7f9af09c865ac8503d07afd7e6bf782d76bb83cbbdba15b7a0db807", size = 388198, upload-time = "2026-04-24T14:36:29.791Z" }, + { url = "https://files.pythonhosted.org/packages/0e/bc/fe5731d6724d978694fbd3196bc1c0d7cab3fd0766e9551c40c39f798b52/fastavro-1.12.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0e331896e8efffc72fa03e63b87ebfc37960113127da8e0f5152d91664ffed68", size = 964331, upload-time = "2026-04-24T14:36:31.297Z" }, + { url = "https://files.pythonhosted.org/packages/98/36/50abf1145e4f1c4f418cd4b5f2ac806643d0b14e360b60e953826edf1b34/fastavro-1.12.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f01ebaada59d74fdf6d28e5031a961a413b3752e9edb0c03866fa18480cf4c8", size = 3340170, upload-time = "2026-04-24T14:36:33.364Z" }, + { url = "https://files.pythonhosted.org/packages/fc/8c/76ef4641e6c1c1aa3e6bb3c9efb5533ffda5dd975c8b5ae54e794322d9e3/fastavro-1.12.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25ef6855935f67582740ffa6bb978e40ec51be876117a3555c36fa2488dcdf25", size = 3425061, upload-time = "2026-04-24T14:36:35.497Z" }, + { url = "https://files.pythonhosted.org/packages/31/10/379ff23425b2b470d5209cbc6736a6e5cbc34392ff17bb7355b8fd4aa0ca/fastavro-1.12.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:84a4f76a0aece0aa72b5ed8162ba2ff8c78908b8361b5a5d92ddd161977ccb74", size = 3243618, upload-time = "2026-04-24T14:36:37.969Z" }, + { url = "https://files.pythonhosted.org/packages/88/29/4c8f9e7cd78f932f0d82823899e67a6d7f7e8f2524992db03956f9d9f5ef/fastavro-1.12.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:81e8da77d201916f6771fc357fda8267c2a256d7aa11923d43bc5f2fc155878b", size = 3378427, upload-time = "2026-04-24T14:36:40.278Z" }, + { url = "https://files.pythonhosted.org/packages/e2/a1/eafeb302aaaea6055d4a9c11272b4aeaf713e43fe8eaf782f43a1fee2b44/fastavro-1.12.2-cp312-cp312-win_amd64.whl", hash = "sha256:1924349c74666c89417bd5cc2749f598e2f15f1d56ee81428b2317ab02c88aae", size = 441077, upload-time = "2026-04-24T14:36:41.791Z" }, + { url = "https://files.pythonhosted.org/packages/56/9d/67e831041ba8efc16265c65bd71ba92e1095bba19b91be99e102f19d9be6/fastavro-1.12.2-cp312-cp312-win_arm64.whl", hash = "sha256:4c346cf449baf3b113e997c34151ad205e7135bc429469b005b180ade7e65e28", size = 378205, upload-time = "2026-04-24T14:36:43.679Z" }, + { url = "https://files.pythonhosted.org/packages/83/39/f489a441d41cc9c0a8449fb1325d7a9c9eb57a5634e6ab19dfb0a1105324/fastavro-1.12.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:57bb6b908cb2e05baab63b04c3a31be3b4545a10bfab9748b8763016b5256704", size = 958566, upload-time = "2026-04-24T14:36:45.49Z" }, + { url = "https://files.pythonhosted.org/packages/31/69/776cc025aee2d02acacb734cf690d2fbc295eaadde1b5d47caf8c77a6a2b/fastavro-1.12.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a007f95cc682f56e6d83f1d17c29c00bf719d6fe8e003282b535af3a1ba09c0", size = 3276390, upload-time = "2026-04-24T14:36:47.875Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bc/b7e15fa788f42cbe65827af2ec06c9ad91bb9f72c213110dbef61b53a5b0/fastavro-1.12.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e90460b0cd21f62be3cb26087e706e2cebb7b3fcef9e05b4473b61bb0415b5e", size = 3372779, upload-time = "2026-04-24T14:36:50.122Z" }, + { url = "https://files.pythonhosted.org/packages/79/c2/98993ca810231fc1397212f48c3d46626983722a24bbaaa5c27ee0963751/fastavro-1.12.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7ccd15966b8218d41b06ec3e7c2556be89a8a693026c771e6564d2e40bbaf8ea", size = 3187591, upload-time = "2026-04-24T14:36:52.451Z" }, + { url = "https://files.pythonhosted.org/packages/c6/bb/c180f340eba6478f1b20deccdd17e2b4a4d5074dafd812e3c4254fd035f7/fastavro-1.12.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:06b6971d3dae10cb34353b857d16ad21ebd6f0ea394e86c96abdcad109005d6e", size = 3320589, upload-time = "2026-04-24T14:36:54.647Z" }, + { url = "https://files.pythonhosted.org/packages/4d/e9/aca0456216b5b8992e7b0a8542711b66799c05bfe24c8e32ef6f56e7eb93/fastavro-1.12.2-cp313-cp313-win_amd64.whl", hash = "sha256:98dfcdfaf1498ae2f0e2fafe900a82e8320cc81d8ae5a95b8b8879eaa3298c39", size = 440883, upload-time = "2026-04-24T14:36:56.585Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7e/984896e716af504927be71b80a1e9661aa96c6f9e1e777d52823aacb99f2/fastavro-1.12.2-cp313-cp313-win_arm64.whl", hash = "sha256:3888ef7a51adc77cdf07251bc762566a1be36211e1cff689f13980f3776a2f36", size = 377536, upload-time = "2026-04-24T14:36:58.274Z" }, + { url = "https://files.pythonhosted.org/packages/e9/42/09a1e1f8d9998d73848a6ff0aad6713ae6abf0dbf99918776f8ef33344a7/fastavro-1.12.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:283dcd3129b632021894425974bedd0eb6db3bbf5994e448ccad10db4d803d31", size = 1049506, upload-time = "2026-04-24T14:36:59.797Z" }, + { url = "https://files.pythonhosted.org/packages/52/ef/80cc16f43919d532f25a707f34b275cccc09dca87a05b000fbbfc8e8f255/fastavro-1.12.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d125e210d5a0a1f701f12c0ecad9a03f1b04b5eddbce6ca36a1fc217da977ef", size = 3495899, upload-time = "2026-04-24T14:37:02.306Z" }, + { url = "https://files.pythonhosted.org/packages/c1/54/a0817d1d0236e9e0233f5c996f450cc795b056b8e06edb531f24b9df82ed/fastavro-1.12.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2d4d66afad78e8f47feaa307728a6b71fe3effc63ba2b9eeb109ee687c9bd397", size = 3399232, upload-time = "2026-04-24T14:37:04.837Z" }, + { url = "https://files.pythonhosted.org/packages/38/0a/650f256c15f5875b6081544b9ba7ed8254329213e7e49e3db0aec68b5bee/fastavro-1.12.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2328ec07925c04c89719e3971c9068a165c7fd474ea87675b1204de0440e71ff", size = 3320222, upload-time = "2026-04-24T14:37:07.281Z" }, + { url = "https://files.pythonhosted.org/packages/f5/54/8351d388f94fbb0870e8cffaae41d3cc607acc8d6a8a6a217e2794829593/fastavro-1.12.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:55dea7e74b834d4b70467fc19c5b9ccb5509fe39abc4d26891187c1b22176423", size = 3337096, upload-time = "2026-04-24T14:37:09.452Z" }, + { url = "https://files.pythonhosted.org/packages/da/eb/b36ba9a88826e8c272df02e2f8b5da717e88b6eb508fddca3ca450043731/fastavro-1.12.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8d37c87826ae7195cfbd20fcd448801f2f563bb38f2691ec6574e39cb9eca6c8", size = 963119, upload-time = "2026-04-24T14:37:11.557Z" }, + { url = "https://files.pythonhosted.org/packages/e1/02/3d7f540fb26ba4ea1f4ebd2783c586614da9ac00906a3092e92fd3f104a2/fastavro-1.12.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c463a3701f293e30d3d62e71e1989f112028d07f87432baf4507eeb57ec3831", size = 3266238, upload-time = "2026-04-24T14:37:13.84Z" }, + { url = "https://files.pythonhosted.org/packages/4c/0b/b77be56c5109da0fc7dcfd7e6b6752fe0a61d0a5c58c6a65e38b4501946a/fastavro-1.12.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f604ba83498e209fff4c7ecc5063a39421dc538dace694bc592f9f338254f3dc", size = 3324020, upload-time = "2026-04-24T14:37:16.096Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6e/951d41f244107e91bf2f59245b71783c03eaab4bdbc960d58316c19652bb/fastavro-1.12.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bfac2dada8ddc002e8b7d8289d6fad4f070bc1fec20371cec684a7d10d932e96", size = 3170160, upload-time = "2026-04-24T14:37:18.168Z" }, + { url = "https://files.pythonhosted.org/packages/94/6f/2adb571fda448d4afd2466e1cef2963fefdc6b37847da05249983e415f17/fastavro-1.12.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bc44ba6289fb1f5ee318335958dde6ad6d742dcb4bb8930de843e9024c64b68c", size = 3281842, upload-time = "2026-04-24T14:37:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/17/07/4bad2e96c4c6bae40253be2573cc09c1e5b9ccf821e1ff74e0d33b64bf90/fastavro-1.12.2-cp314-cp314-win_amd64.whl", hash = "sha256:a475418f71c5aed69899813ecccf392429c08c3a63df3030129db71760b0db8f", size = 450903, upload-time = "2026-04-24T14:37:23.059Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b7/180f67ba9a46ba23a1ff6432f48d3087d4f2048579ecc262b00426cb1c63/fastavro-1.12.2-cp314-cp314-win_arm64.whl", hash = "sha256:daec9f9655a1d4636613c47d6d3343f6e039150d66cdce62543e20ca36612a8a", size = 391076, upload-time = "2026-04-24T14:37:24.756Z" }, + { url = "https://files.pythonhosted.org/packages/dd/8f/18f60329b627d2118a4a2b19e8741fbd807d60bf0470554e1bbfb7f1bca3/fastavro-1.12.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:57594b72cf663bbd0f3ad8a319a999fc3d7c71065a6799b2c1d1a6a137894c5b", size = 1055430, upload-time = "2026-05-09T21:53:14.364Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ac/a1fa1fc29df0efc89d4946a743b09bdc9500591b5b92083eaf8e93664916/fastavro-1.12.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74412132bbfb153cbf704517f2c89f7d3e170feb681b13bceace690f66f8d5fa", size = 3503075, upload-time = "2026-04-24T14:37:26.826Z" }, + { url = "https://files.pythonhosted.org/packages/82/bf/4f669e10b6bc38a731ee3400aed1a1e2d0a3e3cf411e72f6b320d3af0eaf/fastavro-1.12.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e367a84c9133018e0a3bc822abe78d7f1f9a6092991a0ec409468cf4ef260282", size = 3410900, upload-time = "2026-04-24T14:37:29.233Z" }, + { url = "https://files.pythonhosted.org/packages/10/39/ecb19fdae4158a7730b5963fbf1b6d38d74678392d73083be518642af0c1/fastavro-1.12.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:044fafca0853e9ae14009de7763ac9e8e8f8b96f8a4e90bd58b695443266a370", size = 3335637, upload-time = "2026-04-24T14:37:31.472Z" }, + { url = "https://files.pythonhosted.org/packages/32/f1/f21bd5319113e89ceceed2df840df21e9c5150d181db74b6ba80400f9f48/fastavro-1.12.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:afede7324822800e4f90e96b9514188a237a60f35e8e7a10b2129c10c78f6e4d", size = 3356664, upload-time = "2026-04-24T14:37:34.231Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, +] + +[[package]] +name = "idna" +version = "3.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/5c/f3aedc83549aae71cd52b9e9687fe896e3dc6e966ba20eba04718605d198/markdown_it_py-4.1.0.tar.gz", hash = "sha256:760e3f87b2787c044c5138a5ba107b7c2be26c03b13cc7f8fe42756b65b1df6c", size = 81613, upload-time = "2026-05-06T16:32:13.649Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/88/802c82060c54bc7dde21eb0033e337838b8181a1323254aa9ec41cbfc3d1/markdown_it_py-4.1.0-py3-none-any.whl", hash = "sha256:d4939a62a2dd0cd9cb80a191a711ba1d39bac8ed5ef9e9966895b0171c01c46d", size = 90955, upload-time = "2026-05-06T16:32:12.184Z" }, +] + +[[package]] +name = "maturin" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" }, + { url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" }, + { url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" }, + { url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" }, + { url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" }, + { url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" }, + { url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" }, + { url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" }, + { url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mmh3" +version = "5.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/1a/edb23803a168f070ded7a3014c6d706f63b90c84ccc024f89d794a3b7a6d/mmh3-5.2.1.tar.gz", hash = "sha256:bbea5b775f0ac84945191fb83f845a6fd9a21a03ea7f2e187defac7e401616ad", size = 33775, upload-time = "2026-03-05T15:55:57.716Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/bb/88ee54afa5644b0f35ab5b435f208394feb963e5bb47c4e404deb625ffa4/mmh3-5.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5d87a3584093e1a89987e3d36d82c98d9621b2cb944e22a420aa1401e096758f", size = 56080, upload-time = "2026-03-05T15:53:40.452Z" }, + { url = "https://files.pythonhosted.org/packages/cc/bf/5404c2fd6ac84819e8ff1b7e34437b37cf55a2b11318894909e7bb88de3f/mmh3-5.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:30e4d2084df019880d55f6f7bea35328d9b464ebee090baa372c096dc77556fb", size = 40462, upload-time = "2026-03-05T15:53:41.751Z" }, + { url = "https://files.pythonhosted.org/packages/de/0b/52bffad0b52ae4ea53e222b594bd38c08ecac1fc410323220a7202e43da5/mmh3-5.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0bbc17250b10d3466875a40a52520a6bac3c02334ca709207648abd3c223ed5c", size = 40077, upload-time = "2026-03-05T15:53:42.753Z" }, + { url = "https://files.pythonhosted.org/packages/a0/9e/326c93d425b9fa4cbcdc71bc32aaba520db37577d632a24d25d927594eca/mmh3-5.2.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:76219cd1eefb9bf4af7856e3ae563d15158efa145c0aab01e9933051a1954045", size = 95302, upload-time = "2026-03-05T15:53:43.867Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b1/e20d5f0d19c4c0f3df213fa7dcfa0942c4fb127d38e11f398ae8ddf6cccc/mmh3-5.2.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb9d44c25244e11c8be3f12c938ca8ba8404620ef8092245d2093c6ab3df260f", size = 101174, upload-time = "2026-03-05T15:53:45.194Z" }, + { url = "https://files.pythonhosted.org/packages/7f/4a/1a9bb3e33c18b1e1cee2c249a3053c4d4d9c93ecb30738f39a62249a7e86/mmh3-5.2.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d5d542bf2abd0fd0361e8017d03f7cb5786214ceb4a40eef1539d6585d93386", size = 103979, upload-time = "2026-03-05T15:53:46.334Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/dab9ee7545429e7acdd38d23d0104471d31de09a0c695f1b751e0ff34532/mmh3-5.2.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:08043f7cb1fb9467c3fbbbaea7896986e7fbc81f4d3fd9289a73d9110ab6207a", size = 110898, upload-time = "2026-03-05T15:53:47.443Z" }, + { url = "https://files.pythonhosted.org/packages/72/08/408f11af7fe9e76b883142bb06536007cc7f237be2a5e9ad4e837716e627/mmh3-5.2.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:add7ac388d1e0bf57259afbcf9ed05621a3bf11ce5ee337e7536f1e1aaf056b0", size = 118308, upload-time = "2026-03-05T15:53:49.1Z" }, + { url = "https://files.pythonhosted.org/packages/86/2d/0551be7fe0000736d9ad12ffa1f130d7a0c17b49193d6dc41c82bd9404c6/mmh3-5.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41105377f6282e8297f182e393a79cfffd521dde37ace52b106373bdcd9ca5cb", size = 101671, upload-time = "2026-03-05T15:53:50.317Z" }, + { url = "https://files.pythonhosted.org/packages/44/17/6e4f80c4e6ad590139fa2017c3aeca54e7cc9ef68e08aa142a0c90f40a97/mmh3-5.2.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3cb61db880ec11e984348227b333259994c2c85caa775eb7875decb3768db890", size = 96682, upload-time = "2026-03-05T15:53:51.48Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a7/b82fccd38c1fa815de72e94ebe9874562964a10e21e6c1bc3b01d3f15a0e/mmh3-5.2.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b5378de2b139c3a830f0209c1e91f7705919a4b3e563a10955104f5097a70a", size = 110287, upload-time = "2026-03-05T15:53:52.68Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a1/2644069031c8cec0be46f0346f568a53f42fddd843f03cc890306699c1e2/mmh3-5.2.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e904f2417f0d6f6d514f3f8b836416c360f306ddaee1f84de8eef1e722d212e5", size = 111899, upload-time = "2026-03-05T15:53:53.791Z" }, + { url = "https://files.pythonhosted.org/packages/51/7b/6614f3eb8fb33f931fa7616c6d477247e48ec6c5082b02eeeee998cffa94/mmh3-5.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f1fbb0a99125b1287c6d9747f937dc66621426836d1a2d50d05aecfc81911b57", size = 100078, upload-time = "2026-03-05T15:53:55.234Z" }, + { url = "https://files.pythonhosted.org/packages/27/9a/dd4d5a5fb893e64f71b42b69ecae97dd78db35075412488b24036bc5599c/mmh3-5.2.1-cp310-cp310-win32.whl", hash = "sha256:b4cce60d0223074803c9dbe0721ad3fa51dafe7d462fee4b656a1aa01ee07518", size = 40756, upload-time = "2026-03-05T15:53:56.319Z" }, + { url = "https://files.pythonhosted.org/packages/c9/34/0b25889450f8aeffcec840aa73251e853f059c1b72ed1d1c027b956f95f5/mmh3-5.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:6f01f044112d43a20be2f13a11683666d87151542ad627fe41a18b9791d2802f", size = 41519, upload-time = "2026-03-05T15:53:57.41Z" }, + { url = "https://files.pythonhosted.org/packages/fd/31/8fd42e3c526d0bcb1db7f569c0de6729e180860a0495e387a53af33c2043/mmh3-5.2.1-cp310-cp310-win_arm64.whl", hash = "sha256:7501e9be34cb21e72fcfe672aafd0eee65c16ba2afa9dcb5500a587d3a0580f0", size = 39285, upload-time = "2026-03-05T15:53:58.697Z" }, + { url = "https://files.pythonhosted.org/packages/65/d7/3312a59df3c1cdd783f4cf0c4ee8e9decff9c5466937182e4cc7dbbfe6c5/mmh3-5.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:dae0f0bd7d30c0ad61b9a504e8e272cb8391eed3f1587edf933f4f6b33437450", size = 56082, upload-time = "2026-03-05T15:53:59.702Z" }, + { url = "https://files.pythonhosted.org/packages/61/96/6f617baa098ca0d2989bfec6d28b5719532cd8d8848782662f5b755f657f/mmh3-5.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9aeaf53eaa075dd63e81512522fd180097312fb2c9f476333309184285c49ce0", size = 40458, upload-time = "2026-03-05T15:54:01.548Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b4/9cd284bd6062d711e13d26c04d4778ab3f690c1c38a4563e3c767ec8802e/mmh3-5.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0634581290e6714c068f4aa24020acf7880927d1f0084fa753d9799ae9610082", size = 40079, upload-time = "2026-03-05T15:54:02.743Z" }, + { url = "https://files.pythonhosted.org/packages/f6/09/a806334ce1d3d50bf782b95fcee8b3648e1e170327d4bb7b4bad2ad7d956/mmh3-5.2.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e080c0637aea036f35507e803a4778f119a9b436617694ae1c5c366805f1e997", size = 97242, upload-time = "2026-03-05T15:54:04.536Z" }, + { url = "https://files.pythonhosted.org/packages/ee/93/723e317dd9e041c4dc4566a2eb53b01ad94de31750e0b834f1643905e97c/mmh3-5.2.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:db0562c5f71d18596dcd45e854cf2eeba27d7543e1a3acdafb7eef728f7fe85d", size = 103082, upload-time = "2026-03-05T15:54:06.387Z" }, + { url = "https://files.pythonhosted.org/packages/61/b5/f96121e69cc48696075071531cf574f112e1ffd08059f4bffb41210e6fc5/mmh3-5.2.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d9f9a3ce559a5267014b04b82956993270f63ec91765e13e9fd73daf2d2738e", size = 106054, upload-time = "2026-03-05T15:54:07.506Z" }, + { url = "https://files.pythonhosted.org/packages/82/49/192b987ec48d0b2aecf8ac285a9b11fbc00030f6b9c694664ae923458dde/mmh3-5.2.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:960b1b3efa39872ac8b6cc3a556edd6fb90ed74f08c9c45e028f1005b26aa55d", size = 112910, upload-time = "2026-03-05T15:54:09.403Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a1/03e91fd334ed0144b83343a76eb11f17434cd08f746401488cfeafb2d241/mmh3-5.2.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d30b650595fdbe32366b94cb14f30bb2b625e512bd4e1df00611f99dc5c27fd4", size = 120551, upload-time = "2026-03-05T15:54:10.587Z" }, + { url = "https://files.pythonhosted.org/packages/93/b9/b89a71d2ff35c3a764d1c066c7313fc62c7cc48fa48a4b3b0304a4a0146f/mmh3-5.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:82f3802bfc4751f420d591c5c864de538b71cea117fce67e4595c2afede08a15", size = 99096, upload-time = "2026-03-05T15:54:11.76Z" }, + { url = "https://files.pythonhosted.org/packages/36/b5/613772c1c6ed5f7b63df55eb131e887cc43720fec392777b95a79d34e640/mmh3-5.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:915e7a2418f10bd1151b1953df06d896db9783c9cfdb9a8ee1f9b3a4331ab503", size = 98524, upload-time = "2026-03-05T15:54:13.122Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0e/1524566fe8eaf871e4f7bc44095929fcd2620488f402822d848df19d679c/mmh3-5.2.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fc78739b5ec6e4fb02301984a3d442a91406e7700efbe305071e7fd1c78278f2", size = 106239, upload-time = "2026-03-05T15:54:14.601Z" }, + { url = "https://files.pythonhosted.org/packages/04/94/21adfa7d90a7a697137ad6de33eeff6445420ca55e433a5d4919c79bc3b5/mmh3-5.2.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:41aac7002a749f08727cb91babff1daf8deac317c0b1f317adc69be0e6c375d1", size = 109797, upload-time = "2026-03-05T15:54:15.819Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e6/1aacc3a219e1aa62fa65669995d4a3562b35be5200ec03680c7e4bec9676/mmh3-5.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9d8089d853c7963a8ce87fff93e2a67075c0bc08684a08ea6ad13577c38ffc38", size = 97228, upload-time = "2026-03-05T15:54:16.992Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b9/5e4cca8dcccf298add0a27f3c357bc8cf8baf821d35cdc6165e4bd5a48b0/mmh3-5.2.1-cp311-cp311-win32.whl", hash = "sha256:baeb47635cb33375dee4924cd93d7f5dcaa786c740b08423b0209b824a1ee728", size = 40751, upload-time = "2026-03-05T15:54:18.714Z" }, + { url = "https://files.pythonhosted.org/packages/72/fc/5b11d49247f499bcda591171e9cf3b6ee422b19e70aa2cef2e0ae65ca3b9/mmh3-5.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:1e4ecee40ba19e6975e1120829796770325841c2f153c0e9aecca927194c6a2a", size = 41517, upload-time = "2026-03-05T15:54:19.764Z" }, + { url = "https://files.pythonhosted.org/packages/8a/5f/2a511ee8a1c2a527c77726d5231685b72312c5a1a1b7639ad66a9652aa84/mmh3-5.2.1-cp311-cp311-win_arm64.whl", hash = "sha256:c302245fd6c33d96bd169c7ccf2513c20f4c1e417c07ce9dce107c8bc3f8411f", size = 39287, upload-time = "2026-03-05T15:54:20.904Z" }, + { url = "https://files.pythonhosted.org/packages/92/94/bc5c3b573b40a328c4d141c20e399039ada95e5e2a661df3425c5165fd84/mmh3-5.2.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0cc21533878e5586b80d74c281d7f8da7932bc8ace50b8d5f6dbf7e3935f63f1", size = 56087, upload-time = "2026-03-05T15:54:21.92Z" }, + { url = "https://files.pythonhosted.org/packages/f6/80/64a02cc3e95c3af0aaa2590849d9ed24a9f14bb93537addde688e039b7c3/mmh3-5.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4eda76074cfca2787c8cf1bec603eaebdddd8b061ad5502f85cddae998d54f00", size = 40500, upload-time = "2026-03-05T15:54:22.953Z" }, + { url = "https://files.pythonhosted.org/packages/8b/72/e6d6602ce18adf4ddcd0e48f2e13590cc92a536199e52109f46f259d3c46/mmh3-5.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eee884572b06bbe8a2b54f424dbd996139442cf83c76478e1ec162512e0dd2c7", size = 40034, upload-time = "2026-03-05T15:54:23.943Z" }, + { url = "https://files.pythonhosted.org/packages/59/c2/bf4537a8e58e21886ef16477041238cab5095c836496e19fafc34b7445d2/mmh3-5.2.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0d0b7e803191db5f714d264044e06189c8ccd3219e936cc184f07106bd17fd7b", size = 97292, upload-time = "2026-03-05T15:54:25.335Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e2/51ed62063b44d10b06d975ac87af287729eeb5e3ed9772f7584a17983e90/mmh3-5.2.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e6c219e375f6341d0959af814296372d265a8ca1af63825f65e2e87c618f006", size = 103274, upload-time = "2026-03-05T15:54:26.44Z" }, + { url = "https://files.pythonhosted.org/packages/75/ce/12a7524dca59eec92e5b31fdb13ede1e98eda277cf2b786cf73bfbc24e81/mmh3-5.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26fb5b9c3946bf7f1daed7b37e0c03898a6f062149127570f8ede346390a0825", size = 106158, upload-time = "2026-03-05T15:54:28.578Z" }, + { url = "https://files.pythonhosted.org/packages/86/1f/d3ba6dd322d01ab5d44c46c8f0c38ab6bbbf9b5e20e666dfc05bf4a23604/mmh3-5.2.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3c38d142c706201db5b2345166eeef1e7740e3e2422b470b8ba5c8727a9b4c7a", size = 113005, upload-time = "2026-03-05T15:54:29.767Z" }, + { url = "https://files.pythonhosted.org/packages/b6/a9/15d6b6f913294ea41b44d901741298e3718e1cb89ee626b3694625826a43/mmh3-5.2.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50885073e2909251d4718634a191c49ae5f527e5e1736d738e365c3e8be8f22b", size = 120744, upload-time = "2026-03-05T15:54:30.931Z" }, + { url = "https://files.pythonhosted.org/packages/76/b3/70b73923fd0284c439860ff5c871b20210dfdbe9a6b9dd0ee6496d77f174/mmh3-5.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b3f99e1756fc48ad507b95e5d86f2fb21b3d495012ff13e6592ebac14033f166", size = 99111, upload-time = "2026-03-05T15:54:32.353Z" }, + { url = "https://files.pythonhosted.org/packages/dd/38/99f7f75cd27d10d8b899a1caafb9d531f3903e4d54d572220e3d8ac35e89/mmh3-5.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62815d2c67f2dd1be76a253d88af4e1da19aeaa1820146dec52cf8bee2958b16", size = 98623, upload-time = "2026-03-05T15:54:33.801Z" }, + { url = "https://files.pythonhosted.org/packages/fd/68/6e292c0853e204c44d2f03ea5f090be3317a0e2d9417ecb62c9eb27687df/mmh3-5.2.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8f767ba0911602ddef289404e33835a61168314ebd3c729833db2ed685824211", size = 106437, upload-time = "2026-03-05T15:54:35.177Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c6/fedd7284c459cfb58721d461fcf5607a4c1f5d9ab195d113d51d10164d16/mmh3-5.2.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:67e41a497bac88cc1de96eeba56eeb933c39d54bc227352f8455aa87c4ca4000", size = 110002, upload-time = "2026-03-05T15:54:36.673Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ac/ca8e0c19a34f5b71390171d2ff0b9f7f187550d66801a731bb68925126a4/mmh3-5.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3d74a03fb57757ece25aa4b3c1c60157a1cece37a020542785f942e2f827eed5", size = 97507, upload-time = "2026-03-05T15:54:37.804Z" }, + { url = "https://files.pythonhosted.org/packages/df/94/6ebb9094cfc7ac5e7950776b9d13a66bb4a34f83814f32ba2abc9494fc68/mmh3-5.2.1-cp312-cp312-win32.whl", hash = "sha256:7374d6e3ef72afe49697ecd683f3da12f4fc06af2d75433d0580c6746d2fa025", size = 40773, upload-time = "2026-03-05T15:54:40.077Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/cd3527198cf159495966551c84a5f36805a10ac17b294f41f67b83f6a4d6/mmh3-5.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:3a9fed49c6ce4ed7e73f13182760c65c816da006debe67f37635580dfb0fae00", size = 41560, upload-time = "2026-03-05T15:54:41.148Z" }, + { url = "https://files.pythonhosted.org/packages/15/96/6fe5ebd0f970a076e3ed5512871ce7569447b962e96c125528a2f9724470/mmh3-5.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:bbfcb95d9a744e6e2827dfc66ad10e1020e0cac255eb7f85652832d5a264c2fc", size = 39313, upload-time = "2026-03-05T15:54:42.171Z" }, + { url = "https://files.pythonhosted.org/packages/25/a5/9daa0508a1569a54130f6198d5462a92deda870043624aa3ea72721aa765/mmh3-5.2.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:723b2681ed4cc07d3401bbea9c201ad4f2a4ca6ba8cddaff6789f715dd2b391e", size = 40832, upload-time = "2026-03-05T15:54:43.212Z" }, + { url = "https://files.pythonhosted.org/packages/0a/6b/3230c6d80c1f4b766dedf280a92c2241e99f87c1504ff74205ec8cebe451/mmh3-5.2.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:3619473a0e0d329fd4aec8075628f8f616be2da41605300696206d6f36920c3d", size = 41964, upload-time = "2026-03-05T15:54:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/62/fb/648bfddb74a872004b6ee751551bfdda783fe6d70d2e9723bad84dbe5311/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e48d4dbe0f88e53081da605ae68644e5182752803bbc2beb228cca7f1c4454d6", size = 39114, upload-time = "2026-03-05T15:54:45.205Z" }, + { url = "https://files.pythonhosted.org/packages/95/c2/ab7901f87af438468b496728d11264cb397b3574d41506e71b92128e0373/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a482ac121de6973897c92c2f31defc6bafb11c83825109275cffce54bb64933f", size = 39819, upload-time = "2026-03-05T15:54:46.509Z" }, + { url = "https://files.pythonhosted.org/packages/2f/ed/6f88dda0df67de1612f2e130ffea34cf84aaee5bff5b0aff4dbff2babe34/mmh3-5.2.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:17fbb47f0885ace8327ce1235d0416dc86a211dcd8cc1e703f41523be32cfec8", size = 40330, upload-time = "2026-03-05T15:54:47.864Z" }, + { url = "https://files.pythonhosted.org/packages/3d/66/7516d23f53cdf90f43fce24ab80c28f45e6851d78b46bef8c02084edf583/mmh3-5.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d51fde50a77f81330523562e3c2734ffdca9c4c9e9d355478117905e1cfe16c6", size = 56078, upload-time = "2026-03-05T15:54:48.9Z" }, + { url = "https://files.pythonhosted.org/packages/bc/34/4d152fdf4a91a132cb226b671f11c6b796eada9ab78080fb5ce1e95adaab/mmh3-5.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:19bbd3b841174ae6ed588536ab5e1b1fe83d046e668602c20266547298d939a9", size = 40498, upload-time = "2026-03-05T15:54:49.942Z" }, + { url = "https://files.pythonhosted.org/packages/d4/4c/8e3af1b6d85a299767ec97bd923f12b06267089c1472c27c1696870d1175/mmh3-5.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be77c402d5e882b6fbacfd90823f13da8e0a69658405a39a569c6b58fdb17b03", size = 40033, upload-time = "2026-03-05T15:54:50.994Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f2/966ea560e32578d453c9e9db53d602cbb1d0da27317e232afa7c38ceba11/mmh3-5.2.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fd96476f04db5ceba1cfa0f21228f67c1f7402296f0e73fee3513aa680ad237b", size = 97320, upload-time = "2026-03-05T15:54:52.072Z" }, + { url = "https://files.pythonhosted.org/packages/bb/0d/2c5f9893b38aeb6b034d1a44ecd55a010148054f6a516abe53b5e4057297/mmh3-5.2.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:707151644085dd0f20fe4f4b573d28e5130c4aaa5f587e95b60989c5926653b5", size = 103299, upload-time = "2026-03-05T15:54:53.569Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fc/2ebaef4a4d4376f89761274dc274035ffd96006ab496b4ee5af9b08f21a9/mmh3-5.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3737303ca9ea0f7cb83028781148fcda4f1dac7821db0c47672971dabcf63593", size = 106222, upload-time = "2026-03-05T15:54:55.092Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/ea7ffe126d0ba0406622602a2d05e1e1a6841cc92fc322eb576c95b27fad/mmh3-5.2.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2778fed822d7db23ac5008b181441af0c869455b2e7d001f4019636ac31b6fe4", size = 113048, upload-time = "2026-03-05T15:54:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/85/57/9447032edf93a64aa9bef4d9aa596400b1756f40411890f77a284f6293ca/mmh3-5.2.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d57dea657357230cc780e13920d7fa7db059d58fe721c80020f94476da4ca0a1", size = 120742, upload-time = "2026-03-05T15:54:57.453Z" }, + { url = "https://files.pythonhosted.org/packages/53/82/a86cc87cc88c92e9e1a598fee509f0409435b57879a6129bf3b3e40513c7/mmh3-5.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:169e0d178cb59314456ab30772429a802b25d13227088085b0d49b9fe1533104", size = 99132, upload-time = "2026-03-05T15:54:58.583Z" }, + { url = "https://files.pythonhosted.org/packages/54/f7/6b16eb1b40ee89bb740698735574536bc20d6cdafc65ae702ea235578e05/mmh3-5.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7e4e1f580033335c6f76d1e0d6b56baf009d1a64d6a4816347e4271ba951f46d", size = 98686, upload-time = "2026-03-05T15:55:00.078Z" }, + { url = "https://files.pythonhosted.org/packages/e8/88/a601e9f32ad1410f438a6d0544298ea621f989bd34a0731a7190f7dec799/mmh3-5.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2bd9f19f7f1fcebd74e830f4af0f28adad4975d40d80620be19ffb2b2af56c9f", size = 106479, upload-time = "2026-03-05T15:55:01.532Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5c/ce29ae3dfc4feec4007a437a1b7435fb9507532a25147602cd5b52be86db/mmh3-5.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:c88653877aeb514c089d1b3d473451677b8b9a6d1497dbddf1ae7934518b06d2", size = 110030, upload-time = "2026-03-05T15:55:02.934Z" }, + { url = "https://files.pythonhosted.org/packages/13/30/ae444ef2ff87c805d525da4fa63d27cda4fe8a48e77003a036b8461cfd5c/mmh3-5.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fceef7fe67c81e1585198215e42ad3fdba3a25644beda8fbdaf85f4d7b93175a", size = 97536, upload-time = "2026-03-05T15:55:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f9/dc3787ee5c813cc27fe79f45ad4500d9b5437f23a7402435cc34e07c7718/mmh3-5.2.1-cp313-cp313-win32.whl", hash = "sha256:54b64fb2433bc71488e7a449603bf8bd31fbcf9cb56fbe1eb6d459e90b86c37b", size = 40769, upload-time = "2026-03-05T15:55:05.277Z" }, + { url = "https://files.pythonhosted.org/packages/43/67/850e0b5a1e97799822ebfc4ca0e8c6ece3ed8baf7dcdf64de817dfdda2ca/mmh3-5.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:cae6383181f1e345317742d2ddd88f9e7d2682fa4c9432e3a74e47d92dce0229", size = 41563, upload-time = "2026-03-05T15:55:06.283Z" }, + { url = "https://files.pythonhosted.org/packages/c0/cc/98c90b28e1da5458e19fbfaf4adb5289208d3bfccd45dd14eab216a2f0bb/mmh3-5.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:022aa1a528604e6c83d0a7705fdef0b5355d897a9e0fa3a8d26709ceaa06965d", size = 39310, upload-time = "2026-03-05T15:55:07.323Z" }, + { url = "https://files.pythonhosted.org/packages/63/b4/65bc1fb2bb7f83e91c30865023b1847cf89a5f237165575e8c83aa536584/mmh3-5.2.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:d771f085fcdf4035786adfb1d8db026df1eb4b41dac1c3d070d1e49512843227", size = 40794, upload-time = "2026-03-05T15:55:09.773Z" }, + { url = "https://files.pythonhosted.org/packages/c4/86/7168b3d83be8eb553897b1fac9da8bbb06568e5cfe555ffc329ebb46f59d/mmh3-5.2.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:7f196cd7910d71e9d9860da0ff7a77f64d22c1ad931f1dd18559a06e03109fc0", size = 41923, upload-time = "2026-03-05T15:55:10.924Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9b/b653ab611c9060ce8ff0ba25c0226757755725e789292f3ca138a58082cd/mmh3-5.2.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b1f12bd684887a0a5d55e6363ca87056f361e45451105012d329b86ec19dbe0b", size = 39131, upload-time = "2026-03-05T15:55:11.961Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b4/5a2e0d34ab4d33543f01121e832395ea510132ea8e52cdf63926d9d81754/mmh3-5.2.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d106493a60dcb4aef35a0fac85105e150a11cf8bc2b0d388f5a33272d756c966", size = 39825, upload-time = "2026-03-05T15:55:13.013Z" }, + { url = "https://files.pythonhosted.org/packages/bd/69/81699a8f39a3f8d368bec6443435c0c392df0d200ad915bf0d222b588e03/mmh3-5.2.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:44983e45310ee5b9f73397350251cdf6e63a466406a105f1d16cb5baa659270b", size = 40344, upload-time = "2026-03-05T15:55:14.026Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b3/71c8c775807606e8fd8acc5c69016e1caf3200d50b50b6dd4b40ce10b76c/mmh3-5.2.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:368625fb01666655985391dbad3860dc0ba7c0d6b9125819f3121ee7292b4ac8", size = 56291, upload-time = "2026-03-05T15:55:15.137Z" }, + { url = "https://files.pythonhosted.org/packages/6f/75/2c24517d4b2ce9e4917362d24f274d3d541346af764430249ddcc4cb3a08/mmh3-5.2.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:72d1cc63bcc91e14933f77d51b3df899d6a07d184ec515ea7f56bff659e124d7", size = 40575, upload-time = "2026-03-05T15:55:16.518Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b9/e4a360164365ac9f07a25f0f7928e3a66eb9ecc989384060747aa170e6aa/mmh3-5.2.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e8b4b5580280b9265af3e0409974fb79c64cf7523632d03fbf11df18f8b0181e", size = 40052, upload-time = "2026-03-05T15:55:17.735Z" }, + { url = "https://files.pythonhosted.org/packages/97/ca/120d92223a7546131bbbc31c9174168ee7a73b1366f5463ffe69d9e691fe/mmh3-5.2.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4cbbde66f1183db040daede83dd86c06d663c5bb2af6de1142b7c8c37923dd74", size = 97311, upload-time = "2026-03-05T15:55:18.959Z" }, + { url = "https://files.pythonhosted.org/packages/b6/71/c1a60c1652b8813ef9de6d289784847355417ee0f2980bca002fe87f4ae5/mmh3-5.2.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8ff038d52ef6aa0f309feeba00c5095c9118d0abf787e8e8454d6048db2037fc", size = 103279, upload-time = "2026-03-05T15:55:20.448Z" }, + { url = "https://files.pythonhosted.org/packages/48/29/ad97f4be1509cdcb28ae32c15593ce7c415db47ace37f8fad35b493faa9a/mmh3-5.2.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4130d0b9ce5fad6af07421b1aecc7e079519f70d6c05729ab871794eded8617", size = 106290, upload-time = "2026-03-05T15:55:21.6Z" }, + { url = "https://files.pythonhosted.org/packages/77/29/1f86d22e281bd8827ba373600a4a8b0c0eae5ca6aa55b9a8c26d2a34decc/mmh3-5.2.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6e0bfe77d238308839699944164b96a2eeccaf55f2af400f54dc20669d8d5f2", size = 113116, upload-time = "2026-03-05T15:55:22.826Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7c/339971ea7ed4c12d98f421f13db3ea576a9114082ccb59d2d1a0f00ccac1/mmh3-5.2.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f963eafc0a77a6c0562397da004f5876a9bcf7265a7bcc3205e29636bc4a1312", size = 120740, upload-time = "2026-03-05T15:55:24.3Z" }, + { url = "https://files.pythonhosted.org/packages/e4/92/3c7c4bdb8e926bb3c972d1e2907d77960c1c4b250b41e8366cf20c6e4373/mmh3-5.2.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:92883836caf50d5255be03d988d75bc93e3f86ba247b7ca137347c323f731deb", size = 99143, upload-time = "2026-03-05T15:55:25.456Z" }, + { url = "https://files.pythonhosted.org/packages/df/0a/33dd8706e732458c8375eae63c981292de07a406bad4ec03e5269654aa2c/mmh3-5.2.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:57b52603e89355ff318025dd55158f6e71396c0f1f609d548e9ea9c94cc6ce0a", size = 98703, upload-time = "2026-03-05T15:55:26.723Z" }, + { url = "https://files.pythonhosted.org/packages/51/04/76bbce05df76cbc3d396f13b2ea5b1578ef02b6a5187e132c6c33f99d596/mmh3-5.2.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f40a95186a72fa0b67d15fef0f157bfcda00b4f59c8a07cbe5530d41ac35d105", size = 106484, upload-time = "2026-03-05T15:55:28.214Z" }, + { url = "https://files.pythonhosted.org/packages/d3/8f/c6e204a2c70b719c1f62ffd9da27aef2dddcba875ea9c31ca0e87b975a46/mmh3-5.2.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:58370d05d033ee97224c81263af123dea3d931025030fd34b61227a768a8858a", size = 110012, upload-time = "2026-03-05T15:55:29.532Z" }, + { url = "https://files.pythonhosted.org/packages/e3/37/7181efd8e39db386c1ebc3e6b7d1f702a09d7c1197a6f2742ed6b5c16597/mmh3-5.2.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7be6dfb49e48fd0a7d91ff758a2b51336f1cd21f9d44b20f6801f072bd080cdd", size = 97508, upload-time = "2026-03-05T15:55:31.01Z" }, + { url = "https://files.pythonhosted.org/packages/42/0f/afa7ca2615fd85e1469474bb860e381443d0b868c083b62b41cb1d7ca32f/mmh3-5.2.1-cp314-cp314-win32.whl", hash = "sha256:54fe8518abe06a4c3852754bfd498b30cc58e667f376c513eac89a244ce781a4", size = 41387, upload-time = "2026-03-05T15:55:32.403Z" }, + { url = "https://files.pythonhosted.org/packages/71/0d/46d42a260ee1357db3d486e6c7a692e303c017968e14865e00efa10d09fc/mmh3-5.2.1-cp314-cp314-win_amd64.whl", hash = "sha256:3f796b535008708846044c43302719c6956f39ca2d93f2edda5319e79a29efbb", size = 42101, upload-time = "2026-03-05T15:55:33.646Z" }, + { url = "https://files.pythonhosted.org/packages/a4/7b/848a8378059d96501a41159fca90d6a99e89736b0afbe8e8edffeac8c74b/mmh3-5.2.1-cp314-cp314-win_arm64.whl", hash = "sha256:cd471ede0d802dd936b6fab28188302b2d497f68436025857ca72cd3810423fe", size = 39836, upload-time = "2026-03-05T15:55:35.026Z" }, + { url = "https://files.pythonhosted.org/packages/27/61/1dabea76c011ba8547c25d30c91c0ec22544487a8750997a27a0c9e1180b/mmh3-5.2.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5174a697ce042fa77c407e05efe41e03aa56dae9ec67388055820fb48cf4c3ba", size = 57727, upload-time = "2026-03-05T15:55:36.162Z" }, + { url = "https://files.pythonhosted.org/packages/b7/32/731185950d1cf2d5e28979cc8593016ba1619a295faba10dda664a4931b5/mmh3-5.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0a3984146e414684a6be2862d84fcb1035f4984851cb81b26d933bab6119bf00", size = 41308, upload-time = "2026-03-05T15:55:37.254Z" }, + { url = "https://files.pythonhosted.org/packages/76/aa/66c76801c24b8c9418b4edde9b5e57c75e72c94e29c48f707e3962534f18/mmh3-5.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bd6e7d363aa93bd3421b30b6af97064daf47bc96005bddba67c5ffbc6df426b8", size = 40758, upload-time = "2026-03-05T15:55:38.61Z" }, + { url = "https://files.pythonhosted.org/packages/9e/bb/79a1f638a02f0ae389f706d13891e2fbf7d8c0a22ecde67ba828951bb60a/mmh3-5.2.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:113f78e7463a36dbbcea05bfe688efd7fa759d0f0c56e73c974d60dcfec3dfcc", size = 109670, upload-time = "2026-03-05T15:55:40.13Z" }, + { url = "https://files.pythonhosted.org/packages/26/94/8cd0e187a288985bcfc79bf5144d1d712df9dee74365f59d26e3a1865be6/mmh3-5.2.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7e8ec5f606e0809426d2440e0683509fb605a8820a21ebd120dcdba61b74ef7f", size = 117399, upload-time = "2026-03-05T15:55:42.076Z" }, + { url = "https://files.pythonhosted.org/packages/42/94/dfea6059bd5c5beda565f58a4096e43f4858fb6d2862806b8bbd12cbb284/mmh3-5.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22b0f9971ec4e07e8223f2beebe96a6cfc779d940b6f27d26604040dd74d3a44", size = 120386, upload-time = "2026-03-05T15:55:43.481Z" }, + { url = "https://files.pythonhosted.org/packages/47/cb/f9c45e62aaa67220179f487772461d891bb582bb2f9783c944832c60efd9/mmh3-5.2.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85ffc9920ffc39c5eee1e3ac9100c913a0973996fbad5111f939bbda49204bb7", size = 125924, upload-time = "2026-03-05T15:55:44.638Z" }, + { url = "https://files.pythonhosted.org/packages/a5/83/fe54a4a7c11bc9f623dfc1707decd034245602b076dfc1dcc771a4163170/mmh3-5.2.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7aec798c2b01aaa65a55f1124f3405804184373abb318a3091325aece235f67c", size = 135280, upload-time = "2026-03-05T15:55:45.866Z" }, + { url = "https://files.pythonhosted.org/packages/97/67/fe7e9e9c143daddd210cd22aef89cbc425d58ecf238d2b7d9eb0da974105/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:55dbbd8ffbc40d1697d5e2d0375b08599dae8746b0b08dea05eee4ce81648fac", size = 110050, upload-time = "2026-03-05T15:55:47.074Z" }, + { url = "https://files.pythonhosted.org/packages/43/c4/6d4b09fcbef80794de447c9378e39eefc047156b290fa3dd2d5257ca8227/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:6c85c38a279ca9295a69b9b088a2e48aa49737bb1b34e6a9dc6297c110e8d912", size = 111158, upload-time = "2026-03-05T15:55:48.239Z" }, + { url = "https://files.pythonhosted.org/packages/81/a6/ca51c864bdb30524beb055a6d8826db3906af0834ec8c41d097a6e8573d5/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:6290289fa5fb4c70fd7f72016e03633d60388185483ff3b162912c81205ae2cf", size = 116890, upload-time = "2026-03-05T15:55:49.405Z" }, + { url = "https://files.pythonhosted.org/packages/cc/04/5a1fe2e2ad843d03e89af25238cbc4f6840a8bb6c4329a98ab694c71deda/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:4fc6cd65dc4d2fdb2625e288939a3566e36127a84811a4913f02f3d5931da52d", size = 123121, upload-time = "2026-03-05T15:55:50.61Z" }, + { url = "https://files.pythonhosted.org/packages/af/4d/3c820c6f4897afd25905270a9f2330a23f77a207ea7356f7aadace7273c0/mmh3-5.2.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:623f938f6a039536cc02b7582a07a080f13fdfd48f87e63201d92d7e34d09a18", size = 110187, upload-time = "2026-03-05T15:55:52.143Z" }, + { url = "https://files.pythonhosted.org/packages/21/54/1d71cd143752361c0aebef16ad3f55926a6faf7b112d355745c1f8a25f7f/mmh3-5.2.1-cp314-cp314t-win32.whl", hash = "sha256:29bc3973676ae334412efdd367fcd11d036b7be3efc1ce2407ef8676dabfeb82", size = 41934, upload-time = "2026-03-05T15:55:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e4/63a2a88f31d93dea03947cccc2a076946857e799ea4f7acdecbf43b324aa/mmh3-5.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:28cfab66577000b9505a0d068c731aee7ca85cd26d4d63881fab17857e0fe1fb", size = 43036, upload-time = "2026-03-05T15:55:55.252Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0f/59204bf136d1201f8d7884cfbaf7498c5b4674e87a4c693f9bde63741ce1/mmh3-5.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:dfd51b4c56b673dfbc43d7d27ef857dd91124801e2806c69bb45585ce0fa019b", size = 40391, upload-time = "2026-03-05T15:55:56.697Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pyarrow" +version = "25.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/e3/27f57f80141379d60defe6703eb50a707325706f07fedfd1312c7a751995/pyarrow-25.0.1.tar.gz", hash = "sha256:9150a83248bfed9813ea3c3af74c3856c1984d444aa28e58bf7733b9750ddf6a", size = 1201653, upload-time = "2026-08-10T12:40:53.904Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/3e/5cd70becb51e1d044c54ba5e627424a6e87df5b98008cbd22cc6abd409ca/pyarrow-25.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0b1edbb2f385a6a65e9711b62ba86ac54a7816a3f8d17bb3e8a5929d65fb2485", size = 35954271, upload-time = "2026-08-10T12:36:33.857Z" }, + { url = "https://files.pythonhosted.org/packages/64/be/17599e086df264ea7dc221d1101e3131e181e00da428a2f9bd0358f0d06b/pyarrow-25.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:a4dd8bf99a8fac133efc0ed6a92f5fddbe2adba0d0f6dd720e39ba9855cea85c", size = 37647543, upload-time = "2026-08-10T12:36:39.486Z" }, + { url = "https://files.pythonhosted.org/packages/42/34/e138b451fd3970a6eda4599f68ae3b2b32b661bc958de3239d54a0bf6575/pyarrow-25.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bddd0c4f7630c2a3ddf6347c1bdaa79d97bcf6bd445f9e60c816b7d77c85a5ae", size = 46837120, upload-time = "2026-08-10T12:36:46.58Z" }, + { url = "https://files.pythonhosted.org/packages/57/5c/f8fc0eb2de03464a557d5a4d0c15e972d73362414696618833b771f7eddd/pyarrow-25.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a4d6d5e9a3d1879a97c08ded0c797579b7965eafd0f0c26c30b45ccc06db939b", size = 50066460, upload-time = "2026-08-10T12:36:53.702Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d1/0dd64fd06de0333b808a02f60981635f067b71aad3a30698a9a104fae778/pyarrow-25.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:514ddb60285631af068875550c90eddc181db3e8e63a032b1559be189e82f056", size = 49937892, upload-time = "2026-08-10T12:37:00.349Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3c/f89d1bd76d5f3284c2a44d7d7ebbd8204535e5ae2b41f4077069b4ff2ec6/pyarrow-25.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cab40b1edfef0262e0e5251aa2c58d75630f24d06dd7794480243acc001a1d7d", size = 53107240, upload-time = "2026-08-10T12:37:07.205Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/b554a8e09f3f3decccf405eb8fbe86696321cbcb5b62d18b4a5057a4c113/pyarrow-25.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:60e89d8f13861a1f7f8d950fa54aebb8023b30734d0ac51ffa80beabe2df4bba", size = 27848683, upload-time = "2026-08-10T12:37:12.058Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8b/0d23b47702fcfe8b3618d5292035099675c5a1c48258932350c08020f7b5/pyarrow-25.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:51093dd9e10325fbdb3c10a2ae7c4806e5c822d94e74ae4938b26524a3323fee", size = 35946180, upload-time = "2026-08-10T12:37:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/d8/17/707d17a5476c55a9541fde0db8213ac30979a792864d72415f176ba50c45/pyarrow-25.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:eb6203482ff3746a5632303a7279ae0b5a304c46985b49ed1378cb350ea6728d", size = 37644787, upload-time = "2026-08-10T12:37:25.795Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b2/cdc98ecf1a6408280bc3a6a07054cdd99a3f4670acc0545d383ce113e87d/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:880523be3d29efcf83d3998835d206118ccf35e3871dbd2fb60408cf6b007a80", size = 46834633, upload-time = "2026-08-10T12:37:33.604Z" }, + { url = "https://files.pythonhosted.org/packages/c8/6e/d3fafc41f378b2c65be43b827798c0fae42049a641c8526633ed3eb573e2/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:25f8720bf6387d5dc2ebd2622112de630760419e4b66134405dd24110d15f37e", size = 50065507, upload-time = "2026-08-10T12:37:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/d5/12/8d0698954b8c3001844a898e0a6900bebe83d7ee40c11195174c5122f324/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4facd65742a024a4a366328a1d2292062d72d6e023c1b7dda8d4c37544933a25", size = 49955690, upload-time = "2026-08-10T12:37:46.644Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0b/1ecb936ac6409e90a34d58eea1c7cec09a9ae6d2141b9e49ad01a2b1ea47/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa0559502e1cd6254d6814614085dd9c5a3dd0419362978a936a3f68a9e5c3df", size = 53128198, upload-time = "2026-08-10T12:37:52.531Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/5236033550633c9b7377b2a53660b2bbb06cb06dc09c4356332d67643ca1/pyarrow-25.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:62cd0d785b8aa6675ee355f9fc02252a340f4441257c42674937826fd7594325", size = 27857263, upload-time = "2026-08-10T12:37:56.943Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e2/9ab15b88cbfac28e16419ce5439ec29234c5172cb8259301b4ba639bdec0/pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:df961f2e7ae9cf496459259d798652c70625f6c080650d6952f8c04053c58ee9", size = 35861559, upload-time = "2026-08-10T12:38:02.567Z" }, + { url = "https://files.pythonhosted.org/packages/58/79/a0036dbe1eabe1f73127427342f1d99982584c4a2cde2651d6c93499c6f6/pyarrow-25.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:cc4aa407fde9fc660be3939e49ea31f50f3e9fec17c0ec63159f7711edd3efc9", size = 37628383, upload-time = "2026-08-10T12:38:09.083Z" }, + { url = "https://files.pythonhosted.org/packages/13/49/d93a57d375f4bf0cf82913dd6bb54acafde83dd993be2282c81ac5616cad/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:4340f0ba6c1d2e13f21658de1d7c662ca2545018568d0030a1e9afca159d87e3", size = 46820190, upload-time = "2026-08-10T12:38:15.458Z" }, + { url = "https://files.pythonhosted.org/packages/60/c9/711ca85d79f1ec98f29a5eae2b051e25b4ecec5de3e3c0e2d5c5dcb15664/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5389cdf79447ed1515c9e31620e6e1e2302249564d603f2ad727d4f6d313e4c3", size = 50102437, upload-time = "2026-08-10T12:38:22.487Z" }, + { url = "https://files.pythonhosted.org/packages/80/53/8fb8359ff17cfb6263a1cf3ebf7caec9fe197de118719e84fcb1d0618026/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d51592cb7561e87877c506113e7adbf1342ab579e6c21f0ef44b8ba41cb74c80", size = 49942424, upload-time = "2026-08-10T12:38:28.755Z" }, + { url = "https://files.pythonhosted.org/packages/e8/83/4e5ae02a9341571b18a6fca380ac7a58ce6ddae7ab3c060208c0a1e79f02/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6109c94d8b9f3b17a041daca16cacb2f651ad8f1ef70a4232c2c0f37a23da2a8", size = 53144206, upload-time = "2026-08-10T12:38:34.862Z" }, + { url = "https://files.pythonhosted.org/packages/65/ee/197cbf47e49f83e6ebeb946a5259a48a638dea27ac774db42fe78022179d/pyarrow-25.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:8858d7bfc22e3f51529aeaa4077225029724623e4595dc9eff8c793935c34140", size = 27953934, upload-time = "2026-08-10T12:38:39.808Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8d/8f271a7a034c834910ec925d56fa4b29733b1380f5289419f5aaa3b02777/pyarrow-25.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:c7c534ec03c358a76ea3e505e74c1b6aef290af90c444dfd092dbfe23e755b85", size = 35855328, upload-time = "2026-08-10T12:38:45.489Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cd/5bac242f4e841b9971d5eb94fdfe2577e2b70be983e27401e72055786037/pyarrow-25.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:dda9470024204d7bbf2042b47c6e8a0e47a3eeb8e34405882dfaea6577e0c153", size = 37622415, upload-time = "2026-08-10T12:38:51.107Z" }, + { url = "https://files.pythonhosted.org/packages/63/1f/96d03b4e1506524f7087adb0fd6b2f69f0c9c7aaff1ec36d8030082e15a5/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:44a9120ce5bd81936b8ab9a88076e3fd47c2c6838e0e43630fed83626aca81d9", size = 46813813, upload-time = "2026-08-10T12:38:57.773Z" }, + { url = "https://files.pythonhosted.org/packages/98/d6/33a411115b61dbfc16ad6ad73e71730f6fea654ee3667673bc53ab0e2fe7/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0befcf816e45a1af33ac775a9970b749e4868a230c7372f0ae5e932bee27039f", size = 50104452, upload-time = "2026-08-10T12:39:04.579Z" }, + { url = "https://files.pythonhosted.org/packages/33/ae/b1b97c9ca87f9f9ddbb5230c798df94eccce61bd79b9b45458c69a478588/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f89685964f46e4216103c75483aac0c0692a5f72212d7ca835adba5ede56ce3", size = 49951343, upload-time = "2026-08-10T12:39:11.8Z" }, + { url = "https://files.pythonhosted.org/packages/98/9e/a112df5cfd5a68cb1d9fc31cfe38c28d5aec9f10865ce37ecef2e4450873/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6943e2fe7954d29d84de45d29d34c8dc36ce96570e67d89aa9976e650a4a9138", size = 53144784, upload-time = "2026-08-10T12:39:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/31/24/97e8bd98f1e3b07e2ba08bcdff690674fbe16d69a7d2712cc3884665e615/pyarrow-25.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:31e49a7888fcdf3a835da33ae777f6bb9a866334e5a789282fc26dcf426f7f15", size = 27870159, upload-time = "2026-08-10T12:39:26.161Z" }, + { url = "https://files.pythonhosted.org/packages/36/4c/b525824ad3094076919273cd97db61fb3d78252dee76fa3b8dc8f76774aa/pyarrow-25.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:bf0b672390cdcb640d7288f96b826d71ff4e9abb254a86c89890baf51a29cee6", size = 35885255, upload-time = "2026-08-10T12:39:32.366Z" }, + { url = "https://files.pythonhosted.org/packages/08/62/448bb0e940de41aec31d1a956e63ad9c54afdf122a103cc3ab20c2a3ce33/pyarrow-25.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:38a9a4b4b9613380e200641891495a56c3d5a98a092db4a870af9975e220471d", size = 37644461, upload-time = "2026-08-10T12:39:38.142Z" }, + { url = "https://files.pythonhosted.org/packages/6e/9a/13587e38bd4806fd218f50fd13b8903fab60588a699ff0c406372e5b4043/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b726ad7e7b669be982b0c71c07fe4b037d654354130da79a7902a669e93a66b", size = 46877146, upload-time = "2026-08-10T12:39:43.722Z" }, + { url = "https://files.pythonhosted.org/packages/8d/61/1c5d1229fa21da4cff5365e41e57177aaac57c563c727f35419b8513d1c1/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:9171748cdf796972d85a4b60157c279913e242992e350c90c7450182a9838b2a", size = 50131616, upload-time = "2026-08-10T12:39:49.304Z" }, + { url = "https://files.pythonhosted.org/packages/43/20/291e1d65cc0b09aa19f03cf25cf51a2f5fa94b5db315178f2d254ed5cad4/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b7a296aac7a71fa0886c08e155ddb6c636a50013f801f6178daafa0f9e726188", size = 50008879, upload-time = "2026-08-10T12:39:56.891Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7c/1b7c9ec28e76576337e4f97b31141c9a181b89b6d1d6221e9d8205621a58/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0fe7c8b6c03969b49c8c66182e4a18e3819ab92d07cfab5d8370c531b9369ef0", size = 53170864, upload-time = "2026-08-10T12:40:04.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/75/f3d789dc06011a765d14d86bda799cf72ac1d715b6a6edecaa0d73d95062/pyarrow-25.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:f729cfdbd36fd99d543b67a914d2de044c84ebe45be8b34902b299b608c15c8f", size = 28620729, upload-time = "2026-08-10T12:40:51.41Z" }, + { url = "https://files.pythonhosted.org/packages/fc/05/647a8ee6f7c2662feb6921315617bc04dcd6034763fb61b1199720bf6162/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:59a2de54c0cbd954da861eee4d1d330f8e909c45b53455baef696380f2c55033", size = 36130288, upload-time = "2026-08-10T12:40:11.014Z" }, + { url = "https://files.pythonhosted.org/packages/93/f8/c9ee997554d7bea94520667dd1933f109ac1da3ee3556d2b49381e023484/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:35935cd5de130aa5cf4dea052a63e6bf2e17006c35c3a468194242b9b2bf5956", size = 37762187, upload-time = "2026-08-10T12:40:16.592Z" }, + { url = "https://files.pythonhosted.org/packages/a2/08/a28c01c7fe9e96e8233ce2d13df1d402f4f999f848f51d2daacd6bb4c036/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f3831aaa25c67a99f99dc8b05873cb9d64560390372e2aa197ce9dd4a3f06a44", size = 46888003, upload-time = "2026-08-10T12:40:23.242Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b9/58612e977d28dc58c878448866838369ee8da2f1e7cc8ed2c84b952aafee/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6a1fdfc6659b6b19022f2e50627fb5cf7156a66c46bf4299379955cbe742382a", size = 50079036, upload-time = "2026-08-10T12:40:29.169Z" }, + { url = "https://files.pythonhosted.org/packages/72/13/66e1402dcc860e1dc2760b1e0292c9a569b62b3bccab69def1b3e907d006/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:169d3429d5be7c752125890620f75a60776d38b0035eddae939651640822332e", size = 50040226, upload-time = "2026-08-10T12:40:35.186Z" }, + { url = "https://files.pythonhosted.org/packages/78/10/3f1a5497a7ef732ab0f03ecca3e66d89d9c0f57fdc61b4794c456b781f01/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:119297a6dc197e45d9c6d4415f7814a67ffa36c180d26f68c154c58067ae782d", size = 53149035, upload-time = "2026-08-10T12:40:41.454Z" }, + { url = "https://files.pythonhosted.org/packages/93/c0/37d4a7e8e2f7a6076283673d5298018ca26478b934c6ee369e10505ab32c/pyarrow-25.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4288f27577352d608ca08553b0865e4a9b3aa14820c5d95b53337218d609835b", size = 28753071, upload-time = "2026-08-10T12:40:46.623Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, + { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, + { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, + { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, + { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, + { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, + { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, + { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, + { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, + { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pyiceberg" +version = "0.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "click" }, + { name = "fsspec" }, + { name = "mmh3" }, + { name = "pydantic" }, + { name = "pyparsing" }, + { name = "pyroaring" }, + { name = "requests" }, + { name = "rich" }, + { name = "strictyaml" }, + { name = "tenacity" }, + { name = "zstandard" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/08/bde71e0bbcf1a62c92d7fa457b508691596c65fa7e52c1982c78c461cd1c/pyiceberg-0.12.0.tar.gz", hash = "sha256:19f165d298054f9436108691098b60fa0fa99d0eff5fb884700c43b29334a39d", size = 1212830, upload-time = "2026-09-01T17:28:42.378Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/a4/32d35527d48b3d569d7cc0a4319a52c431d91e0a82f8855059167426b741/pyiceberg-0.12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f7462f4777bcda2af0a7c270a51525542a79ff37cf1334a82e0dd80717aa92", size = 567107, upload-time = "2026-09-01T17:27:57.111Z" }, + { url = "https://files.pythonhosted.org/packages/f7/9f/5d1c08d172a5ed6f2eef466435576fb11efdcdb04c8d7eb0de1eb5d9363d/pyiceberg-0.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:97132565255b94831ce858837cc333a1ac837a116e1ce233299e0b6a67f82418", size = 568086, upload-time = "2026-09-01T17:27:58.794Z" }, + { url = "https://files.pythonhosted.org/packages/02/01/79e2fe2b35f2cff510a795fffc2eb9f82c84a96a3e29ed3fca40bc3a1f63/pyiceberg-0.12.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82e7164eb23c46b567d7fafb38058d7ca13aec5d4ef59d0ee701cfdeda004dfe", size = 761523, upload-time = "2026-09-01T17:28:00.02Z" }, + { url = "https://files.pythonhosted.org/packages/e7/34/6e635511ef60d26f036efbea60bb0820ede0ca311743a4f6bac085a0f394/pyiceberg-0.12.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c6484af01afa37a3f3cfb299b4fd8b14b138f95955b57cb01c10acd88171d28", size = 758400, upload-time = "2026-09-01T17:28:01.291Z" }, + { url = "https://files.pythonhosted.org/packages/d2/69/d3c12cb3ff64e1616c538651731bed18a7734e7f51bcf8f5f5aa4296072b/pyiceberg-0.12.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b54c151610a4f9e198fa8c13c881c4c4d4f1beaa4571023dd79428aa1bbe7f5e", size = 757281, upload-time = "2026-09-01T17:28:02.366Z" }, + { url = "https://files.pythonhosted.org/packages/ba/f7/fe179589ba419a082e7c29c995cf7bbe6d69080d1feb6c0806bbf1551d44/pyiceberg-0.12.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5a667fd37415626fbfc7b01e373ef66a09f0ddd185875a11e868a5ef128da1f6", size = 757202, upload-time = "2026-09-01T17:28:03.688Z" }, + { url = "https://files.pythonhosted.org/packages/7c/5f/735d67ab9ced4dfc1396755a19c4ec286fc78702f81ad5c8720ec407d19e/pyiceberg-0.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:2c0bcdff87f92b35c4374a6f680b3c6e25e5a87e40fa61df49a563dc8ae13e44", size = 564549, upload-time = "2026-09-01T17:28:05.169Z" }, + { url = "https://files.pythonhosted.org/packages/5e/07/aab0770651e9dd70902ff170d7db3875caa2f53c0df1103035022721d54d/pyiceberg-0.12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ae611854967d691f3158ae06407622072a9c442e29f5fba073ecbedb4d0b8b66", size = 566935, upload-time = "2026-09-01T17:28:06.564Z" }, + { url = "https://files.pythonhosted.org/packages/6a/11/c14909196946ab59677d8c814bd9d811691b62a794bb27895d406dd34119/pyiceberg-0.12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:63929b5664512c67bb89603c5ca8c568a08b4a9922ec86675b339214d362ea81", size = 567919, upload-time = "2026-09-01T17:28:07.642Z" }, + { url = "https://files.pythonhosted.org/packages/88/05/33f6ef4315e6210dcd7e98dbd4c367390ca24b71820f782df0fb5df49aee/pyiceberg-0.12.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cd913759564d08444567690c27a187b408d06e6c82a17ba4e4f98aaf90031e86", size = 773938, upload-time = "2026-09-01T17:28:08.672Z" }, + { url = "https://files.pythonhosted.org/packages/f0/38/11579d59d3d91288e4b805d13642417ebadae3548c554b2474cb8229b616/pyiceberg-0.12.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2dec70218c8e81f1630b9d7184b760cd161eff936127745355ba94100a4e7380", size = 771959, upload-time = "2026-09-01T17:28:10.099Z" }, + { url = "https://files.pythonhosted.org/packages/84/77/17168e83f5216a57b74c1c64875f5469239d16ea1ad60902aed7a2fdc2d3/pyiceberg-0.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:29fb5032276aaac1ad01f98125622012067be075cc70053a0dbd0d61e4a6e8e5", size = 770200, upload-time = "2026-09-01T17:28:11.432Z" }, + { url = "https://files.pythonhosted.org/packages/25/0c/88fcbf58f0c659b7f714ceed56ffd42b26f85fafbf5eea61056d9c10e654/pyiceberg-0.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0b432f97a7774a211980f8d135c12a75221d0696204a0dc67f9b750a5c1a3332", size = 770216, upload-time = "2026-09-01T17:28:12.637Z" }, + { url = "https://files.pythonhosted.org/packages/60/80/5e3c42eb35c88ed5b73bae31e1fc0cca69f7c09a604aafc09bec1feab68d/pyiceberg-0.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:bb4d04f93c1b98c19c365dc2bc8115e8396a1b2e1eb99ec9e04ade262c82c565", size = 564293, upload-time = "2026-09-01T17:28:13.888Z" }, + { url = "https://files.pythonhosted.org/packages/b1/45/9fc0692dd081ab34e53731b0cf3eba0f948e5a54948840fa06a4ed4cbf9b/pyiceberg-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0079c44d065fc70df09deb03dc7297314c31b8d9708559c0bec2b8f851ffbf21", size = 567622, upload-time = "2026-09-01T17:28:15.199Z" }, + { url = "https://files.pythonhosted.org/packages/0e/67/b11334fe6af2a5729bfd30c0e03dee78887e9874a634888aa81d91dc82e9/pyiceberg-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:baf35f45ed5ee1a14c8db8264de31fb942b9d0d21913262f0d444d2df45e8176", size = 568214, upload-time = "2026-09-01T17:28:16.459Z" }, + { url = "https://files.pythonhosted.org/packages/1d/24/eccd190e358514e7e0d9a5c7591e44e71eb0e2ab5cff6cb90adb9ecaa963/pyiceberg-0.12.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b20c36d08b4b12da572b10c20594be78eee7e51845040443c45231d2ad57c28a", size = 779432, upload-time = "2026-09-01T17:28:17.775Z" }, + { url = "https://files.pythonhosted.org/packages/ed/75/046692b5ae4330d251974a428fdd82c9d7840715580d332c7cbcd65a13db/pyiceberg-0.12.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17dca4377c76b7b047a2e807f009595b8f0b090c792a84ecb427c2fa683a96e2", size = 781395, upload-time = "2026-09-01T17:28:18.906Z" }, + { url = "https://files.pythonhosted.org/packages/5f/38/f8b0dc8cbc53459c6780a3ebf78382960c78fd80db54d0eeb3f7a63c9c7f/pyiceberg-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5902178a7d46bc4b783a4026c474178a8cb4c9c413b47cdb2e520df4082ed255", size = 773096, upload-time = "2026-09-01T17:28:20.236Z" }, + { url = "https://files.pythonhosted.org/packages/d9/4e/ef4265f3b7108d1591ff233edf3ef6ca50d2639026b4a1cb62ddda3398bd/pyiceberg-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:cbb9f060d170b1b072e2ba2e821f5c76302e5731452f37fa63e104334f3feba1", size = 778402, upload-time = "2026-09-01T17:28:21.79Z" }, + { url = "https://files.pythonhosted.org/packages/f9/2c/858c329a4e93897568ab73e95892dde0bc32bf4a76a6133096ef6b660854/pyiceberg-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:0fcce46f5633491b50ebf8ed94fbf5c8d3b6bed76902cf28361368e9169e16f2", size = 564568, upload-time = "2026-09-01T17:28:23.048Z" }, + { url = "https://files.pythonhosted.org/packages/44/7d/c04a65b08ba272bfcbb31638222d71a9f1c7f12c0d8b659530e14817afaf/pyiceberg-0.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:498763380220a1d8881d52c218318e884be28c3ea5824cbbcb22a042c12a9ad3", size = 567243, upload-time = "2026-09-01T17:28:24.149Z" }, + { url = "https://files.pythonhosted.org/packages/7d/bc/73277e56a30234afed4405bbe874a5410dbf9fa6c22c2352cc2615f7ed78/pyiceberg-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:307e46f46ffd48e0b270acf10bc5892f09e8c9fa2c828e9ffbf32ad480504bae", size = 567739, upload-time = "2026-09-01T17:28:25.376Z" }, + { url = "https://files.pythonhosted.org/packages/a2/72/8e09e90fd556af1ea90b993287da7997423a99784f4b7ccc77bd96a28f39/pyiceberg-0.12.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e055cc459d7b6eba21bd62eedbda0d0845ade0c2161a04c0f8eb252ec3e2d7d1", size = 773284, upload-time = "2026-09-01T17:28:26.452Z" }, + { url = "https://files.pythonhosted.org/packages/83/f1/cb542e8a46690d9cd2112eafd052f7bfdcff33f2027a344126d07b5b681a/pyiceberg-0.12.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:893e35df750644dab19bb873522d15914335e22368a711703176ea187c26b655", size = 776034, upload-time = "2026-09-01T17:28:27.823Z" }, + { url = "https://files.pythonhosted.org/packages/52/62/9e41c64c9bd741da75b408379ce3175af9f2dbf453c0a2bd9e5bd404e068/pyiceberg-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f53afc4ae649d35d43eee2b32515f22bc64fa0d4921c359ed9e7744ca1101ae0", size = 768107, upload-time = "2026-09-01T17:28:28.94Z" }, + { url = "https://files.pythonhosted.org/packages/9a/39/18af56141c920e62dcd4dc4f91aa058c7361e8f2e8dd45f73cf3f0c25b64/pyiceberg-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd57f73a55dc9439f183e13aa1d9c79ac4274564bdd412da735314e4e26b77ff", size = 774220, upload-time = "2026-09-01T17:28:30.094Z" }, + { url = "https://files.pythonhosted.org/packages/13/98/50a2a45e451df14ce0da864b1fd869a950c9cea0877800baa0ff287c5991/pyiceberg-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:934c30733c3debf9b13bbcdb85c4cfdaf4b72a808f7af1dd027a38e4e2baef07", size = 563797, upload-time = "2026-09-01T17:28:31.388Z" }, + { url = "https://files.pythonhosted.org/packages/19/e3/49ca88aff0dd74560acdcbf1d23d33a1e4e3a2d6d17f03399e7401faf084/pyiceberg-0.12.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7bb480f6ac06e4afb3de1e1a46781d071e3f1cfcc5802de9e48e6289b3546c0a", size = 567458, upload-time = "2026-09-01T17:28:32.72Z" }, + { url = "https://files.pythonhosted.org/packages/5e/90/18a84508ec3bae4e90b5f631a4f4ad7de477729b8fa90014c46550b8e1e6/pyiceberg-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:16f2f03c20d01ced43198aedfffb620163e7debc21d33ff02cc18e107aa43e69", size = 568050, upload-time = "2026-09-01T17:28:33.963Z" }, + { url = "https://files.pythonhosted.org/packages/54/a4/e3281a6e98645c179652c7a9c04bdc3fb2d388f26a75dbf3225891804a11/pyiceberg-0.12.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:115ecdabd4c47d64b4eda1a4271a4c4eea0514411832f11adc7cdf6630e57a66", size = 772948, upload-time = "2026-09-01T17:28:35.211Z" }, + { url = "https://files.pythonhosted.org/packages/70/d7/0b4fcd024b938dceeb941626ef465cc1d17d023ca461db58a636701b1482/pyiceberg-0.12.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb3db80cd510ddc34246059f4631661aa5b5d94c36b131499c56e746f22db3c5", size = 774232, upload-time = "2026-09-01T17:28:36.297Z" }, + { url = "https://files.pythonhosted.org/packages/97/bf/2c2d14235fcb1a28eba3e576e3472b77ee9408b52b13a2d33937a6189ea6/pyiceberg-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:82e0250f6c9baa11644efa44da5f968205c158c75126deb1b1730ce158cb3f76", size = 769045, upload-time = "2026-09-01T17:28:37.858Z" }, + { url = "https://files.pythonhosted.org/packages/ed/fb/c16e5aa6810a1e2dc0907b7a7d4605701b9da537d4175736cbec0c86ffb5/pyiceberg-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e73a396826d745b2a6ac0d3b5151c8cdcbb090e81bfc31f07139cb463e62dbe3", size = 772590, upload-time = "2026-09-01T17:28:39.132Z" }, + { url = "https://files.pythonhosted.org/packages/a4/57/2ae640b6220a321958484162cfbdd1a2772198bac0c4588a990b8afd22d5/pyiceberg-0.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:ee7b572d209a39224a093661f462f95551b2bd3982a07aecd80c7dd560be15ea", size = 561552, upload-time = "2026-09-01T17:28:41.156Z" }, +] + +[[package]] +name = "pyiceberg-core" +source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "fastavro" }, + { name = "maturin" }, + { name = "pyarrow" }, + { name = "pyiceberg" }, + { name = "pytest" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "fastavro", specifier = ">=1.11.1" }, + { name = "maturin", specifier = ">=1.9.4,<2.0" }, + { name = "pyarrow", specifier = ">=17" }, + { name = "pyiceberg", specifier = ">=0.11" }, + { name = "pytest", specifier = ">=8.3.2" }, +] + +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + +[[package]] +name = "pyroaring" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/46/a50510d080f8cb089303ec0f7cd80736b2949ca3d148f48f1cc90c49e345/pyroaring-1.1.0.tar.gz", hash = "sha256:f02e4021397ae02a139defdc6813b9942ab163de90affddd4ce4efbac299f619", size = 200298, upload-time = "2026-04-24T21:29:25.212Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/b0/ab1c3b9227aadc326138cbbb973549399ece103d35b8c11952405a252a4d/pyroaring-1.1.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:19c9bb8b32f17fe63e54e78f88a7a67fe2dca352bf79d7d3b43f0a901db5ea3b", size = 332486, upload-time = "2026-04-24T21:27:35.332Z" }, + { url = "https://files.pythonhosted.org/packages/b9/02/a3a916ac9b6de70ac8696f767eb3487c41ca37b7f991216abbebe528045c/pyroaring-1.1.0-cp310-cp310-macosx_14_0_universal2.whl", hash = "sha256:212f471da113eba24a0fedbd1ef078bfed34b792e8f48dba3f7301f1bca2678a", size = 707625, upload-time = "2026-04-24T21:27:36.809Z" }, + { url = "https://files.pythonhosted.org/packages/22/53/e845f27b6e6e23cb322f834fcf871e27e4e2e9284360d63e68bfcf9088dd/pyroaring-1.1.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:a31276323fdd355505883162f1dca4d7acc3d6ec2159c85ddef863f1fe9e8f05", size = 382766, upload-time = "2026-04-24T21:27:38.337Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d3/4c9439ff556091d93c591ac466ab526991fa8745c1a15e9ca49081f215da/pyroaring-1.1.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55c9d0b708a65308eaa95c53d9de56a2b4425812395f6e57a4d9ad8efb081bf9", size = 1970617, upload-time = "2026-04-24T21:27:39.722Z" }, + { url = "https://files.pythonhosted.org/packages/b6/4a/568684cc6d5294f0ec183bacf38f5a27b2c37908d0e0ac6f4423ae7c7c5f/pyroaring-1.1.0-cp310-cp310-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5599b691e5e993f7fdd9880429d1c485eae03dc392ba5b2dbe2eef0434bea1a5", size = 1853502, upload-time = "2026-04-24T21:27:41.087Z" }, + { url = "https://files.pythonhosted.org/packages/87/f0/82c4335651ae243eed411daa8e5acc1c62179a3e90350e4785c487e643a7/pyroaring-1.1.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8376c62e60a49e8fe51ffce569f74beed6e53fb58e5d6739f954c8c51ec0bec3", size = 2171600, upload-time = "2026-04-24T21:27:42.79Z" }, + { url = "https://files.pythonhosted.org/packages/62/b6/e6859ba41ad4c421248c2baaa4aa6c434369991200c4809cd389a4b1688f/pyroaring-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cc998e81748b72659802a7b434b39292308a5f4ab8dd4bf4739d65de1d4ed2bf", size = 2887912, upload-time = "2026-04-24T21:27:43.864Z" }, + { url = "https://files.pythonhosted.org/packages/07/c2/3f31ebc60ab094c1f2ff72f60aa0b5650d49c7653b5b575648cfb22b5441/pyroaring-1.1.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:fe0b9f0baf190663df37571cbb8a77370e7bd3a66068276d48f813c53ceba530", size = 2691997, upload-time = "2026-04-24T21:27:44.996Z" }, + { url = "https://files.pythonhosted.org/packages/00/6e/c1d09230ebe0088ca011be2879e7c2f956cbdbc530c6427f6d046658b1df/pyroaring-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:83815b3a4cb9c3b17096c70833373610d5c93e89c134548cf43436c90326bfb3", size = 3143377, upload-time = "2026-04-24T21:27:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/cd/fa/665ad8ce092e24b4d3b1f5e34b032d73c6a3f4792f229a70b702cd050408/pyroaring-1.1.0-cp310-cp310-win32.whl", hash = "sha256:e9caf67c0c05d27bab0d88cf193e73023f624c6714f1ff807c7d4b135b19d36a", size = 210064, upload-time = "2026-04-24T21:27:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/f9/8e/802d6edca2dfdca841ecaafb1d60c05657f2d3948b7b78eb892cf8195dac/pyroaring-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:a6343249ce9401dd90c3b934fd9a4a618b6fc455f27a9aa11d198eebd4743137", size = 259507, upload-time = "2026-04-24T21:27:48.788Z" }, + { url = "https://files.pythonhosted.org/packages/c9/4c/b57b59a7d7f913eb6d1ce2578fbd82a3209805a33a673f562ca1035839c1/pyroaring-1.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:8f5194c0544b08e56e108dad096c468c67f8ac60eec42763aed36ccbd3565346", size = 217549, upload-time = "2026-04-24T21:27:50.129Z" }, + { url = "https://files.pythonhosted.org/packages/4e/aa/574f153feb89856010092c33cafe4a52f4da8cea19d441cbc4cbcb8ccb1f/pyroaring-1.1.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:462b0952277d3100a90ae890ae641d3fb3561b10cfea542e02468f0bef7700a5", size = 332142, upload-time = "2026-04-24T21:27:51.258Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a1/938a9fbdd41699ac9419ca922fcf80eb58c0c62aa34539aa540abaee9a63/pyroaring-1.1.0-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:9cf8608c9d6cb6bff9c624744f7a2ba8ab12276f097a07930490fe0e2219e9be", size = 706441, upload-time = "2026-04-24T21:27:52.226Z" }, + { url = "https://files.pythonhosted.org/packages/06/06/5120acc9918223ccb647dfadb430da1ec08d7ada818469ad3a2f1574b8c7/pyroaring-1.1.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:44a9f9719ed18e86627286f90057f9b7e22f6ba1d952c9793f9600b5c14e8680", size = 382003, upload-time = "2026-04-24T21:27:53.322Z" }, + { url = "https://files.pythonhosted.org/packages/88/73/adc7951e0d4e65ab7a84fa5562bc28ba2dcc6f461a9156edc0d5aefda7f6/pyroaring-1.1.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c14fc6bd65e5624f76b90297b081222261476978f795f60d48745553617ddceb", size = 2034493, upload-time = "2026-04-24T21:27:54.739Z" }, + { url = "https://files.pythonhosted.org/packages/c7/28/992f2f2ef573b54559ed4bc62b5cf0ea095a59173521cfbe78dfefcc08ad/pyroaring-1.1.0-cp311-cp311-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:01212da3752d6486adcca98c9d353f8fb8e36513e05062cdd0feebc4211dbe70", size = 1901479, upload-time = "2026-04-24T21:27:57.115Z" }, + { url = "https://files.pythonhosted.org/packages/10/4b/38bded4ca17af6359c1766c25e942435332d26e2cb3321d9d081546b75f8/pyroaring-1.1.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:100585f438b293112e2c52e45a442835837c8a0267dd1e513bafec35628f8ecb", size = 2235338, upload-time = "2026-04-24T21:27:58.66Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ff/8e7da41468a33fc9d0fbee949e8f3f734930b5173020a478686558388f31/pyroaring-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:532e53191d8dd29dedfc5202cbb45632f7df751b207a7f6d6860fb7067c7fe11", size = 2951300, upload-time = "2026-04-24T21:28:00.12Z" }, + { url = "https://files.pythonhosted.org/packages/23/a8/f8a4b55b0817aac3caf6b3b51f2e93cf152f32784949919512bd38feb0f6/pyroaring-1.1.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:731a7a9e050758986d5757eea10f9ccb08f9c3ef514ce0335f4a90e126f81131", size = 2752747, upload-time = "2026-04-24T21:28:01.281Z" }, + { url = "https://files.pythonhosted.org/packages/b7/26/b5a29c0f38c581ac290245150d4d14b627f110eac208f3569d5d8739c78f/pyroaring-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e9864e19109e76111befc75d799d334e7365eb4189607aa734053c12e7840fa5", size = 3203933, upload-time = "2026-04-24T21:28:02.595Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/2b3661d9a9e12dac02d2c3ef4545bd2236fcd964ba8fdd96e308ca621153/pyroaring-1.1.0-cp311-cp311-win32.whl", hash = "sha256:7caf95de39ce869ea0978068521cf6faa7350574fd1734ad6c63e5ed8cd06baa", size = 209205, upload-time = "2026-04-24T21:28:03.811Z" }, + { url = "https://files.pythonhosted.org/packages/be/1f/416a8cf29738d2e8c552fcaff7951c2a9a3bac0faffbb88b888287665834/pyroaring-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a23cd023985b5f2ba23e84e1fadaeacde3c8a59e1d2adb3fe782e99db1e22387", size = 260338, upload-time = "2026-04-24T21:28:04.752Z" }, + { url = "https://files.pythonhosted.org/packages/a9/0d/fd3f57014e98ffd20a3db7fc07157be8abb6cc5a356ccb20e1ea2493c397/pyroaring-1.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:a98d1147fe1d3195053b67b474bccc0be5021506765d27f613a943c8c99f9e4c", size = 217570, upload-time = "2026-04-24T21:28:06.126Z" }, + { url = "https://files.pythonhosted.org/packages/96/e9/d8dcccfdac1657e6a53b6ade0c0c71d59244316810c82537a5d634f8b7fd/pyroaring-1.1.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:fdf484d26016e0c016f23f2b635d2899daec034565fdcc062ed6b10f3b26a3f4", size = 334166, upload-time = "2026-04-24T21:28:07.184Z" }, + { url = "https://files.pythonhosted.org/packages/1a/16/70f8268c9bac4f6d91a82df254332edfa07020fe02e97c6d3d0295ee4db3/pyroaring-1.1.0-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:e9c2b9aa8decdcf40ed8f4c887092c20a272f8c32215c3fee65e9db92ecf418e", size = 711970, upload-time = "2026-04-24T21:28:08.929Z" }, + { url = "https://files.pythonhosted.org/packages/6e/a6/0b88a8a8e4ffdd0bf53765c3ea17ce3b747f7de42b1f10d5c50a13ba3ecc/pyroaring-1.1.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:5eb031237e9d39cbdfc9276facacdd88e27aefb58940bd8b56b878dfd38d6022", size = 385825, upload-time = "2026-04-24T21:28:10.077Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a7/f06a899b896bb74a031201fbba707abf23e2485f44ead28d2e91977ee204/pyroaring-1.1.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:66aa6a321fbc598f26d5e66050a7f145c2253f3fe5737b589841ff0cbe5cb177", size = 2000799, upload-time = "2026-04-24T21:28:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/eb/16/b13e0727ef5c91c84ac5d9b5c4af43cb3f28d8822d96d44aa4aeefc10b37/pyroaring-1.1.0-cp312-cp312-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cff06d18c9a30f8547a92757078aa345db1ba5b22e3082a05f64e50b384e27a", size = 1843405, upload-time = "2026-04-24T21:28:12.394Z" }, + { url = "https://files.pythonhosted.org/packages/c5/4b/ff77d6eb4747c65aba49d345318059f1ccfbd206bbcd34686cea819187e9/pyroaring-1.1.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:043dbbaa905f7288c515ac06a96b67a3763f35e9ae06f0c0278c0d9964d16760", size = 2224307, upload-time = "2026-04-24T21:28:13.569Z" }, + { url = "https://files.pythonhosted.org/packages/50/d1/9b4e2175a9bd07592ef1c6f4692ba0cfe3abd54f33ebd574aa2b2ab88c2c/pyroaring-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:71fec09bd42f8c33ac3b762cd00c5db842eb583ffd0e361739ce1c17ad078a6a", size = 2902586, upload-time = "2026-04-24T21:28:15.485Z" }, + { url = "https://files.pythonhosted.org/packages/67/24/904952d6bfe2e4946f361958157774230cb1ac171d306e2460620274c58a/pyroaring-1.1.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c9f30ca28b991a920b446ed3ee19c7ecafcc49c46db592abf89cf239a7bb45f4", size = 2741581, upload-time = "2026-04-24T21:28:17.029Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f5/8ad5605870fbdcb568f0b847e1fea24adea15e07c90231fb62f339c08b14/pyroaring-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:67650460c65bdd7b4f5078d9c955aa38f64627d02cb48f9cfb24eae84bca2aba", size = 3182906, upload-time = "2026-04-24T21:28:18.435Z" }, + { url = "https://files.pythonhosted.org/packages/cc/5d/264414a1b1bea72b2a7756e2b1dca709e5c695b0cd332a8f90c297ae3b33/pyroaring-1.1.0-cp312-cp312-win32.whl", hash = "sha256:61a8eabee99104ca197b6e7cce05dc4f27f503be52881800cd370eb5a5152d3f", size = 211231, upload-time = "2026-04-24T21:28:20.414Z" }, + { url = "https://files.pythonhosted.org/packages/ea/39/dd3341e235a3794c613ea32bd35618a88ff2ae067dbe9dd7c382c8c146d2/pyroaring-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:51ebe5e6f48e3dc9df91a4cb62137ef72e1469acd6f37479abd9991f6d945cc9", size = 263337, upload-time = "2026-04-24T21:28:21.371Z" }, + { url = "https://files.pythonhosted.org/packages/12/5d/bb8e93dd7412180c621086ed46014a0f09f9a71d9370ce8cf607c5a2cf00/pyroaring-1.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:9882a204178cc8c915e0ce30abb4bdd1668e383c571b06649d5ed272d9625877", size = 216727, upload-time = "2026-04-24T21:28:22.536Z" }, + { url = "https://files.pythonhosted.org/packages/7d/75/1d39ecb04e6cd96d191eb8884864355051df80928dd5096a9dea43fbf63b/pyroaring-1.1.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:72f68a16b00b35481d9b3bfe897ecd8a1f7da69efd92ba5b17347ca11c21cb0d", size = 333363, upload-time = "2026-04-24T21:28:23.838Z" }, + { url = "https://files.pythonhosted.org/packages/20/3e/65cd0871e86d11c5c5cfd0f5abb0ca80eb2b6b5dbe5a2433f315a9ebd90c/pyroaring-1.1.0-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:4c443e9f942b6089efe8c9b264576e9d116f90be28a315679375bba2d8a915d6", size = 710573, upload-time = "2026-04-24T21:28:24.884Z" }, + { url = "https://files.pythonhosted.org/packages/f9/a2/f8f23515f41414332e60cd86e4957e2a6838070b2ad5fe25e80f136de635/pyroaring-1.1.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:3beb40eb1220d1ce4fb3661bb019e9a21857e5bb294fe8c1c5016aeb6e82318c", size = 384880, upload-time = "2026-04-24T21:28:25.864Z" }, + { url = "https://files.pythonhosted.org/packages/b0/5b/82dc44b5074a1ff62e702d12611272d1711a60d5518dab23f94e1f7a9b3d/pyroaring-1.1.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f1f56004e8f1c1489bf279c25f1fa4764252cd9af5fb35675774268a4a615ba", size = 1999529, upload-time = "2026-04-24T21:28:26.859Z" }, + { url = "https://files.pythonhosted.org/packages/11/40/b07bac8cdc4b709a05f5c55bb52d4f684e5ea1fadfa0b6d9decf477a9d2a/pyroaring-1.1.0-cp313-cp313-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:13660386ea8905ee4d42c21a6275463e2dc7d31e0b5d65eec210aa7043ad96f4", size = 1842927, upload-time = "2026-04-24T21:28:28.056Z" }, + { url = "https://files.pythonhosted.org/packages/0d/60/c4b511965802dfc77978a9e16f2813f47fb3083db1822019ba1bb169c685/pyroaring-1.1.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0dfb6cf50fd8898179e460e699a6b8326ca508c627d083f7bf62f769fe1717d5", size = 2199538, upload-time = "2026-04-24T21:28:29.425Z" }, + { url = "https://files.pythonhosted.org/packages/e8/12/38f6b50b3f3f41a8b752d3e9efcf105b18eb2c66811831059f25613734ac/pyroaring-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:81ebbc0c880c8a10f13118632e5c0d59159ceada8b651bba18f2e6dc70efdeda", size = 2896904, upload-time = "2026-04-24T21:28:30.67Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b6/b5436e4b93c6bf2bd3dd6ccb88cbdc64b12084151a43e2f5c94be50eb710/pyroaring-1.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:370d191b0d1b32bbd99452ef5f0485f22fcc4bf7404d33b821d0ce2459951152", size = 2733819, upload-time = "2026-04-24T21:28:31.882Z" }, + { url = "https://files.pythonhosted.org/packages/ab/8f/f392f268de9607a5c7a95aaed6b9c8a81f00c14d85c33855e9f492095478/pyroaring-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8b3bfad0ae3ef0e67b40c193863dce8b7d79de545dadbe53c19acc3ace38f66", size = 3161730, upload-time = "2026-04-24T21:28:33.244Z" }, + { url = "https://files.pythonhosted.org/packages/9e/a1/03250fd4834b6a5c13e6600bca47ea20fda579f80bce3551d4985185d164/pyroaring-1.1.0-cp313-cp313-win32.whl", hash = "sha256:eead129046822cb0fd47c78740b81bdaffd0515c0bb0306a2318acf0f0540b58", size = 211194, upload-time = "2026-04-24T21:28:35.001Z" }, + { url = "https://files.pythonhosted.org/packages/70/63/d9b307462cddc82fe94a67d6810e5c802818690e131ba690c1de674d8558/pyroaring-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:90ab2f00c09eed5bd986a80c8641e2dc10e7aca1a2d892d89a44b396e39c08ea", size = 263110, upload-time = "2026-04-24T21:28:35.976Z" }, + { url = "https://files.pythonhosted.org/packages/d9/4a/aa6e9833a6ba9a630efdbec8783b63da6602f763b37a5b5fbc01d73a1af1/pyroaring-1.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:51dd2490a64ad4ed53c4fb58ef1ee3f84f6cbd97cdb47abd9065c9f714ab72ef", size = 216546, upload-time = "2026-04-24T21:28:37.065Z" }, + { url = "https://files.pythonhosted.org/packages/93/ab/2260fd567a2d5d957393b932ea940dc31146bd509c88164c1b786eee7836/pyroaring-1.1.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:5e337f8c5b3c2e0c27da83fc2cb702684a47eee907a960cfee964fcb5344515b", size = 335093, upload-time = "2026-04-24T21:28:38.325Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6c/df82a832ff3760c7c7653b80d030fb43b18eb88bfa604e7de3e84457286e/pyroaring-1.1.0-cp314-cp314-macosx_14_0_universal2.whl", hash = "sha256:53acecba8f898e96b84d4139356e30719c70358177e270055901d3ec1cb0e34c", size = 712387, upload-time = "2026-04-24T21:28:39.404Z" }, + { url = "https://files.pythonhosted.org/packages/12/b9/a94d6b2d7a1be2fa5009ecfc345bacb2ee0b536020aeb23e92c6bb7e70f2/pyroaring-1.1.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:986efb3aec7655d69c14db2309a2072dbf181bdb906091fede83ad18e316cdaf", size = 385413, upload-time = "2026-04-24T21:28:40.563Z" }, + { url = "https://files.pythonhosted.org/packages/60/6a/3658eadbe28a5a2093c27857dd21441f1ea1cede2ddbe367df76e3018859/pyroaring-1.1.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92643c9dd303de8960c3dbed93a28b8d87da5ed0a7776568979f379d7bc8a885", size = 1995135, upload-time = "2026-04-24T21:28:41.931Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/4706ec770790d3433520eb0ea98fc662ccb1533164fd00b01f3413c3425c/pyroaring-1.1.0-cp314-cp314-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6a1d4c59d5b23c01d62f86d57ceefd0c0977de0425aafa7069f2d70563fed3b8", size = 1833652, upload-time = "2026-04-24T21:28:43.381Z" }, + { url = "https://files.pythonhosted.org/packages/2e/38/b8b861738e49fd4c4a54bebe257dced603999365629b4e10cb85fac940b0/pyroaring-1.1.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8ac1bc26223befbca986551521f37f4c1670dfe26fccb2f0fc2775e75be99c1", size = 2188218, upload-time = "2026-04-24T21:28:44.487Z" }, + { url = "https://files.pythonhosted.org/packages/01/8c/96afa9b5f509a5c607deaf30538edb3bdf026447a864cfe3f2c3d7484875/pyroaring-1.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b490f2d22df30affbfdcbe4f7896f321edb72a8dc0cbe5f38adec3de5b947c25", size = 2898243, upload-time = "2026-04-24T21:28:45.9Z" }, + { url = "https://files.pythonhosted.org/packages/f9/1e/86f8720525250fe742fc77ea5c2a2074a1ea830efe84a79112ce6fe113d3/pyroaring-1.1.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:56a67794188275f8897a8f1fa64d6313c48241bebbdef38833063e7281b29ef8", size = 2715091, upload-time = "2026-04-24T21:28:47.721Z" }, + { url = "https://files.pythonhosted.org/packages/9d/21/29e8f58c8af2ce016904a7e6aa61be675945224971cd70f3f698e584a23f/pyroaring-1.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d9f196007f0b15ea19c21732faacaea83cbf5946b6db4949b3b98cf871c93f0", size = 3149470, upload-time = "2026-04-24T21:28:48.918Z" }, + { url = "https://files.pythonhosted.org/packages/ab/e1/f67ef1c9de461a80707a2f2981320b1b30632720bac426a9bfd51e4744b6/pyroaring-1.1.0-cp314-cp314-win32.whl", hash = "sha256:abc0f0ce22464864fea208315d25e999e45cb5ee646ac1ca11d314a6a51dbe4a", size = 216552, upload-time = "2026-04-24T21:28:50.652Z" }, + { url = "https://files.pythonhosted.org/packages/e8/83/a8d9fee17e6eedf2a2281b2aabcdde86930408486381ec48d1f7d3404521/pyroaring-1.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:532ae6bb1d3431d9956ef07589dd5c8dd918301a83d937c7dc6e511b1364d76a", size = 270712, upload-time = "2026-04-24T21:28:51.817Z" }, + { url = "https://files.pythonhosted.org/packages/42/50/ab2bf3fe45e4c2952690d657321a8470558f92cf93cb197fe5d31f7110d2/pyroaring-1.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:d2706a89242a347be20805147d58a38f4f4d8f6846228c4ee8dfd3587113719c", size = 224783, upload-time = "2026-04-24T21:28:53.183Z" }, + { url = "https://files.pythonhosted.org/packages/d4/0c/9eb48ac698280170f184045814b7bd44829af37c1c6de79a4d7b5ea0c8b8/pyroaring-1.1.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:39eff7dd06c163c22d0a9f9fd72d27e671457bea8cdb71215382a10512539e1d", size = 345689, upload-time = "2026-04-24T21:28:54.494Z" }, + { url = "https://files.pythonhosted.org/packages/f6/61/66b18f8ed17e70f88a410dcfac21e5964c2ad01bd4d6a25024a87522c8a9/pyroaring-1.1.0-cp314-cp314t-macosx_14_0_universal2.whl", hash = "sha256:562fa04bbfd41144d1276ed79505007557c161371450d68a1d71fc83dc01d083", size = 732373, upload-time = "2026-04-24T21:28:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/04/7a/976482874ea5e4476f9dd84e7d0274e480446b1b6ab45dfe301281814b3b/pyroaring-1.1.0-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:591e2ed4d60443dafd9075c1f72e9aaf359ccf5120e32a8c340c2b2ae3da45e7", size = 392985, upload-time = "2026-04-24T21:28:56.629Z" }, + { url = "https://files.pythonhosted.org/packages/ea/22/1eed09ff3aa792865dd52ef447cbe52dbc5901ed88bf1cf4513f7220150e/pyroaring-1.1.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:381eda673442c389993f8b0db2dbf5d02ea8ea9aac6ba736f64cc1ffb6c96885", size = 2045479, upload-time = "2026-04-24T21:28:58.008Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/de43464f9b28c445868e4bc8f0e6c6dcd51103bd9a757e3dcd9af25a4a69/pyroaring-1.1.0-cp314-cp314t-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d9127feb5356ba3a92bdffa04c1bf6bcbc8d436369f78badf441018c3029dd63", size = 1853013, upload-time = "2026-04-24T21:28:59.265Z" }, + { url = "https://files.pythonhosted.org/packages/03/17/29b128a580ec43905fb766b934e7dcb1095059e99e38e941edf50152207b/pyroaring-1.1.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:650db21c10f42ff2b09ef02c10a779a3d59d0c7512552f3844738b30adbcb8a5", size = 2222628, upload-time = "2026-04-24T21:29:00.792Z" }, + { url = "https://files.pythonhosted.org/packages/bd/d8/bb7d69978a5fcac95da48bedb114554d8345b50b77f042e8cd2a8277bb4b/pyroaring-1.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a5fbcb86e44f1c0c9c052917eee67a04cbac9de7392fb4bc77c140ff4a7e471", size = 2931231, upload-time = "2026-04-24T21:29:02.275Z" }, + { url = "https://files.pythonhosted.org/packages/0b/12/d44d144352a4586544313a97b2576f8f8673b98c02ae7fed77d38751cc1f/pyroaring-1.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:0f1d76ef29034017eb2cceebd5fa0504d6ced218ce6432f99da5adecbe038269", size = 2729546, upload-time = "2026-04-24T21:29:03.414Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f9/0d01c6ba01c0d01609fddb1d46138a7ae95b7db386bac4afb0ff082d5c0e/pyroaring-1.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19d0c81c865d63791fe20e5b38733b66f4f962e677ae7e8b3d3c4947ac6e752f", size = 3177123, upload-time = "2026-04-24T21:29:04.619Z" }, + { url = "https://files.pythonhosted.org/packages/43/6d/f991526fdef3cf7739f6db0cdf12b157e840e0ddd4a7e1c2a477da9072d6/pyroaring-1.1.0-cp314-cp314t-win32.whl", hash = "sha256:1fc112b9a9890f89cc645a16604783ed7fa25299f149b0ef7b45a5e2e3c1f31f", size = 241484, upload-time = "2026-04-24T21:29:06.114Z" }, + { url = "https://files.pythonhosted.org/packages/2e/6e/fb9876940acb50df355a473c087b9924e7b3368070403683941653b6fabc/pyroaring-1.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d92a0f4c7e6bb7deeafac68c79c92ef9340895fe825cf1a31078443753ab6756", size = 304537, upload-time = "2026-04-24T21:29:07.312Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e0/39afe4bddbed6276c54e35e310aa345fbeb00f8890e96e7f48cdc2be9c66/pyroaring-1.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:99c42fe1449acfbf130da65e66b4d5b2726aba4497be359bae7672e38a15fc62", size = 234615, upload-time = "2026-04-24T21:29:08.751Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "requests" +version = "2.33.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5f/a4/98b9c7c6428a668bf7e42ebb7c79d576a1c3c1e3ae2d47e674b468388871/requests-2.33.1.tar.gz", hash = "sha256:18817f8c57c6263968bc123d237e3b8b08ac046f5456bd1e307ee8f4250d3517", size = 134120, upload-time = "2026-03-30T16:09:15.531Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/8e/7540e8a2036f79a125c1d2ebadf69ed7901608859186c856fa0388ef4197/requests-2.33.1-py3-none-any.whl", hash = "sha256:4e6d1ef462f3626a1f0a0a9c42dd93c63bad33f9f1c1937509b8c5c8718ab56a", size = 64947, upload-time = "2026-03-30T16:09:13.83Z" }, +] + +[[package]] +name = "rich" +version = "14.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/67/cae617f1351490c25a4b8ac3b8b63a4dda609295d8222bad12242dfdc629/rich-14.3.4.tar.gz", hash = "sha256:817e02727f2b25b40ef56f5aa2217f400c8489f79ca8f46ea2b70dd5e14558a9", size = 230524, upload-time = "2026-04-11T02:57:45.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/76/6d163cfac87b632216f71879e6b2cf17163f773ff59c00b5ff4900a80fa3/rich-14.3.4-py3-none-any.whl", hash = "sha256:07e7adb4690f68864777b1450859253bed81a99a31ac321ac1817b2313558952", size = 310480, upload-time = "2026-04-11T02:57:47.484Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "strictyaml" +version = "1.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/08/efd28d49162ce89c2ad61a88bd80e11fb77bc9f6c145402589112d38f8af/strictyaml-1.7.3.tar.gz", hash = "sha256:22f854a5fcab42b5ddba8030a0e4be51ca89af0267961c8d6cfa86395586c407", size = 115206, upload-time = "2023-03-10T12:50:27.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/7c/a81ef5ef10978dd073a854e0fa93b5d8021d0594b639cc8f6453c3c78a1d/strictyaml-1.7.3-py3-none-any.whl", hash = "sha256:fb5c8a4edb43bebb765959e420f9b3978d7f1af88c80606c03fb420888f5d1c7", size = 123917, upload-time = "2023-03-10T12:50:17.242Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/7a/28efd1d371f1acd037ac64ed1c5e2b41514a6cc937dd6ab6a13ab9f0702f/zstandard-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd", size = 795256, upload-time = "2025-09-14T22:15:56.415Z" }, + { url = "https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7", size = 640565, upload-time = "2025-09-14T22:15:58.177Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1b/4fdb2c12eb58f31f28c4d28e8dc36611dd7205df8452e63f52fb6261d13e/zstandard-0.25.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550", size = 5345306, upload-time = "2025-09-14T22:16:00.165Z" }, + { url = "https://files.pythonhosted.org/packages/73/28/a44bdece01bca027b079f0e00be3b6bd89a4df180071da59a3dd7381665b/zstandard-0.25.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d", size = 5055561, upload-time = "2025-09-14T22:16:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/e9/74/68341185a4f32b274e0fc3410d5ad0750497e1acc20bd0f5b5f64ce17785/zstandard-0.25.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b", size = 5402214, upload-time = "2025-09-14T22:16:04.109Z" }, + { url = "https://files.pythonhosted.org/packages/8b/67/f92e64e748fd6aaffe01e2b75a083c0c4fd27abe1c8747fee4555fcee7dd/zstandard-0.25.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0", size = 5449703, upload-time = "2025-09-14T22:16:06.312Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0", size = 5556583, upload-time = "2025-09-14T22:16:08.457Z" }, + { url = "https://files.pythonhosted.org/packages/d7/83/41939e60d8d7ebfe2b747be022d0806953799140a702b90ffe214d557638/zstandard-0.25.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd", size = 5045332, upload-time = "2025-09-14T22:16:10.444Z" }, + { url = "https://files.pythonhosted.org/packages/b3/87/d3ee185e3d1aa0133399893697ae91f221fda79deb61adbe998a7235c43f/zstandard-0.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701", size = 5572283, upload-time = "2025-09-14T22:16:12.128Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1d/58635ae6104df96671076ac7d4ae7816838ce7debd94aecf83e30b7121b0/zstandard-0.25.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1", size = 4959754, upload-time = "2025-09-14T22:16:14.225Z" }, + { url = "https://files.pythonhosted.org/packages/75/d6/57e9cb0a9983e9a229dd8fd2e6e96593ef2aa82a3907188436f22b111ccd/zstandard-0.25.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150", size = 5266477, upload-time = "2025-09-14T22:16:16.343Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a9/ee891e5edf33a6ebce0a028726f0bbd8567effe20fe3d5808c42323e8542/zstandard-0.25.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab", size = 5440914, upload-time = "2025-09-14T22:16:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/58/08/a8522c28c08031a9521f27abc6f78dbdee7312a7463dd2cfc658b813323b/zstandard-0.25.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e", size = 5819847, upload-time = "2025-09-14T22:16:20.559Z" }, + { url = "https://files.pythonhosted.org/packages/6f/11/4c91411805c3f7b6f31c60e78ce347ca48f6f16d552fc659af6ec3b73202/zstandard-0.25.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74", size = 5363131, upload-time = "2025-09-14T22:16:22.206Z" }, + { url = "https://files.pythonhosted.org/packages/ef/d6/8c4bd38a3b24c4c7676a7a3d8de85d6ee7a983602a734b9f9cdefb04a5d6/zstandard-0.25.0-cp310-cp310-win32.whl", hash = "sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa", size = 436469, upload-time = "2025-09-14T22:16:25.002Z" }, + { url = "https://files.pythonhosted.org/packages/93/90/96d50ad417a8ace5f841b3228e93d1bb13e6ad356737f42e2dde30d8bd68/zstandard-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e", size = 506100, upload-time = "2025-09-14T22:16:23.569Z" }, + { url = "https://files.pythonhosted.org/packages/2a/83/c3ca27c363d104980f1c9cee1101cc8ba724ac8c28a033ede6aab89585b1/zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c", size = 795254, upload-time = "2025-09-14T22:16:26.137Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4d/e66465c5411a7cf4866aeadc7d108081d8ceba9bc7abe6b14aa21c671ec3/zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f", size = 640559, upload-time = "2025-09-14T22:16:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/12/56/354fe655905f290d3b147b33fe946b0f27e791e4b50a5f004c802cb3eb7b/zstandard-0.25.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431", size = 5348020, upload-time = "2025-09-14T22:16:29.523Z" }, + { url = "https://files.pythonhosted.org/packages/3b/13/2b7ed68bd85e69a2069bcc72141d378f22cae5a0f3b353a2c8f50ef30c1b/zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a", size = 5058126, upload-time = "2025-09-14T22:16:31.811Z" }, + { url = "https://files.pythonhosted.org/packages/c9/dd/fdaf0674f4b10d92cb120ccff58bbb6626bf8368f00ebfd2a41ba4a0dc99/zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc", size = 5405390, upload-time = "2025-09-14T22:16:33.486Z" }, + { url = "https://files.pythonhosted.org/packages/0f/67/354d1555575bc2490435f90d67ca4dd65238ff2f119f30f72d5cde09c2ad/zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6", size = 5452914, upload-time = "2025-09-14T22:16:35.277Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/e9cfd801a3f9190bf3e759c422bbfd2247db9d7f3d54a56ecde70137791a/zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072", size = 5559635, upload-time = "2025-09-14T22:16:37.141Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/5ba550f797ca953a52d708c8e4f380959e7e3280af029e38fbf47b55916e/zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277", size = 5048277, upload-time = "2025-09-14T22:16:38.807Z" }, + { url = "https://files.pythonhosted.org/packages/46/c0/ca3e533b4fa03112facbe7fbe7779cb1ebec215688e5df576fe5429172e0/zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313", size = 5574377, upload-time = "2025-09-14T22:16:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/12/9b/3fb626390113f272abd0799fd677ea33d5fc3ec185e62e6be534493c4b60/zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097", size = 4961493, upload-time = "2025-09-14T22:16:43.3Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d3/23094a6b6a4b1343b27ae68249daa17ae0651fcfec9ed4de09d14b940285/zstandard-0.25.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778", size = 5269018, upload-time = "2025-09-14T22:16:45.292Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a7/bb5a0c1c0f3f4b5e9d5b55198e39de91e04ba7c205cc46fcb0f95f0383c1/zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065", size = 5443672, upload-time = "2025-09-14T22:16:47.076Z" }, + { url = "https://files.pythonhosted.org/packages/27/22/503347aa08d073993f25109c36c8d9f029c7d5949198050962cb568dfa5e/zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa", size = 5822753, upload-time = "2025-09-14T22:16:49.316Z" }, + { url = "https://files.pythonhosted.org/packages/e2/be/94267dc6ee64f0f8ba2b2ae7c7a2df934a816baaa7291db9e1aa77394c3c/zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7", size = 5366047, upload-time = "2025-09-14T22:16:51.328Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a3/732893eab0a3a7aecff8b99052fecf9f605cf0fb5fb6d0290e36beee47a4/zstandard-0.25.0-cp311-cp311-win32.whl", hash = "sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4", size = 436484, upload-time = "2025-09-14T22:16:55.005Z" }, + { url = "https://files.pythonhosted.org/packages/43/a3/c6155f5c1cce691cb80dfd38627046e50af3ee9ddc5d0b45b9b063bfb8c9/zstandard-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2", size = 506183, upload-time = "2025-09-14T22:16:52.753Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3e/8945ab86a0820cc0e0cdbf38086a92868a9172020fdab8a03ac19662b0e5/zstandard-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137", size = 462533, upload-time = "2025-09-14T22:16:53.878Z" }, + { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, + { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +] diff --git a/crates/catalog/glue/Cargo.toml b/crates/catalog/glue/Cargo.toml index b6126021f5..ac912bd405 100644 --- a/crates/catalog/glue/Cargo.toml +++ b/crates/catalog/glue/Cargo.toml @@ -19,6 +19,7 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg-catalog-glue" +publish = true rust-version = { workspace = true } version = { workspace = true } @@ -34,11 +35,13 @@ async-trait = { workspace = true } aws-config = { workspace = true } aws-sdk-glue = { workspace = true } iceberg = { workspace = true } +iceberg-property-macro = { workspace = true } +iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] } serde_json = { workspace = true } tokio = { workspace = true } -tracing = { workspace = true } [dev-dependencies] -ctor = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } -port_scanner = { workspace = true } + +[lints] +workspace = true diff --git a/crates/catalog/glue/DEPENDENCIES.rust.tsv b/crates/catalog/glue/DEPENDENCIES.rust.tsv deleted file mode 100644 index 2d9f686262..0000000000 --- a/crates/catalog/glue/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,380 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayvec@0.7.6 X X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-ord@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-config@1.8.6 X -aws-credential-types@1.2.6 X -aws-runtime@1.5.10 X -aws-sdk-glue@1.119.0 X -aws-sdk-sso@1.83.0 X -aws-sdk-ssooidc@1.84.0 X -aws-sdk-sts@1.85.0 X -aws-sigv4@1.3.4 X -aws-smithy-async@1.2.5 X -aws-smithy-http@0.62.3 X -aws-smithy-http-client@1.1.1 X -aws-smithy-json@0.61.5 X -aws-smithy-observability@0.1.3 X -aws-smithy-query@0.60.7 X -aws-smithy-runtime@1.9.1 X -aws-smithy-runtime-api@1.9.0 X -aws-smithy-types@1.3.2 X -aws-smithy-xml@0.60.10 X -aws-types@1.3.8 X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.21.7 X X -base64@0.22.1 X X -base64-simd@0.8.0 X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -bytes-utils@0.1.4 X X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation@0.10.1 X X -core-foundation@0.9.4 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -deranged@0.5.3 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -h2@0.3.27 X -h2@0.4.12 X -half@2.6.0 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@0.2.12 X X -http@1.3.1 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hyper@0.14.32 X -hyper@1.7.0 X -hyper-rustls@0.24.2 X X X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-catalog-glue@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-conv@0.1.0 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -openssl-probe@0.1.6 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustls@0.21.12 X X X -rustls@0.23.31 X X X -rustls-native-certs@0.6.3 X X X -rustls-native-certs@0.8.1 X X X -rustls-pemfile@1.0.4 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.101.7 X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -schannel@0.1.27 X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -sct@0.7.1 X X X -security-framework@2.11.1 X X -security-framework@3.4.0 X X -security-framework-sys@2.15.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.6 X X -simdutf8@0.1.5 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.5.10 X X -socket2@0.6.0 X X -stable_deref_trait@1.2.0 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -time@0.3.43 X X -time-core@0.1.6 X X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.24.1 X X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.7 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -vsimd@0.8.0 X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -webpki-roots@1.0.2 X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -xmlparser@0.13.6 X X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/glue/public-api.txt b/crates/catalog/glue/public-api.txt new file mode 100644 index 0000000000..eafce20540 --- /dev/null +++ b/crates/catalog/glue/public-api.txt @@ -0,0 +1,42 @@ +pub mod iceberg_catalog_glue +pub macro iceberg_catalog_glue::with_catalog_id! +pub struct iceberg_catalog_glue::GlueCatalog +impl iceberg_catalog_glue::GlueCatalog +pub fn iceberg_catalog_glue::GlueCatalog::file_io(&self) -> iceberg::io::file_io::FileIO +impl core::fmt::Debug for iceberg_catalog_glue::GlueCatalog +pub fn iceberg_catalog_glue::GlueCatalog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::Catalog for iceberg_catalog_glue::GlueCatalog +pub fn iceberg_catalog_glue::GlueCatalog::create_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::create_table<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, creation: iceberg::catalog::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::drop_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::drop_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::get_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::list_namespaces<'life0, 'life1, 'async_trait>(&'life0 self, parent: core::option::Option<&'life1 iceberg::catalog::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::list_tables<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::load_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::namespace_exists<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::purge_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::register_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::catalog::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::rename_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, src: &'life1 iceberg::catalog::TableIdent, dest: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::table_exists<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::update_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_glue::GlueCatalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::catalog::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub struct iceberg_catalog_glue::GlueCatalogBuilder +impl core::default::Default for iceberg_catalog_glue::GlueCatalogBuilder +pub fn iceberg_catalog_glue::GlueCatalogBuilder::default() -> iceberg_catalog_glue::GlueCatalogBuilder +impl core::fmt::Debug for iceberg_catalog_glue::GlueCatalogBuilder +pub fn iceberg_catalog_glue::GlueCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::CatalogBuilder for iceberg_catalog_glue::GlueCatalogBuilder +pub type iceberg_catalog_glue::GlueCatalogBuilder::C = iceberg_catalog_glue::GlueCatalog +pub fn iceberg_catalog_glue::GlueCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg_catalog_glue::GlueCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg_catalog_glue::GlueCatalogBuilder::with_runtime(self, runtime: iceberg::runtime::Runtime) -> Self +pub fn iceberg_catalog_glue::GlueCatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +pub const iceberg_catalog_glue::AWS_ACCESS_KEY_ID: &str +pub const iceberg_catalog_glue::AWS_PROFILE_NAME: &str +pub const iceberg_catalog_glue::AWS_REGION_NAME: &str +pub const iceberg_catalog_glue::AWS_SECRET_ACCESS_KEY: &str +pub const iceberg_catalog_glue::AWS_SESSION_TOKEN: &str +pub const iceberg_catalog_glue::GLUE_CATALOG_PROP_CATALOG_ID: &str +pub const iceberg_catalog_glue::GLUE_CATALOG_PROP_URI: &str +pub const iceberg_catalog_glue::GLUE_CATALOG_PROP_WAREHOUSE: &str diff --git a/crates/catalog/glue/src/catalog.rs b/crates/catalog/glue/src/catalog.rs index 9b383df2b1..d6a05000f9 100644 --- a/crates/catalog/glue/src/catalog.rs +++ b/crates/catalog/glue/src/catalog.rs @@ -17,26 +17,32 @@ use std::collections::HashMap; use std::fmt::Debug; +use std::str::FromStr; +use std::sync::Arc; use anyhow::anyhow; use async_trait::async_trait; use aws_sdk_glue::operation::create_table::CreateTableError; use aws_sdk_glue::operation::update_table::UpdateTableError; use aws_sdk_glue::types::TableInput; +use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; use iceberg::io::{ - FileIO, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, + FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, + S3_SESSION_TOKEN, StorageFactory, }; use iceberg::spec::{TableMetadata, TableMetadataBuilder}; use iceberg::table::Table; use iceberg::{ Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, - TableCommit, TableCreation, TableIdent, + Runtime, TableCommit, TableCreation, TableIdent, delete_removed_metadata_files, }; +use iceberg_property_macro::Properties; +use iceberg_storage_opendal::OpenDalStorageFactory; use crate::error::{from_aws_build_error, from_aws_sdk_error}; use crate::utils::{ convert_to_database, convert_to_glue_table, convert_to_namespace, create_sdk_config, - get_default_table_location, get_metadata_location, validate_namespace, + get_default_table_location, get_metadata_location, is_iceberg_table, validate_namespace, }; use crate::{ AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, with_catalog_id, @@ -50,152 +56,226 @@ pub const GLUE_CATALOG_PROP_CATALOG_ID: &str = "catalog_id"; pub const GLUE_CATALOG_PROP_WAREHOUSE: &str = "warehouse"; /// Builder for [`GlueCatalog`]. -#[derive(Debug)] -pub struct GlueCatalogBuilder(GlueCatalogConfig); - -impl Default for GlueCatalogBuilder { - fn default() -> Self { - Self(GlueCatalogConfig { - name: None, - uri: None, - catalog_id: None, - warehouse: "".to_string(), - props: HashMap::new(), - }) - } +#[derive(Debug, Default)] +pub struct GlueCatalogBuilder { + storage_factory: Option>, + kms_client_factory: Option>, + runtime: Option, } impl CatalogBuilder for GlueCatalogBuilder { type C = GlueCatalog; + fn with_storage_factory(mut self, storage_factory: Arc) -> Self { + self.storage_factory = Some(storage_factory); + self + } + + fn with_kms_client_factory(mut self, kms_client_factory: Arc) -> Self { + self.kms_client_factory = Some(kms_client_factory); + self + } + + fn with_runtime(mut self, runtime: Runtime) -> Self { + self.runtime = Some(runtime); + self + } + fn load( - mut self, + self, name: impl Into, props: HashMap, ) -> impl Future> + Send { - self.0.name = Some(name.into()); - - if props.contains_key(GLUE_CATALOG_PROP_URI) { - self.0.uri = props.get(GLUE_CATALOG_PROP_URI).cloned() - } - - if props.contains_key(GLUE_CATALOG_PROP_CATALOG_ID) { - self.0.catalog_id = props.get(GLUE_CATALOG_PROP_CATALOG_ID).cloned() - } - - if props.contains_key(GLUE_CATALOG_PROP_WAREHOUSE) { - self.0.warehouse = props - .get(GLUE_CATALOG_PROP_WAREHOUSE) - .cloned() - .unwrap_or_default(); - } - - // Collect other remaining properties - self.0.props = props - .into_iter() - .filter(|(k, _)| { - k != GLUE_CATALOG_PROP_URI - && k != GLUE_CATALOG_PROP_CATALOG_ID - && k != GLUE_CATALOG_PROP_WAREHOUSE - }) - .collect(); + let name = name.into(); async move { - if self.0.name.is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Catalog name is required", - )); - } - if self.0.warehouse.is_empty() { + let catalog_properties = GlueCatalogProperties::from_properties(&props)?; + if catalog_properties.warehouse.is_empty() { return Err(Error::new( ErrorKind::DataInvalid, "Catalog warehouse is required", )); } - GlueCatalog::new(self.0).await + let runtime = match self.runtime { + Some(rt) => rt, + None => Runtime::try_current()?, + }; + let kms_client = match self.kms_client_factory { + Some(factory) => Some(factory.create_kms_client(&props).await?), + None => None, + }; + GlueCatalog::new( + name, + catalog_properties, + props, + self.storage_factory, + runtime, + kms_client, + ) + .await } } } -#[derive(Debug)] -/// Glue Catalog configuration -pub(crate) struct GlueCatalogConfig { - name: Option, +/// Glue catalog properties parsed from the `load` property map. +#[derive(Debug, Properties)] +pub(crate) struct GlueCatalogProperties { + #[property(key = GLUE_CATALOG_PROP_URI, default = None)] uri: Option, + #[property(key = GLUE_CATALOG_PROP_CATALOG_ID, default = None)] catalog_id: Option, + #[property(key = GLUE_CATALOG_PROP_WAREHOUSE, default = "")] warehouse: String, - props: HashMap, } struct GlueClient(aws_sdk_glue::Client); /// Glue Catalog pub struct GlueCatalog { - config: GlueCatalogConfig, + name: String, + properties: GlueCatalogProperties, client: GlueClient, file_io: FileIO, + runtime: Runtime, + kms_client: Option>, } impl Debug for GlueCatalog { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("GlueCatalog") - .field("config", &self.config) + .field("name", &self.name) + .field("properties", &self.properties) .finish_non_exhaustive() } } impl GlueCatalog { /// Create a new glue catalog - async fn new(config: GlueCatalogConfig) -> Result { - let sdk_config = create_sdk_config(&config.props, config.uri.as_ref()).await; - let mut file_io_props = config.props.clone(); - if !file_io_props.contains_key(S3_ACCESS_KEY_ID) { - if let Some(access_key_id) = file_io_props.get(AWS_ACCESS_KEY_ID) { - file_io_props.insert(S3_ACCESS_KEY_ID.to_string(), access_key_id.to_string()); - } + async fn new( + name: String, + properties: GlueCatalogProperties, + props: HashMap, + storage_factory: Option>, + runtime: Runtime, + kms_client: Option>, + ) -> Result { + let sdk_config = create_sdk_config(&props, properties.uri.as_ref()).await; + let mut file_io_props = props; + if !file_io_props.contains_key(S3_ACCESS_KEY_ID) + && let Some(access_key_id) = file_io_props.get(AWS_ACCESS_KEY_ID) + { + file_io_props.insert(S3_ACCESS_KEY_ID.to_string(), access_key_id.to_string()); } - if !file_io_props.contains_key(S3_SECRET_ACCESS_KEY) { - if let Some(secret_access_key) = file_io_props.get(AWS_SECRET_ACCESS_KEY) { - file_io_props.insert( - S3_SECRET_ACCESS_KEY.to_string(), - secret_access_key.to_string(), - ); - } + if !file_io_props.contains_key(S3_SECRET_ACCESS_KEY) + && let Some(secret_access_key) = file_io_props.get(AWS_SECRET_ACCESS_KEY) + { + file_io_props.insert( + S3_SECRET_ACCESS_KEY.to_string(), + secret_access_key.to_string(), + ); } - if !file_io_props.contains_key(S3_REGION) { - if let Some(region) = file_io_props.get(AWS_REGION_NAME) { - file_io_props.insert(S3_REGION.to_string(), region.to_string()); - } + if !file_io_props.contains_key(S3_REGION) + && let Some(region) = file_io_props.get(AWS_REGION_NAME) + { + file_io_props.insert(S3_REGION.to_string(), region.to_string()); } - if !file_io_props.contains_key(S3_SESSION_TOKEN) { - if let Some(session_token) = file_io_props.get(AWS_SESSION_TOKEN) { - file_io_props.insert(S3_SESSION_TOKEN.to_string(), session_token.to_string()); - } + if !file_io_props.contains_key(S3_SESSION_TOKEN) + && let Some(session_token) = file_io_props.get(AWS_SESSION_TOKEN) + { + file_io_props.insert(S3_SESSION_TOKEN.to_string(), session_token.to_string()); } - if !file_io_props.contains_key(S3_ENDPOINT) { - if let Some(aws_endpoint) = config.uri.as_ref() { - file_io_props.insert(S3_ENDPOINT.to_string(), aws_endpoint.to_string()); - } + if !file_io_props.contains_key(S3_ENDPOINT) + && let Some(aws_endpoint) = properties.uri.as_ref() + { + file_io_props.insert(S3_ENDPOINT.to_string(), aws_endpoint.to_string()); } let client = aws_sdk_glue::Client::new(&sdk_config); - let file_io = FileIO::from_path(&config.warehouse)? + // Use provided factory or default to OpenDalStorageFactory::S3 + let factory = storage_factory.unwrap_or_else(|| { + Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + }) + }); + let file_io = FileIOBuilder::new(factory) .with_props(file_io_props) - .build()?; + .build(); Ok(GlueCatalog { - config, + name, + properties, client: GlueClient(client), file_io, + runtime, + kms_client, }) } /// Get the catalogs `FileIO` pub fn file_io(&self) -> FileIO { self.file_io.clone() } + + /// Loads a table from the Glue Catalog along with its version_id for optimistic locking. + /// + /// # Returns + /// A `Result` wrapping a tuple of (`Table`, `Option`) where the String is the version_id + /// from Glue that should be used for optimistic concurrency control when updating the table. + /// + /// # Errors + /// This function may return an error in several scenarios, including: + /// - Failure to validate the namespace. + /// - Failure to retrieve the table from the Glue Catalog. + /// - Absence of metadata location information in the table's properties. + /// - Issues reading or deserializing the table's metadata file. + async fn load_table_with_version_id( + &self, + table: &TableIdent, + ) -> Result<(Table, Option)> { + let db_name = validate_namespace(table.namespace())?; + let table_name = table.name(); + + let builder = self + .client + .0 + .get_table() + .database_name(&db_name) + .name(table_name); + let builder = with_catalog_id!(builder, self.properties); + + let glue_table_output = builder.send().await.map_err(from_aws_sdk_error)?; + + let glue_table = glue_table_output.table().ok_or_else(|| { + Error::new( + ErrorKind::TableNotFound, + format!( + "Table object for database: {db_name} and table: {table_name} does not exist" + ), + ) + })?; + + let version_id = glue_table.version_id.clone(); + let metadata_location = get_metadata_location(&glue_table.parameters)?; + + let metadata = TableMetadata::read_from(&self.file_io, &metadata_location).await?; + + let mut builder = Table::builder() + .file_io(self.file_io()) + .metadata_location(metadata_location) + .metadata(metadata) + .identifier(TableIdent::new( + NamespaceIdent::new(db_name), + table_name.to_owned(), + )) + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + let table = builder.build()?; + + Ok((table, version_id)) + } } #[async_trait] @@ -220,7 +300,7 @@ impl Catalog for GlueCatalog { Some(token) => self.client.0.get_databases().next_token(token), None => self.client.0.get_databases(), }; - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); let resp = builder.send().await.map_err(from_aws_sdk_error)?; let dbs: Vec = resp @@ -258,10 +338,17 @@ impl Catalog for GlueCatalog { namespace: &NamespaceIdent, properties: HashMap, ) -> Result { + if self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceAlreadyExists, + format!("Namespace {namespace:?} already exists"), + )); + } + let db_input = convert_to_database(namespace, &properties)?; let builder = self.client.0.create_database().database_input(db_input); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); builder.send().await.map_err(from_aws_sdk_error)?; @@ -282,9 +369,21 @@ impl Catalog for GlueCatalog { let db_name = validate_namespace(namespace)?; let builder = self.client.0.get_database().name(&db_name); - let builder = with_catalog_id!(builder, self.config); - - let resp = builder.send().await.map_err(from_aws_sdk_error)?; + let builder = with_catalog_id!(builder, self.properties); + + let resp = builder.send().await.map_err(|err| { + if err + .as_service_error() + .map(|e| e.is_entity_not_found_exception()) + == Some(true) + { + return Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + ); + } + from_aws_sdk_error(err) + })?; match resp.database() { Some(db) => { @@ -292,8 +391,8 @@ impl Catalog for GlueCatalog { Ok(namespace) } None => Err(Error::new( - ErrorKind::DataInvalid, - format!("Database with name: {} does not exist", db_name), + ErrorKind::NamespaceNotFound, + format!("Database with name: {db_name} does not exist"), )), } } @@ -314,7 +413,7 @@ impl Catalog for GlueCatalog { let db_name = validate_namespace(namespace)?; let builder = self.client.0.get_database().name(&db_name); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); let resp = builder.send().await; @@ -348,6 +447,13 @@ impl Catalog for GlueCatalog { namespace: &NamespaceIdent, properties: HashMap, ) -> Result<()> { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + let db_name = validate_namespace(namespace)?; let db_input = convert_to_database(namespace, &properties)?; @@ -357,7 +463,7 @@ impl Catalog for GlueCatalog { .update_database() .name(&db_name) .database_input(db_input); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); builder.send().await.map_err(from_aws_sdk_error)?; @@ -375,30 +481,53 @@ impl Catalog for GlueCatalog { /// - `Err(...)` signifies failure to drop the namespace due to validation /// errors, connectivity issues, or Glue Catalog constraints. async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + let db_name = validate_namespace(namespace)?; - let table_list = self.list_tables(namespace).await?; - if !table_list.is_empty() { + // Check for ANY Glue table in the database, not just Iceberg tables. + // Glue's `delete_database` will fail if any table (Iceberg or not) is + // still present, and `list_tables` only returns Iceberg tables, so we + // query Glue directly here. + let builder = self + .client + .0 + .get_tables() + .database_name(&db_name) + .max_results(1); + let builder = with_catalog_id!(builder, self.properties); + let resp = builder.send().await.map_err(from_aws_sdk_error)?; + + if !resp.table_list().is_empty() { return Err(Error::new( ErrorKind::DataInvalid, - format!("Database with name: {} is not empty", &db_name), + format!("Database with name: {} is not empty", db_name), )); } let builder = self.client.0.delete_database().name(db_name); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); builder.send().await.map_err(from_aws_sdk_error)?; Ok(()) } - /// Asynchronously lists all tables within a specified namespace. + /// Asynchronously lists all Iceberg tables within a specified namespace. + /// + /// Glue databases may contain a mix of Iceberg and non-Iceberg tables + /// (e.g. plain Hive tables). Only tables whose `table_type` parameter is + /// set to `ICEBERG` (case-insensitive) are returned /// /// # Returns /// A `Result>`, which is: /// - `Ok(vec![...])` containing a vector of `TableIdent` instances, each - /// representing a table within the specified namespace. + /// representing an Iceberg table within the specified namespace. /// - `Err(...)` if an error occurs during namespace validation or while /// querying the database. async fn list_tables(&self, namespace: &NamespaceIdent) -> Result> { @@ -417,12 +546,13 @@ impl Catalog for GlueCatalog { .next_token(token), None => self.client.0.get_tables().database_name(&db_name), }; - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); let resp = builder.send().await.map_err(from_aws_sdk_error)?; let tables: Vec<_> = resp .table_list() .iter() + .filter(|tbl| is_iceberg_table(&tbl.parameters)) .map(|tbl| TableIdent::new(namespace.clone(), tbl.name().to_string())) .collect(); @@ -457,27 +587,23 @@ impl Catalog for GlueCatalog { let db_name = validate_namespace(namespace)?; let table_name = creation.name.clone(); - let location = match &creation.location { - Some(location) => location.clone(), - None => { - let ns = self.get_namespace(namespace).await?; - let location = - get_default_table_location(&ns, &db_name, &table_name, &self.config.warehouse); - creation.location = Some(location.clone()); - location - } - }; + if creation.location.is_none() { + let ns = self.get_namespace(namespace).await?; + let location = + get_default_table_location(&ns, &db_name, &table_name, &self.properties.warehouse); + creation.location = Some(location); + } let metadata = TableMetadataBuilder::from_table_creation(creation)? .build()? .metadata; - let metadata_location = - MetadataLocation::new_with_table_location(location.clone()).to_string(); + let metadata_location = MetadataLocation::try_new_with_metadata(&metadata)?; metadata.write_to(&self.file_io, &metadata_location).await?; + let metadata_location_str = metadata_location.to_string(); let glue_table = convert_to_glue_table( &table_name, - metadata_location.clone(), + metadata_location_str.clone(), &metadata, metadata.properties(), None, @@ -489,16 +615,20 @@ impl Catalog for GlueCatalog { .create_table() .database_name(&db_name) .table_input(glue_table); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); builder.send().await.map_err(from_aws_sdk_error)?; - Table::builder() + let mut builder = Table::builder() .file_io(self.file_io()) - .metadata_location(metadata_location) + .metadata_location(metadata_location_str) .metadata(metadata) .identifier(TableIdent::new(NamespaceIdent::new(db_name), table_name)) - .build() + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + builder.build() } /// Loads a table from the Glue Catalog and constructs a `Table` object @@ -514,43 +644,8 @@ impl Catalog for GlueCatalog { /// - Absence of metadata location information in the table's properties. /// - Issues reading or deserializing the table's metadata file. async fn load_table(&self, table: &TableIdent) -> Result { - let db_name = validate_namespace(table.namespace())?; - let table_name = table.name(); - - let builder = self - .client - .0 - .get_table() - .database_name(&db_name) - .name(table_name); - let builder = with_catalog_id!(builder, self.config); - - let glue_table_output = builder.send().await.map_err(from_aws_sdk_error)?; - - match glue_table_output.table() { - None => Err(Error::new( - ErrorKind::TableNotFound, - format!( - "Table object for database: {} and table: {} does not exist", - db_name, table_name - ), - )), - Some(table) => { - let metadata_location = get_metadata_location(&table.parameters)?; - - let metadata = TableMetadata::read_from(&self.file_io, &metadata_location).await?; - - Table::builder() - .file_io(self.file_io()) - .metadata_location(metadata_location) - .metadata(metadata) - .identifier(TableIdent::new( - NamespaceIdent::new(db_name), - table_name.to_owned(), - )) - .build() - } - } + let (table, _) = self.load_table_with_version_id(table).await?; + Ok(table) } /// Asynchronously drops a table from the database. @@ -573,13 +668,19 @@ impl Catalog for GlueCatalog { .delete_table() .database_name(&db_name) .name(table_name); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); builder.send().await.map_err(from_aws_sdk_error)?; Ok(()) } + async fn purge_table(&self, table: &TableIdent) -> Result<()> { + let table_info = self.load_table(table).await?; + self.drop_table(table).await?; + iceberg::drop_table_data(&table_info).await + } + /// Asynchronously checks the existence of a specified table /// in the database. /// @@ -597,7 +698,7 @@ impl Catalog for GlueCatalog { .get_table() .database_name(&db_name) .name(table_name); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); let resp = builder.send().await; @@ -635,7 +736,7 @@ impl Catalog for GlueCatalog { .get_table() .database_name(&src_db_name) .name(src_table_name); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); let glue_table_output = builder.send().await.map_err(from_aws_sdk_error)?; @@ -643,8 +744,7 @@ impl Catalog for GlueCatalog { None => Err(Error::new( ErrorKind::TableNotFound, format!( - "'Table' object for database: {} and table: {} does not exist", - src_db_name, src_table_name + "'Table' object for database: {src_db_name} and table: {src_table_name} does not exist" ), )), Some(table) => { @@ -663,7 +763,7 @@ impl Catalog for GlueCatalog { .create_table() .database_name(&dest_db_name) .table_input(rename_table_input); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); builder.send().await.map_err(from_aws_sdk_error)?; @@ -672,10 +772,8 @@ impl Catalog for GlueCatalog { match drop_src_table_result { Ok(_) => Ok(()), Err(_) => { - let err_msg_src_table = format!( - "Failed to drop old table {}.{}.", - src_db_name, src_table_name - ); + let err_msg_src_table = + format!("Failed to drop old table {src_db_name}.{src_table_name}."); let drop_dest_table_result = self.drop_table(dest).await; @@ -683,15 +781,13 @@ impl Catalog for GlueCatalog { Ok(_) => Err(Error::new( ErrorKind::Unexpected, format!( - "{} Rolled back table creation for {}.{}.", - err_msg_src_table, dest_db_name, dest_table_name + "{err_msg_src_table} Rolled back table creation for {dest_db_name}.{dest_table_name}." ), )), Err(_) => Err(Error::new( ErrorKind::Unexpected, format!( - "{} Failed to roll back table creation for {}.{}. Please clean up manually.", - err_msg_src_table, dest_db_name, dest_table_name + "{err_msg_src_table} Failed to roll back table creation for {dest_db_name}.{dest_table_name}. Please clean up manually." ), )), } @@ -735,7 +831,7 @@ impl Catalog for GlueCatalog { .create_table() .database_name(&db_name) .table_input(table_input); - let builder = with_catalog_id!(builder, self.config); + let builder = with_catalog_id!(builder, self.properties); builder.send().await.map_err(|e| { let error = e.into_service_error(); @@ -753,34 +849,42 @@ impl Catalog for GlueCatalog { format!("Failed to register table {table_ident} due to AWS SDK error"), ), } - .with_source(anyhow!("aws sdk error: {:?}", error)) + .with_source(anyhow!("aws sdk error: {error:?}")) })?; - Ok(Table::builder() + let mut builder = Table::builder() .identifier(table_ident.clone()) .metadata_location(metadata_location) .metadata(metadata) .file_io(self.file_io()) - .build()?) + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + Ok(builder.build()?) } async fn update_table(&self, commit: TableCommit) -> Result
{ let table_ident = commit.identifier().clone(); let table_namespace = validate_namespace(table_ident.namespace())?; - let current_table = self.load_table(&table_ident).await?; + + let (current_table, current_version_id) = + self.load_table_with_version_id(&table_ident).await?; let current_metadata_location = current_table.metadata_location_result()?.to_string(); + let base_metadata = current_table.metadata_ref(); let staged_table = commit.apply(current_table)?; - let staged_metadata_location = staged_table.metadata_location_result()?; + let staged_metadata_location_str = staged_table.metadata_location_result()?; + let staged_metadata_location = MetadataLocation::from_str(staged_metadata_location_str)?; // Write new metadata staged_table .metadata() - .write_to(staged_table.file_io(), staged_metadata_location) + .write_to(staged_table.file_io(), &staged_metadata_location) .await?; - // Persist staged table to Glue - let builder = self + // Persist staged table to Glue with optimistic locking + let mut builder = self .client .0 .update_table() @@ -793,7 +897,13 @@ impl Catalog for GlueCatalog { staged_table.metadata().properties(), Some(current_metadata_location), )?); - let builder = with_catalog_id!(builder, self.config); + + // Add VersionId for optimistic locking + if let Some(version_id) = current_version_id { + builder = builder.version_id(version_id); + } + + let builder = with_catalog_id!(builder, self.properties); let _ = builder.send().await.map_err(|e| { let error = e.into_service_error(); match error { @@ -811,9 +921,126 @@ impl Catalog for GlueCatalog { format!("Operation failed for table: {table_ident} for hitting aws sdk error"), ), } - .with_source(anyhow!("aws sdk error: {:?}", error)) + .with_source(anyhow!("aws sdk error: {error:?}")) })?; + delete_removed_metadata_files( + staged_table.file_io(), + &base_metadata, + staged_table.metadata(), + ) + .await; + Ok(staged_table) } } + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use super::*; + + #[test] + fn test_catalog_properties() { + let properties = GlueCatalogProperties::from_properties(&HashMap::from([ + ( + GLUE_CATALOG_PROP_URI.to_string(), + "http://localhost".to_string(), + ), + ( + GLUE_CATALOG_PROP_CATALOG_ID.to_string(), + "catalog".to_string(), + ), + ( + GLUE_CATALOG_PROP_WAREHOUSE.to_string(), + "s3://warehouse".to_string(), + ), + ])) + .unwrap(); + + assert_eq!(properties.uri.as_deref(), Some("http://localhost")); + assert_eq!(properties.catalog_id.as_deref(), Some("catalog")); + assert_eq!(properties.warehouse, "s3://warehouse"); + } + + #[test] + fn test_catalog_properties_defaults() { + let properties = GlueCatalogProperties::from_properties(&HashMap::new()).unwrap(); + + assert_eq!(properties.uri, None); + assert_eq!(properties.catalog_id, None); + assert_eq!(properties.warehouse, ""); + } + + #[tokio::test] + async fn test_load_rejects_missing_or_empty_warehouse() { + for (case, props) in [ + ("missing", HashMap::new()), + ( + "empty", + HashMap::from([(GLUE_CATALOG_PROP_WAREHOUSE.to_string(), String::new())]), + ), + ] { + let error = GlueCatalogBuilder::default() + .load("glue", props) + .await + .unwrap_err(); + + assert_eq!( + (error.kind(), error.message()), + (ErrorKind::DataInvalid, "Catalog warehouse is required"), + "{case}" + ); + } + } + + #[tokio::test] + async fn test_load_forwards_all_properties_to_kms_factory() { + #[derive(Debug, Default)] + struct RecordingKmsClientFactory(Mutex>); + + #[async_trait] + impl KmsClientFactory for RecordingKmsClientFactory { + async fn create_kms_client( + &self, + properties: &HashMap, + ) -> Result> { + *self.0.lock().unwrap() = properties.clone(); + Err(Error::new( + ErrorKind::Unexpected, + "KMS initialization failed", + )) + } + } + + let properties = HashMap::from([ + ( + GLUE_CATALOG_PROP_URI.to_string(), + "http://localhost".to_string(), + ), + ( + GLUE_CATALOG_PROP_CATALOG_ID.to_string(), + "catalog".to_string(), + ), + ( + GLUE_CATALOG_PROP_WAREHOUSE.to_string(), + "s3://warehouse".to_string(), + ), + (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), + ("custom.property".to_string(), "value".to_string()), + ]); + let factory = Arc::new(RecordingKmsClientFactory::default()); + let error = GlueCatalogBuilder::default() + .with_kms_client_factory(factory.clone()) + .load("glue", properties.clone()) + .await + .unwrap_err(); + + assert_eq!(*factory.0.lock().unwrap(), properties); + assert_eq!( + (error.kind(), error.message()), + (ErrorKind::Unexpected, "KMS initialization failed") + ); + } +} diff --git a/crates/catalog/glue/src/error.rs b/crates/catalog/glue/src/error.rs index 741d8c90c3..c936ff4edd 100644 --- a/crates/catalog/glue/src/error.rs +++ b/crates/catalog/glue/src/error.rs @@ -27,7 +27,7 @@ where T: Debug { ErrorKind::Unexpected, "Operation failed for hitting aws sdk error".to_string(), ) - .with_source(anyhow!("aws sdk error: {:?}", error)) + .with_source(anyhow!("aws sdk error: {error:?}")) } /// Format AWS Build error into iceberg error @@ -36,5 +36,5 @@ pub(crate) fn from_aws_build_error(error: aws_sdk_glue::error::BuildError) -> Er ErrorKind::Unexpected, "Operation failed for hitting aws build error".to_string(), ) - .with_source(anyhow!("aws build error: {:?}", error)) + .with_source(anyhow!("aws build error: {error:?}")) } diff --git a/crates/catalog/glue/src/schema.rs b/crates/catalog/glue/src/schema.rs index 43918484d5..95c1d8bab0 100644 --- a/crates/catalog/glue/src/schema.rs +++ b/crates/catalog/glue/src/schema.rs @@ -25,7 +25,7 @@ pub(crate) const ICEBERG_FIELD_CURRENT: &str = "iceberg.field.current"; use std::collections::HashMap; use aws_sdk_glue::types::Column; -use iceberg::spec::{PrimitiveType, SchemaVisitor, TableMetadata, visit_schema}; +use iceberg::spec::{PrimitiveType, SchemaVisitor, TableMetadata, VariantType, visit_schema}; use iceberg::{Error, ErrorKind, Result}; use crate::error::from_aws_build_error; @@ -79,11 +79,7 @@ impl GlueSchemaBuilder { impl SchemaVisitor for GlueSchemaBuilder { type T = String; - fn schema( - &mut self, - _schema: &iceberg::spec::Schema, - value: Self::T, - ) -> iceberg::Result { + fn schema(&mut self, _schema: &iceberg::spec::Schema, value: Self::T) -> Result { Ok(value) } @@ -96,7 +92,7 @@ impl SchemaVisitor for GlueSchemaBuilder { &mut self, r#_struct: &iceberg::spec::StructType, results: Vec, - ) -> iceberg::Result { + ) -> Result { Ok(format!("struct<{}>", results.join(", "))) } @@ -105,13 +101,9 @@ impl SchemaVisitor for GlueSchemaBuilder { Ok(()) } - fn field( - &mut self, - field: &iceberg::spec::NestedFieldRef, - value: String, - ) -> iceberg::Result { + fn field(&mut self, field: &iceberg::spec::NestedFieldRef, value: String) -> Result { if self.is_inside_struct() { - return Ok(format!("{}:{}", field.name, &value)); + return Ok(format!("{}:{}", field.name, value)); } let parameters = HashMap::from([ @@ -142,8 +134,8 @@ impl SchemaVisitor for GlueSchemaBuilder { Ok(value) } - fn list(&mut self, _list: &iceberg::spec::ListType, value: String) -> iceberg::Result { - Ok(format!("array<{}>", value)) + fn list(&mut self, _list: &iceberg::spec::ListType, value: String) -> Result { + Ok(format!("array<{value}>")) } fn map( @@ -151,12 +143,22 @@ impl SchemaVisitor for GlueSchemaBuilder { _map: &iceberg::spec::MapType, key_value: String, value: String, - ) -> iceberg::Result { - Ok(format!("map<{},{}>", key_value, value)) + ) -> Result { + Ok(format!("map<{key_value},{value}>")) } - fn primitive(&mut self, p: &iceberg::spec::PrimitiveType) -> iceberg::Result { + fn primitive(&mut self, p: &PrimitiveType) -> Result { + // Glue column types are informational for Iceberg tables; readers use Iceberg metadata as + // the source of truth. Keep the existing Glue-compatible spellings and use the Iceberg + // display representation for geospatial types. A non-failing fallback for all other types + // is tracked in https://github.com/apache/iceberg-rust/issues/3375. let glue_type = match p { + PrimitiveType::Unknown => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Conversion from {p:?} is not supported"), + )); + } PrimitiveType::Boolean => "boolean".to_string(), PrimitiveType::Int => "int".to_string(), PrimitiveType::Long => "bigint".to_string(), @@ -165,38 +167,52 @@ impl SchemaVisitor for GlueSchemaBuilder { PrimitiveType::Date => "date".to_string(), PrimitiveType::Timestamp => "timestamp".to_string(), PrimitiveType::TimestampNs => "timestamp_ns".to_string(), - PrimitiveType::TimestamptzNs => "timestamptz_ns".to_string(), + PrimitiveType::Timestamptz | PrimitiveType::TimestamptzNs => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Conversion from {p:?} is not supported"), + )); + } PrimitiveType::Time | PrimitiveType::String | PrimitiveType::Uuid => { "string".to_string() } PrimitiveType::Binary | PrimitiveType::Fixed(_) => "binary".to_string(), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => p.to_string(), PrimitiveType::Decimal { precision, scale } => { - format!("decimal({},{})", precision, scale) - } - _ => { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "Conversion from 'Timestamptz' is not supported", - )); + format!("decimal({precision},{scale})") } }; Ok(glue_type) } + + fn variant(&mut self, _v: &VariantType) -> Result { + Ok("variant".to_string()) + } } #[cfg(test)] mod tests { use iceberg::TableCreation; - use iceberg::spec::{Schema, TableMetadataBuilder}; + use iceberg::spec::{ + FormatVersion, GeometryType, NestedField, PrimitiveType, Schema, TableMetadataBuilder, Type, + }; use super::*; fn create_metadata(schema: Schema) -> Result { + create_metadata_with_format_version(schema, FormatVersion::V2) + } + + fn create_metadata_with_format_version( + schema: Schema, + format_version: FormatVersion, + ) -> Result { let table_creation = TableCreation::builder() .name("my_table".to_string()) .location("my_location".to_string()) .schema(schema) + .format_version(format_version) .build(); let metadata = TableMetadataBuilder::from_table_creation(table_creation)? .build()? @@ -524,4 +540,53 @@ mod tests { assert_eq!(result, expected); Ok(()) } + + #[test] + fn test_schema_with_variant() -> Result<()> { + let record = r#"{ + "type": "struct", + "schema-id": 1, + "fields": [ + { + "id": 1, + "name": "v", + "required": true, + "type": "variant" + } + ] + }"#; + + let schema = serde_json::from_str::(record)?; + // Variant requires format version 3. + let metadata = create_metadata_with_format_version(schema, FormatVersion::V3)?; + + let result = GlueSchemaBuilder::from_iceberg(&metadata)?.build(); + + let expected = vec![create_column("v", "variant", "1", false)?]; + + assert_eq!(result, expected); + Ok(()) + } + + #[test] + fn test_schema_with_geospatial_type() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(GeometryType::default())), + ) + .into(), + ]) + .build() + .unwrap(); + let metadata = create_metadata_with_format_version(schema, FormatVersion::V3).unwrap(); + + let result = GlueSchemaBuilder::from_iceberg(&metadata).unwrap().build(); + + assert_eq!(result, vec![ + create_column("geom", "geometry(OGC:CRS84)", "1", false,).unwrap() + ]); + } } diff --git a/crates/catalog/glue/src/utils.rs b/crates/catalog/glue/src/utils.rs index d2c21b4d74..75f9fac86d 100644 --- a/crates/catalog/glue/src/utils.rs +++ b/crates/catalog/glue/src/utils.rs @@ -185,8 +185,7 @@ pub(crate) fn validate_namespace(namespace: &NamespaceIdent) -> Result { return Err(Error::new( ErrorKind::DataInvalid, format!( - "Invalid database name: {:?}, hierarchical namespaces are not supported", - namespace + "Invalid database name: {namespace:?}, hierarchical namespaces are not supported" ), )); } @@ -227,6 +226,17 @@ pub(crate) fn get_default_table_location( } } +/// Returns `true` if the given Glue table is an Iceberg table. +/// +/// Iceberg tables are identified by the `table_type=ICEBERG` parameter +/// (case-insensitive). +pub(crate) fn is_iceberg_table(parameters: &Option>) -> bool { + parameters + .as_ref() + .and_then(|p| p.get(TABLE_TYPE)) + .is_some_and(|v| v.eq_ignore_ascii_case(ICEBERG)) +} + /// Get metadata location from `GlueTable` parameters pub(crate) fn get_metadata_location( parameters: &Option>, @@ -236,7 +246,7 @@ pub(crate) fn get_metadata_location( Some(location) => Ok(location.to_string()), None => Err(Error::new( ErrorKind::DataInvalid, - format!("No '{}' set on table", METADATA_LOCATION), + format!("No '{METADATA_LOCATION}' set on table"), )), }, None => Err(Error::new( @@ -303,12 +313,39 @@ mod tests { Ok(()) } + #[test] + fn test_is_iceberg_table() { + // table_type=ICEBERG -> iceberg + let params_table_type = Some(HashMap::from([( + TABLE_TYPE.to_string(), + ICEBERG.to_string(), + )])); + assert!(is_iceberg_table(¶ms_table_type)); + + // table_type is case-insensitive + let params_table_type_lower = Some(HashMap::from([( + TABLE_TYPE.to_string(), + "iceberg".to_string(), + )])); + assert!(is_iceberg_table(¶ms_table_type_lower)); + + // Plain Hive table -> not iceberg + let params_hive = Some(HashMap::from([( + TABLE_TYPE.to_string(), + "EXTERNAL_TABLE".to_string(), + )])); + assert!(!is_iceberg_table(¶ms_hive)); + + // No parameters at all -> not iceberg + assert!(!is_iceberg_table(&None)); + + // Empty parameters -> not iceberg + assert!(!is_iceberg_table(&Some(HashMap::new()))); + } + #[test] fn test_convert_to_glue_table() -> Result<()> { let table_name = "my_table".to_string(); - let location = "s3a://warehouse/hive".to_string(); - let metadata_location = MetadataLocation::new_with_table_location(location).to_string(); - let properties = HashMap::new(); let schema = Schema::builder() .with_schema_id(1) .with_fields(vec![ @@ -317,6 +354,7 @@ mod tests { .build()?; let metadata = create_metadata(schema)?; + let metadata_location = MetadataLocation::try_new_with_metadata(&metadata)?.to_string(); let parameters = HashMap::from([ (ICEBERG_FIELD_ID.to_string(), "1".to_string()), @@ -337,8 +375,13 @@ mod tests { .location(metadata.location()) .build(); - let result = - convert_to_glue_table(&table_name, metadata_location, &metadata, &properties, None)?; + let result = convert_to_glue_table( + &table_name, + metadata_location, + &metadata, + metadata.properties(), + None, + )?; assert_eq!(result.name(), &table_name); assert_eq!(result.description(), None); @@ -436,7 +479,7 @@ mod tests { #[tokio::test] async fn test_config_with_custom_endpoint() { let properties = HashMap::new(); - let endpoint_url = "http://custom_url:5000"; + let endpoint_url = "http://custom_url:5001"; let sdk_config = create_sdk_config(&properties, Some(&endpoint_url.to_string())).await; diff --git a/crates/catalog/glue/testdata/glue_catalog/docker-compose.yaml b/crates/catalog/glue/testdata/glue_catalog/docker-compose.yaml deleted file mode 100644 index bb77d5f8b5..0000000000 --- a/crates/catalog/glue/testdata/glue_catalog/docker-compose.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -services: - minio: - image: minio/minio:RELEASE.2025-05-24T17-08-30Z - expose: - - 9000 - - 9001 - environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio - command: [ "server", "/data", "--console-address", ":9001" ] - - mc: - depends_on: - - minio - image: minio/mc:RELEASE.2025-05-21T01-59-54Z - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - entrypoint: > - /bin/sh -c " until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; /usr/bin/mc mb minio/warehouse; /usr/bin/mc policy set public minio/warehouse; tail -f /dev/null " - - moto: - image: motoserver/moto:5.0.3 - expose: - - 5000 diff --git a/crates/catalog/glue/tests/glue_catalog_test.rs b/crates/catalog/glue/tests/glue_catalog_test.rs deleted file mode 100644 index 548fc4f0f3..0000000000 --- a/crates/catalog/glue/tests/glue_catalog_test.rs +++ /dev/null @@ -1,503 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for glue catalog. - -use std::collections::HashMap; -use std::net::SocketAddr; -use std::sync::RwLock; - -use ctor::{ctor, dtor}; -use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY}; -use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; -use iceberg::{ - Catalog, CatalogBuilder, Namespace, NamespaceIdent, Result, TableCreation, TableIdent, -}; -use iceberg_catalog_glue::{ - AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, GLUE_CATALOG_PROP_URI, - GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder, -}; -use iceberg_test_utils::docker::DockerCompose; -use iceberg_test_utils::{normalize_test_name, set_up}; -use port_scanner::scan_port_addr; -use tokio::time::sleep; -use tracing::info; - -const GLUE_CATALOG_PORT: u16 = 5000; -const MINIO_PORT: u16 = 9000; -static DOCKER_COMPOSE_ENV: RwLock> = RwLock::new(None); - -#[ctor] -fn before_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - let docker_compose = DockerCompose::new( - normalize_test_name(module_path!()), - format!("{}/testdata/glue_catalog", env!("CARGO_MANIFEST_DIR")), - ); - docker_compose.up(); - guard.replace(docker_compose); -} - -#[dtor] -fn after_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - guard.take(); -} - -async fn get_catalog() -> GlueCatalog { - set_up(); - - let (glue_catalog_ip, minio_ip) = { - let guard = DOCKER_COMPOSE_ENV.read().unwrap(); - let docker_compose = guard.as_ref().unwrap(); - ( - docker_compose.get_container_ip("moto"), - docker_compose.get_container_ip("minio"), - ) - }; - let glue_socket_addr = SocketAddr::new(glue_catalog_ip, GLUE_CATALOG_PORT); - let minio_socket_addr = SocketAddr::new(minio_ip, MINIO_PORT); - while !scan_port_addr(glue_socket_addr) { - info!("Waiting for 1s glue catalog to ready..."); - sleep(std::time::Duration::from_millis(1000)).await; - } - - while !scan_port_addr(minio_socket_addr) { - info!("Waiting for 1s minio to ready..."); - sleep(std::time::Duration::from_millis(1000)).await; - } - - let props = HashMap::from([ - (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()), - ( - AWS_SECRET_ACCESS_KEY.to_string(), - "my_secret_key".to_string(), - ), - (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), - ( - S3_ENDPOINT.to_string(), - format!("http://{}", minio_socket_addr), - ), - (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), - (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), - (S3_REGION.to_string(), "us-east-1".to_string()), - ]); - - // Wait for bucket to actually exist - let file_io = iceberg::io::FileIO::from_path("s3a://") - .unwrap() - .with_props(props.clone()) - .build() - .unwrap(); - - let mut retries = 0; - while retries < 30 { - if file_io.exists("s3a://warehouse/").await.unwrap_or(false) { - info!("S3 bucket 'warehouse' is ready"); - break; - } - info!("Waiting for bucket creation... (attempt {})", retries + 1); - sleep(std::time::Duration::from_millis(1000)).await; - retries += 1; - } - - let mut glue_props = HashMap::from([ - ( - GLUE_CATALOG_PROP_URI.to_string(), - format!("http://{}", glue_socket_addr), - ), - ( - GLUE_CATALOG_PROP_WAREHOUSE.to_string(), - "s3a://warehouse/hive".to_string(), - ), - ]); - glue_props.extend(props.clone()); - - GlueCatalogBuilder::default() - .load("glue", glue_props) - .await - .unwrap() -} - -async fn set_test_namespace(catalog: &GlueCatalog, namespace: &NamespaceIdent) -> Result<()> { - let properties = HashMap::new(); - catalog.create_namespace(namespace, properties).await?; - - Ok(()) -} - -fn set_table_creation(location: Option, name: impl ToString) -> Result { - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - let builder = TableCreation::builder() - .name(name.to_string()) - .properties(HashMap::new()) - .location_opt(location) - .schema(schema); - - Ok(builder.build()) -} - -#[tokio::test] -async fn test_rename_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_rename_table".into())); - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let table = catalog.create_table(namespace.name(), creation).await?; - - let dest = TableIdent::new(namespace.name().clone(), "my_table_rename".to_string()); - - catalog.rename_table(table.identifier(), &dest).await?; - - let table = catalog.load_table(&dest).await?; - assert_eq!(table.identifier(), &dest); - - let src = TableIdent::new(namespace.name().clone(), "my_table".to_string()); - - let src_table_exists = catalog.table_exists(&src).await?; - assert!(!src_table_exists); - - Ok(()) -} - -#[tokio::test] -async fn test_table_exists() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_table_exists".into())); - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let ident = TableIdent::new(namespace.name().clone(), "my_table".to_string()); - - let exists = catalog.table_exists(&ident).await?; - assert!(!exists); - - let table = catalog.create_table(namespace.name(), creation).await?; - - let exists = catalog.table_exists(table.identifier()).await?; - - assert!(exists); - - Ok(()) -} - -#[tokio::test] -async fn test_drop_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_drop_table".into())); - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let table = catalog.create_table(namespace.name(), creation).await?; - - catalog.drop_table(table.identifier()).await?; - - let result = catalog.table_exists(table.identifier()).await?; - - assert!(!result); - - Ok(()) -} - -#[tokio::test] -async fn test_load_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_load_table".into())); - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let expected = catalog.create_table(namespace.name(), creation).await?; - - let result = catalog - .load_table(&TableIdent::new( - namespace.name().clone(), - "my_table".to_string(), - )) - .await?; - - assert_eq!(result.identifier(), expected.identifier()); - assert_eq!(result.metadata_location(), expected.metadata_location()); - assert_eq!(result.metadata(), expected.metadata()); - - Ok(()) -} - -#[tokio::test] -async fn test_create_table() -> Result<()> { - let catalog = get_catalog().await; - let namespace = NamespaceIdent::new("test_create_table".to_string()); - set_test_namespace(&catalog, &namespace).await?; - // inject custom location, ignore the namespace prefix - let creation = set_table_creation(Some("s3a://warehouse/hive".into()), "my_table")?; - let result = catalog.create_table(&namespace, creation).await?; - - assert_eq!(result.identifier().name(), "my_table"); - assert!( - result - .metadata_location() - .is_some_and(|location| location.starts_with("s3a://warehouse/hive/metadata/00000-")) - ); - assert!( - catalog - .file_io() - .exists("s3a://warehouse/hive/metadata/") - .await? - ); - - Ok(()) -} - -#[tokio::test] -async fn test_list_tables() -> Result<()> { - let catalog = get_catalog().await; - let namespace = NamespaceIdent::new("test_list_tables".to_string()); - set_test_namespace(&catalog, &namespace).await?; - - let expected = vec![]; - let result = catalog.list_tables(&namespace).await?; - - assert_eq!(result, expected); - - Ok(()) -} - -#[tokio::test] -async fn test_drop_namespace() -> Result<()> { - let catalog = get_catalog().await; - let namespace = NamespaceIdent::new("test_drop_namespace".to_string()); - set_test_namespace(&catalog, &namespace).await?; - - let exists = catalog.namespace_exists(&namespace).await?; - assert!(exists); - - catalog.drop_namespace(&namespace).await?; - - let exists = catalog.namespace_exists(&namespace).await?; - assert!(!exists); - - Ok(()) -} - -#[tokio::test] -async fn test_update_namespace() -> Result<()> { - let catalog = get_catalog().await; - let namespace = NamespaceIdent::new("test_update_namespace".into()); - set_test_namespace(&catalog, &namespace).await?; - - let before_update = catalog.get_namespace(&namespace).await?; - let before_update = before_update.properties().get("description"); - - assert_eq!(before_update, None); - - let properties = HashMap::from([("description".to_string(), "my_update".to_string())]); - - catalog.update_namespace(&namespace, properties).await?; - - let after_update = catalog.get_namespace(&namespace).await?; - let after_update = after_update.properties().get("description"); - - assert_eq!(after_update, Some("my_update".to_string()).as_ref()); - - Ok(()) -} - -#[tokio::test] -async fn test_namespace_exists() -> Result<()> { - let catalog = get_catalog().await; - - let namespace = NamespaceIdent::new("test_namespace_exists".into()); - - let exists = catalog.namespace_exists(&namespace).await?; - assert!(!exists); - - set_test_namespace(&catalog, &namespace).await?; - - let exists = catalog.namespace_exists(&namespace).await?; - assert!(exists); - - Ok(()) -} - -#[tokio::test] -async fn test_get_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let namespace = NamespaceIdent::new("test_get_namespace".into()); - - let does_not_exist = catalog.get_namespace(&namespace).await; - assert!(does_not_exist.is_err()); - - set_test_namespace(&catalog, &namespace).await?; - - let result = catalog.get_namespace(&namespace).await?; - let expected = Namespace::new(namespace); - - assert_eq!(result, expected); - - Ok(()) -} - -#[tokio::test] -async fn test_create_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let properties = HashMap::new(); - let namespace = NamespaceIdent::new("test_create_namespace".into()); - - let expected = Namespace::new(namespace.clone()); - - let result = catalog.create_namespace(&namespace, properties).await?; - - assert_eq!(result, expected); - - Ok(()) -} - -#[tokio::test] -async fn test_list_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let namespace = NamespaceIdent::new("test_list_namespace".to_string()); - set_test_namespace(&catalog, &namespace).await?; - - let result = catalog.list_namespaces(None).await?; - assert!(result.contains(&namespace)); - - let empty_result = catalog.list_namespaces(Some(&namespace)).await?; - assert!(empty_result.is_empty()); - - Ok(()) -} - -#[tokio::test] -async fn test_update_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_update_table".into())); - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let expected = catalog.create_table(namespace.name(), creation).await?; - - let table = catalog - .load_table(&TableIdent::new( - namespace.name().clone(), - "my_table".to_string(), - )) - .await?; - - assert_eq!(table.identifier(), expected.identifier()); - assert_eq!(table.metadata_location(), expected.metadata_location()); - assert_eq!(table.metadata(), expected.metadata()); - - // Store the original metadata location for comparison - let original_metadata_location = table.metadata_location(); - - // Update table properties using the transaction - let tx = Transaction::new(&table); - let tx = tx - .update_table_properties() - .set("test_property".to_string(), "test_value".to_string()) - .apply(tx)?; - - // Commit the transaction to the catalog - let updated_table = tx.commit(&catalog).await?; - - // Verify the update was successful - assert_eq!( - updated_table.metadata().properties().get("test_property"), - Some(&"test_value".to_string()) - ); - - // Verify the metadata location has been updated - assert_ne!( - updated_table.metadata_location(), - original_metadata_location, - "Metadata location should be updated after commit" - ); - - // Load the table again from the catalog to verify changes were persisted - let reloaded_table = catalog.load_table(table.identifier()).await?; - - // Verify the reloaded table matches the updated table - assert_eq!( - reloaded_table.metadata().properties().get("test_property"), - Some(&"test_value".to_string()) - ); - assert_eq!( - reloaded_table.metadata_location(), - updated_table.metadata_location(), - "Reloaded table should have the same metadata location as the updated table" - ); - - Ok(()) -} - -#[tokio::test] -async fn test_register_table() -> Result<()> { - let catalog = get_catalog().await; - let namespace = NamespaceIdent::new("test_register_table".into()); - set_test_namespace(&catalog, &namespace).await?; - - let creation = set_table_creation( - Some("s3a://warehouse/hive/test_register_table".into()), - "my_table", - )?; - let table = catalog.create_table(&namespace, creation).await?; - let metadata_location = table - .metadata_location() - .expect("Expected metadata location to be set") - .to_string(); - - catalog.drop_table(table.identifier()).await?; - let ident = TableIdent::new(namespace.clone(), "my_table".to_string()); - - let registered = catalog - .register_table(&ident, metadata_location.clone()) - .await?; - - assert_eq!(registered.identifier(), &ident); - assert_eq!( - registered.metadata_location(), - Some(metadata_location.as_str()) - ); - - Ok(()) -} diff --git a/crates/catalog/hms/Cargo.toml b/crates/catalog/hms/Cargo.toml index 549dbb9c02..ecd3a89cbe 100644 --- a/crates/catalog/hms/Cargo.toml +++ b/crates/catalog/hms/Cargo.toml @@ -19,6 +19,7 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg-catalog-hms" +publish = true rust-version = { workspace = true } version = { workspace = true } @@ -54,10 +55,12 @@ motore-macros = { workspace = true } volo = { workspace = true } [dev-dependencies] -ctor = { workspace = true } +iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } -port_scanner = { workspace = true } [package.metadata.cargo-machete] # These dependencies are added to ensure minimal dependency version ignored = ["faststr", "linkedbytes", "metainfo", "volo", "motore-macros"] + +[lints] +workspace = true diff --git a/crates/catalog/hms/DEPENDENCIES.rust.tsv b/crates/catalog/hms/DEPENDENCIES.rust.tsv deleted file mode 100644 index cef38cabcd..0000000000 --- a/crates/catalog/hms/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,362 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayvec@0.7.6 X X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-ord@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-broadcast@0.7.2 X X -async-lock@3.4.1 X X -async-recursion@1.1.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -cc@1.2.36 X X -cfg-if@1.0.3 X X -cfg_aliases@0.2.1 X -chrono@0.4.42 X X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -dashmap@6.1.0 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -faststr@0.2.32 X X -find-msvc-tools@0.1.1 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -half@2.6.0 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hive_metastore@0.2.0 X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-catalog-hms@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -integer-encoding@4.0.2 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -linked-hash-map@0.5.6 X X -linkedbytes@0.1.16 X X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -memoffset@0.9.1 X -metainfo@0.7.14 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -motore@0.4.1 X X -motore-macros@0.4.3 X X -mur3@0.1.0 X -murmur3@0.5.2 X X -nix@0.29.0 X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -num_enum@0.7.4 X X X -num_enum_derive@0.7.4 X X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pilota@0.11.10 X X -pin-project@1.1.10 X X -pin-project-internal@1.1.10 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro-crate@3.3.0 X X -proc-macro2@1.0.101 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -redox_syscall@0.5.17 X -ref-cast@1.0.24 X X -ref-cast-impl@1.0.24 X X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc-hash@2.1.1 X X -rustc_version@0.4.1 X X -rustls@0.23.31 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.6 X X -simdutf8@0.1.5 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.5.10 X X -socket2@0.6.0 X X -sonic-number@0.1.0 X -sonic-rs@0.3.17 X -sonic-simd@0.1.1 X -stable_deref_trait@1.2.0 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-stream@0.1.17 X -tokio-util@0.7.16 X -toml_datetime@0.6.11 X X -toml_edit@0.22.27 X X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -volo@0.10.7 X X -volo-thrift@0.10.8 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -webpki-roots@1.0.2 X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -winnow@0.7.13 X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/hms/public-api.txt b/crates/catalog/hms/public-api.txt new file mode 100644 index 0000000000..9ea5f553b3 --- /dev/null +++ b/crates/catalog/hms/public-api.txt @@ -0,0 +1,45 @@ +pub mod iceberg_catalog_hms +pub enum iceberg_catalog_hms::HmsThriftTransport +pub iceberg_catalog_hms::HmsThriftTransport::Buffered +pub iceberg_catalog_hms::HmsThriftTransport::Framed +impl core::default::Default for iceberg_catalog_hms::HmsThriftTransport +pub fn iceberg_catalog_hms::HmsThriftTransport::default() -> iceberg_catalog_hms::HmsThriftTransport +impl core::fmt::Debug for iceberg_catalog_hms::HmsThriftTransport +pub fn iceberg_catalog_hms::HmsThriftTransport::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_catalog_hms::HmsCatalog +impl iceberg_catalog_hms::HmsCatalog +pub fn iceberg_catalog_hms::HmsCatalog::file_io(&self) -> iceberg::io::file_io::FileIO +impl core::fmt::Debug for iceberg_catalog_hms::HmsCatalog +pub fn iceberg_catalog_hms::HmsCatalog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::Catalog for iceberg_catalog_hms::HmsCatalog +pub fn iceberg_catalog_hms::HmsCatalog::create_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::create_table<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, creation: iceberg::catalog::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::drop_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::drop_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::get_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::list_namespaces<'life0, 'life1, 'async_trait>(&'life0 self, parent: core::option::Option<&'life1 iceberg::catalog::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::list_tables<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::load_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::namespace_exists<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::purge_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::register_table<'life0, 'life1, 'async_trait>(&'life0 self, _table_ident: &'life1 iceberg::catalog::TableIdent, _metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::rename_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, src: &'life1 iceberg::catalog::TableIdent, dest: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::table_exists<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::update_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_hms::HmsCatalog::update_table<'life0, 'async_trait>(&'life0 self, _commit: iceberg::catalog::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub struct iceberg_catalog_hms::HmsCatalogBuilder +impl core::default::Default for iceberg_catalog_hms::HmsCatalogBuilder +pub fn iceberg_catalog_hms::HmsCatalogBuilder::default() -> Self +impl core::fmt::Debug for iceberg_catalog_hms::HmsCatalogBuilder +pub fn iceberg_catalog_hms::HmsCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::CatalogBuilder for iceberg_catalog_hms::HmsCatalogBuilder +pub type iceberg_catalog_hms::HmsCatalogBuilder::C = iceberg_catalog_hms::HmsCatalog +pub fn iceberg_catalog_hms::HmsCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg_catalog_hms::HmsCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg_catalog_hms::HmsCatalogBuilder::with_runtime(self, runtime: iceberg::runtime::Runtime) -> Self +pub fn iceberg_catalog_hms::HmsCatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +pub const iceberg_catalog_hms::HMS_CATALOG_PROP_THRIFT_TRANSPORT: &str +pub const iceberg_catalog_hms::HMS_CATALOG_PROP_URI: &str +pub const iceberg_catalog_hms::HMS_CATALOG_PROP_WAREHOUSE: &str +pub const iceberg_catalog_hms::THRIFT_TRANSPORT_BUFFERED: &str +pub const iceberg_catalog_hms::THRIFT_TRANSPORT_FRAMED: &str diff --git a/crates/catalog/hms/src/catalog.rs b/crates/catalog/hms/src/catalog.rs index c8f046cb7e..5ea493a5f3 100644 --- a/crates/catalog/hms/src/catalog.rs +++ b/crates/catalog/hms/src/catalog.rs @@ -18,19 +18,21 @@ use std::collections::HashMap; use std::fmt::{Debug, Formatter}; use std::net::ToSocketAddrs; +use std::sync::Arc; use anyhow::anyhow; use async_trait::async_trait; use hive_metastore::{ - ThriftHiveMetastoreClient, ThriftHiveMetastoreClientBuilder, - ThriftHiveMetastoreGetDatabaseException, ThriftHiveMetastoreGetTableException, + GetTableRequest, ThriftHiveMetastoreClient, ThriftHiveMetastoreClientBuilder, + ThriftHiveMetastoreGetDatabaseException, ThriftHiveMetastoreGetTableReqException, }; -use iceberg::io::FileIO; +use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use iceberg::io::{FileIO, FileIOBuilder, StorageFactory}; use iceberg::spec::{TableMetadata, TableMetadataBuilder}; use iceberg::table::Table; use iceberg::{ Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, - TableCommit, TableCreation, TableIdent, + Runtime, TableCommit, TableCreation, TableIdent, }; use volo_thrift::MaybeException; @@ -50,38 +52,63 @@ pub const THRIFT_TRANSPORT_BUFFERED: &str = "buffered"; /// HMS Catalog warehouse location pub const HMS_CATALOG_PROP_WAREHOUSE: &str = "warehouse"; -/// Builder for [`RestCatalog`]. +/// Builder for [`HmsCatalog`]. #[derive(Debug)] -pub struct HmsCatalogBuilder(HmsCatalogConfig); +pub struct HmsCatalogBuilder { + config: HmsCatalogConfig, + storage_factory: Option>, + kms_client_factory: Option>, + runtime: Option, +} impl Default for HmsCatalogBuilder { fn default() -> Self { - Self(HmsCatalogConfig { - name: None, - address: "".to_string(), - thrift_transport: HmsThriftTransport::default(), - warehouse: "".to_string(), - props: HashMap::new(), - }) + Self { + config: HmsCatalogConfig { + name: None, + address: "".to_string(), + thrift_transport: HmsThriftTransport::default(), + warehouse: "".to_string(), + props: HashMap::new(), + }, + storage_factory: None, + kms_client_factory: None, + runtime: None, + } } } impl CatalogBuilder for HmsCatalogBuilder { type C = HmsCatalog; + fn with_storage_factory(mut self, storage_factory: Arc) -> Self { + self.storage_factory = Some(storage_factory); + self + } + + fn with_kms_client_factory(mut self, kms_client_factory: Arc) -> Self { + self.kms_client_factory = Some(kms_client_factory); + self + } + + fn with_runtime(mut self, runtime: Runtime) -> Self { + self.runtime = Some(runtime); + self + } + fn load( mut self, name: impl Into, props: HashMap, ) -> impl Future> + Send { - self.0.name = Some(name.into()); + self.config.name = Some(name.into()); if props.contains_key(HMS_CATALOG_PROP_URI) { - self.0.address = props.get(HMS_CATALOG_PROP_URI).cloned().unwrap_or_default(); + self.config.address = props.get(HMS_CATALOG_PROP_URI).cloned().unwrap_or_default(); } if let Some(tt) = props.get(HMS_CATALOG_PROP_THRIFT_TRANSPORT) { - self.0.thrift_transport = match tt.to_lowercase().as_str() { + self.config.thrift_transport = match tt.to_lowercase().as_str() { THRIFT_TRANSPORT_FRAMED => HmsThriftTransport::Framed, THRIFT_TRANSPORT_BUFFERED => HmsThriftTransport::Buffered, _ => HmsThriftTransport::default(), @@ -89,13 +116,13 @@ impl CatalogBuilder for HmsCatalogBuilder { } if props.contains_key(HMS_CATALOG_PROP_WAREHOUSE) { - self.0.warehouse = props + self.config.warehouse = props .get(HMS_CATALOG_PROP_WAREHOUSE) .cloned() .unwrap_or_default(); } - self.0.props = props + self.config.props = props .into_iter() .filter(|(k, _)| { k != HMS_CATALOG_PROP_URI @@ -104,28 +131,36 @@ impl CatalogBuilder for HmsCatalogBuilder { }) .collect(); - let result = { - if self.0.name.is_none() { - Err(Error::new( + async move { + let kms_client = match self.kms_client_factory { + Some(factory) => Some(factory.create_kms_client(&self.config.props).await?), + None => None, + }; + + if self.config.name.is_none() { + return Err(Error::new( ErrorKind::DataInvalid, "Catalog name is required", - )) - } else if self.0.address.is_empty() { - Err(Error::new( + )); + } + if self.config.address.is_empty() { + return Err(Error::new( ErrorKind::DataInvalid, "Catalog address is required", - )) - } else if self.0.warehouse.is_empty() { - Err(Error::new( + )); + } + if self.config.warehouse.is_empty() { + return Err(Error::new( ErrorKind::DataInvalid, "Catalog warehouse is required", - )) - } else { - HmsCatalog::new(self.0) + )); } - }; - - std::future::ready(result) + let runtime = match self.runtime { + Some(rt) => rt, + None => Runtime::try_current()?, + }; + HmsCatalog::new(self.config, self.storage_factory, runtime, kms_client) + } } } @@ -157,6 +192,8 @@ pub struct HmsCatalog { config: HmsCatalogConfig, client: HmsClient, file_io: FileIO, + runtime: Runtime, + kms_client: Option>, } impl Debug for HmsCatalog { @@ -169,7 +206,12 @@ impl Debug for HmsCatalog { impl HmsCatalog { /// Create a new hms catalog. - fn new(config: HmsCatalogConfig) -> Result { + fn new( + config: HmsCatalogConfig, + storage_factory: Option>, + runtime: Runtime, + kms_client: Option>, + ) -> Result { let address = config .address .as_str() @@ -194,14 +236,22 @@ impl HmsCatalog { .build(), }; - let file_io = FileIO::from_path(&config.warehouse)? + let factory = storage_factory.ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "StorageFactory must be provided for HmsCatalog. Use `with_storage_factory` to configure it.", + ) + })?; + let file_io = FileIOBuilder::new(factory) .with_props(&config.props) - .build()?; + .build(); Ok(Self { config, client: HmsClient(client), file_io, + runtime, + kms_client, }) } /// Get the catalogs `FileIO` @@ -258,6 +308,12 @@ impl Catalog for HmsCatalog { namespace: &NamespaceIdent, properties: HashMap, ) -> Result { + if self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceAlreadyExists, + format!("Namespace {namespace:?} already exists"), + )); + } let database = convert_to_database(namespace, &properties)?; self.client @@ -282,13 +338,29 @@ impl Catalog for HmsCatalog { async fn get_namespace(&self, namespace: &NamespaceIdent) -> Result { let name = validate_namespace(namespace)?; - let db = self + let resp = self .client .0 .get_database(name.into()) .await - .map(from_thrift_exception) - .map_err(from_thrift_error)??; + .map_err(from_thrift_error)?; + + let db = match resp { + MaybeException::Ok(db) => db, + MaybeException::Exception(ThriftHiveMetastoreGetDatabaseException::O1(_)) => { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} not found"), + )); + } + MaybeException::Exception(exception) => { + return Err(Error::new( + ErrorKind::Unexpected, + "Operation failed for hitting thrift error".to_string(), + ) + .with_source(anyhow!("thrift error: {exception:?}"))); + } + }; let ns = convert_to_namespace(&db)?; @@ -321,7 +393,7 @@ impl Catalog for HmsCatalog { ErrorKind::Unexpected, "Operation failed for hitting thrift error".to_string(), ) - .with_source(anyhow!("thrift error: {:?}", exception))), + .with_source(anyhow!("thrift error: {exception:?}"))), Err(err) => Err(from_thrift_error(err)), } } @@ -341,6 +413,12 @@ impl Catalog for HmsCatalog { namespace: &NamespaceIdent, properties: HashMap, ) -> Result<()> { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } let db = convert_to_database(namespace, &properties)?; let name = match &db.name { @@ -372,6 +450,13 @@ impl Catalog for HmsCatalog { async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> { let name = validate_namespace(namespace)?; + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + self.client .0 .drop_database(name.into(), false, false) @@ -392,6 +477,12 @@ impl Catalog for HmsCatalog { /// querying the database. async fn list_tables(&self, namespace: &NamespaceIdent) -> Result> { let name = validate_namespace(namespace)?; + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } let tables = self .client @@ -442,17 +533,17 @@ impl Catalog for HmsCatalog { .build()? .metadata; - let metadata_location = - MetadataLocation::new_with_table_location(location.clone()).to_string(); + let metadata_location = MetadataLocation::try_new_with_metadata(&metadata)?; metadata.write_to(&self.file_io, &metadata_location).await?; + let metadata_location_str = metadata_location.to_string(); let hive_table = convert_to_hive_table( db_name.clone(), metadata.current_schema(), table_name.clone(), location, - metadata_location.clone(), + metadata_location_str.clone(), metadata.properties(), )?; @@ -462,12 +553,16 @@ impl Catalog for HmsCatalog { .await .map_err(from_thrift_error)?; - Table::builder() + let mut builder = Table::builder() .file_io(self.file_io()) - .metadata_location(metadata_location) + .metadata_location(metadata_location_str) .metadata(metadata) .identifier(TableIdent::new(NamespaceIdent::new(db_name), table_name)) - .build() + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + builder.build() } /// Loads a table from the Hive Metastore and constructs a `Table` object @@ -488,16 +583,21 @@ impl Catalog for HmsCatalog { let hive_table = self .client .0 - .get_table(db_name.clone().into(), table.name.clone().into()) + .get_table_req(GetTableRequest { + db_name: db_name.clone().into(), + tbl_name: table.name.clone().into(), + ..Default::default() + }) .await .map(from_thrift_exception) - .map_err(from_thrift_error)??; + .map_err(from_thrift_error)?? + .table; let metadata_location = get_metadata_location(&hive_table.parameters)?; let metadata = TableMetadata::read_from(&self.file_io, &metadata_location).await?; - Table::builder() + let mut builder = Table::builder() .file_io(self.file_io()) .metadata_location(metadata_location) .metadata(metadata) @@ -505,7 +605,11 @@ impl Catalog for HmsCatalog { NamespaceIdent::new(db_name), table.name.clone(), )) - .build() + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + builder.build() } /// Asynchronously drops a table from the database. @@ -520,6 +624,18 @@ impl Catalog for HmsCatalog { /// - Any network or communication error occurs with the database backend. async fn drop_table(&self, table: &TableIdent) -> Result<()> { let db_name = validate_namespace(table.namespace())?; + if !self.namespace_exists(table.namespace()).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {:?} does not exist", table.namespace()), + )); + } + if !self.table_exists(table).await? { + return Err(Error::new( + ErrorKind::TableNotFound, + format!("Table {table:?} does not exist"), + )); + } self.client .0 @@ -530,6 +646,12 @@ impl Catalog for HmsCatalog { Ok(()) } + async fn purge_table(&self, table: &TableIdent) -> Result<()> { + let table_info = self.load_table(table).await?; + self.drop_table(table).await?; + iceberg::drop_table_data(&table_info).await + } + /// Asynchronously checks the existence of a specified table /// in the database. /// @@ -544,17 +666,23 @@ impl Catalog for HmsCatalog { let resp = self .client .0 - .get_table(db_name.into(), table_name.into()) + .get_table_req(GetTableRequest { + db_name: db_name.into(), + tbl_name: table_name.into(), + ..Default::default() + }) .await; match resp { Ok(MaybeException::Ok(_)) => Ok(true), - Ok(MaybeException::Exception(ThriftHiveMetastoreGetTableException::O2(_))) => Ok(false), + Ok(MaybeException::Exception(ThriftHiveMetastoreGetTableReqException::O2(_))) => { + Ok(false) + } Ok(MaybeException::Exception(exception)) => Err(Error::new( ErrorKind::Unexpected, "Operation failed for hitting thrift error".to_string(), ) - .with_source(anyhow!("thrift error: {:?}", exception))), + .with_source(anyhow!("thrift error: {exception:?}"))), Err(err) => Err(from_thrift_error(err)), } } @@ -568,6 +696,12 @@ impl Catalog for HmsCatalog { async fn rename_table(&self, src: &TableIdent, dest: &TableIdent) -> Result<()> { let src_dbname = validate_namespace(src.namespace())?; let dest_dbname = validate_namespace(dest.namespace())?; + if self.table_exists(dest).await? { + return Err(Error::new( + ErrorKind::TableAlreadyExists, + format!("Destination table {dest:?} already exists"), + )); + } let src_tbl_name = src.name.clone(); let dest_tbl_name = dest.name.clone(); @@ -575,10 +709,15 @@ impl Catalog for HmsCatalog { let mut tbl = self .client .0 - .get_table(src_dbname.clone().into(), src_tbl_name.clone().into()) + .get_table_req(GetTableRequest { + db_name: src_dbname.clone().into(), + tbl_name: src_tbl_name.clone().into(), + ..Default::default() + }) .await .map(from_thrift_exception) - .map_err(from_thrift_error)??; + .map_err(from_thrift_error)?? + .table; tbl.db_name = Some(dest_dbname.into()); tbl.table_name = Some(dest_tbl_name.into()); diff --git a/crates/catalog/hms/src/error.rs b/crates/catalog/hms/src/error.rs index 15da3eaf68..ad3cca070c 100644 --- a/crates/catalog/hms/src/error.rs +++ b/crates/catalog/hms/src/error.rs @@ -30,7 +30,7 @@ pub fn from_thrift_error(error: impl std::error::Error) -> Error { ErrorKind::Unexpected, "Operation failed for hitting thrift error".to_string(), ) - .with_source(anyhow!("thrift error: {:?}", error)) + .with_source(anyhow!("thrift error: {error:?}")) } /// Format a thrift exception into iceberg error. @@ -41,7 +41,7 @@ pub fn from_thrift_exception(value: MaybeException) -> Result ErrorKind::Unexpected, "Operation failed for hitting thrift error".to_string(), ) - .with_source(anyhow!("thrift error: {:?}", err))), + .with_source(anyhow!("thrift error: {err:?}"))), } } diff --git a/crates/catalog/hms/src/schema.rs b/crates/catalog/hms/src/schema.rs index 6b7f71c6a0..5909454bf9 100644 --- a/crates/catalog/hms/src/schema.rs +++ b/crates/catalog/hms/src/schema.rs @@ -16,7 +16,7 @@ // under the License. use hive_metastore::FieldSchema; -use iceberg::spec::{PrimitiveType, Schema, SchemaVisitor, visit_schema}; +use iceberg::spec::{PrimitiveType, Schema, SchemaVisitor, VariantType, visit_schema}; use iceberg::{Error, ErrorKind, Result}; type HiveSchema = Vec; @@ -49,18 +49,11 @@ impl HiveSchemaBuilder { impl SchemaVisitor for HiveSchemaBuilder { type T = String; - fn schema( - &mut self, - _schema: &iceberg::spec::Schema, - value: String, - ) -> iceberg::Result { + fn schema(&mut self, _schema: &Schema, value: String) -> Result { Ok(value) } - fn before_struct_field( - &mut self, - _field: &iceberg::spec::NestedFieldRef, - ) -> iceberg::Result<()> { + fn before_struct_field(&mut self, _field: &iceberg::spec::NestedFieldRef) -> Result<()> { self.depth += 1; Ok(()) } @@ -69,23 +62,16 @@ impl SchemaVisitor for HiveSchemaBuilder { &mut self, r#_struct: &iceberg::spec::StructType, results: Vec, - ) -> iceberg::Result { + ) -> Result { Ok(format!("struct<{}>", results.join(", "))) } - fn after_struct_field( - &mut self, - _field: &iceberg::spec::NestedFieldRef, - ) -> iceberg::Result<()> { + fn after_struct_field(&mut self, _field: &iceberg::spec::NestedFieldRef) -> Result<()> { self.depth -= 1; Ok(()) } - fn field( - &mut self, - field: &iceberg::spec::NestedFieldRef, - value: String, - ) -> iceberg::Result { + fn field(&mut self, field: &iceberg::spec::NestedFieldRef, value: String) -> Result { if self.is_inside_struct() { return Ok(format!("{}:{}", field.name, value)); } @@ -99,8 +85,8 @@ impl SchemaVisitor for HiveSchemaBuilder { Ok(value) } - fn list(&mut self, _list: &iceberg::spec::ListType, value: String) -> iceberg::Result { - Ok(format!("array<{}>", value)) + fn list(&mut self, _list: &iceberg::spec::ListType, value: String) -> Result { + Ok(format!("array<{value}>")) } fn map( @@ -108,12 +94,18 @@ impl SchemaVisitor for HiveSchemaBuilder { _map: &iceberg::spec::MapType, key_value: String, value: String, - ) -> iceberg::Result { - Ok(format!("map<{},{}>", key_value, value)) + ) -> Result { + Ok(format!("map<{key_value},{value}>")) } - fn primitive(&mut self, p: &iceberg::spec::PrimitiveType) -> iceberg::Result { + fn primitive(&mut self, p: &PrimitiveType) -> Result { let hive_type = match p { + PrimitiveType::Unknown => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Conversion from {p:?} is not supported"), + )); + } PrimitiveType::Boolean => "boolean".to_string(), PrimitiveType::Int => "int".to_string(), PrimitiveType::Long => "bigint".to_string(), @@ -122,30 +114,41 @@ impl SchemaVisitor for HiveSchemaBuilder { PrimitiveType::Date => "date".to_string(), PrimitiveType::Timestamp => "timestamp".to_string(), PrimitiveType::TimestampNs => "timestamp_ns".to_string(), - PrimitiveType::TimestamptzNs => "timestamptz_ns".to_string(), + PrimitiveType::Timestamptz | PrimitiveType::TimestamptzNs => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Conversion from {p:?} is not supported"), + )); + } PrimitiveType::Time | PrimitiveType::String | PrimitiveType::Uuid => { "string".to_string() } PrimitiveType::Binary | PrimitiveType::Fixed(_) => "binary".to_string(), - PrimitiveType::Decimal { precision, scale } => { - format!("decimal({},{})", precision, scale) - } - _ => { + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { return Err(Error::new( ErrorKind::FeatureUnsupported, - "Conversion from 'Timestamptz' is not supported", + format!("Conversion from {p:?} is not supported"), )); } + PrimitiveType::Decimal { precision, scale } => { + format!("decimal({precision},{scale})") + } }; Ok(hive_type) } + + fn variant(&mut self, _v: &VariantType) -> Result { + // Match iceberg-java's HiveSchemaUtil, which maps VARIANT to "unknown" + // (apache/iceberg#15964). + Ok("unknown".to_string()) + } } #[cfg(test)] mod tests { use iceberg::Result; - use iceberg::spec::Schema; + use iceberg::spec::{GeometryType, NestedField, PrimitiveType, Schema, Type}; use super::*; @@ -457,4 +460,54 @@ mod tests { Ok(()) } + + #[test] + fn test_schema_with_variant() -> Result<()> { + // VARIANT maps to Hive "unknown", matching iceberg-java's HiveSchemaUtil + // (apache/iceberg#15964). + let record = r#"{ + "type": "struct", + "schema-id": 1, + "fields": [ + { + "id": 1, + "name": "v", + "required": true, + "type": "variant" + } + ] + }"#; + + let schema = serde_json::from_str::(record)?; + + let result = HiveSchemaBuilder::from_iceberg(&schema)?.build(); + + let expected = vec![FieldSchema { + name: Some("v".into()), + r#type: Some("unknown".into()), + comment: None, + }]; + + assert_eq!(result, expected); + + Ok(()) + } + + #[test] + fn test_schema_with_geospatial_type_is_unsupported() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(GeometryType::default())), + ) + .into(), + ]) + .build() + .unwrap(); + + let err = HiveSchemaBuilder::from_iceberg(&schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + } } diff --git a/crates/catalog/hms/src/utils.rs b/crates/catalog/hms/src/utils.rs index 1b65bf0ccb..e5c342c72f 100644 --- a/crates/catalog/hms/src/utils.rs +++ b/crates/catalog/hms/src/utils.rs @@ -37,7 +37,7 @@ const OWNER: &str = "owner"; const COMMENT: &str = "comment"; /// hive metatore `location` property const LOCATION: &str = "location"; -/// hive metatore `metadat_location` property +/// hive metatore `metadata_location` property const METADATA_LOCATION: &str = "metadata_location"; /// hive metatore `external` property const EXTERNAL: &str = "EXTERNAL"; @@ -128,7 +128,7 @@ pub(crate) fn convert_to_database( _ => { return Err(Error::new( ErrorKind::DataInvalid, - format!("Invalid value for setting 'owner_type': {}", v), + format!("Invalid value for setting 'owner_type': {v}"), )); } }; @@ -214,8 +214,7 @@ pub(crate) fn validate_namespace(namespace: &NamespaceIdent) -> Result { return Err(Error::new( ErrorKind::DataInvalid, format!( - "Invalid database name: {:?}, hierarchical namespaces are not supported", - namespace + "Invalid database name: {namespace:?}, hierarchical namespaces are not supported" ), )); } @@ -257,7 +256,7 @@ pub(crate) fn get_metadata_location( Some(location) => Ok(location.to_string()), None => Err(Error::new( ErrorKind::DataInvalid, - format!("No '{}' set on table", METADATA_LOCATION), + format!("No '{METADATA_LOCATION}' set on table"), )), }, None => Err(Error::new( @@ -272,7 +271,7 @@ fn format_location_uri(location: String) -> String { let mut location = location; if !location.starts_with('/') { - location = format!("/{}", location); + location = format!("/{location}"); } if location.ends_with('/') && location.len() > 1 { @@ -292,8 +291,7 @@ fn validate_owner_settings(properties: &HashMap) -> Result<()> { return Err(Error::new( ErrorKind::DataInvalid, format!( - "Setting '{}' without setting '{}' is not allowed", - HMS_DB_OWNER_TYPE, HMS_DB_OWNER + "Setting '{HMS_DB_OWNER_TYPE}' without setting '{HMS_DB_OWNER}' is not allowed" ), )); } @@ -313,8 +311,8 @@ fn get_current_time() -> Result { #[cfg(test)] mod tests { - use iceberg::spec::{NestedField, PrimitiveType, Type}; - use iceberg::{MetadataLocation, Namespace, NamespaceIdent}; + use iceberg::spec::{NestedField, PrimitiveType, TableMetadataBuilder, Type}; + use iceberg::{MetadataLocation, Namespace, NamespaceIdent, TableCreation}; use super::*; @@ -345,8 +343,6 @@ mod tests { let db_name = "my_db".to_string(); let table_name = "my_table".to_string(); let location = "s3a://warehouse/hms".to_string(); - let metadata_location = - MetadataLocation::new_with_table_location(location.clone()).to_string(); let properties = HashMap::new(); let schema = Schema::builder() .with_schema_id(1) @@ -356,6 +352,17 @@ mod tests { ]) .build()?; + let table_creation = TableCreation::builder() + .name(table_name.clone()) + .location(location.clone()) + .schema(schema.clone()) + .properties(properties.clone()) + .build(); + let metadata = TableMetadataBuilder::from_table_creation(table_creation)? + .build()? + .metadata; + let metadata_location = MetadataLocation::try_new_with_metadata(&metadata)?.to_string(); + let result = convert_to_hive_table( db_name.clone(), &schema, diff --git a/crates/catalog/hms/testdata/hms_catalog/Dockerfile b/crates/catalog/hms/testdata/hms_catalog/Dockerfile deleted file mode 100644 index 8392e174a8..0000000000 --- a/crates/catalog/hms/testdata/hms_catalog/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM --platform=$BUILDPLATFORM openjdk:8-jre-slim AS build - -ARG BUILDPLATFORM - -RUN apt-get update -qq && apt-get -qq -y install curl - -ENV AWSSDK_VERSION=2.20.18 -ENV HADOOP_VERSION=3.1.0 - -RUN curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.271/aws-java-sdk-bundle-1.11.271.jar -Lo /tmp/aws-java-sdk-bundle-1.11.271.jar -RUN curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar -Lo /tmp/hadoop-aws-${HADOOP_VERSION}.jar - - -FROM apache/hive:3.1.3 - -ENV AWSSDK_VERSION=2.20.18 -ENV HADOOP_VERSION=3.1.0 - -COPY --from=build /tmp/hadoop-aws-${HADOOP_VERSION}.jar /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar -COPY --from=build /tmp/aws-java-sdk-bundle-1.11.271.jar /opt/hive/lib/aws-java-sdk-bundle-1.11.271.jar -COPY core-site.xml /opt/hadoop/etc/hadoop/core-site.xml \ No newline at end of file diff --git a/crates/catalog/hms/testdata/hms_catalog/docker-compose.yaml b/crates/catalog/hms/testdata/hms_catalog/docker-compose.yaml deleted file mode 100644 index 2729f562c4..0000000000 --- a/crates/catalog/hms/testdata/hms_catalog/docker-compose.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -services: - minio: - image: minio/minio:RELEASE.2025-05-24T17-08-30Z - expose: - - 9000 - - 9001 - environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio - command: [ "server", "/data", "--console-address", ":9001" ] - - mc: - depends_on: - - minio - image: minio/mc:RELEASE.2025-05-21T01-59-54Z - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - entrypoint: > - /bin/sh -c " until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; /usr/bin/mc mb minio/warehouse; /usr/bin/mc policy set public minio/warehouse; tail -f /dev/null " - - hive-metastore: - image: iceberg-hive-metastore - build: ./ - platform: ${DOCKER_DEFAULT_PLATFORM} - expose: - - 9083 - environment: - SERVICE_NAME: "metastore" - SERVICE_OPTS: "-Dmetastore.warehouse.dir=s3a://warehouse/hive/" diff --git a/crates/catalog/hms/tests/hms_catalog_test.rs b/crates/catalog/hms/tests/hms_catalog_test.rs index 2bf12779b1..b64184ce77 100644 --- a/crates/catalog/hms/tests/hms_catalog_test.rs +++ b/crates/catalog/hms/tests/hms_catalog_test.rs @@ -16,76 +16,37 @@ // under the License. //! Integration tests for hms catalog. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. +//! Each test uses unique namespaces based on module path to avoid conflicts. use std::collections::HashMap; -use std::net::SocketAddr; -use std::sync::RwLock; +use std::sync::Arc; -use ctor::{ctor, dtor}; -use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY}; -use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; -use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent, TableCreation, TableIdent}; +use iceberg::io::{ + FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, + S3_SECRET_ACCESS_KEY, +}; +use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent}; use iceberg_catalog_hms::{ HMS_CATALOG_PROP_THRIFT_TRANSPORT, HMS_CATALOG_PROP_URI, HMS_CATALOG_PROP_WAREHOUSE, HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED, }; -use iceberg_test_utils::docker::DockerCompose; -use iceberg_test_utils::{normalize_test_name, set_up}; -use port_scanner::scan_port_addr; +use iceberg_storage_opendal::OpenDalStorageFactory; +use iceberg_test_utils::{get_hms_endpoint, get_object_store_endpoint, set_up}; use tokio::time::sleep; use tracing::info; -const HMS_CATALOG_PORT: u16 = 9083; -const MINIO_PORT: u16 = 9000; -static DOCKER_COMPOSE_ENV: RwLock> = RwLock::new(None); type Result = std::result::Result; -#[ctor] -fn before_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - let docker_compose = DockerCompose::new( - normalize_test_name(module_path!()), - format!("{}/testdata/hms_catalog", env!("CARGO_MANIFEST_DIR")), - ); - docker_compose.up(); - guard.replace(docker_compose); -} - -#[dtor] -fn after_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - guard.take(); -} - async fn get_catalog() -> HmsCatalog { set_up(); - let (hms_catalog_ip, minio_ip) = { - let guard = DOCKER_COMPOSE_ENV.read().unwrap(); - let docker_compose = guard.as_ref().unwrap(); - ( - docker_compose.get_container_ip("hive-metastore"), - docker_compose.get_container_ip("minio"), - ) - }; - let hms_socket_addr = SocketAddr::new(hms_catalog_ip, HMS_CATALOG_PORT); - let minio_socket_addr = SocketAddr::new(minio_ip, MINIO_PORT); - while !scan_port_addr(hms_socket_addr) { - info!("scan hms_socket_addr {} check", hms_socket_addr); - info!("Waiting for 1s hms catalog to ready..."); - sleep(std::time::Duration::from_millis(1000)).await; - } - - while !scan_port_addr(minio_socket_addr) { - info!("Waiting for 1s minio to ready..."); - sleep(std::time::Duration::from_millis(1000)).await; - } + let hms_endpoint = get_hms_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); let props = HashMap::from([ - ( - HMS_CATALOG_PROP_URI.to_string(), - hms_socket_addr.to_string(), - ), + (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), ( HMS_CATALOG_PROP_THRIFT_TRANSPORT.to_string(), THRIFT_TRANSPORT_BUFFERED.to_string(), @@ -94,21 +55,19 @@ async fn get_catalog() -> HmsCatalog { HMS_CATALOG_PROP_WAREHOUSE.to_string(), "s3a://warehouse/hive".to_string(), ), - ( - S3_ENDPOINT.to_string(), - format!("http://{}", minio_socket_addr), - ), + (S3_ENDPOINT.to_string(), object_store_endpoint), (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), ]); // Wait for bucket to actually exist - let file_io = iceberg::io::FileIO::from_path("s3a://") - .unwrap() - .with_props(props.clone()) - .build() - .unwrap(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .with_props(props.clone()) + .build(); let mut retries = 0; while retries < 30 { @@ -122,207 +81,14 @@ async fn get_catalog() -> HmsCatalog { } HmsCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) .load("hms", props) .await .unwrap() } -async fn set_test_namespace(catalog: &HmsCatalog, namespace: &NamespaceIdent) -> Result<()> { - let properties = HashMap::new(); - - catalog.create_namespace(namespace, properties).await?; - - Ok(()) -} - -fn set_table_creation(location: Option, name: impl ToString) -> Result { - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - let builder = TableCreation::builder() - .name(name.to_string()) - .properties(HashMap::new()) - .location_opt(location) - .schema(schema); - - Ok(builder.build()) -} - -#[tokio::test] -async fn test_rename_table() -> Result<()> { - let catalog = get_catalog().await; - let creation: TableCreation = set_table_creation(None, "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_rename_table".into())); - set_test_namespace(&catalog, namespace.name()).await?; - - let table: iceberg::table::Table = catalog.create_table(namespace.name(), creation).await?; - - let dest = TableIdent::new(namespace.name().clone(), "my_table_rename".to_string()); - - catalog.rename_table(table.identifier(), &dest).await?; - - let result = catalog.table_exists(&dest).await?; - - assert!(result); - - Ok(()) -} - -#[tokio::test] -async fn test_table_exists() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_table_exists".into())); - set_test_namespace(&catalog, namespace.name()).await?; - - let table = catalog.create_table(namespace.name(), creation).await?; - - let result = catalog.table_exists(table.identifier()).await?; - - assert!(result); - - Ok(()) -} - -#[tokio::test] -async fn test_drop_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_drop_table".into())); - set_test_namespace(&catalog, namespace.name()).await?; - - let table = catalog.create_table(namespace.name(), creation).await?; - - catalog.drop_table(table.identifier()).await?; - - let result = catalog.table_exists(table.identifier()).await?; - - assert!(!result); - - Ok(()) -} - -#[tokio::test] -async fn test_load_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_load_table".into())); - set_test_namespace(&catalog, namespace.name()).await?; - - let expected = catalog.create_table(namespace.name(), creation).await?; - - let result = catalog - .load_table(&TableIdent::new( - namespace.name().clone(), - "my_table".to_string(), - )) - .await?; - - assert_eq!(result.identifier(), expected.identifier()); - assert_eq!(result.metadata_location(), expected.metadata_location()); - assert_eq!(result.metadata(), expected.metadata()); - - Ok(()) -} - -#[tokio::test] -async fn test_create_table() -> Result<()> { - let catalog = get_catalog().await; - // inject custom location, ignore the namespace prefix - let creation = set_table_creation(Some("s3a://warehouse/hive".into()), "my_table")?; - let namespace = Namespace::new(NamespaceIdent::new("test_create_table".into())); - set_test_namespace(&catalog, namespace.name()).await?; - - let result = catalog.create_table(namespace.name(), creation).await?; - - assert_eq!(result.identifier().name(), "my_table"); - assert!( - result - .metadata_location() - .is_some_and(|location| location.starts_with("s3a://warehouse/hive/metadata/00000-")) - ); - assert!( - catalog - .file_io() - .exists("s3a://warehouse/hive/metadata/") - .await? - ); - - Ok(()) -} - -#[tokio::test] -async fn test_list_tables() -> Result<()> { - let catalog = get_catalog().await; - let ns = Namespace::new(NamespaceIdent::new("test_list_tables".into())); - let result = catalog.list_tables(ns.name()).await?; - set_test_namespace(&catalog, ns.name()).await?; - - assert_eq!(result, vec![]); - - let creation = set_table_creation(None, "my_table")?; - catalog.create_table(ns.name(), creation).await?; - let result = catalog.list_tables(ns.name()).await?; - - assert_eq!(result, vec![TableIdent::new( - ns.name().clone(), - "my_table".to_string() - )]); - - Ok(()) -} - -#[tokio::test] -async fn test_list_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let result_no_parent = catalog.list_namespaces(None).await?; - - let result_with_parent = catalog - .list_namespaces(Some(&NamespaceIdent::new("parent".into()))) - .await?; - - assert!(result_no_parent.contains(&NamespaceIdent::new("default".into()))); - assert!(result_with_parent.is_empty()); - - Ok(()) -} - -#[tokio::test] -async fn test_create_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let properties = HashMap::from([ - ("comment".to_string(), "my_description".to_string()), - ("location".to_string(), "my_location".to_string()), - ( - "hive.metastore.database.owner".to_string(), - "apache".to_string(), - ), - ( - "hive.metastore.database.owner-type".to_string(), - "user".to_string(), - ), - ("key1".to_string(), "value1".to_string()), - ]); - - let ns = Namespace::with_properties( - NamespaceIdent::new("test_create_namespace".into()), - properties.clone(), - ); - - let result = catalog.create_namespace(ns.name(), properties).await?; - - assert_eq!(result, ns); - - Ok(()) -} - #[tokio::test] async fn test_get_default_namespace() -> Result<()> { let catalog = get_catalog().await; @@ -349,58 +115,3 @@ async fn test_get_default_namespace() -> Result<()> { Ok(()) } - -#[tokio::test] -async fn test_namespace_exists() -> Result<()> { - let catalog = get_catalog().await; - - let ns_exists = Namespace::new(NamespaceIdent::new("default".into())); - let ns_not_exists = Namespace::new(NamespaceIdent::new("test_namespace_exists".into())); - - let result_exists = catalog.namespace_exists(ns_exists.name()).await?; - let result_not_exists = catalog.namespace_exists(ns_not_exists.name()).await?; - - assert!(result_exists); - assert!(!result_not_exists); - - Ok(()) -} - -#[tokio::test] -async fn test_update_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let ns = NamespaceIdent::new("test_update_namespace".into()); - set_test_namespace(&catalog, &ns).await?; - let properties = HashMap::from([("comment".to_string(), "my_update".to_string())]); - - catalog.update_namespace(&ns, properties).await?; - - let db = catalog.get_namespace(&ns).await?; - - assert_eq!( - db.properties().get("comment"), - Some(&"my_update".to_string()) - ); - - Ok(()) -} - -#[tokio::test] -async fn test_drop_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let ns = Namespace::new(NamespaceIdent::new("delete_me".into())); - - catalog.create_namespace(ns.name(), HashMap::new()).await?; - - let result = catalog.namespace_exists(ns.name()).await?; - assert!(result); - - catalog.drop_namespace(ns.name()).await?; - - let result = catalog.namespace_exists(ns.name()).await?; - assert!(!result); - - Ok(()) -} diff --git a/crates/catalog/loader/Cargo.toml b/crates/catalog/loader/Cargo.toml index 014f84a3a1..28e9615ec2 100644 --- a/crates/catalog/loader/Cargo.toml +++ b/crates/catalog/loader/Cargo.toml @@ -19,20 +19,33 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg-catalog-loader" +publish = true rust-version = { workspace = true } version = { workspace = true } categories = ["database"] description = "Apache Iceberg Catalog Loader API" -keywords = ["iceberg", "catalog"] +keywords = ["iceberg", "catalog", "loader"] license = { workspace = true } repository = { workspace = true } [dependencies] +async-trait = { workspace = true } iceberg = { workspace = true } -iceberg-catalog-rest = { workspace = true } iceberg-catalog-glue = { workspace = true } -iceberg-catalog-s3tables = { workspace = true } iceberg-catalog-hms = { workspace = true } +iceberg-catalog-rest = { workspace = true } +iceberg-catalog-s3tables = { workspace = true } +iceberg-catalog-sql = { workspace = true } tokio = { workspace = true } -async-trait = { workspace = true } + +[dev-dependencies] +iceberg-storage-opendal = { workspace = true } +iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } +reqwest = { workspace = true } +rstest = { workspace = true } +sqlx = { workspace = true, features = ["runtime-tokio", "sqlite", "migrate"] } +tempfile = { workspace = true } + +[lints] +workspace = true diff --git a/crates/catalog/loader/DEPENDENCIES.rust.tsv b/crates/catalog/loader/DEPENDENCIES.rust.tsv deleted file mode 100644 index d809a30ca4..0000000000 --- a/crates/catalog/loader/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,419 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayvec@0.7.6 X X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-ord@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-broadcast@0.7.2 X X -async-lock@3.4.1 X X -async-recursion@1.1.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-config@1.8.6 X -aws-credential-types@1.2.6 X -aws-runtime@1.5.10 X -aws-sdk-glue@1.119.0 X -aws-sdk-s3tables@1.37.0 X -aws-sdk-sso@1.83.0 X -aws-sdk-ssooidc@1.84.0 X -aws-sdk-sts@1.85.0 X -aws-sigv4@1.3.4 X -aws-smithy-async@1.2.5 X -aws-smithy-http@0.62.3 X -aws-smithy-http-client@1.1.1 X -aws-smithy-json@0.61.5 X -aws-smithy-observability@0.1.3 X -aws-smithy-query@0.60.7 X -aws-smithy-runtime@1.9.1 X -aws-smithy-runtime-api@1.9.0 X -aws-smithy-types@1.3.2 X -aws-smithy-xml@0.60.10 X -aws-types@1.3.8 X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.21.7 X X -base64@0.22.1 X X -base64-simd@0.8.0 X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -bytes-utils@0.1.4 X X -cc@1.2.36 X X -cfg-if@1.0.3 X X -cfg_aliases@0.2.1 X -chrono@0.4.42 X X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation@0.10.1 X X -core-foundation@0.9.4 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -dashmap@6.1.0 X -deranged@0.5.3 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -faststr@0.2.32 X X -find-msvc-tools@0.1.1 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -h2@0.3.27 X -h2@0.4.12 X -half@2.6.0 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hive_metastore@0.2.0 X -hmac@0.12.1 X X -home@0.5.11 X X -http@0.2.12 X X -http@1.3.1 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hyper@0.14.32 X -hyper@1.7.0 X -hyper-rustls@0.24.2 X X X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-catalog-glue@0.7.0 X -iceberg-catalog-hms@0.7.0 X -iceberg-catalog-loader@0.7.0 X -iceberg-catalog-rest@0.7.0 X -iceberg-catalog-s3tables@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -integer-encoding@4.0.2 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -linked-hash-map@0.5.6 X X -linkedbytes@0.1.16 X X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -memoffset@0.9.1 X -metainfo@0.7.14 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -motore@0.4.1 X X -motore-macros@0.4.3 X X -mur3@0.1.0 X -murmur3@0.5.2 X X -nix@0.29.0 X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-conv@0.1.0 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -num_enum@0.7.4 X X X -num_enum_derive@0.7.4 X X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -openssl-probe@0.1.6 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pilota@0.11.10 X X -pin-project@1.1.10 X X -pin-project-internal@1.1.10 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro-crate@3.3.0 X X -proc-macro2@1.0.101 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -redox_syscall@0.5.17 X -ref-cast@1.0.24 X X -ref-cast-impl@1.0.24 X X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc-hash@2.1.1 X X -rustc_version@0.4.1 X X -rustls@0.21.12 X X X -rustls@0.23.31 X X X -rustls-native-certs@0.6.3 X X X -rustls-native-certs@0.8.1 X X X -rustls-pemfile@1.0.4 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.101.7 X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -schannel@0.1.27 X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -sct@0.7.1 X X X -security-framework@2.11.1 X X -security-framework@3.4.0 X X -security-framework-sys@2.15.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.6 X X -simdutf8@0.1.5 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.5.10 X X -socket2@0.6.0 X X -sonic-number@0.1.0 X -sonic-rs@0.3.17 X -sonic-simd@0.1.1 X -stable_deref_trait@1.2.0 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -time@0.3.43 X X -time-core@0.1.6 X X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.24.1 X X -tokio-rustls@0.26.2 X X -tokio-stream@0.1.17 X -tokio-util@0.7.16 X -toml_datetime@0.6.11 X X -toml_edit@0.22.27 X X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.7 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -volo@0.10.7 X X -volo-thrift@0.10.8 X X -vsimd@0.8.0 X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -webpki-roots@1.0.2 X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -winnow@0.7.13 X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -xmlparser@0.13.6 X X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integrations/datafusion/LICENSE b/crates/catalog/loader/LICENSE similarity index 100% rename from crates/integrations/datafusion/LICENSE rename to crates/catalog/loader/LICENSE diff --git a/crates/integrations/datafusion/NOTICE b/crates/catalog/loader/NOTICE similarity index 100% rename from crates/integrations/datafusion/NOTICE rename to crates/catalog/loader/NOTICE diff --git a/crates/catalog/loader/public-api.txt b/crates/catalog/loader/public-api.txt new file mode 100644 index 0000000000..cf4fb8d73f --- /dev/null +++ b/crates/catalog/loader/public-api.txt @@ -0,0 +1,21 @@ +pub mod iceberg_catalog_loader +pub struct iceberg_catalog_loader::CatalogLoader<'a> +impl iceberg_catalog_loader::CatalogLoader<'_> +pub async fn iceberg_catalog_loader::CatalogLoader<'_>::load(self, name: alloc::string::String, props: std::collections::hash::map::HashMap) -> iceberg::error::Result> +pub fn iceberg_catalog_loader::CatalogLoader<'_>::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg_catalog_loader::CatalogLoader<'_>::with_runtime(self, runtime: iceberg::runtime::Runtime) -> Self +pub fn iceberg_catalog_loader::CatalogLoader<'_>::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +impl<'a> core::convert::From<&'a str> for iceberg_catalog_loader::CatalogLoader<'a> +pub fn iceberg_catalog_loader::CatalogLoader<'a>::from(s: &'a str) -> Self +pub trait iceberg_catalog_loader::BoxedCatalogBuilder: core::marker::Send +pub fn iceberg_catalog_loader::BoxedCatalogBuilder::load<'async_trait>(self: alloc::boxed::Box, name: alloc::string::String, props: std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait +pub fn iceberg_catalog_loader::BoxedCatalogBuilder::with_kms_client_factory(self: alloc::boxed::Box, kms_client_factory: alloc::sync::Arc) -> alloc::boxed::Box +pub fn iceberg_catalog_loader::BoxedCatalogBuilder::with_runtime(self: alloc::boxed::Box, runtime: iceberg::runtime::Runtime) -> alloc::boxed::Box +pub fn iceberg_catalog_loader::BoxedCatalogBuilder::with_storage_factory(self: alloc::boxed::Box, storage_factory: alloc::sync::Arc) -> alloc::boxed::Box +impl iceberg_catalog_loader::BoxedCatalogBuilder for T +pub fn T::load<'async_trait>(self: alloc::boxed::Box, name: alloc::string::String, props: std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait +pub fn T::with_kms_client_factory(self: alloc::boxed::Box, kms_client_factory: alloc::sync::Arc) -> alloc::boxed::Box +pub fn T::with_runtime(self: alloc::boxed::Box, runtime: iceberg::runtime::Runtime) -> alloc::boxed::Box +pub fn T::with_storage_factory(self: alloc::boxed::Box, storage_factory: alloc::sync::Arc) -> alloc::boxed::Box +pub fn iceberg_catalog_loader::load(type: &str) -> iceberg::error::Result> +pub fn iceberg_catalog_loader::supported_types() -> alloc::vec::Vec<&'static str> diff --git a/crates/catalog/loader/src/lib.rs b/crates/catalog/loader/src/lib.rs index 9c18ab4e5f..63e405c97b 100644 --- a/crates/catalog/loader/src/lib.rs +++ b/crates/catalog/loader/src/lib.rs @@ -19,21 +19,25 @@ use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; -use iceberg::{Catalog, CatalogBuilder, Error, ErrorKind, Result}; +use iceberg::encryption::kms::KmsClientFactory; +use iceberg::io::StorageFactory; +use iceberg::{Catalog, CatalogBuilder, Error, ErrorKind, Result, Runtime}; use iceberg_catalog_glue::GlueCatalogBuilder; use iceberg_catalog_hms::HmsCatalogBuilder; use iceberg_catalog_rest::RestCatalogBuilder; use iceberg_catalog_s3tables::S3TablesCatalogBuilder; +use iceberg_catalog_sql::SqlCatalogBuilder; -/// A CatalogBuilderFactory creating a new catalog builder. -type CatalogBuilderFactory = fn() -> Box; +/// Creates a new catalog builder, boxed as a [`BoxedCatalogBuilder`]. +type BoxedCatalogBuilderFactory = fn() -> Box; -/// A registry of catalog builders. -static CATALOG_REGISTRY: &[(&str, CatalogBuilderFactory)] = &[ +/// Maps each catalog type to the factory for its builder. +static CATALOG_REGISTRY: &[(&str, BoxedCatalogBuilderFactory)] = &[ ("rest", || Box::new(RestCatalogBuilder::default())), ("glue", || Box::new(GlueCatalogBuilder::default())), ("s3tables", || Box::new(S3TablesCatalogBuilder::default())), ("hms", || Box::new(HmsCatalogBuilder::default())), + ("sql", || Box::new(SqlCatalogBuilder::default())), ]; /// Return the list of supported catalog types. @@ -42,7 +46,25 @@ pub fn supported_types() -> Vec<&'static str> { } #[async_trait] -pub trait BoxedCatalogBuilder { +pub trait BoxedCatalogBuilder: Send { + /// Sets the storage factory used to build the catalog's `FileIO`; see + /// [`CatalogBuilder::with_storage_factory`]. + fn with_storage_factory( + self: Box, + storage_factory: Arc, + ) -> Box; + + /// Sets the KMS client factory used to enable table encryption; see + /// [`CatalogBuilder::with_kms_client_factory`]. + fn with_kms_client_factory( + self: Box, + kms_client_factory: Arc, + ) -> Box; + + /// Sets the runtime the catalog, and the tables it creates, spawn their + /// tasks on; see [`CatalogBuilder::with_runtime`]. + fn with_runtime(self: Box, runtime: Runtime) -> Box; + async fn load( self: Box, name: String, @@ -52,6 +74,27 @@ pub trait BoxedCatalogBuilder { #[async_trait] impl BoxedCatalogBuilder for T { + fn with_storage_factory( + self: Box, + storage_factory: Arc, + ) -> Box { + Box::new(CatalogBuilder::with_storage_factory(*self, storage_factory)) + } + + fn with_kms_client_factory( + self: Box, + kms_client_factory: Arc, + ) -> Box { + Box::new(CatalogBuilder::with_kms_client_factory( + *self, + kms_client_factory, + )) + } + + fn with_runtime(self: Box, runtime: Runtime) -> Box { + Box::new(CatalogBuilder::with_runtime(*self, runtime)) + } + async fn load( self: Box, name: String, @@ -85,30 +128,84 @@ pub fn load(r#type: &str) -> Result> { /// Ergonomic catalog loader builder pattern. pub struct CatalogLoader<'a> { catalog_type: &'a str, + storage_factory: Option>, + kms_client_factory: Option>, + runtime: Option, } impl<'a> From<&'a str> for CatalogLoader<'a> { fn from(s: &'a str) -> Self { - Self { catalog_type: s } + Self { + catalog_type: s, + storage_factory: None, + kms_client_factory: None, + runtime: None, + } } } impl CatalogLoader<'_> { + /// Sets the storage factory used to build the catalog's `FileIO`; see + /// [`CatalogBuilder::with_storage_factory`]. + pub fn with_storage_factory(mut self, storage_factory: Arc) -> Self { + self.storage_factory = Some(storage_factory); + self + } + + /// Sets the KMS client factory used to enable table encryption; see + /// [`CatalogBuilder::with_kms_client_factory`]. + pub fn with_kms_client_factory( + mut self, + kms_client_factory: Arc, + ) -> Self { + self.kms_client_factory = Some(kms_client_factory); + self + } + + /// Sets the runtime the catalog, and the tables it creates, spawn their + /// tasks on; see [`CatalogBuilder::with_runtime`]. + pub fn with_runtime(mut self, runtime: Runtime) -> Self { + self.runtime = Some(runtime); + self + } + pub async fn load( self, name: String, props: HashMap, ) -> Result> { let builder = load(self.catalog_type)?; - builder.load(name, props).await + self.configure(builder).load(name, props).await + } + + /// Applies the settings given to this loader to `builder`. + fn configure(self, mut builder: Box) -> Box { + if let Some(storage_factory) = self.storage_factory { + builder = builder.with_storage_factory(storage_factory); + } + if let Some(kms_client_factory) = self.kms_client_factory { + builder = builder.with_kms_client_factory(kms_client_factory); + } + if let Some(runtime) = self.runtime { + builder = builder.with_runtime(runtime); + } + builder } } #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::{Arc, Mutex}; + + use iceberg::encryption::kms::{KmsClientFactory, MemoryKmsClientFactory}; + use iceberg::io::{LocalFsStorageFactory, StorageFactory}; + use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalog, MemoryCatalogBuilder}; + use iceberg::{CatalogBuilder, Result, Runtime}; + use sqlx::migrate::MigrateDatabase; + use tempfile::TempDir; - use crate::{CatalogLoader, load}; + use crate::{BoxedCatalogBuilder, CatalogLoader, load}; #[tokio::test] async fn test_load_unsupported_catalog() { @@ -204,6 +301,7 @@ mod tests { let catalog_loader = load("hms").unwrap(); let catalog = catalog_loader + .with_storage_factory(Arc::new(LocalFsStorageFactory)) .load( "hms".to_string(), HashMap::from([ @@ -220,6 +318,197 @@ mod tests { assert!(catalog.is_ok()); } + fn temp_path() -> String { + let temp_dir = TempDir::new().unwrap(); + temp_dir.path().to_str().unwrap().to_string() + } + + #[tokio::test] + async fn test_catalog_loader_pattern_sql_catalog() { + use iceberg_catalog_sql::{SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE}; + + let uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&uri).await.unwrap(); + + let catalog_loader = load("sql").unwrap(); + let catalog = catalog_loader + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "sql".to_string(), + HashMap::from([ + (SQL_CATALOG_PROP_URI.to_string(), uri), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + "s3://warehouse".to_string(), + ), + ]), + ) + .await; + + assert!(catalog.is_ok()); + } + + #[tokio::test] + async fn test_catalog_loader_with_storage_factory_loads_sql_catalog() { + use iceberg_catalog_sql::{SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE}; + + let uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&uri).await.unwrap(); + + // A SQL catalog can't be loaded without a storage factory. + CatalogLoader::from("sql") + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "sql".to_string(), + HashMap::from([ + (SQL_CATALOG_PROP_URI.to_string(), uri), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + "s3://warehouse".to_string(), + ), + ]), + ) + .await + .unwrap(); + } + + /// The settings a [`RecordingBuilder`] was given. + #[derive(Debug, Default)] + struct Recorded { + storage_factory: Option>, + kms_client_factory: Option>, + runtime: Option, + } + + /// A memory catalog builder that records the settings it is given. + #[derive(Debug, Default)] + struct RecordingBuilder { + recorded: Arc>, + } + + impl CatalogBuilder for RecordingBuilder { + type C = MemoryCatalog; + + fn with_storage_factory(self, storage_factory: Arc) -> Self { + self.recorded.lock().unwrap().storage_factory = Some(storage_factory); + self + } + + fn with_kms_client_factory(self, kms_client_factory: Arc) -> Self { + self.recorded.lock().unwrap().kms_client_factory = Some(kms_client_factory); + self + } + + fn with_runtime(self, runtime: Runtime) -> Self { + self.recorded.lock().unwrap().runtime = Some(runtime); + self + } + + fn load( + self, + name: impl Into, + props: HashMap, + ) -> impl Future> + Send { + MemoryCatalogBuilder::default().load(name, props) + } + } + + /// A boxed [`RecordingBuilder`], and the settings it records. + fn recording_builder() -> (Box, Arc>) { + let recorded = Arc::new(Mutex::new(Recorded::default())); + let builder = Box::new(RecordingBuilder { + recorded: recorded.clone(), + }); + (builder, recorded) + } + + /// A tokio runtime whose worker thread is named `loader-test-runtime`. + fn named_tokio_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .thread_name("loader-test-runtime") + .enable_all() + .build() + .unwrap() + } + + /// Asserts that tasks spawned on `runtime` run on a thread of + /// [`named_tokio_runtime`]. + async fn assert_spawns_on_named_runtime(runtime: &Runtime) { + let thread = runtime + .io() + .spawn(async { std::thread::current().name().map(str::to_string) }) + .await + .unwrap(); + assert!( + thread + .as_deref() + .is_some_and(|n| n.starts_with("loader-test-runtime")), + "got: {thread:?}" + ); + } + + #[test] + fn test_with_runtime_reaches_the_catalog_builder() { + let tokio_runtime = named_tokio_runtime(); + let (builder, recorded) = recording_builder(); + + tokio_runtime.block_on(async { + builder + .with_runtime(Runtime::new(&tokio_runtime)) + .load( + "memory".to_string(), + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), temp_path())]), + ) + .await + .unwrap(); + + // The builder got the runtime passed to the boxed builder: its + // tasks run on that runtime's threads. + let runtime = recorded.lock().unwrap().runtime.take().unwrap(); + assert_spawns_on_named_runtime(&runtime).await; + }); + } + + #[test] + fn test_catalog_loader_settings_reach_the_catalog_builder() { + let tokio_runtime = named_tokio_runtime(); + let storage_factory: Arc = Arc::new(LocalFsStorageFactory); + let kms_client_factory: Arc = + Arc::new(MemoryKmsClientFactory::default()); + let (builder, recorded) = recording_builder(); + + CatalogLoader::from("memory") + .with_storage_factory(storage_factory.clone()) + .with_kms_client_factory(kms_client_factory.clone()) + .with_runtime(Runtime::new(&tokio_runtime)) + .configure(builder); + + let recorded = std::mem::take(&mut *recorded.lock().unwrap()); + assert!(Arc::ptr_eq( + &recorded.storage_factory.unwrap(), + &storage_factory + )); + assert!(Arc::ptr_eq( + &recorded.kms_client_factory.unwrap(), + &kms_client_factory + )); + tokio_runtime.block_on(assert_spawns_on_named_runtime(&recorded.runtime.unwrap())); + } + + #[test] + fn test_catalog_loader_without_settings_leaves_the_builder_default() { + let (builder, recorded) = recording_builder(); + + CatalogLoader::from("memory").configure(builder); + + assert!(matches!(*recorded.lock().unwrap(), Recorded { + storage_factory: None, + kms_client_factory: None, + runtime: None, + })); + } + #[tokio::test] async fn test_error_message_includes_supported_types() { let err = match load("does-not-exist") { diff --git a/crates/catalog/loader/tests/common/mod.rs b/crates/catalog/loader/tests/common/mod.rs new file mode 100644 index 0000000000..0b7e22cea1 --- /dev/null +++ b/crates/catalog/loader/tests/common/mod.rs @@ -0,0 +1,391 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared helpers for catalog integration suites. + +#![allow(dead_code)] + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use iceberg::encryption::kms::{KmsClientFactory, MemoryKmsClientFactory}; +use iceberg::io::{ + FileIOBuilder, LocalFsStorageFactory, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, + S3_REGION, S3_SECRET_ACCESS_KEY, +}; +use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; +use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, TableProperties, Type}; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation}; +use iceberg_catalog_glue::{ + AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, GLUE_CATALOG_PROP_URI, + GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder, +}; +use iceberg_catalog_hms::{ + HMS_CATALOG_PROP_THRIFT_TRANSPORT, HMS_CATALOG_PROP_URI, HMS_CATALOG_PROP_WAREHOUSE, + HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED, +}; +use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalog, RestCatalogBuilder}; +use iceberg_catalog_s3tables::{ + S3TABLES_CATALOG_PROP_ENDPOINT_URL, S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN, + S3TablesCatalogBuilder, +}; +use iceberg_catalog_sql::{ + SQL_CATALOG_PROP_BIND_STYLE, SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE, SqlBindStyle, + SqlCatalogBuilder, +}; +use iceberg_storage_opendal::OpenDalStorageFactory; +use iceberg_test_utils::{ + get_glue_endpoint, get_hms_endpoint, get_object_store_endpoint, get_rest_catalog_endpoint, + set_up, +}; +use sqlx::migrate::MigrateDatabase; +use tempfile::TempDir; +use tokio::time::sleep; + +#[derive(Debug, Clone, Copy)] +pub enum CatalogKind { + Rest, + Glue, + Hms, + Sql, + S3Tables, + Memory, +} + +impl fmt::Display for CatalogKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + CatalogKind::Rest => "rest_catalog", + CatalogKind::Glue => "glue_catalog", + CatalogKind::Hms => "hms_catalog", + CatalogKind::Sql => "sql_catalog", + CatalogKind::S3Tables => "s3tables_catalog", + CatalogKind::Memory => "memory_catalog", + }; + f.write_str(name) + } +} + +pub struct CatalogHarness { + pub catalog: Arc, + pub label: &'static str, + _tempdirs: Vec, +} + +// Shared setup for each catalog implementation so the suites exercise +// the same behavior against all backends. +pub async fn load_catalog(kind: CatalogKind) -> Option { + set_up(); + let factory = MemoryKmsClientFactory::new(); + factory.add_master_key(TEST_MASTER_KEY_ID).unwrap(); + let kms_client_factory = Arc::new(factory) as Arc; + + match kind { + CatalogKind::Rest => Some(CatalogHarness { + catalog: Arc::new(rest_catalog(Arc::clone(&kms_client_factory)).await) + as Arc, + label: "rest", + _tempdirs: Vec::new(), + }), + CatalogKind::Glue => Some(CatalogHarness { + catalog: Arc::new(glue_catalog(Arc::clone(&kms_client_factory)).await) + as Arc, + label: "glue", + _tempdirs: Vec::new(), + }), + CatalogKind::Hms => Some(CatalogHarness { + catalog: Arc::new(hms_catalog(Arc::clone(&kms_client_factory)).await) + as Arc, + label: "hms", + _tempdirs: Vec::new(), + }), + CatalogKind::Sql => { + let warehouse_dir = TempDir::new().unwrap(); + let db_dir = TempDir::new().unwrap(); + let db_path = db_dir.path().join("catalog.db"); + let db_uri = format!("sqlite:{}", db_path.to_str().unwrap()); + sqlx::Sqlite::create_database(&db_uri).await.unwrap(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .with_kms_client_factory(Arc::clone(&kms_client_factory)) + .load( + "sql", + HashMap::from([ + (SQL_CATALOG_PROP_URI.to_string(), db_uri), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + warehouse_dir.path().to_str().unwrap().to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]), + ) + .await + .unwrap(); + + Some(CatalogHarness { + catalog: Arc::new(catalog) as Arc, + label: "sql", + _tempdirs: vec![warehouse_dir, db_dir], + }) + } + CatalogKind::S3Tables => { + let table_bucket_arn = match std::env::var("TABLE_BUCKET_ARN").ok() { + Some(value) => value, + None => return None, + }; + + let mut props = HashMap::from([( + S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN.to_string(), + table_bucket_arn, + )]); + + if let Ok(endpoint_url) = std::env::var("S3TABLES_ENDPOINT_URL") { + props.insert(S3TABLES_CATALOG_PROP_ENDPOINT_URL.to_string(), endpoint_url); + } + + let catalog = S3TablesCatalogBuilder::default() + .with_kms_client_factory(Arc::clone(&kms_client_factory)) + .load("s3tables", props) + .await + .unwrap(); + + Some(CatalogHarness { + catalog: Arc::new(catalog) as Arc, + label: "s3tables", + _tempdirs: Vec::new(), + }) + } + CatalogKind::Memory => { + let warehouse_dir = TempDir::new().unwrap(); + let props = HashMap::from([( + MEMORY_CATALOG_WAREHOUSE.to_string(), + warehouse_dir.path().to_str().unwrap().to_string(), + )]); + let catalog = MemoryCatalogBuilder::default() + .with_kms_client_factory(Arc::clone(&kms_client_factory)) + .load("memory", props) + .await + .unwrap(); + + Some(CatalogHarness { + catalog: Arc::new(catalog) as Arc, + label: "memory", + _tempdirs: vec![warehouse_dir], + }) + } + } +} + +pub const TEST_MASTER_KEY_ID: &str = "test-master-key"; + +// Catalog-specific setup is intentionally isolated here so the suites +// remain implementation-agnostic. +async fn rest_catalog(kms_client_factory: Arc) -> RestCatalog { + let rest_endpoint = get_rest_catalog_endpoint(); + + let client = reqwest::Client::new(); + let mut retries = 0; + while retries < 30 { + if client + .get(format!("{rest_endpoint}/v1/config")) + .send() + .await + .map(|resp| resp.status().is_success()) + .unwrap_or(false) + { + break; + } + sleep(std::time::Duration::from_millis(1000)).await; + retries += 1; + } + + RestCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .with_kms_client_factory(kms_client_factory) + .load( + "rest", + HashMap::from([(REST_CATALOG_PROP_URI.to_string(), rest_endpoint)]), + ) + .await + .unwrap() +} + +async fn glue_catalog(kms_client_factory: Arc) -> GlueCatalog { + let glue_endpoint = get_glue_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); + + let props = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()), + ( + AWS_SECRET_ACCESS_KEY.to_string(), + "my_secret_key".to_string(), + ), + (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), + (S3_ENDPOINT.to_string(), object_store_endpoint), + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + ]); + + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .with_props(props.clone()) + .build(); + + let mut retries = 0; + while retries < 30 { + if file_io.exists("s3a://warehouse/").await.unwrap_or(false) { + break; + } + sleep(std::time::Duration::from_millis(1000)).await; + retries += 1; + } + + let mut glue_props = HashMap::from([ + (GLUE_CATALOG_PROP_URI.to_string(), glue_endpoint), + ( + GLUE_CATALOG_PROP_WAREHOUSE.to_string(), + "s3a://warehouse/hive".to_string(), + ), + ]); + glue_props.extend(props); + + GlueCatalogBuilder::default() + .with_kms_client_factory(kms_client_factory) + .load("glue", glue_props) + .await + .unwrap() +} + +async fn hms_catalog(kms_client_factory: Arc) -> HmsCatalog { + let hms_endpoint = get_hms_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); + + let props = HashMap::from([ + (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), + ( + HMS_CATALOG_PROP_THRIFT_TRANSPORT.to_string(), + THRIFT_TRANSPORT_BUFFERED.to_string(), + ), + ( + HMS_CATALOG_PROP_WAREHOUSE.to_string(), + "s3a://warehouse/hive".to_string(), + ), + (S3_ENDPOINT.to_string(), object_store_endpoint), + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + ]); + + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .with_props(props.clone()) + .build(); + + let mut retries = 0; + while retries < 30 { + if file_io.exists("s3a://warehouse/").await.unwrap_or(false) { + break; + } + sleep(std::time::Duration::from_millis(1000)).await; + retries += 1; + } + + HmsCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .with_kms_client_factory(kms_client_factory) + .load("hms", props) + .await + .unwrap() +} + +// Common table schema used across suites to validate shared behavior. +pub fn table_creation(name: impl ToString) -> TableCreation { + let schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "bar", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + TableCreation::builder() + .name(name.to_string()) + .properties(HashMap::new()) + .schema(schema) + .build() +} + +pub fn encrypted_table_creation(name: impl ToString) -> TableCreation { + TableCreation { + format_version: FormatVersion::V3, + properties: HashMap::from([( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + TEST_MASTER_KEY_ID.to_string(), + )]), + ..table_creation(name) + } +} + +#[derive(Clone, Copy, Debug)] +pub enum TableMode { + Plain, + Encrypted, +} + +impl TableMode { + pub fn table_creation(self, name: impl ToString) -> TableCreation { + match self { + Self::Plain => table_creation(name), + Self::Encrypted => encrypted_table_creation(name), + } + } + + pub const fn label(self) -> &'static str { + match self { + Self::Plain => "plain", + Self::Encrypted => "encrypted", + } + } +} + +pub fn assert_map_contains(expected: &HashMap, actual: &HashMap) { + for (key, value) in expected { + assert_eq!(actual.get(key), Some(value)); + } +} + +pub async fn cleanup_namespace_dyn(catalog: &dyn Catalog, namespace: &NamespaceIdent) { + if let Ok(tables) = catalog.list_tables(namespace).await { + for table in tables { + let _ = catalog.purge_table(&table).await; + } + } + let _ = catalog.drop_namespace(namespace).await; +} diff --git a/crates/catalog/loader/tests/namespace_suite.rs b/crates/catalog/loader/tests/namespace_suite.rs new file mode 100644 index 0000000000..024e8a62b8 --- /dev/null +++ b/crates/catalog/loader/tests/namespace_suite.rs @@ -0,0 +1,355 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Common namespace behavior across catalogs. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; + +use common::{CatalogKind, assert_map_contains, cleanup_namespace_dyn, load_catalog}; +use iceberg::{ErrorKind, NamespaceIdent, Result}; +use iceberg_test_utils::normalize_test_name_with_parts; +use rstest::rstest; + +// Common behavior: querying a missing namespace should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_namespace_missing_returns_error(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_namespace_missing_returns_error", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + let err = catalog.get_namespace(&namespace).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::NamespaceNotFound); + + Ok(()) +} + +// Common behavior: namespace lifecycle CRUD. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_namespace_lifecycle(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_namespace_lifecycle", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + assert!(!catalog.namespace_exists(&namespace).await?); + + let props = HashMap::from([ + ("owner".to_string(), "rust".to_string()), + ("purpose".to_string(), "catalog_suite".to_string()), + ]); + let created = catalog.create_namespace(&namespace, props.clone()).await?; + assert_eq!(created.name(), &namespace); + assert_map_contains(&props, created.properties()); + + let fetched = catalog.get_namespace(&namespace).await?; + assert_eq!(fetched.name(), &namespace); + assert_map_contains(&props, fetched.properties()); + + let namespaces = catalog.list_namespaces(None).await?; + assert!(namespaces.contains(&namespace)); + + catalog.drop_namespace(&namespace).await?; + assert!(!catalog.namespace_exists(&namespace).await?); + + Ok(()) +} + +// Common behavior: update_namespace persists changes when supported. +#[rstest] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_update_namespace_supported(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_namespace_supported", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let updated_props = HashMap::from([("owner".to_string(), "updated".to_string())]); + catalog + .update_namespace(&namespace, updated_props.clone()) + .await?; + + let updated = catalog.get_namespace(&namespace).await?; + assert_map_contains(&updated_props, updated.properties()); + + Ok(()) +} + +// Common behavior: update_namespace returns FeatureUnsupported when not implemented. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[tokio::test] +async fn test_catalog_update_namespace_unsupported(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_namespace_unsupported", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let err = catalog + .update_namespace( + &namespace, + HashMap::from([("key".to_string(), "value".to_string())]), + ) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + + Ok(()) +} + +// Common behavior: listing namespaces under a parent returns its children. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_namespace_listing_with_parent(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let parent_name = + normalize_test_name_with_parts!("catalog_namespace_listing_with_parent", harness.label); + let parent = NamespaceIdent::new(parent_name.clone()); + let child1 = NamespaceIdent::from_strs([&parent_name, "child1"]).unwrap(); + let child2 = NamespaceIdent::from_strs([&parent_name, "child2"]).unwrap(); + + cleanup_namespace_dyn(catalog.as_ref(), &child1).await; + cleanup_namespace_dyn(catalog.as_ref(), &child2).await; + cleanup_namespace_dyn(catalog.as_ref(), &parent).await; + + catalog.create_namespace(&parent, HashMap::new()).await?; + + catalog.create_namespace(&child1, HashMap::new()).await?; + catalog.create_namespace(&child2, HashMap::new()).await?; + + let top_level = catalog.list_namespaces(None).await?; + assert!(top_level.contains(&parent)); + + let children = catalog.list_namespaces(Some(&parent)).await?; + assert!(children.contains(&child1)); + assert!(children.contains(&child2)); + + Ok(()) +} + +// Common behavior: hierarchical namespaces are rejected when unsupported. +#[rstest] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[tokio::test] +async fn test_catalog_namespace_listing_with_parent_unsupported( + #[case] kind: CatalogKind, +) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let parent_name = normalize_test_name_with_parts!( + "catalog_namespace_listing_with_parent_unsupported", + harness.label + ); + let parent = NamespaceIdent::new(parent_name.clone()); + let child = NamespaceIdent::from_strs([&parent_name, "child"]).unwrap(); + + cleanup_namespace_dyn(catalog.as_ref(), &child).await; + cleanup_namespace_dyn(catalog.as_ref(), &parent).await; + + catalog.create_namespace(&parent, HashMap::new()).await?; + + let err = catalog + .create_namespace(&child, HashMap::new()) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + + Ok(()) +} + +// Common behavior: listing top-level namespaces includes created namespaces. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_list_namespaces_contains_created(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let ns_one = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_list_namespaces_contains_created", + harness.label, + "one" + )); + let ns_two = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_list_namespaces_contains_created", + harness.label, + "two" + )); + + cleanup_namespace_dyn(catalog.as_ref(), &ns_one).await; + cleanup_namespace_dyn(catalog.as_ref(), &ns_two).await; + + catalog.create_namespace(&ns_one, HashMap::new()).await?; + catalog.create_namespace(&ns_two, HashMap::new()).await?; + + let namespaces = catalog.list_namespaces(None).await?; + assert!(namespaces.contains(&ns_one)); + assert!(namespaces.contains(&ns_two)); + + Ok(()) +} + +// Common behavior: creating an existing namespace should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_create_namespace_duplicate_fails(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_create_namespace_duplicate_fails", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let err = catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::NamespaceAlreadyExists); + Ok(()) +} + +// Common behavior: update on a missing namespace should return NamespaceNotFound. +#[rstest] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_update_namespace_missing_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_namespace_missing_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + let err = catalog + .update_namespace( + &namespace, + HashMap::from([("key".to_string(), "value".to_string())]), + ) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::NamespaceNotFound); + + Ok(()) +} + +// Common behavior: dropping a missing namespace should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_drop_namespace_missing_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_drop_namespace_missing_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + let err = catalog.drop_namespace(&namespace).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::NamespaceNotFound); + Ok(()) +} diff --git a/crates/catalog/loader/tests/rest_pagination_test.rs b/crates/catalog/loader/tests/rest_pagination_test.rs new file mode 100644 index 0000000000..bf1889c35d --- /dev/null +++ b/crates/catalog/loader/tests/rest_pagination_test.rs @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! REST pagination against the Docker fixture started by `make docker-up`. + +mod common; + +use std::collections::{HashMap, HashSet}; +use std::num::NonZeroU32; +use std::sync::Arc; + +use common::{cleanup_namespace_dyn, table_creation}; +use iceberg::io::LocalFsStorageFactory; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, Result}; +use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalogBuilder}; +use iceberg_test_utils::{get_rest_catalog_endpoint, normalize_test_name_with_parts, set_up}; + +#[tokio::test] +async fn test_rest_lists_all_namespaces_and_tables_with_page_size() -> Result<()> { + set_up(); + let catalog = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .with_page_size(NonZeroU32::MIN) + .load( + "rest", + HashMap::from([( + REST_CATALOG_PROP_URI.to_string(), + get_rest_catalog_endpoint(), + )]), + ) + .await?; + let parent_name = normalize_test_name_with_parts!("pagination"); + let parent = NamespaceIdent::new(parent_name.clone()); + let expected_namespaces = ["child1", "child2", "child3"] + .into_iter() + .map(|name| NamespaceIdent::from_strs([parent_name.as_str(), name])) + .collect::>>()?; + + // Isolate the listing from other suites and clean up remnants of failed runs. + for child in &expected_namespaces { + cleanup_namespace_dyn(&catalog, child).await; + } + cleanup_namespace_dyn(&catalog, &parent).await; + catalog.create_namespace(&parent, HashMap::new()).await?; + for child in &expected_namespaces { + catalog.create_namespace(child, HashMap::new()).await?; + } + let mut expected_tables = Vec::new(); + for name in ["table1", "table2", "table3"] { + let table = catalog.create_table(&parent, table_creation(name)).await?; + expected_tables.push(table.identifier().clone()); + } + + let namespaces = catalog.list_namespaces(Some(&parent)).await; + let tables = catalog.list_tables(&parent).await; + + // Clean up before assertions so a pagination regression does not leave data behind. + for table in &expected_tables { + catalog.purge_table(table).await?; + } + for child in &expected_namespaces { + catalog.drop_namespace(child).await?; + } + catalog.drop_namespace(&parent).await?; + + let namespaces = namespaces?; + let tables = tables?; + + assert_eq!(namespaces.len(), expected_namespaces.len()); + assert_eq!(tables.len(), expected_tables.len()); + assert_eq!( + namespaces.into_iter().collect::>(), + expected_namespaces.into_iter().collect::>() + ); + assert_eq!( + tables.into_iter().collect::>(), + expected_tables.into_iter().collect::>() + ); + Ok(()) +} diff --git a/crates/catalog/loader/tests/schema_update_suite.rs b/crates/catalog/loader/tests/schema_update_suite.rs new file mode 100644 index 0000000000..9421bbf0ee --- /dev/null +++ b/crates/catalog/loader/tests/schema_update_suite.rs @@ -0,0 +1,284 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Common schema-update behavior across catalogs. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; + +use common::{CatalogKind, cleanup_namespace_dyn, load_catalog}; +use iceberg::spec::{NestedField, PrimitiveType, Schema, StructType, Type}; +use iceberg::transaction::{AddColumn, ApplyTransactionAction, Transaction}; +use iceberg::{ErrorKind, NamespaceIdent, Result, TableCreation, TableIdent}; +use iceberg_test_utils::normalize_test_name_with_parts; +use rstest::rstest; + +fn base_schema() -> Schema { + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), + ]) + .with_identifier_field_ids(vec![2]) + .build() + .unwrap() +} + +// Common behavior: adding a top-level column appends it to the schema. +// HMS is excluded because update_table is not yet supported. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_schema_add_column(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_schema_add_column", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = + normalize_test_name_with_parts!("catalog_schema_add_column", harness.label, "table"); + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name(table_name) + .schema(base_schema()) + .build(), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx + .update_schema() + .add_column(AddColumn::optional( + "a", + Type::Primitive(PrimitiveType::Int), + )) + .apply(tx)?; + let updated = tx.commit(catalog.as_ref()).await?; + + let schema = updated.metadata().current_schema(); + let field_a = schema.field_by_name("a").expect("field 'a' should exist"); + assert_eq!(field_a.id, 4); + assert_eq!(*field_a.field_type, Type::Primitive(PrimitiveType::Int)); + + Ok(()) +} + +// Common behavior: adding a nested struct column then a sub-field within it, +// and deleting another top-level column, all persist correctly. +// HMS is excluded because update_table is not yet supported. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_schema_add_nested_and_delete_column(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_schema_add_nested_and_delete_column", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = normalize_test_name_with_parts!( + "catalog_schema_add_nested_and_delete_column", + harness.label, + "table" + ); + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name(table_name) + .schema(base_schema()) + .build(), + ) + .await?; + + // First transaction: add a nested struct column. + let tx = Transaction::new(&table); + let tx = tx + .update_schema() + .add_column(AddColumn::optional( + "info", + Type::Struct(StructType::new(vec![ + NestedField::optional(0, "city", Type::Primitive(PrimitiveType::String)).into(), + ])), + )) + .apply(tx)?; + let table = tx.commit(catalog.as_ref()).await?; + + // Second transaction: add a sub-field to the nested struct and delete a top-level column. + let tx = Transaction::new(&table); + let tx = tx + .update_schema() + .add_column( + AddColumn::builder() + .name("zip") + .field_type(Type::Primitive(PrimitiveType::String)) + .parent("info") + .build(), + ) + .delete_column("baz") + .apply(tx)?; + let table = tx.commit(catalog.as_ref()).await?; + + let schema = table.metadata().current_schema(); + assert!(schema.field_by_name("info").is_some()); + assert!(schema.field_by_name("info.city").is_some()); + assert!(schema.field_by_name("info.zip").is_some()); + assert!(schema.field_by_name("baz").is_none()); + + Ok(()) +} + +// Common behavior: deleting an identifier field or a nonexistent field must fail. +// HMS is excluded because update_table is not yet supported. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_schema_delete_invalid_column_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_schema_delete_invalid_column_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = normalize_test_name_with_parts!( + "catalog_schema_delete_invalid_column_errors", + harness.label, + "table" + ); + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name(table_name) + .schema(base_schema()) + .build(), + ) + .await?; + + // Deleting an identifier field must fail. + let tx = Transaction::new(&table); + let tx = tx.update_schema().delete_column("bar").apply(tx)?; + let err = tx.commit(catalog.as_ref()).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + + // Deleting a nonexistent field must fail. + let tx = Transaction::new(&table); + let tx = tx.update_schema().delete_column("nonexistent").apply(tx)?; + let err = tx.commit(catalog.as_ref()).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + + Ok(()) +} + +// Common behavior: loading a table after a schema update reflects the new schema. +// HMS is excluded because update_table is not yet supported. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_schema_update_persisted_after_reload( + #[case] kind: CatalogKind, +) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_schema_update_persisted_after_reload", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = normalize_test_name_with_parts!( + "catalog_schema_update_persisted_after_reload", + harness.label, + "table" + ); + let table_ident = TableIdent::new(namespace.clone(), table_name.clone()); + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name(table_name) + .schema(base_schema()) + .build(), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx + .update_schema() + .add_column(AddColumn::optional( + "new_field", + Type::Primitive(PrimitiveType::Long), + )) + .apply(tx)?; + tx.commit(catalog.as_ref()).await?; + + let reloaded = catalog.load_table(&table_ident).await?; + assert!( + reloaded + .metadata() + .current_schema() + .field_by_name("new_field") + .is_some() + ); + + Ok(()) +} diff --git a/crates/catalog/loader/tests/table_register_suite.rs b/crates/catalog/loader/tests/table_register_suite.rs new file mode 100644 index 0000000000..b43054b833 --- /dev/null +++ b/crates/catalog/loader/tests/table_register_suite.rs @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Common register-table behavior across catalogs. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; + +use common::{CatalogKind, cleanup_namespace_dyn, load_catalog, table_creation}; +use iceberg::{ErrorKind, NamespaceIdent, Result, TableIdent}; +use iceberg_test_utils::normalize_test_name_with_parts; +use rstest::rstest; + +// Common behavior: register_table rehydrates a dropped table. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_register_table_roundtrip(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_register_table_roundtrip", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table = catalog + .create_table( + &namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_register_table_roundtrip", + harness.label, + "table" + )), + ) + .await?; + let table_ident = table.identifier().clone(); + let metadata_location = table + .metadata_location() + .ok_or_else(|| iceberg::Error::new(ErrorKind::Unexpected, "Missing metadata location"))? + .to_string(); + + catalog.drop_table(&table_ident).await?; + + let registered = catalog + .register_table(&table_ident, metadata_location.clone()) + .await?; + assert_eq!(registered.identifier(), &table_ident); + assert_eq!( + registered.metadata_location(), + Some(metadata_location.as_str()) + ); + + Ok(()) +} + +// HMS and S3Tables do not support register_table yet. +#[rstest] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[tokio::test] +async fn test_catalog_register_table_unsupported(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_register_table_unsupported", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table = catalog + .create_table( + &namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_register_table_unsupported", + harness.label, + "table" + )), + ) + .await?; + let table_ident = table.identifier().clone(); + let metadata_location = table + .metadata_location() + .ok_or_else(|| iceberg::Error::new(ErrorKind::Unexpected, "Missing metadata location"))? + .to_string(); + + let err = catalog + .register_table(&table_ident, metadata_location) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + + Ok(()) +} + +// Common behavior: registering a table with an existing name should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_register_table_conflict_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_register_table_conflict_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_ident = TableIdent::new( + namespace.clone(), + normalize_test_name_with_parts!( + "catalog_register_table_conflict_errors", + harness.label, + "table" + ), + ); + let table = catalog + .create_table(&namespace, table_creation(table_ident.name.clone())) + .await?; + let metadata_location = table + .metadata_location() + .ok_or_else(|| iceberg::Error::new(ErrorKind::Unexpected, "Missing metadata location"))? + .to_string(); + + assert!( + catalog + .register_table(&table_ident, metadata_location) + .await + .is_err() + ); + Ok(()) +} diff --git a/crates/catalog/loader/tests/table_rename_suite.rs b/crates/catalog/loader/tests/table_rename_suite.rs new file mode 100644 index 0000000000..757d1cd044 --- /dev/null +++ b/crates/catalog/loader/tests/table_rename_suite.rs @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Common rename behavior across catalogs. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; + +use common::{CatalogKind, cleanup_namespace_dyn, load_catalog, table_creation}; +use iceberg::{NamespaceIdent, Result, TableIdent}; +use iceberg_test_utils::normalize_test_name_with_parts; +use rstest::rstest; + +// Common behavior: renaming across namespaces moves the table. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_rename_table_across_namespaces(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let src_namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_rename_table_across_namespaces", + harness.label, + "src" + )); + let dst_namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_rename_table_across_namespaces", + harness.label, + "dst" + )); + + cleanup_namespace_dyn(catalog.as_ref(), &src_namespace).await; + cleanup_namespace_dyn(catalog.as_ref(), &dst_namespace).await; + catalog + .create_namespace(&src_namespace, HashMap::new()) + .await?; + catalog + .create_namespace(&dst_namespace, HashMap::new()) + .await?; + + let table = catalog + .create_table( + &src_namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_rename_table_across_namespaces", + harness.label, + "table" + )), + ) + .await?; + let src_ident = table.identifier().clone(); + let dst_ident = TableIdent::new(dst_namespace.clone(), src_ident.name.clone()); + + catalog.rename_table(&src_ident, &dst_ident).await?; + assert!(catalog.table_exists(&dst_ident).await?); + assert!(!catalog.table_exists(&src_ident).await?); + + Ok(()) +} + +// Common behavior: renaming a missing table should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_rename_table_missing_source_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_rename_table_missing_source_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let src_ident = TableIdent::new(namespace.clone(), "missing".to_string()); + let dst_ident = TableIdent::new(namespace.clone(), "dest".to_string()); + + assert!(catalog.rename_table(&src_ident, &dst_ident).await.is_err()); + Ok(()) +} + +// Common behavior: renaming to an existing destination should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_rename_table_dest_exists_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_rename_table_dest_exists_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let src = catalog + .create_table( + &namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_rename_table_dest_exists_errors", + harness.label, + "src" + )), + ) + .await? + .identifier() + .clone(); + let dst = catalog + .create_table( + &namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_rename_table_dest_exists_errors", + harness.label, + "dst" + )), + ) + .await? + .identifier() + .clone(); + + assert!(catalog.rename_table(&src, &dst).await.is_err()); + Ok(()) +} diff --git a/crates/catalog/loader/tests/table_suite.rs b/crates/catalog/loader/tests/table_suite.rs new file mode 100644 index 0000000000..fd836d4b25 --- /dev/null +++ b/crates/catalog/loader/tests/table_suite.rs @@ -0,0 +1,507 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Common table behavior across catalogs. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; + +use common::{CatalogKind, TableMode, cleanup_namespace_dyn, load_catalog, table_creation}; +use iceberg::spec::{DataContentType, DataFileBuilder, DataFileFormat, Struct, TableProperties}; +use iceberg::transaction::{ApplyTransactionAction, Transaction}; +use iceberg::{ErrorKind, NamespaceIdent, Result, TableIdent}; +use iceberg_test_utils::normalize_test_name_with_parts; +use rstest::rstest; + +// Common behavior: table lifecycle CRUD. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_table_lifecycle(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_table_lifecycle", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = + normalize_test_name_with_parts!("catalog_table_lifecycle", harness.label, "table"); + let table = catalog + .create_table(&namespace, table_creation(table_name)) + .await?; + let ident = table.identifier().clone(); + + assert!(catalog.table_exists(&ident).await?); + let loaded = catalog.load_table(&ident).await?; + assert_eq!(loaded.identifier(), &ident); + + let tables = catalog.list_tables(&namespace).await?; + assert!(tables.contains(&ident)); + + let dest = TableIdent::new(ident.namespace.clone(), format!("{}_renamed", ident.name)); + catalog.rename_table(&ident, &dest).await?; + assert!(catalog.table_exists(&dest).await?); + assert!(!catalog.table_exists(&ident).await?); + + catalog.drop_table(&dest).await?; + assert!(!catalog.table_exists(&dest).await?); + + catalog.drop_namespace(&namespace).await?; + + Ok(()) +} + +// Common behavior: listing tables for a missing namespace should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_list_tables_missing_namespace_errors( + #[case] kind: CatalogKind, +) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_list_tables_missing_namespace_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + assert!(catalog.list_tables(&namespace).await.is_err()); + Ok(()) +} + +// Common behavior: listing tables in an empty namespace returns empty. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_list_tables_empty_namespace(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_list_tables_empty_namespace", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let tables = catalog.list_tables(&namespace).await?; + assert!(tables.is_empty()); + + Ok(()) +} + +// Common behavior: created tables expose the requested schema. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_create_table_schema(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_create_table_schema", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = + normalize_test_name_with_parts!("catalog_create_table_schema", harness.label, "table"); + let creation = table_creation(table_name); + let expected_schema = creation.schema.clone(); + + let table = catalog.create_table(&namespace, creation).await?; + assert_eq!(table.identifier().namespace, namespace); + assert_eq!(table.metadata().current_schema().as_ref(), &expected_schema); + + Ok(()) +} + +// Common behavior: updating table properties persists through the catalog. +// HMS is excluded because update_table is not supported yet. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_update_table_properties(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_table_properties", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = + normalize_test_name_with_parts!("catalog_update_table_properties", harness.label, "table"); + let table = catalog + .create_table(&namespace, table_creation(table_name)) + .await?; + + let tx = Transaction::new(&table); + let tx = tx + .update_table_properties() + .set("test_property".to_string(), "test_value".to_string()) + .apply(tx)?; + let updated = tx.commit(catalog.as_ref()).await?; + + assert_eq!( + updated.metadata().properties().get("test_property"), + Some(&"test_value".to_string()) + ); + + Ok(()) +} + +// Common behavior: catalogs that write metadata files delete the ones that fall out of the +// metadata log when `write.metadata.delete-after-commit.enabled` is set. +#[rstest] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_update_table_deletes_metadata_files_removed_from_log( + #[case] kind: CatalogKind, +) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_table_deletes_metadata_files", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = normalize_test_name_with_parts!( + "catalog_update_table_deletes_metadata_files", + harness.label, + "table" + ); + let mut creation = table_creation(table_name); + creation.properties = HashMap::from([ + ( + TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(), + "true".to_string(), + ), + ( + TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), + "1".to_string(), + ), + ]); + let mut table = catalog.create_table(&namespace, creation).await?; + + let mut locations = vec![table.metadata_location_result()?.to_string()]; + for i in 0..3 { + let tx = Transaction::new(&table); + let tx = tx + .update_table_properties() + .set("test_property".to_string(), i.to_string()) + .apply(tx)?; + table = tx.commit(catalog.as_ref()).await?; + locations.push(table.metadata_location_result()?.to_string()); + } + + let mut existing = Vec::new(); + for location in &locations { + if table.file_io().exists(location).await? { + existing.push(location.clone()); + } + } + // The current metadata file and the one entry left in the metadata log remain. + assert_eq!(existing, locations[2..].to_vec()); + + Ok(()) +} + +// Common behavior: update_table_properties is rejected when unsupported. +#[rstest] +#[case::hms_catalog(CatalogKind::Hms)] +#[tokio::test] +async fn test_catalog_update_table_properties_unsupported(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_table_properties_unsupported", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = normalize_test_name_with_parts!( + "catalog_update_table_properties_unsupported", + harness.label, + "table" + ); + let table = catalog + .create_table(&namespace, table_creation(table_name)) + .await?; + + let tx = Transaction::new(&table); + let tx = tx + .update_table_properties() + .set("test_property".to_string(), "test_value".to_string()) + .apply(tx)?; + + let err = tx.commit(catalog.as_ref()).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + + Ok(()) +} + +// Common behavior: dropping a missing table should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_drop_table_missing_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_drop_table_missing_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_ident = TableIdent::new(namespace.clone(), "missing".to_string()); + assert!(catalog.drop_table(&table_ident).await.is_err()); + Ok(()) +} + +// Common behavior: purge_table removes the table from the catalog. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_purge_table(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_purge_table", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = normalize_test_name_with_parts!("catalog_purge_table", harness.label, "table"); + let table = catalog + .create_table(&namespace, table_creation(table_name)) + .await?; + let ident = table.identifier().clone(); + + assert!(catalog.table_exists(&ident).await?); + + // Capture metadata location and file_io before purge so we can verify + // that the underlying files are actually deleted. + let metadata_location = table.metadata_location().map(|s| s.to_string()); + let file_io = table.file_io().clone(); + + catalog.purge_table(&ident).await?; + assert!(!catalog.table_exists(&ident).await?); + + if let Some(location) = &metadata_location { + assert!( + !file_io.exists(location).await?, + "Metadata file should have been deleted after purge" + ); + } + + catalog.drop_namespace(&namespace).await?; + + Ok(()) +} + +#[rstest] +// REST cannot purge external manifests, and HMS cannot update tables. +#[case::glue_plain(CatalogKind::Glue, TableMode::Plain)] +#[case::glue_encrypted(CatalogKind::Glue, TableMode::Encrypted)] +#[case::sql_plain(CatalogKind::Sql, TableMode::Plain)] +#[case::sql_encrypted(CatalogKind::Sql, TableMode::Encrypted)] +#[case::s3tables_plain(CatalogKind::S3Tables, TableMode::Plain)] +#[case::s3tables_encrypted(CatalogKind::S3Tables, TableMode::Encrypted)] +#[case::memory_plain(CatalogKind::Memory, TableMode::Plain)] +#[case::memory_encrypted(CatalogKind::Memory, TableMode::Encrypted)] +#[tokio::test] +async fn test_catalog_purge_table_with_data( + #[case] kind: CatalogKind, + #[case] mode: TableMode, +) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_purge_table_with_data", + harness.label, + mode.label() + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = normalize_test_name_with_parts!( + "catalog_purge_table_with_data", + harness.label, + mode.label(), + "table" + ); + let table = catalog + .create_table(&namespace, mode.table_creation(table_name)) + .await?; + let ident = table.identifier().clone(); + let file_io = table.file_io().clone(); + let data_file_path = format!("{}/data/00000.parquet", table.metadata().location()); + + file_io + .new_output(&data_file_path)? + .write(vec![0; 16].into()) + .await?; + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(data_file_path.clone()) + .file_format(DataFileFormat::Parquet) + .partition(Struct::empty()) + .record_count(1) + .file_size_in_bytes(16) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap(); + + let tx = Transaction::new(&table); + tx.fast_append() + .add_data_files(vec![data_file]) + .apply(tx)? + .commit(catalog.as_ref()) + .await?; + let table = catalog.load_table(&ident).await?; + + let metadata_location = table.metadata_location().map(str::to_string); + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list_location = snapshot.manifest_list().to_string(); + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + let manifest_location = manifest_list.entries()[0].manifest_path.clone(); + + assert!(file_io.exists(&data_file_path).await?); + assert!(file_io.exists(&manifest_location).await?); + assert!(file_io.exists(&manifest_list_location).await?); + + catalog.purge_table(&ident).await?; + + assert!(!catalog.table_exists(&ident).await?); + assert!(!file_io.exists(&data_file_path).await?); + assert!(!file_io.exists(&manifest_location).await?); + assert!(!file_io.exists(&manifest_list_location).await?); + if let Some(location) = &metadata_location { + assert!(!file_io.exists(location).await?); + } + + catalog.drop_namespace(&namespace).await?; + + Ok(()) +} + +// Common behavior: purging a missing table should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_purge_table_missing_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_purge_table_missing_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_ident = TableIdent::new(namespace.clone(), "missing".to_string()); + assert!(catalog.purge_table(&table_ident).await.is_err()); + Ok(()) +} diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index 916b5ccf75..62b1e8998e 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -19,6 +19,7 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg-catalog-rest" +publish = true rust-version = { workspace = true } version = { workspace = true } @@ -28,24 +29,39 @@ keywords = ["iceberg", "rest", "catalog"] license = { workspace = true } repository = { workspace = true } +[features] +default = [] +# SigV4 request signing. Off by default to keep the AWS crates out. +sigv4 = ["dep:aws-credential-types", "dep:aws-sigv4", "dep:base64", "dep:sha2"] + [dependencies] async-trait = { workspace = true } +aws-credential-types = { workspace = true, optional = true } +aws-sigv4 = { workspace = true, optional = true } +base64 = { workspace = true, optional = true } chrono = { workspace = true } http = { workspace = true } iceberg = { workspace = true } itertools = { workspace = true } -reqwest = { workspace = true } +moka = { workspace = true, features = ["sync"] } +reqwest = { workspace = true, features = ["form", "query"] } serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } -tokio = { workspace = true, features = ["sync"] } +sha2 = { workspace = true, optional = true } +tokio = { workspace = true } tracing = { workspace = true } typed-builder = { workspace = true } uuid = { workspace = true, features = ["v4"] } [dev-dependencies] -ctor = { workspace = true } +bytes = { workspace = true } +futures = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } mockito = { workspace = true } -port_scanner = { workspace = true } +# `stream` lets tests build a streaming body to exercise HttpRequestBody::Streaming. +reqwest = { workspace = true, features = ["stream"] } tokio = { workspace = true } + +[lints] +workspace = true diff --git a/crates/catalog/rest/DEPENDENCIES.rust.tsv b/crates/catalog/rest/DEPENDENCIES.rust.tsv deleted file mode 100644 index cf238f4b4c..0000000000 --- a/crates/catalog/rest/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,328 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayvec@0.7.6 X X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-ord@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -h2@0.4.12 X -half@2.6.0 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-catalog-rest@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustls@0.23.31 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simdutf8@0.1.5 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.0 X X -stable_deref_trait@1.2.0 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -webpki-roots@1.0.2 X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt new file mode 100644 index 0000000000..a0c6fd03ce --- /dev/null +++ b/crates/catalog/rest/public-api.txt @@ -0,0 +1,426 @@ +pub mod iceberg_catalog_rest +pub use iceberg_catalog_rest::Credentials +pub enum iceberg_catalog_rest::HttpRequestBody<'a> +pub iceberg_catalog_rest::HttpRequestBody::Buffered(&'a [u8]) +pub iceberg_catalog_rest::HttpRequestBody::Empty +pub iceberg_catalog_rest::HttpRequestBody::Streaming +impl<'a> iceberg_catalog_rest::HttpRequestBody<'a> +pub fn iceberg_catalog_rest::HttpRequestBody<'a>::as_bytes(&self) -> core::option::Option<&'a [u8]> +impl<'a> core::clone::Clone for iceberg_catalog_rest::HttpRequestBody<'a> +pub fn iceberg_catalog_rest::HttpRequestBody<'a>::clone(&self) -> iceberg_catalog_rest::HttpRequestBody<'a> +impl<'a> core::cmp::Eq for iceberg_catalog_rest::HttpRequestBody<'a> +impl<'a> core::cmp::PartialEq for iceberg_catalog_rest::HttpRequestBody<'a> +pub fn iceberg_catalog_rest::HttpRequestBody<'a>::eq(&self, other: &iceberg_catalog_rest::HttpRequestBody<'a>) -> bool +impl<'a> core::fmt::Debug for iceberg_catalog_rest::HttpRequestBody<'a> +pub fn iceberg_catalog_rest::HttpRequestBody<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl<'a> core::marker::Copy for iceberg_catalog_rest::HttpRequestBody<'a> +impl<'a> core::marker::StructuralPartialEq for iceberg_catalog_rest::HttpRequestBody<'a> +#[non_exhaustive] pub enum iceberg_catalog_rest::PayloadHashMode +pub iceberg_catalog_rest::PayloadHashMode::IcebergRest +pub iceberg_catalog_rest::PayloadHashMode::StandardAws +impl core::clone::Clone for iceberg_catalog_rest::PayloadHashMode +pub fn iceberg_catalog_rest::PayloadHashMode::clone(&self) -> iceberg_catalog_rest::PayloadHashMode +impl core::cmp::Eq for iceberg_catalog_rest::PayloadHashMode +impl core::cmp::PartialEq for iceberg_catalog_rest::PayloadHashMode +pub fn iceberg_catalog_rest::PayloadHashMode::eq(&self, other: &iceberg_catalog_rest::PayloadHashMode) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::PayloadHashMode +pub fn iceberg_catalog_rest::PayloadHashMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg_catalog_rest::PayloadHashMode +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::PayloadHashMode +pub struct iceberg_catalog_rest::CommitTableRequest +pub iceberg_catalog_rest::CommitTableRequest::identifier: core::option::Option +pub iceberg_catalog_rest::CommitTableRequest::requirements: alloc::vec::Vec +pub iceberg_catalog_rest::CommitTableRequest::updates: alloc::vec::Vec +impl core::clone::Clone for iceberg_catalog_rest::CommitTableRequest +pub fn iceberg_catalog_rest::CommitTableRequest::clone(&self) -> iceberg_catalog_rest::CommitTableRequest +impl core::cmp::PartialEq for iceberg_catalog_rest::CommitTableRequest +pub fn iceberg_catalog_rest::CommitTableRequest::eq(&self, other: &iceberg_catalog_rest::CommitTableRequest) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::CommitTableRequest +pub fn iceberg_catalog_rest::CommitTableRequest::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::CommitTableRequest +impl serde_core::ser::Serialize for iceberg_catalog_rest::CommitTableRequest +pub fn iceberg_catalog_rest::CommitTableRequest::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::CommitTableRequest +pub fn iceberg_catalog_rest::CommitTableRequest::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::CommitTableResponse +pub iceberg_catalog_rest::CommitTableResponse::metadata: iceberg::spec::table_metadata::TableMetadata +pub iceberg_catalog_rest::CommitTableResponse::metadata_location: alloc::string::String +impl core::clone::Clone for iceberg_catalog_rest::CommitTableResponse +pub fn iceberg_catalog_rest::CommitTableResponse::clone(&self) -> iceberg_catalog_rest::CommitTableResponse +impl core::cmp::Eq for iceberg_catalog_rest::CommitTableResponse +impl core::cmp::PartialEq for iceberg_catalog_rest::CommitTableResponse +pub fn iceberg_catalog_rest::CommitTableResponse::eq(&self, other: &iceberg_catalog_rest::CommitTableResponse) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::CommitTableResponse +pub fn iceberg_catalog_rest::CommitTableResponse::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::CommitTableResponse +impl serde_core::ser::Serialize for iceberg_catalog_rest::CommitTableResponse +pub fn iceberg_catalog_rest::CommitTableResponse::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::CommitTableResponse +pub fn iceberg_catalog_rest::CommitTableResponse::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::CreateNamespaceRequest +pub iceberg_catalog_rest::CreateNamespaceRequest::namespace: iceberg::catalog::NamespaceIdent +pub iceberg_catalog_rest::CreateNamespaceRequest::properties: std::collections::hash::map::HashMap +impl core::clone::Clone for iceberg_catalog_rest::CreateNamespaceRequest +pub fn iceberg_catalog_rest::CreateNamespaceRequest::clone(&self) -> iceberg_catalog_rest::CreateNamespaceRequest +impl core::cmp::Eq for iceberg_catalog_rest::CreateNamespaceRequest +impl core::cmp::PartialEq for iceberg_catalog_rest::CreateNamespaceRequest +pub fn iceberg_catalog_rest::CreateNamespaceRequest::eq(&self, other: &iceberg_catalog_rest::CreateNamespaceRequest) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::CreateNamespaceRequest +pub fn iceberg_catalog_rest::CreateNamespaceRequest::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::CreateNamespaceRequest +impl serde_core::ser::Serialize for iceberg_catalog_rest::CreateNamespaceRequest +pub fn iceberg_catalog_rest::CreateNamespaceRequest::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::CreateNamespaceRequest +pub fn iceberg_catalog_rest::CreateNamespaceRequest::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::CreateTableRequest +pub iceberg_catalog_rest::CreateTableRequest::location: core::option::Option +pub iceberg_catalog_rest::CreateTableRequest::name: alloc::string::String +pub iceberg_catalog_rest::CreateTableRequest::partition_spec: core::option::Option +pub iceberg_catalog_rest::CreateTableRequest::properties: std::collections::hash::map::HashMap +pub iceberg_catalog_rest::CreateTableRequest::schema: iceberg::spec::schema::Schema +pub iceberg_catalog_rest::CreateTableRequest::stage_create: core::option::Option +pub iceberg_catalog_rest::CreateTableRequest::write_order: core::option::Option +impl core::clone::Clone for iceberg_catalog_rest::CreateTableRequest +pub fn iceberg_catalog_rest::CreateTableRequest::clone(&self) -> iceberg_catalog_rest::CreateTableRequest +impl core::cmp::Eq for iceberg_catalog_rest::CreateTableRequest +impl core::cmp::PartialEq for iceberg_catalog_rest::CreateTableRequest +pub fn iceberg_catalog_rest::CreateTableRequest::eq(&self, other: &iceberg_catalog_rest::CreateTableRequest) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::CreateTableRequest +pub fn iceberg_catalog_rest::CreateTableRequest::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::CreateTableRequest +impl serde_core::ser::Serialize for iceberg_catalog_rest::CreateTableRequest +pub fn iceberg_catalog_rest::CreateTableRequest::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::CreateTableRequest +pub fn iceberg_catalog_rest::CreateTableRequest::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::Endpoint +impl iceberg_catalog_rest::Endpoint +pub fn iceberg_catalog_rest::Endpoint::method(&self) -> &str +pub fn iceberg_catalog_rest::Endpoint::path(&self) -> &str +impl core::clone::Clone for iceberg_catalog_rest::Endpoint +pub fn iceberg_catalog_rest::Endpoint::clone(&self) -> iceberg_catalog_rest::Endpoint +impl core::cmp::Eq for iceberg_catalog_rest::Endpoint +impl core::cmp::PartialEq for iceberg_catalog_rest::Endpoint +pub fn iceberg_catalog_rest::Endpoint::eq(&self, other: &iceberg_catalog_rest::Endpoint) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::Endpoint +pub fn iceberg_catalog_rest::Endpoint::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg_catalog_rest::Endpoint +pub fn iceberg_catalog_rest::Endpoint::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg_catalog_rest::Endpoint +pub fn iceberg_catalog_rest::Endpoint::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::Endpoint +impl core::str::traits::FromStr for iceberg_catalog_rest::Endpoint +pub type iceberg_catalog_rest::Endpoint::Err = iceberg::error::Error +pub fn iceberg_catalog_rest::Endpoint::from_str(s: &str) -> core::result::Result +impl serde_core::ser::Serialize for iceberg_catalog_rest::Endpoint +pub fn iceberg_catalog_rest::Endpoint::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::Endpoint +pub fn iceberg_catalog_rest::Endpoint::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::ErrorModel +pub iceberg_catalog_rest::ErrorModel::code: u16 +pub iceberg_catalog_rest::ErrorModel::message: alloc::string::String +pub iceberg_catalog_rest::ErrorModel::stack: core::option::Option> +pub iceberg_catalog_rest::ErrorModel::type: alloc::string::String +impl core::convert::From for iceberg::error::Error +pub fn iceberg::error::Error::from(value: iceberg_catalog_rest::ErrorModel) -> Self +impl core::fmt::Debug for iceberg_catalog_rest::ErrorModel +pub fn iceberg_catalog_rest::ErrorModel::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg_catalog_rest::ErrorModel +pub fn iceberg_catalog_rest::ErrorModel::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::ErrorModel +pub fn iceberg_catalog_rest::ErrorModel::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::ErrorResponse +impl core::convert::From for iceberg::error::Error +pub fn iceberg::error::Error::from(resp: iceberg_catalog_rest::ErrorResponse) -> iceberg::error::Error +impl core::fmt::Debug for iceberg_catalog_rest::ErrorResponse +pub fn iceberg_catalog_rest::ErrorResponse::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg_catalog_rest::ErrorResponse +pub fn iceberg_catalog_rest::ErrorResponse::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::ErrorResponse +pub fn iceberg_catalog_rest::ErrorResponse::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::HttpClient +impl iceberg_catalog_rest::HttpClient +pub async fn iceberg_catalog_rest::HttpClient::post_form(&self, url: &str, headers: &http::header::map::HeaderMap, form: &std::collections::hash::map::HashMap<&str, &str>) -> iceberg::error::Result +impl core::clone::Clone for iceberg_catalog_rest::HttpClient +pub fn iceberg_catalog_rest::HttpClient::clone(&self) -> iceberg_catalog_rest::HttpClient +impl core::fmt::Debug for iceberg_catalog_rest::HttpClient +pub fn iceberg_catalog_rest::HttpClient::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_catalog_rest::HttpRequest +impl iceberg_catalog_rest::HttpRequest +pub fn iceberg_catalog_rest::HttpRequest::body(&self) -> iceberg_catalog_rest::HttpRequestBody<'_> +pub fn iceberg_catalog_rest::HttpRequest::headers(&self) -> &http::header::map::HeaderMap +pub fn iceberg_catalog_rest::HttpRequest::headers_mut(&mut self) -> &mut http::header::map::HeaderMap +pub fn iceberg_catalog_rest::HttpRequest::method(&self) -> &http::method::Method +pub fn iceberg_catalog_rest::HttpRequest::new(inner: reqwest::async_impl::request::Request) -> Self +pub fn iceberg_catalog_rest::HttpRequest::url_str(&self) -> &str +pub struct iceberg_catalog_rest::HttpResponse +impl iceberg_catalog_rest::HttpResponse +pub fn iceberg_catalog_rest::HttpResponse::body(&self) -> &[u8] +pub fn iceberg_catalog_rest::HttpResponse::headers(&self) -> &http::header::map::HeaderMap +pub fn iceberg_catalog_rest::HttpResponse::new(status: http::status::StatusCode, headers: http::header::map::HeaderMap, body: alloc::vec::Vec) -> Self +pub fn iceberg_catalog_rest::HttpResponse::status(&self) -> http::status::StatusCode +impl core::clone::Clone for iceberg_catalog_rest::HttpResponse +pub fn iceberg_catalog_rest::HttpResponse::clone(&self) -> iceberg_catalog_rest::HttpResponse +impl core::fmt::Debug for iceberg_catalog_rest::HttpResponse +pub fn iceberg_catalog_rest::HttpResponse::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_catalog_rest::ListNamespaceResponse +pub iceberg_catalog_rest::ListNamespaceResponse::namespaces: alloc::vec::Vec +pub iceberg_catalog_rest::ListNamespaceResponse::next_page_token: core::option::Option +impl core::clone::Clone for iceberg_catalog_rest::ListNamespaceResponse +pub fn iceberg_catalog_rest::ListNamespaceResponse::clone(&self) -> iceberg_catalog_rest::ListNamespaceResponse +impl core::cmp::Eq for iceberg_catalog_rest::ListNamespaceResponse +impl core::cmp::PartialEq for iceberg_catalog_rest::ListNamespaceResponse +pub fn iceberg_catalog_rest::ListNamespaceResponse::eq(&self, other: &iceberg_catalog_rest::ListNamespaceResponse) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::ListNamespaceResponse +pub fn iceberg_catalog_rest::ListNamespaceResponse::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::ListNamespaceResponse +impl serde_core::ser::Serialize for iceberg_catalog_rest::ListNamespaceResponse +pub fn iceberg_catalog_rest::ListNamespaceResponse::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::ListNamespaceResponse +pub fn iceberg_catalog_rest::ListNamespaceResponse::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::ListTablesResponse +pub iceberg_catalog_rest::ListTablesResponse::identifiers: alloc::vec::Vec +pub iceberg_catalog_rest::ListTablesResponse::next_page_token: core::option::Option +impl core::clone::Clone for iceberg_catalog_rest::ListTablesResponse +pub fn iceberg_catalog_rest::ListTablesResponse::clone(&self) -> iceberg_catalog_rest::ListTablesResponse +impl core::cmp::Eq for iceberg_catalog_rest::ListTablesResponse +impl core::cmp::PartialEq for iceberg_catalog_rest::ListTablesResponse +pub fn iceberg_catalog_rest::ListTablesResponse::eq(&self, other: &iceberg_catalog_rest::ListTablesResponse) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::ListTablesResponse +pub fn iceberg_catalog_rest::ListTablesResponse::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::ListTablesResponse +impl serde_core::ser::Serialize for iceberg_catalog_rest::ListTablesResponse +pub fn iceberg_catalog_rest::ListTablesResponse::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::ListTablesResponse +pub fn iceberg_catalog_rest::ListTablesResponse::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::LoadTableResult +pub iceberg_catalog_rest::LoadTableResult::config: std::collections::hash::map::HashMap +pub iceberg_catalog_rest::LoadTableResult::metadata: iceberg::spec::table_metadata::TableMetadata +pub iceberg_catalog_rest::LoadTableResult::metadata_location: core::option::Option +pub iceberg_catalog_rest::LoadTableResult::storage_credentials: core::option::Option> +impl core::clone::Clone for iceberg_catalog_rest::LoadTableResult +pub fn iceberg_catalog_rest::LoadTableResult::clone(&self) -> iceberg_catalog_rest::LoadTableResult +impl core::cmp::Eq for iceberg_catalog_rest::LoadTableResult +impl core::cmp::PartialEq for iceberg_catalog_rest::LoadTableResult +pub fn iceberg_catalog_rest::LoadTableResult::eq(&self, other: &iceberg_catalog_rest::LoadTableResult) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::LoadTableResult +pub fn iceberg_catalog_rest::LoadTableResult::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::LoadTableResult +impl serde_core::ser::Serialize for iceberg_catalog_rest::LoadTableResult +pub fn iceberg_catalog_rest::LoadTableResult::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::LoadTableResult +pub fn iceberg_catalog_rest::LoadTableResult::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::NamespaceResponse +pub iceberg_catalog_rest::NamespaceResponse::namespace: iceberg::catalog::NamespaceIdent +pub iceberg_catalog_rest::NamespaceResponse::properties: std::collections::hash::map::HashMap +impl core::clone::Clone for iceberg_catalog_rest::NamespaceResponse +pub fn iceberg_catalog_rest::NamespaceResponse::clone(&self) -> iceberg_catalog_rest::NamespaceResponse +impl core::cmp::Eq for iceberg_catalog_rest::NamespaceResponse +impl core::cmp::PartialEq for iceberg_catalog_rest::NamespaceResponse +pub fn iceberg_catalog_rest::NamespaceResponse::eq(&self, other: &iceberg_catalog_rest::NamespaceResponse) -> bool +impl core::convert::From<&iceberg::catalog::Namespace> for iceberg_catalog_rest::NamespaceResponse +pub fn iceberg_catalog_rest::NamespaceResponse::from(value: &iceberg::catalog::Namespace) -> Self +impl core::convert::From for iceberg::catalog::Namespace +pub fn iceberg::catalog::Namespace::from(value: iceberg_catalog_rest::NamespaceResponse) -> Self +impl core::fmt::Debug for iceberg_catalog_rest::NamespaceResponse +pub fn iceberg_catalog_rest::NamespaceResponse::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::NamespaceResponse +impl serde_core::ser::Serialize for iceberg_catalog_rest::NamespaceResponse +pub fn iceberg_catalog_rest::NamespaceResponse::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::NamespaceResponse +pub fn iceberg_catalog_rest::NamespaceResponse::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::NoopAuthManager +impl core::fmt::Debug for iceberg_catalog_rest::NoopAuthManager +pub fn iceberg_catalog_rest::NoopAuthManager::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::NoopAuthManager +pub fn iceberg_catalog_rest::NoopAuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::NoopAuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::NoopAuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub struct iceberg_catalog_rest::OAuth2Manager +impl iceberg_catalog_rest::OAuth2Manager +pub fn iceberg_catalog_rest::OAuth2Manager::new(token_endpoint: impl core::convert::Into) -> Self +pub fn iceberg_catalog_rest::OAuth2Manager::with_credential(self, client_id: core::option::Option, client_secret: alloc::string::String) -> Self +pub fn iceberg_catalog_rest::OAuth2Manager::with_extra_headers(self, headers: http::header::map::HeaderMap) -> Self +pub fn iceberg_catalog_rest::OAuth2Manager::with_extra_oauth_params(self, params: std::collections::hash::map::HashMap) -> Self +pub fn iceberg_catalog_rest::OAuth2Manager::with_token(self, token: impl core::convert::Into) -> Self +impl core::fmt::Debug for iceberg_catalog_rest::OAuth2Manager +pub fn iceberg_catalog_rest::OAuth2Manager::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::OAuth2Manager +pub fn iceberg_catalog_rest::OAuth2Manager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::OAuth2Manager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::OAuth2Manager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub struct iceberg_catalog_rest::RegisterTableRequest +pub iceberg_catalog_rest::RegisterTableRequest::metadata_location: alloc::string::String +pub iceberg_catalog_rest::RegisterTableRequest::name: alloc::string::String +pub iceberg_catalog_rest::RegisterTableRequest::overwrite: core::option::Option +impl core::clone::Clone for iceberg_catalog_rest::RegisterTableRequest +pub fn iceberg_catalog_rest::RegisterTableRequest::clone(&self) -> iceberg_catalog_rest::RegisterTableRequest +impl core::cmp::Eq for iceberg_catalog_rest::RegisterTableRequest +impl core::cmp::PartialEq for iceberg_catalog_rest::RegisterTableRequest +pub fn iceberg_catalog_rest::RegisterTableRequest::eq(&self, other: &iceberg_catalog_rest::RegisterTableRequest) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::RegisterTableRequest +pub fn iceberg_catalog_rest::RegisterTableRequest::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::RegisterTableRequest +impl serde_core::ser::Serialize for iceberg_catalog_rest::RegisterTableRequest +pub fn iceberg_catalog_rest::RegisterTableRequest::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::RegisterTableRequest +pub fn iceberg_catalog_rest::RegisterTableRequest::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::RenameTableRequest +pub iceberg_catalog_rest::RenameTableRequest::destination: iceberg::catalog::TableIdent +pub iceberg_catalog_rest::RenameTableRequest::source: iceberg::catalog::TableIdent +impl core::clone::Clone for iceberg_catalog_rest::RenameTableRequest +pub fn iceberg_catalog_rest::RenameTableRequest::clone(&self) -> iceberg_catalog_rest::RenameTableRequest +impl core::cmp::Eq for iceberg_catalog_rest::RenameTableRequest +impl core::cmp::PartialEq for iceberg_catalog_rest::RenameTableRequest +pub fn iceberg_catalog_rest::RenameTableRequest::eq(&self, other: &iceberg_catalog_rest::RenameTableRequest) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::RenameTableRequest +pub fn iceberg_catalog_rest::RenameTableRequest::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::RenameTableRequest +impl serde_core::ser::Serialize for iceberg_catalog_rest::RenameTableRequest +pub fn iceberg_catalog_rest::RenameTableRequest::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::RenameTableRequest +pub fn iceberg_catalog_rest::RenameTableRequest::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::RestCatalog +impl core::fmt::Debug for iceberg_catalog_rest::RestCatalog +pub fn iceberg_catalog_rest::RestCatalog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::Catalog for iceberg_catalog_rest::RestCatalog +pub fn iceberg_catalog_rest::RestCatalog::create_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::create_table<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, creation: iceberg::catalog::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::drop_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::drop_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::get_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::list_namespaces<'life0, 'life1, 'async_trait>(&'life0 self, parent: core::option::Option<&'life1 iceberg::catalog::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::list_tables<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::load_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::namespace_exists<'life0, 'life1, 'async_trait>(&'life0 self, ns: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::purge_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::register_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::catalog::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::rename_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, src: &'life1 iceberg::catalog::TableIdent, dest: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::table_exists<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::update_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::RestCatalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::catalog::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub struct iceberg_catalog_rest::RestCatalogBuilder +impl iceberg_catalog_rest::RestCatalogBuilder +pub fn iceberg_catalog_rest::RestCatalogBuilder::with_auth_manager(self, auth_manager: M) -> Self where M: iceberg_catalog_rest::AuthManager + 'static +pub fn iceberg_catalog_rest::RestCatalogBuilder::with_client(self, client: reqwest::async_impl::client::Client) -> Self +pub fn iceberg_catalog_rest::RestCatalogBuilder::with_page_size(self, page_size: core::num::nonzero::NonZeroU32) -> Self +pub fn iceberg_catalog_rest::RestCatalogBuilder::with_session_context(self, context: iceberg::catalog::session::SessionContext) -> Self +impl core::default::Default for iceberg_catalog_rest::RestCatalogBuilder +pub fn iceberg_catalog_rest::RestCatalogBuilder::default() -> iceberg_catalog_rest::RestCatalogBuilder +impl core::fmt::Debug for iceberg_catalog_rest::RestCatalogBuilder +pub fn iceberg_catalog_rest::RestCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::CatalogBuilder for iceberg_catalog_rest::RestCatalogBuilder +pub type iceberg_catalog_rest::RestCatalogBuilder::C = iceberg_catalog_rest::RestCatalog +pub fn iceberg_catalog_rest::RestCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg_catalog_rest::RestCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg_catalog_rest::RestCatalogBuilder::with_runtime(self, runtime: iceberg::runtime::Runtime) -> Self +pub fn iceberg_catalog_rest::RestCatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +pub struct iceberg_catalog_rest::RestSessionCatalog +impl core::fmt::Debug for iceberg_catalog_rest::RestSessionCatalog +pub fn iceberg_catalog_rest::RestSessionCatalog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::session::SessionCatalog for iceberg_catalog_rest::RestSessionCatalog +pub fn iceberg_catalog_rest::RestSessionCatalog::create_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::create_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent, creation: iceberg::catalog::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::drop_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::drop_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::get_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::list_namespaces<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, parent: core::option::Option<&'life2 iceberg::catalog::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::list_tables<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, namespace: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::load_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table_ident: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::namespace_exists<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, ns: &'life2 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::purge_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::register_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table_ident: &'life2 iceberg::catalog::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::rename_table<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, src: &'life2 iceberg::catalog::TableIdent, dest: &'life3 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::table_exists<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, table: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::update_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _context: &'life1 iceberg::catalog::session::SessionContext, _namespace: &'life2 iceberg::catalog::NamespaceIdent, _properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::RestSessionCatalog::update_table<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, commit: iceberg::catalog::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub struct iceberg_catalog_rest::RestSessionCatalogBuilder +impl iceberg_catalog_rest::RestSessionCatalogBuilder +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_auth_manager(self, auth_manager: M) -> Self where M: iceberg_catalog_rest::AuthManager + 'static +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_client(self, client: reqwest::async_impl::client::Client) -> Self +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_page_size(self, page_size: core::num::nonzero::NonZeroU32) -> Self +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_runtime(self, runtime: iceberg::runtime::Runtime) -> Self +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +impl core::default::Default for iceberg_catalog_rest::RestSessionCatalogBuilder +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::default() -> Self +impl core::fmt::Debug for iceberg_catalog_rest::RestSessionCatalogBuilder +pub fn iceberg_catalog_rest::RestSessionCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_catalog_rest::SigV4Signer +impl iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::sign(&self, request: &mut iceberg_catalog_rest::HttpRequest, credentials: &aws_credential_types::credentials_impl::Credentials) -> iceberg::error::Result<()> +impl core::clone::Clone for iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::clone(&self) -> iceberg_catalog_rest::SigV4Signer +impl core::fmt::Debug for iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg_catalog_rest::SigV4Signer +pub fn iceberg_catalog_rest::SigV4Signer::builder() -> SigV4SignerBuilder<((), (), ())> +pub struct iceberg_catalog_rest::StorageCredential +pub iceberg_catalog_rest::StorageCredential::config: std::collections::hash::map::HashMap +pub iceberg_catalog_rest::StorageCredential::prefix: alloc::string::String +impl core::clone::Clone for iceberg_catalog_rest::StorageCredential +pub fn iceberg_catalog_rest::StorageCredential::clone(&self) -> iceberg_catalog_rest::StorageCredential +impl core::cmp::Eq for iceberg_catalog_rest::StorageCredential +impl core::cmp::PartialEq for iceberg_catalog_rest::StorageCredential +pub fn iceberg_catalog_rest::StorageCredential::eq(&self, other: &iceberg_catalog_rest::StorageCredential) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::StorageCredential +pub fn iceberg_catalog_rest::StorageCredential::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::StorageCredential +impl serde_core::ser::Serialize for iceberg_catalog_rest::StorageCredential +pub fn iceberg_catalog_rest::StorageCredential::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::StorageCredential +pub fn iceberg_catalog_rest::StorageCredential::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::UpdateNamespacePropertiesRequest +pub iceberg_catalog_rest::UpdateNamespacePropertiesRequest::removals: core::option::Option> +pub iceberg_catalog_rest::UpdateNamespacePropertiesRequest::updates: std::collections::hash::map::HashMap +impl core::clone::Clone for iceberg_catalog_rest::UpdateNamespacePropertiesRequest +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesRequest::clone(&self) -> iceberg_catalog_rest::UpdateNamespacePropertiesRequest +impl core::cmp::Eq for iceberg_catalog_rest::UpdateNamespacePropertiesRequest +impl core::cmp::PartialEq for iceberg_catalog_rest::UpdateNamespacePropertiesRequest +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesRequest::eq(&self, other: &iceberg_catalog_rest::UpdateNamespacePropertiesRequest) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::UpdateNamespacePropertiesRequest +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesRequest::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::UpdateNamespacePropertiesRequest +impl serde_core::ser::Serialize for iceberg_catalog_rest::UpdateNamespacePropertiesRequest +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesRequest::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::UpdateNamespacePropertiesRequest +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesRequest::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_catalog_rest::UpdateNamespacePropertiesResponse +pub iceberg_catalog_rest::UpdateNamespacePropertiesResponse::missing: core::option::Option> +pub iceberg_catalog_rest::UpdateNamespacePropertiesResponse::removed: alloc::vec::Vec +pub iceberg_catalog_rest::UpdateNamespacePropertiesResponse::updated: alloc::vec::Vec +impl core::clone::Clone for iceberg_catalog_rest::UpdateNamespacePropertiesResponse +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesResponse::clone(&self) -> iceberg_catalog_rest::UpdateNamespacePropertiesResponse +impl core::cmp::Eq for iceberg_catalog_rest::UpdateNamespacePropertiesResponse +impl core::cmp::PartialEq for iceberg_catalog_rest::UpdateNamespacePropertiesResponse +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesResponse::eq(&self, other: &iceberg_catalog_rest::UpdateNamespacePropertiesResponse) -> bool +impl core::fmt::Debug for iceberg_catalog_rest::UpdateNamespacePropertiesResponse +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesResponse::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_catalog_rest::UpdateNamespacePropertiesResponse +impl serde_core::ser::Serialize for iceberg_catalog_rest::UpdateNamespacePropertiesResponse +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesResponse::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::UpdateNamespacePropertiesResponse +pub fn iceberg_catalog_rest::UpdateNamespacePropertiesResponse::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub const iceberg_catalog_rest::AUTH_TYPE_NONE: &str +pub const iceberg_catalog_rest::AUTH_TYPE_OAUTH2: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_AUTH_TYPE: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_DISABLE_HEADER_REDACTION: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_PAGE_SIZE: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_TABLE_CACHE_MAX_ENTRIES: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_URI: &str +pub const iceberg_catalog_rest::REST_CATALOG_PROP_WAREHOUSE: &str +pub trait iceberg_catalog_rest::AuthManager: core::fmt::Debug + core::marker::Send + core::marker::Sync +pub fn iceberg_catalog_rest::AuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::AuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::AuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::NoopAuthManager +pub fn iceberg_catalog_rest::NoopAuthManager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::NoopAuthManager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::NoopAuthManager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, _client: &'life1 iceberg_catalog_rest::HttpClient, _props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +impl iceberg_catalog_rest::AuthManager for iceberg_catalog_rest::OAuth2Manager +pub fn iceberg_catalog_rest::OAuth2Manager::catalog_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_rest::OAuth2Manager::contextual_session<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::catalog::session::SessionContext, catalog_session: alloc::sync::Arc) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_rest::OAuth2Manager::init_session<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, client: &'life1 iceberg_catalog_rest::HttpClient, props: &'life2 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub trait iceberg_catalog_rest::AuthSession: core::fmt::Debug + core::marker::Send + core::marker::Sync +pub fn iceberg_catalog_rest::AuthSession::authenticate<'life0, 'life1, 'async_trait>(&'life0 self, request: &'life1 mut iceberg_catalog_rest::HttpRequest) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait diff --git a/crates/catalog/rest/src/auth/mod.rs b/crates/catalog/rest/src/auth/mod.rs new file mode 100644 index 0000000000..a3aab4df90 --- /dev/null +++ b/crates/catalog/rest/src/auth/mod.rs @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Pluggable authentication for the REST catalog, mirroring Iceberg Java's +//! `AuthManager`/`AuthSession` API. + +mod oauth2; +#[cfg(feature = "sigv4")] +mod sigv4; + +use std::collections::HashMap; +use std::fmt::Debug; +use std::sync::Arc; + +use async_trait::async_trait; +#[cfg(feature = "sigv4")] +pub use aws_credential_types::Credentials; +use iceberg::{Result, SessionContext}; +pub use oauth2::OAuth2Manager; +#[cfg(feature = "sigv4")] +pub use sigv4::{PayloadHashMode, SigV4Signer}; + +use crate::client::HttpClient; +use crate::request::HttpRequest; + +/// `rest.auth.type` value disabling authentication. +pub const AUTH_TYPE_NONE: &str = "none"; +/// `rest.auth.type` value selecting OAuth2 token authentication. +pub const AUTH_TYPE_OAUTH2: &str = "oauth2"; + +/// Creates the [`AuthSession`]s used to authenticate REST catalog requests. +/// +/// A manager is exclusively scoped to one catalog and must not be reused by +/// other catalogs. It is either created from the `rest.auth.type` property or +/// injected through +/// [`RestCatalogBuilder::with_auth_manager`](crate::RestCatalogBuilder::with_auth_manager) or +/// [`RestSessionCatalogBuilder::with_auth_manager`](crate::RestSessionCatalogBuilder::with_auth_manager). +/// Catalog initialization calls [`AuthManager::catalog_session`] exactly once; +/// the context-specific sessions that [`AuthManager::contextual_session`] +/// derives from it may rely on the state established by that call. +/// +/// [`Self::init_session`] and [`Self::catalog_session`] are handed the +/// catalog's [`HttpClient`], which an implementation may clone and store for +/// its own requests (e.g. a token exchange) so that they share the catalog's +/// connection pool and configuration. +#[async_trait] +pub trait AuthManager: Debug + Send + Sync { + /// Session used for the initial `/v1/config` handshake, given the + /// user-supplied properties. + /// + /// Returns a [`Box`]: an init session is used once and released, unlike + /// the shared [`AuthManager::catalog_session`]. + async fn init_session( + &self, + client: &HttpClient, + props: &HashMap, + ) -> Result>; + + /// Session used for all subsequent catalog requests, given the properties + /// merged from the user configuration and the server's config response. + /// + /// Returns an [`Arc`]: this session is shared by concurrent requests for + /// the rest of the catalog's lifetime. Implementations may carry state + /// (e.g. a cached token) over from the init session. + async fn catalog_session( + &self, + client: &HttpClient, + props: &HashMap, + ) -> Result>; + + /// Returns the authentication session for a specific context. + /// + /// The catalog calls this method only after [`Self::catalog_session`] has + /// succeeded. `catalog_session` is the catalog session returned by this + /// manager. If the context does not require different authentication, + /// implementations should return `catalog_session` unchanged. + /// + /// The catalog does not cache the returned session. Implementations should + /// cache context-specific sessions internally using [`SessionContext::session_id`] + /// and are responsible for eviction. Reusing a session ID with different + /// context may therefore return the previously cached session. + /// + /// The catalog calls this method for every request without serializing + /// calls that share a session ID. Caching implementations must guard + /// against concurrently creating multiple sessions for the same context. + /// + /// No [`HttpClient`] is passed. Implementations that need one, e.g. for a + /// token exchange, should store the client handed to [`Self::catalog_session`]. + async fn contextual_session( + &self, + context: &SessionContext, + catalog_session: Arc, + ) -> Result> { + let _ = context; + Ok(catalog_session) + } +} + +/// Authenticates outgoing REST catalog requests. +#[async_trait] +pub trait AuthSession: Debug + Send + Sync { + /// Applies authentication to the request (adds headers, signs, ...). + async fn authenticate(&self, request: &mut HttpRequest) -> Result<()>; +} + +/// [`AuthManager`] that performs no authentication. +#[derive(Debug)] +pub struct NoopAuthManager; + +/// [`AuthSession`] that performs no authentication. +#[derive(Debug)] +pub(crate) struct NoopSession; + +#[async_trait] +impl AuthManager for NoopAuthManager { + async fn init_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(Box::new(NoopSession)) + } + + async fn catalog_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(Arc::new(NoopSession)) + } +} + +#[async_trait] +impl AuthSession for NoopSession { + async fn authenticate(&self, _request: &mut HttpRequest) -> Result<()> { + Ok(()) + } +} diff --git a/crates/catalog/rest/src/auth/oauth2.rs b/crates/catalog/rest/src/auth/oauth2.rs new file mode 100644 index 0000000000..aad91996a8 --- /dev/null +++ b/crates/catalog/rest/src/auth/oauth2.rs @@ -0,0 +1,383 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; + +use async_trait::async_trait; +use http::StatusCode; +use iceberg::sensitive::SensitiveString; +use iceberg::{Error, ErrorKind, Result}; +use reqwest::header::HeaderMap; +use tokio::sync::Mutex; + +use super::{AuthManager, AuthSession, HttpRequest}; +use crate::catalog::{ + REST_CATALOG_PROP_URI, RestCatalogConfig, credential_from_props, default_token_endpoint, + explicit_headers_from_props, +}; +use crate::client::HttpClient; +use crate::types::{ErrorResponse, TokenResponse}; + +/// The manager's own OAuth2 options, which properties are merged onto. +struct OAuth2Params { + extra_headers: HeaderMap, + token_endpoint: String, + credential: Option<(Option, SensitiveString)>, + extra_oauth_params: HashMap, +} + +/// [`AuthManager`] implementing the OAuth2 client-credentials flow used by +/// Iceberg REST catalogs. +/// +/// A configured `token` is used directly; otherwise `credential` is exchanged +/// for a token at the token endpoint and cached. The cached token is shared +/// across sessions so it survives the config handshake. +pub struct OAuth2Manager { + token: Arc>>, + init_params: OAuth2Params, + /// True when the token endpoint was derived from the catalog URI (not + /// explicitly configured): it is then recomputed from the merged URI in + /// [`Self::catalog_session`], since `/v1/config` may override the URI. + endpoint_is_default: bool, +} + +impl OAuth2Manager { + /// Creates a manager exchanging credentials at `token_endpoint`, with no + /// token or credential configured. Combine with the `with_*` methods: + /// + /// ```rust,ignore + /// let manager = OAuth2Manager::new("https://auth.example.com/v1/oauth/tokens") + /// .with_credential(Some("client-id".into()), "client-secret".into()); + /// ``` + pub fn new(token_endpoint: impl Into) -> Self { + Self { + token: Arc::new(Mutex::new(None)), + init_params: OAuth2Params { + extra_headers: HeaderMap::new(), + token_endpoint: token_endpoint.into(), + credential: None, + // Same default as the configuration path: the catalog scope. + extra_oauth_params: HashMap::from([("scope".to_string(), "catalog".to_string())]), + }, + endpoint_is_default: false, + } + } + + /// Sets a bearer token used directly (takes precedence over `credential`). + pub fn with_token(mut self, token: impl Into) -> Self { + self.token = Arc::new(Mutex::new(Some(SensitiveString::from(token.into())))); + self + } + + /// Sets the client credential exchanged for a token at the token endpoint. + pub fn with_credential(mut self, client_id: Option, client_secret: String) -> Self { + self.init_params.credential = Some((client_id, client_secret.into())); + self + } + + /// Sets extra headers sent with token requests. + pub fn with_extra_headers(mut self, headers: HeaderMap) -> Self { + self.init_params.extra_headers = headers; + self + } + + /// Adds extra OAuth2 form parameters (e.g. `scope`, `audience`), merged + /// onto the defaults: provide a `scope` entry to replace the default + /// `catalog` scope. + pub fn with_extra_oauth_params(mut self, params: HashMap) -> Self { + self.init_params.extra_oauth_params.extend(params); + self + } + + pub(crate) fn from_config(cfg: &RestCatalogConfig) -> Result { + Ok(Self { + token: Arc::new(Mutex::new(cfg.token().map(SensitiveString::from))), + init_params: OAuth2Params { + extra_headers: cfg.extra_headers()?, + token_endpoint: cfg.get_token_endpoint(), + credential: cfg.credential().map(|(id, secret)| (id, secret.into())), + extra_oauth_params: cfg.extra_oauth_params(), + }, + endpoint_is_default: cfg.explicit_oauth2_server_uri().is_none(), + }) + } +} + +impl Debug for OAuth2Manager { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OAuth2Manager") + .field("token_endpoint", &self.init_params.token_endpoint) + .finish_non_exhaustive() + } +} + +#[async_trait] +impl AuthManager for OAuth2Manager { + async fn init_session( + &self, + client: &HttpClient, + props: &HashMap, + ) -> Result> { + Ok(Box::new(self.session_from(client, props).await?)) + } + + async fn catalog_session( + &self, + client: &HttpClient, + props: &HashMap, + ) -> Result> { + Ok(Arc::new(self.session_from(client, props).await?)) + } +} + +impl OAuth2Manager { + /// Builds a session from the manager's options with `props` merged onto + /// them, so an injected manager keeps whatever a property doesn't + /// override. The manager's token cell is shared with every session it + /// builds, so a token cached during the handshake survives it. + async fn session_from( + &self, + client: &HttpClient, + props: &HashMap, + ) -> Result { + // The properties may carry a new token (or restate the user's). + if let Some(token) = props.get("token") { + *self.token.lock().await = Some(SensitiveString::from(token.clone())); + } + + let mut extra_headers = self.init_params.extra_headers.clone(); + extra_headers.extend(explicit_headers_from_props(props)?); + + let mut extra_oauth_params = self.init_params.extra_oauth_params.clone(); + for key in ["scope", "audience", "resource"] { + if let Some(value) = props.get(key) { + extra_oauth_params.insert(key.to_string(), value.to_string()); + } + } + + let token_endpoint = match props.get("oauth2-server-uri") { + Some(uri) if !uri.is_empty() => uri.clone(), + // A default endpoint follows the merged catalog URI (which + // `/v1/config` may have overridden); explicit ones are kept. + _ if self.endpoint_is_default => props + .get(REST_CATALOG_PROP_URI) + .map(|uri| default_token_endpoint(uri)) + .unwrap_or_else(|| self.init_params.token_endpoint.clone()), + _ => self.init_params.token_endpoint.clone(), + }; + + let credential = credential_from_props(props) + .map(|(id, secret)| (id, secret.into())) + .or_else(|| self.init_params.credential.clone()); + + Ok(OAuth2Session { + token: self.token.clone(), + // A configured token takes precedence over the credential: the + // token cell is pre-seeded, and the credential only comes into + // play once that token is gone. + token_source: match credential { + Some(credential) => { + TokenSource::ClientCredentials(Box::new(ClientCredentialsConfig { + client: client.clone(), + credential, + token_endpoint, + extra_headers, + extra_oauth_params, + })) + } + None => TokenSource::StaticToken, + }, + }) + } +} + +/// Attaches `token` as a `Authorization: Bearer ` header, marked +/// sensitive so `Debug`-formatted requests redact it. +fn attach_bearer(req: &mut HttpRequest, token: &SensitiveString) -> Result<()> { + let mut value: http::HeaderValue = + format!("Bearer {}", token.expose()).parse().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + "Invalid token received from catalog server!", + ) + .with_source(e) + })?; + value.set_sensitive(true); + req.headers_mut().insert(http::header::AUTHORIZATION, value); + Ok(()) +} + +/// [`AuthSession`] attaching an OAuth2 bearer token. +/// +/// The token is a configured one (which replaces whatever the cell holds), a +/// token cached by an earlier session (the cell is shared with the owning +/// [`OAuth2Manager`]), or — with [`TokenSource::ClientCredentials`] — one +/// exchanged for the credential on demand. +/// +/// # TODO: Support automatic token refreshing. +struct OAuth2Session { + token: Arc>>, + token_source: TokenSource, +} + +/// How an [`OAuth2Session`] obtains a token once none is cached. +enum TokenSource { + /// Nothing to obtain: the session attaches the configured token, or no + /// authentication at all when there is none. + StaticToken, + /// The credential is exchanged for a token at the token endpoint. + ClientCredentials(Box), +} + +struct ClientCredentialsConfig { + client: HttpClient, + credential: (Option, SensitiveString), + token_endpoint: String, + extra_headers: HeaderMap, + extra_oauth_params: HashMap, +} + +impl Debug for OAuth2Session { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut out = f.debug_struct("OAuth2Session"); + if let TokenSource::ClientCredentials(config) = &self.token_source { + out.field("token_endpoint", &config.token_endpoint); + } + out.finish_non_exhaustive() + } +} + +impl ClientCredentialsConfig { + async fn exchange_credential_for_token(&self) -> Result { + let (client_id, client_secret) = &self.credential; + + let mut params = HashMap::with_capacity(4); + params.insert("grant_type", "client_credentials"); + if let Some(client_id) = client_id { + params.insert("client_id", client_id); + } + params.insert("client_secret", client_secret.expose()); + params.extend( + self.extra_oauth_params + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())), + ); + + let response = self + .client + .post_form(&self.token_endpoint, &self.extra_headers, ¶ms) + .await?; + let status = response.status(); + let body = response.body(); + + let auth_res: TokenResponse = if status == StatusCode::OK { + Ok(serde_json::from_slice(body).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + "Failed to parse response from rest catalog server!", + ) + .with_context("operation", "auth") + .with_context("url", self.token_endpoint.clone()) + .with_context("json", String::from_utf8_lossy(body)) + .with_source(e) + })?) + } else { + let e: ErrorResponse = serde_json::from_slice(body).map_err(|e| { + Error::new(ErrorKind::Unexpected, "Received unexpected response") + .with_context("code", status.to_string()) + .with_context("operation", "auth") + .with_context("url", self.token_endpoint.clone()) + .with_context("json", String::from_utf8_lossy(body)) + .with_source(e) + })?; + Err(Error::from(e)) + }?; + Ok(auth_res.access_token) + } +} + +#[async_trait] +impl AuthSession for OAuth2Session { + /// Uses the cached token when present; otherwise exchanges the credential + /// for one, caches it, then uses it. Without a credential and without a + /// token, no authentication is attached. + async fn authenticate(&self, req: &mut HttpRequest) -> Result<()> { + // The lock is held across the exchange: waiters reuse a successful + // result, and retry themselves after a failure. + let token = { + let mut token = self.token.lock().await; + match (&*token, &self.token_source) { + (Some(token), _) => Some(token.clone()), + (None, TokenSource::StaticToken) => None, + (None, TokenSource::ClientCredentials(config)) => { + let new_token = + SensitiveString::from(config.exchange_credential_for_token().await?); + *token = Some(new_token.clone()); + Some(new_token) + } + } + }; + + match token { + Some(token) => attach_bearer(req, &token), + None => Ok(()), + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use reqwest::Client; + + use super::*; + use crate::RestCatalogConfig; + + fn test_client() -> HttpClient { + HttpClient::new( + &RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .build(), + ) + .unwrap() + } + + #[tokio::test] + async fn test_static_token_session_attaches_token() { + // Token-only config: the token is attached as-is. + let manager = OAuth2Manager::new("http://localhost/unused").with_token("tok-static"); + let session = manager + .init_session(&test_client(), &HashMap::new()) + .await + .unwrap(); + + let mut req = HttpRequest::new( + Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + session.authenticate(&mut req).await.unwrap(); + assert_eq!( + req.headers().get("authorization").unwrap(), + "Bearer tok-static" + ); + } +} diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs new file mode 100644 index 0000000000..b7b4ae5bf1 --- /dev/null +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -0,0 +1,1225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS SigV4 request signing for the REST catalog. + +use chrono::{DateTime, Utc}; +use iceberg::{Error, ErrorKind, Result}; +use sha2::{Digest, Sha256}; +use typed_builder::TypedBuilder; + +use super::Credentials; + +/// How the payload hash is encoded in the `x-amz-content-sha256` header. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum PayloadHashMode { + /// Iceberg Java's style: a base64 header when there is a body, hex when + /// there is none, and hex in the canonical request. The base64 comes from + /// how Java configures the AWS SDK, not from SigV4, so a verifier that + /// trusts the header instead of hashing the body rejects it. + IcebergRest, + /// Standard SigV4: hex everywhere. + StandardAws, +} + +fn encode_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn base64_encode(bytes: &[u8]) -> String { + base64::engine::Engine::encode(&base64::engine::general_purpose::STANDARD, bytes) +} + +/// The content header and canonical payload hash, computed from one digest. +fn payload_hashes(body: Option<&[u8]>, mode: PayloadHashMode) -> (String, String) { + let digest = Sha256::digest(body.unwrap_or_default()); + let hex = encode_hex(&digest); + let header = match mode { + PayloadHashMode::IcebergRest if body.is_some() => base64_encode(&digest), + _ => hex.clone(), + }; + (header, hex) +} + +/// Signs REST catalog requests the way Iceberg Java's `RESTSigV4AuthSession` +/// does. Carries no credentials, so one signer serves every session. +/// +/// Built with [`SigV4Signer::builder`]. +#[derive(Clone, Debug, TypedBuilder)] +pub struct SigV4Signer { + /// The signing region, e.g. `us-east-1`. + #[builder(setter(into))] + region: String, + /// The signing name, e.g. `execute-api`. + #[builder(setter(into))] + service: String, + /// How the payload hash is encoded. + mode: PayloadHashMode, +} + +impl SigV4Signer { + /// Signs `request` in place. An existing `Authorization` moves to + /// `Original-Authorization`, and a caller's `x-amz-date`, + /// `x-amz-content-sha256` or `x-amz-security-token` that the signer + /// overwrites moves to `Original-` (Java keeps a caller's content + /// hash when there is a body). Userinfo leaves the URL, and a `+` in the + /// query becomes `%20`, so write a literal plus as `%2B`. + /// + /// Uses `credentials` as given and never refreshes them: resolve temporary + /// ones from their provider before each call. + /// + /// On error, including for a streaming body or a non-UTF-8 header, the + /// request is left unchanged. + /// + /// Send the result through a client that does not follow redirects: a + /// redirect replays the signature, and across hosts reqwest drops + /// `Authorization` but keeps `Original-Authorization`. + /// + /// `aws_sigv4` traces requests without redacting `Original-Authorization`. + /// A `tracing` subscriber is muted for the call, but the `log` bridge (no + /// subscriber, or `log-always`) still forwards those events, so keep + /// `aws_sigv4` below trace level there. + pub fn sign(&self, request: &mut crate::HttpRequest, credentials: &Credentials) -> Result<()> { + self.sign_at(request, credentials, Utc::now()) + } + + fn sign_at( + &self, + request: &mut crate::HttpRequest, + credentials: &Credentials, + now: DateTime, + ) -> Result<()> { + let headers = request.headers().clone(); + let url = request.url().clone(); + let signed = self.sign_in_place(request, credentials, now); + if signed.is_err() { + *request.headers_mut() = headers; + *request.url_mut() = url; + } + signed + } + + fn sign_in_place( + &self, + request: &mut crate::HttpRequest, + credentials: &Credentials, + now: DateTime, + ) -> Result<()> { + use aws_sigv4::http_request::{SignableBody, SignableRequest, sign}; + use aws_sigv4::sign::v4; + use tracing::dispatcher::Dispatch; + use tracing::level_filters::LevelFilter; + + // Before the content hash is displaced, which takes it out of signing. + signable_headers(request).try_for_each(|header| header.map(|_| ()))?; + let (content_header, payload_hash) = payload_hashes(signable_body(request)?, self.mode); + + convert_headers(request); + + // Relocated after signing, so the `Original-` copy is not signed. + let displaced_content_hash: Vec<_> = request + .headers() + .get_all(CONTENT_SHA256) + .iter() + .filter(|v| v.as_bytes() != content_header.as_bytes()) + .cloned() + .collect(); + let content_value = content_header.parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid x-amz-content-sha256 value").with_source(e) + })?; + request.headers_mut().insert(CONTENT_SHA256, content_value); + + rewrite_url_for_signing(request)?; + + let identity = credentials.clone().into(); + let params = v4::SigningParams::builder() + .identity(&identity) + .region(&self.region) + .name(&self.service) + .time(now.into()) + .settings(signing_settings()) + .build() + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "failed to build SigV4 params").with_source(e) + })? + .into(); + + let headers = signable_headers(request).collect::>>()?; + let signable = SignableRequest::new( + request.method().as_str(), + request.url_str(), + headers.into_iter(), + SignableBody::Precomputed(payload_hash), + ) + .map_err(|e| { + Error::new(ErrorKind::DataInvalid, "request is not signable").with_source(e) + })?; + + // `aws_sigv4` traces the request, `Original-Authorization` included. + // Mute it only when a subscriber could record that (the max level is + // `OFF` until one is registered): `with_default` also sets + // tracing-core's `EXISTS` flag, which nothing clears, and with the + // `log` feature tracing then stops forwarding events to `log` for good. + let signed = if LevelFilter::current() == LevelFilter::TRACE { + tracing::dispatcher::with_default(&Dispatch::none(), || sign(signable, ¶ms)) + } else { + sign(signable, ¶ms) + }; + let (instructions, _signature) = signed + .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 signing failed").with_source(e))? + .into_parts(); + + update_request_headers(request, instructions, displaced_content_hash) + } +} + +/// The body to sign; as in Java, an absent body and an empty one differ. +fn signable_body(request: &crate::HttpRequest) -> Result> { + match request.body() { + crate::HttpRequestBody::Empty => Ok(None), + crate::HttpRequestBody::Buffered(bytes) => Ok(Some(bytes)), + crate::HttpRequestBody::Streaming => Err(Error::new( + ErrorKind::FeatureUnsupported, + "cannot sign a streaming request body", + )), + } +} + +/// The headers to sign. A non-UTF-8 one is an error: skipping it would send +/// it unsigned. +fn signable_headers(request: &crate::HttpRequest) -> impl Iterator> { + request.headers().iter().map(|(n, v)| { + let v = std::str::from_utf8(v.as_bytes()).map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("cannot sign non-UTF-8 header value for `{n}`"), + ) + .with_source(e) + })?; + Ok((n.as_str(), v)) + }) +} + +/// Drops userinfo, which the wire `Host` never carries, and rewrites `+` in the +/// query as `%20`: a space to AWS and Java either way, but unambiguous to any +/// verifier. +fn rewrite_url_for_signing(request: &mut crate::HttpRequest) -> Result<()> { + if !request.url().username().is_empty() || request.url().password().is_some() { + let url = request.url_mut(); + url.set_username("") + .and_then(|()| url.set_password(None)) + .map_err(|()| { + Error::new( + ErrorKind::DataInvalid, + "cannot strip userinfo from the request URL", + ) + })?; + } + if let Some(query) = request.url().query().filter(|q| q.contains('+')) { + let unambiguous = query.replace('+', "%20"); + request.url_mut().set_query(Some(&unambiguous)); + } + Ok(()) +} + +/// `Aws4Signer`'s settings: normalized double-encoded path, and Java's ignore +/// list. +fn signing_settings() -> aws_sigv4::http_request::SigningSettings { + use aws_sigv4::http_request::{ + PayloadChecksumKind, PercentEncodingMode, SigningSettings, UriPathNormalizationMode, + }; + + let mut settings = SigningSettings::default(); + settings.percent_encoding_mode = PercentEncodingMode::Double; + settings.uri_path_normalization_mode = UriPathNormalizationMode::Enabled; + // We set and relocate the content hash ourselves, outside the instructions. + settings.payload_checksum_kind = PayloadChecksumKind::NoHeader; + let mut excluded = settings.excluded_headers.take().unwrap_or_default(); + excluded.extend([ + // Java's list, spelled out rather than left to the crate's defaults. + "connection".into(), + "expect".into(), + "transfer-encoding".into(), + "user-agent".into(), + "x-amzn-trace-id".into(), + // Not Java's, but proxies append to it too. + "x-forwarded-for".into(), + // Relocation appends to these after signing. + "original-x-amz-date".into(), + "original-x-amz-content-sha256".into(), + "original-x-amz-security-token".into(), + ]); + settings.excluded_headers = Some(excluded); + settings +} + +/// Java's `convertHeaders`: moves `Authorization` aside before signing, so the +/// moved copy is signed. +fn convert_headers(request: &mut crate::HttpRequest) { + let displaced: Vec<_> = request + .headers() + .get_all(reqwest::header::AUTHORIZATION) + .iter() + .cloned() + .collect(); + if displaced.is_empty() { + return; + } + request.headers_mut().remove(reqwest::header::AUTHORIZATION); + for mut value in displaced { + value.set_sensitive(true); + request.headers_mut().append(RELOCATED_AUTHORIZATION, value); + } +} + +/// Java's `updateRequestHeaders`: installs the signed headers, moving +/// conflicting caller values aside. +fn update_request_headers( + request: &mut crate::HttpRequest, + instructions: aws_sigv4::http_request::SigningInstructions, + displaced_content_hash: Vec, +) -> Result<()> { + let (signed_headers, _params) = instructions.into_parts(); + let h = request.headers_mut(); + for mut value in displaced_content_hash { + // The original may carry a credential. + value.set_sensitive(true); + h.append(RELOCATED_CONTENT_SHA256, value); + } + for header in signed_headers { + let name: reqwest::header::HeaderName = header.name().parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid signed header name").with_source(e) + })?; + if let Some(relocated) = relocated_name(name.as_str()) { + relocate_conflicting(h, name.as_str(), header.value(), relocated); + } + let mut value: reqwest::header::HeaderValue = header.value().parse().map_err(|e| { + Error::new(ErrorKind::Unexpected, "invalid signed header value").with_source(e) + })?; + if name == reqwest::header::AUTHORIZATION || name == SECURITY_TOKEN { + value.set_sensitive(true); + } + h.insert(name, value); + } + Ok(()) +} + +const CONTENT_SHA256: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-amz-content-sha256"); +const AMZ_DATE: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-amz-date"); +const SECURITY_TOKEN: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("x-amz-security-token"); + +const RELOCATED_AUTHORIZATION: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-authorization"); +const RELOCATED_AMZ_DATE: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-x-amz-date"); +const RELOCATED_CONTENT_SHA256: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-x-amz-content-sha256"); +const RELOCATED_SECURITY_TOKEN: reqwest::header::HeaderName = + reqwest::header::HeaderName::from_static("original-x-amz-security-token"); + +/// The `Original-` counterpart of a header the signer generates. +fn relocated_name(name: &str) -> Option { + match name { + n if n == AMZ_DATE => Some(RELOCATED_AMZ_DATE), + n if n == SECURITY_TOKEN => Some(RELOCATED_SECURITY_TOKEN), + _ => None, + } +} + +/// Moves `name`'s values that differ from `signed` to `relocated`. +fn relocate_conflicting( + headers: &mut reqwest::header::HeaderMap, + name: &str, + signed: &str, + relocated: reqwest::header::HeaderName, +) { + let conflicting: Vec<_> = headers + .get_all(name) + .iter() + .filter(|value| value.as_bytes() != signed.as_bytes()) + .cloned() + .collect(); + for mut value in conflicting { + // It may carry a credential, e.g. a session token. + value.set_sensitive(true); + headers.append(relocated.clone(), value); + } +} + +#[cfg(test)] +mod tests { + use chrono::TimeZone; + + use super::*; + use crate::HttpRequest; + + const EMPTY_HEX: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + #[test] + fn signing_rewrites_an_ambiguous_plus_out_of_the_query() { + // reqwest writes a space as `+`; signing makes it `%20`. + let mut request = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/namespaces") + .query(&[("parent", "my ns")]) + .build() + .unwrap(), + ); + assert!(request.url().query().unwrap().contains("my+ns")); + + let signer = test_signer(PayloadHashMode::StandardAws); + let now = test_time(); + signer + .sign_at(&mut request, &test_credentials(), now) + .unwrap(); + + let query = request.url().query().unwrap(); + assert!(!query.contains('+'), "{query}"); + assert!(query.contains("my%20ns"), "{query}"); + assert_signature_is( + &request, + "b7bb5a323a1ce0ace18454171084deef2dac44c933c3949771fc70179d3cce2b", + ); + } + + #[test] + fn payload_hashes_iceberg_mode() { + let v = payload_hashes(Some(b"hello"), PayloadHashMode::IcebergRest).0; + assert_eq!(v, "LPJNul+wow4m6DsqxbninhsWHlwfp0JecwQzYpOLmCQ="); + let e = payload_hashes(None, PayloadHashMode::IcebergRest).0; + assert_eq!(e, EMPTY_HEX); + } + + /// As in Java, an empty body is hashed, unlike an absent one. + #[test] + fn signing_separates_an_empty_body_from_an_absent_one() { + let signer = test_signer(PayloadHashMode::IcebergRest); + let now = test_time(); + let hash_of = |builder: reqwest::RequestBuilder| { + let mut req = HttpRequest::new(builder.build().unwrap()); + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); + req.headers() + .get("x-amz-content-sha256") + .unwrap() + .to_str() + .unwrap() + .to_string() + }; + + let client = reqwest::Client::new(); + let url = "https://rest.example.com/v1/namespaces"; + assert_eq!( + hash_of(client.post(url).body("")), + "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + ); + assert_eq!(hash_of(client.post(url)), EMPTY_HEX); + } + + #[test] + fn payload_hashes_standard_mode() { + let v = payload_hashes(Some(b"hello"), PayloadHashMode::StandardAws).0; + assert_eq!( + v, + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + ); + } + + /// Pins a signature this crate produced; `signatures_match_iceberg_java` + /// checks against Java. + fn assert_signature_is(req: &HttpRequest, expected: &str) { + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + assert!(auth.ends_with(&format!("Signature={expected}")), "{auth}"); + } + + fn test_signer(mode: PayloadHashMode) -> SigV4Signer { + SigV4Signer::builder() + .region("us-east-1") + .service("execute-api") + .mode(mode) + .build() + } + + fn test_credentials() -> Credentials { + Credentials::new("ak", "sk", None::, None, "test") + } + + fn example_credentials(session_token: Option<&str>) -> Credentials { + Credentials::new( + "AKIDEXAMPLE", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + session_token.map(str::to_string), + None, + "test", + ) + } + + fn test_time() -> DateTime { + Utc.with_ymd_and_hms(2015, 8, 30, 12, 36, 0).unwrap() + } + + fn signed_headers(req: &HttpRequest) -> Vec { + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + let signed = auth.split("SignedHeaders=").nth(1).unwrap(); + let signed = signed.split(',').next().unwrap(); + signed.split(';').map(str::to_string).collect() + } + + /// Every header value as `(name, value)`, sorted. + fn header_list(req: &HttpRequest) -> Vec<(String, String)> { + let mut headers: Vec<_> = req + .headers() + .iter() + .map(|(n, v)| (n.to_string(), v.to_str().unwrap().to_string())) + .collect(); + headers.sort(); + headers + } + + fn sorted_pairs(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + let mut pairs: Vec<_> = pairs + .iter() + .map(|(n, v)| (n.to_string(), v.to_string())) + .collect(); + pairs.sort(); + pairs + } + + /// Records every event field. + #[derive(Clone, Default)] + struct CapturedLog(std::sync::Arc>); + + impl tracing::field::Visit for CapturedLog { + fn record_debug(&mut self, _: &tracing::field::Field, value: &dyn std::fmt::Debug) { + self.0.lock().unwrap().push_str(&format!("{value:?}")); + } + } + + impl tracing::Subscriber for CapturedLog { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::Id { + tracing::Id::from_u64(1) + } + fn record(&self, _: &tracing::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::Id, _: &tracing::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + event.record(&mut self.clone()); + } + fn enter(&self, _: &tracing::Id) {} + fn exit(&self, _: &tracing::Id) {} + } + + /// `aws_sigv4` does not redact `Original-Authorization` itself. + #[test] + fn signing_does_not_trace_a_relocated_bearer_token() { + const TOKEN: &str = "Bearer topsecretdelegatetoken"; + let signer = test_signer(PayloadHashMode::IcebergRest); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header(reqwest::header::AUTHORIZATION, TOKEN) + .build() + .unwrap(), + ); + + let log = CapturedLog::default(); + tracing::subscriber::with_default(log.clone(), || { + // `sign_at` mutes only at this level, which is what is under test. + assert_eq!( + tracing::level_filters::LevelFilter::current(), + tracing::level_filters::LevelFilter::TRACE + ); + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + tracing::trace!(canary = "subscriber-is-live"); + }); + + let captured = log.0.lock().unwrap().clone(); + assert!(captured.contains("subscriber-is-live"), "captured nothing"); + assert!(!captured.contains(TOKEN), "{captured}"); + assert_eq!(req.headers().get(RELOCATED_AUTHORIZATION).unwrap(), TOKEN); + } + + #[test] + fn a_non_utf8_header_is_rejected_without_changing_the_request() { + let signer = test_signer(PayloadHashMode::IcebergRest); + for name in ["x-amz-meta-tenant", "authorization", "x-amz-content-sha256"] { + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://user:pw@rest.example.com/v1/config?warehouse=my+catalog" + .parse() + .unwrap(), + )); + req.headers_mut() + .insert("authorization", "Bearer delegate-token".parse().unwrap()); + req.headers_mut() + .insert("x-amz-content-sha256", "caller-hash".parse().unwrap()); + req.headers_mut().insert( + name, + reqwest::header::HeaderValue::from_bytes(b"acme\xfa").unwrap(), + ); + let headers = req.headers().clone(); + let url = req.url().clone(); + + let err = signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains(name), "{err}"); + assert_eq!(req.headers(), &headers, "{name}"); + assert_eq!(req.url(), &url, "{name}"); + } + } + + #[test] + fn a_non_ascii_utf8_header_is_signed() { + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com/v1/config".parse().unwrap(), + )); + req.headers_mut().insert( + "x-tenant", + reqwest::header::HeaderValue::from_bytes("Zürich".as_bytes()).unwrap(), + ); + + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + + assert_eq!(signed_headers(&req), [ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-tenant" + ]); + } + + #[test] + fn a_signing_failure_after_relocation_restores_the_request() { + let signer = test_signer(PayloadHashMode::IcebergRest); + // Longer than `http::Uri` accepts, so signing fails after the headers + // and query were rewritten. + let url = format!( + "https://rest.example.com/v1/config?warehouse=my+catalog&pad={}", + "a".repeat(70_000) + ); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get(url) + .header("authorization", "Bearer delegate-token") + .header("x-amz-content-sha256", "caller-hash") + .build() + .unwrap(), + ); + let headers = req.headers().clone(); + let url = req.url().clone(); + + let err = signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}"); + assert_eq!(req.headers(), &headers); + assert_eq!(req.url(), &url); + } + + /// Relocation appends to a caller's `Original-x-amz-*` after signing, so it + /// must stay unsigned. + #[test] + fn a_caller_supplied_relocation_header_is_not_signed() { + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("x-amz-content-sha256", "caller-hash") + .header("original-x-amz-content-sha256", "previous") + .build() + .unwrap(), + ); + + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + + assert_eq!(signed_headers(&req), [ + "host", + "x-amz-content-sha256", + "x-amz-date" + ]); + let relocated: Vec<_> = req + .headers() + .get_all("original-x-amz-content-sha256") + .iter() + .map(|v| v.to_str().unwrap()) + .collect(); + assert_eq!(relocated, ["previous", "caller-hash"]); + } + + #[test] + fn userinfo_is_stripped_before_signing() { + // A hand-built request can carry userinfo; the wire `Host` never does. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://user:pw@rest.example.com/v1/config" + .parse() + .unwrap(), + )); + assert_eq!(req.url().username(), "user"); + let now = test_time(); + + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); + + assert_eq!(req.url().username(), ""); + assert_eq!(req.url().password(), None); + let mut plain = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com/v1/config".parse().unwrap(), + )); + signer + .sign_at(&mut plain, &test_credentials(), now) + .unwrap(); + assert_eq!( + req.headers().get("authorization"), + plain.headers().get("authorization"), + ); + } + + #[test] + fn a_doubled_slash_in_the_path_is_normalized() { + // A trailing slash on the catalog URI gives `//v1/...`, which + // `Aws4Signer` collapses. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com//v1//config".parse().unwrap(), + )); + let now = test_time(); + + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); + + let mut plain = HttpRequest::new(reqwest::Request::new( + reqwest::Method::GET, + "https://rest.example.com/v1/config".parse().unwrap(), + )); + signer + .sign_at(&mut plain, &test_credentials(), now) + .unwrap(); + assert_eq!( + req.headers().get("authorization"), + plain.headers().get("authorization"), + ); + } + + #[test] + fn caller_headers_the_signer_overwrites_are_relocated() { + // As in Java, conflicting caller values move to `Original-`. + let creds = Credentials::new( + "ak".to_string(), + "sk".to_string(), + Some("signer-token".to_string()), + None, + "test", + ); + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer caller-token") + .header("x-amz-date", "19700101T000000Z") + .header("x-amz-security-token", "caller-session") + .header("x-amz-content-sha256", "caller-hash") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert_eq!( + header_list(&req), + sorted_pairs(&[ + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=ak/20150830/us-east-1/execute-api/aws4_request, \ + SignedHeaders=host;original-authorization;x-amz-content-sha256;x-amz-date;\ + x-amz-security-token, \ + Signature=87560cb735277b284a547d63f57cfe3a721f9bce0b08a9402a58d754d2ed707c" + ), + ("original-authorization", "Bearer caller-token"), + ("original-x-amz-content-sha256", "caller-hash"), + ("original-x-amz-date", "19700101T000000Z"), + ("original-x-amz-security-token", "caller-session"), + ("x-amz-content-sha256", EMPTY_HEX), + ("x-amz-date", "20150830T123600Z"), + ("x-amz-security-token", "signer-token"), + ]) + ); + // Relocated originals may be credentials themselves. + for relocated in [RELOCATED_SECURITY_TOKEN, RELOCATED_CONTENT_SHA256] { + assert!(req.headers().get(relocated).unwrap().is_sensitive()); + } + } + + #[test] + fn an_existing_authorization_is_never_signed() { + // The signer replaces `authorization`, and a proxy may rewrite + // `user-agent`, so neither is signed. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer caller-token") + .header("user-agent", "example/1.0") + .build() + .unwrap(), + ); + + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + + assert_eq!(signed_headers(&req), [ + "host", + "original-authorization", + "x-amz-content-sha256", + "x-amz-date" + ]); + } + + /// As in Java, every `Authorization` value is relocated, and stays redacted. + #[test] + fn every_repeated_authorization_is_relocated_and_kept_sensitive() { + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("authorization", "Bearer first") + .header("authorization", "Bearer second") + .build() + .unwrap(), + ); + + signer + .sign_at(&mut req, &test_credentials(), test_time()) + .unwrap(); + + let relocated: Vec<_> = req + .headers() + .get_all(RELOCATED_AUTHORIZATION) + .iter() + .collect(); + assert_eq!(relocated.len(), 2, "{relocated:?}"); + assert_eq!(relocated[0], "Bearer first"); + assert_eq!(relocated[1], "Bearer second"); + assert!(relocated.iter().all(|v| v.is_sensitive()), "{relocated:?}"); + } + + #[test] + fn hop_by_hop_headers_are_not_signed() { + // A proxy may drop or rewrite these, so they are not signed. + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .header("expect", "100-continue") + .header("connection", "keep-alive") + .header("x-forwarded-for", "203.0.113.7") + .header("x-tenant", "acme") + .build() + .unwrap(), + ); + let now = test_time(); + + signer.sign_at(&mut req, &test_credentials(), now).unwrap(); + + assert_eq!(signed_headers(&req), [ + "host", + "x-amz-content-sha256", + "x-amz-date", + "x-tenant" + ]); + assert_signature_is( + &req, + "f938221412ed6b55cf3db380ce6ded476419ad3d7db4c76d932031c98465ce79", + ); + } + + #[test] + fn signed_credentials_are_marked_sensitive() { + // Both carry a credential, so `Debug` must not print them. + let creds = Credentials::new( + "ak".to_string(), + "sk".to_string(), + Some("session-token".to_string()), + None, + "test", + ); + let signer = test_signer(PayloadHashMode::StandardAws); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert!(req.headers().get("authorization").unwrap().is_sensitive()); + assert!( + req.headers() + .get("x-amz-security-token") + .unwrap() + .is_sensitive() + ); + let debug = format!("{:?}", req.headers()); + assert!(!debug.contains("session-token"), "{debug}"); + } + + #[test] + fn signs_with_a_non_default_service_and_session_token() { + // As a non-AWS catalog might vend: its own signing name, and STS + // credentials. + let creds = Credentials::new( + "STS.EXAMPLEACCESSKEYID", + "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + Some("example-session-token".to_string()), + None, + "test", + ); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("custom-service") + .mode(PayloadHashMode::IcebergRest) + .build(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://catalog.example.com/v1/config?warehouse=my-catalog") + .build() + .unwrap(), + ); + let now = Utc.with_ymd_and_hms(2026, 8, 26, 12, 0, 0).unwrap(); + + signer.sign_at(&mut req, &creds, now).unwrap(); + + assert_signature_is( + &req, + "6b7065e5f44da4f5c3654126b8d5fe29599905afb6b98f4de65b6ec6e1be783f", + ); + let auth = req + .headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap(); + assert!( + auth.contains("/us-east-1/custom-service/aws4_request"), + "{auth}" + ); + assert_eq!( + req.headers().get("x-amz-security-token").unwrap(), + "example-session-token" + ); + } + + #[test] + fn signs_request_iceberg_mode() { + let creds = example_credentials(Some("SESSIONTOKEN")); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); + let client = reqwest::Client::new(); + let mut req = HttpRequest::new( + client + .post("https://rest.example.com/v1/namespaces") + .body("{}") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert_eq!( + header_list(&req), + sorted_pairs(&[ + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/glue/aws4_request, \ + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + Signature=effad6acde583dd14ba7aff52b2b83776a54421c010fe82067f166819057cb32" + ), + ( + "x-amz-content-sha256", + "RBNvo1WzZ4oRRq0W9+hknpT7T8If536DEMBg9hyq/4o=" + ), + ("x-amz-date", "20150830T123600Z"), + ("x-amz-security-token", "SESSIONTOKEN"), + ]) + ); + } + + /// An empty body hashes to the hex constant, and caller headers are signed + /// (Java's `authenticateWithoutBody`). + #[test] + fn signs_empty_body_and_all_headers() { + let creds = example_credentials(None); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); + let client = reqwest::Client::new(); + let mut req = HttpRequest::new( + client + .get("https://rest.example.com/v1/config") + .header("content-type", "application/json") + .header("content-encoding", "gzip") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert_eq!( + header_list(&req), + sorted_pairs(&[ + ( + "authorization", + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/glue/aws4_request, \ + SignedHeaders=content-encoding;content-type;host;x-amz-content-sha256;x-amz-date, \ + Signature=eaef7eb88d9cd810031684748671d8a3c9394ea5168622a212ed041b670b9777" + ), + ("content-encoding", "gzip"), + ("content-type", "application/json"), + ("x-amz-content-sha256", EMPTY_HEX), + ("x-amz-date", "20150830T123600Z"), + ]) + ); + } + + #[test] + fn iceberg_mode_signs_the_hex_payload_hash_not_the_base64_header() { + // Needs a body: without one the header is the hex constant too. + let creds = example_credentials(None); + let signer = test_signer(PayloadHashMode::IcebergRest); + let body = br#"{"namespace":["ns"]}"#; + let mut req = HttpRequest::new( + reqwest::Client::new() + .post("https://rest.example.com/v1/namespaces") + .body(body.to_vec()) + .build() + .unwrap(), + ); + let now = test_time(); + + signer.sign_at(&mut req, &creds, now).unwrap(); + + assert_eq!( + req.headers().get("x-amz-content-sha256").unwrap(), + payload_hashes(Some(body), PayloadHashMode::IcebergRest) + .0 + .as_str() + ); + assert_ne!( + req.headers().get("x-amz-content-sha256").unwrap(), + encode_hex(&Sha256::digest(body)).as_str() + ); + assert_signature_is( + &req, + "c68682c26cab6a781256f83b0076f50014f4922c3907f4ff09c204a74d61fc1d", + ); + } + + /// What Iceberg Java's `RESTSigV4AuthSession` (iceberg-aws 1.10.1) sent for + /// these requests, given these credentials as `rest.*` properties. + #[test] + fn signatures_match_iceberg_java() { + let creds = example_credentials(Some("example-session-token")); + let signer = test_signer(PayloadHashMode::IcebergRest); + let now = Utc.with_ymd_and_hms(2026, 10, 2, 11, 40, 30).unwrap(); + let client = reqwest::Client::new(); + + // A body, so the header carries base64, and a token to relocate. + let mut post = HttpRequest::new( + client + .post("https://rest.example.com/v1/namespaces") + .header("content-type", "application/json") + .header("authorization", "Bearer delegate-token") + .body(r#"{"namespace":["a","b"],"properties":{}}"#) + .build() + .unwrap(), + ); + signer.sign_at(&mut post, &creds, now).unwrap(); + assert_eq!( + post.headers().get("authorization").unwrap(), + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20261002/us-east-1/execute-api/aws4_request, \ + SignedHeaders=content-type;host;original-authorization;\ + x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + Signature=c5d6bfcecd19c5421e8696c67465f6909b5e09398608d8e79a9eb391e04de556" + ); + assert_eq!( + post.headers().get("x-amz-content-sha256").unwrap(), + "n1CB8Yl4L77sBwHTc5qTpUEhlsHHOXFzg0dV0fbOwXs=" + ); + + // No body, a multi-level namespace and an encoded query. + let mut get = HttpRequest::new( + client + .get("https://rest.example.com/v1/namespaces/a%1Fb/tables/x,y?pageToken=a%20b%2Fc") + .build() + .unwrap(), + ); + signer.sign_at(&mut get, &creds, now).unwrap(); + assert_eq!( + get.headers().get("authorization").unwrap(), + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20261002/us-east-1/execute-api/aws4_request, \ + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, \ + Signature=5f6832ec82fc821b08cd3eac3a86ffc3e0c333571fd0a5796e43f2534f23329f" + ); + } + + /// The signed `host` keeps a non-default port, as on the wire. + #[test] + fn signs_host_with_non_default_port() { + let creds = example_credentials(None); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); + let client = reqwest::Client::new(); + let mut req = HttpRequest::new( + client + .get("https://rest.example.com:8181/v1/config") + .build() + .unwrap(), + ); + let now = test_time(); + + signer.sign_at(&mut req, &creds, now).unwrap(); + assert_signature_is( + &req, + "f7801a4ecac5fe6dcc4ee385223428ec4833f21d0c2fc10d2fb00694b2c0def7", + ); + } + + /// Like `Aws4Signer`, the canonical path is encoded again: `,` becomes `%2C` + /// and `%2C` becomes `%252C`. + #[test] + fn canonical_uri_is_aws_double_encoded() { + let creds = example_credentials(None); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("glue") + .mode(PayloadHashMode::IcebergRest) + .build(); + let client = reqwest::Client::new(); + let mut req = HttpRequest::new( + client + .get("https://rest.example.com/v1/namespaces/a%2Cb/tables/x,y") + .build() + .unwrap(), + ); + let now = test_time(); + + signer.sign_at(&mut req, &creds, now).unwrap(); + assert_signature_is( + &req, + "4d966b6fc2dfb62be5a603e4e07e5dc85b2af1c7e6a181c5646bfbf38ddfa543", + ); + } + + /// The AWS SigV4 test suite's `post-x-www-form-urlencoded`, which signs a + /// hex `x-amz-content-sha256` like this mode. + #[test] + fn standard_mode_matches_the_aws_test_suite() { + let creds = example_credentials(None); + let signer = SigV4Signer::builder() + .region("us-east-1") + .service("service") + .mode(PayloadHashMode::StandardAws) + .build(); + let mut req = HttpRequest::new( + reqwest::Client::new() + .post("https://example.amazonaws.com/") + .header("content-type", "application/x-www-form-urlencoded") + .header("content-length", "13") + .body("Param1=value1") + .build() + .unwrap(), + ); + + signer.sign_at(&mut req, &creds, test_time()).unwrap(); + + assert_eq!( + req.headers().get("x-amz-content-sha256").unwrap(), + "9095672bbd1f56dfc5b65f3e153adc8731a4a654192329106275f4c7b24d0b6e" + ); + assert_eq!( + req.headers().get("authorization").unwrap(), + "AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20150830/us-east-1/service/aws4_request, \ + SignedHeaders=content-length;content-type;host;x-amz-content-sha256;x-amz-date, \ + Signature=d3875051da38690788ef43de4db0d8f280229d82040bfac253562e56c3f20e0b" + ); + } + + /// The one test on the live clock: everything else pins the time. + #[test] + fn sign_stamps_the_current_time() { + use chrono::SubsecRound; + + let signer = test_signer(PayloadHashMode::IcebergRest); + let mut req = HttpRequest::new( + reqwest::Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + let before = Utc::now(); + signer.sign(&mut req, &test_credentials()).unwrap(); + let after = Utc::now(); + + let date = req.headers().get("x-amz-date").unwrap().to_str().unwrap(); + let stamped = chrono::NaiveDateTime::parse_from_str(date, "%Y%m%dT%H%M%SZ") + .unwrap() + .and_utc(); + assert_eq!(date.len(), "YYYYMMDDTHHMMSSZ".len(), "{date}"); + // The header drops sub-seconds, so compare against a truncated start. + assert!( + before.trunc_subsecs(0) <= stamped && stamped <= after, + "{date}" + ); + } +} diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 6cb4d93b99..f8e7c4179b 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -17,17 +17,23 @@ //! This module contains the iceberg REST catalog implementation. -use std::any::Any; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::fmt::{Debug, Formatter}; use std::future::Future; +use std::num::NonZeroU32; use std::str::FromStr; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; use async_trait::async_trait; -use iceberg::io::{self, FileIO}; +use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use iceberg::io::{ + ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageFactory, +}; use iceberg::table::Table; use iceberg::{ - Catalog, CatalogBuilder, Error, ErrorKind, Namespace, NamespaceIdent, Result, TableCommit, - TableCreation, TableIdent, + Catalog, CatalogBuilder, Error, ErrorKind, Namespace, NamespaceIdent, Result, Runtime, + SessionCatalog, SessionContext, TableCommit, TableCreation, TableIdent, }; use itertools::Itertools; use reqwest::header::{ @@ -37,97 +43,146 @@ use reqwest::{Client, Method, StatusCode, Url}; use tokio::sync::OnceCell; use typed_builder::TypedBuilder; +use crate::auth::{ + AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, AuthManager, AuthSession, NoopAuthManager, OAuth2Manager, +}; use crate::client::{ HttpClient, deserialize_catalog_response, deserialize_unexpected_catalog_error, }; +use crate::endpoint::{Endpoint, V1_NAMESPACE_EXISTS, V1_TABLE_EXISTS}; +use crate::request::HttpRequest; +use crate::response::HttpResponse; use crate::types::{ - CatalogConfig, CommitTableRequest, CommitTableResponse, CreateTableRequest, - ListNamespaceResponse, ListTableResponse, LoadTableResponse, NamespaceSerde, - RegisterTableRequest, RenameTableRequest, + CatalogConfig, CommitTableRequest, CommitTableResponse, CreateNamespaceRequest, + CreateTableRequest, ListNamespaceResponse, ListTablesResponse, LoadTableResult, + NamespaceResponse, RegisterTableRequest, RenameTableRequest, }; /// REST catalog URI pub const REST_CATALOG_PROP_URI: &str = "uri"; /// REST catalog warehouse location pub const REST_CATALOG_PROP_WAREHOUSE: &str = "warehouse"; +/// Requested page size for namespace and table listings; see +/// [`RestSessionCatalogBuilder::with_page_size`]. +pub const REST_CATALOG_PROP_PAGE_SIZE: &str = "rest-page-size"; +/// Disable header redaction in error logs and `Debug` output (defaults to +/// false for security) +pub const REST_CATALOG_PROP_DISABLE_HEADER_REDACTION: &str = "disable-header-redaction"; +/// Authentication scheme: `none` or `oauth2`. When unset, `oauth2` is used +/// if a `token`, `credential` or `oauth2-server-uri` is configured, `none` +/// otherwise. +pub const REST_CATALOG_PROP_AUTH_TYPE: &str = "rest.auth.type"; +/// Maximum number of loaded tables kept for freshness-aware loading (defaults +/// to 100; 0 disables the cache). The cache is keyed by session id, so it only +/// helps callers that reuse a [`SessionContext`], and a table changed through +/// this catalog is invalidated only for the session that changed it. Loads that +/// vend short-lived storage credentials are never cached. +pub const REST_CATALOG_PROP_TABLE_CACHE_MAX_ENTRIES: &str = "rest-table-cache.max-entries"; +/// How long a loaded table stays cached for freshness-aware loading, in +/// milliseconds (defaults to 5 minutes; must be positive and at most 1000 years). +pub const REST_CATALOG_PROP_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS: &str = + "rest-table-cache.expire-after-write-ms"; + +const TABLE_CACHE_MAX_ENTRIES_DEFAULT: u64 = 100; +const TABLE_CACHE_EXPIRE_AFTER_WRITE_MS_DEFAULT: u64 = 5 * 60 * 1000; +/// moka panics on a time-to-live beyond 1000 years. +const TABLE_CACHE_EXPIRE_AFTER_WRITE_MAX: Duration = Duration::from_secs(1000 * 365 * 24 * 3600); + +/// Tables loaded with an `ETag`, keyed by session id and identifier. Loads that +/// vend short-lived storage credentials are not cached, since a `304` cannot +/// refresh them. +type TableCache = moka::sync::Cache<(String, TableIdent), (String, Table)>; + +/// Whether a `loadTable` config key carries a short-lived storage credential. +fn is_short_lived_credential(key: &str) -> bool { + let key = key.to_ascii_lowercase(); + key == S3_SESSION_TOKEN || key == GCS_TOKEN || key.starts_with(ADLS_SAS_TOKEN) +} const ICEBERG_REST_SPEC_VERSION: &str = "0.14.1"; const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); const PATH_V1: &str = "v1"; -/// Builder for [`RestCatalog`]. -#[derive(Debug)] -pub struct RestCatalogBuilder(RestCatalogConfig); - -impl Default for RestCatalogBuilder { - fn default() -> Self { - Self(RestCatalogConfig { - name: None, - uri: "".to_string(), - warehouse: None, - props: HashMap::new(), - client: None, - }) - } +/// Builder for [`RestCatalog`], the [`Catalog`]-compatible façade over a +/// [`RestSessionCatalog`]. +/// +/// The resulting catalog binds one [`SessionContext`] to every operation. Use +/// [`RestSessionCatalogBuilder`] when the caller supplies a context per operation. +#[derive(Debug, Default)] +pub struct RestCatalogBuilder { + session_context: Option, + inner: RestSessionCatalogBuilder, } impl CatalogBuilder for RestCatalogBuilder { type C = RestCatalog; + fn with_storage_factory(mut self, storage_factory: Arc) -> Self { + self.inner = self.inner.with_storage_factory(storage_factory); + self + } + + fn with_kms_client_factory(mut self, kms_client_factory: Arc) -> Self { + self.inner = self.inner.with_kms_client_factory(kms_client_factory); + self + } + + fn with_runtime(mut self, runtime: Runtime) -> Self { + self.inner = self.inner.with_runtime(runtime); + self + } + fn load( - mut self, + self, name: impl Into, props: HashMap, ) -> impl Future> + Send { - self.0.name = Some(name.into()); - - if props.contains_key(REST_CATALOG_PROP_URI) { - self.0.uri = props - .get(REST_CATALOG_PROP_URI) - .cloned() - .unwrap_or_default(); - } + let name = name.into(); + async move { + let context = self.session_context.unwrap_or_else(SessionContext::empty); + let session_catalog = Arc::new(self.inner.load(name, props).await?); - if props.contains_key(REST_CATALOG_PROP_WAREHOUSE) { - self.0.warehouse = props.get(REST_CATALOG_PROP_WAREHOUSE).cloned() + Ok(RestCatalog::from_session_catalog(context, session_catalog)) } - - // Collect other remaining properties - self.0.props = props - .into_iter() - .filter(|(k, _)| k != REST_CATALOG_PROP_URI && k != REST_CATALOG_PROP_WAREHOUSE) - .collect(); - - let result = { - if self.0.name.is_none() { - Err(Error::new( - ErrorKind::DataInvalid, - "Catalog name is required", - )) - } else if self.0.uri.is_empty() { - Err(Error::new( - ErrorKind::DataInvalid, - "Catalog uri is required", - )) - } else { - Ok(RestCatalog::new(self.0)) - } - }; - - std::future::ready(result) } } impl RestCatalogBuilder { + /// Sets the requested page size for namespace and table listings; see + /// [`RestSessionCatalogBuilder::with_page_size`]. + pub fn with_page_size(mut self, page_size: NonZeroU32) -> Self { + self.inner = self.inner.with_page_size(page_size); + self + } + /// Configures the catalog with a custom HTTP client. pub fn with_client(mut self, client: Client) -> Self { - self.0.client = Some(client); + self.inner = self.inner.with_client(client); + self + } + + /// Binds the session context forwarded with every catalog operation. + /// + /// If this is not called, [`load`](CatalogBuilder::load) creates a fresh + /// [`SessionContext::empty`] context. + pub fn with_session_context(mut self, context: SessionContext) -> Self { + self.session_context = Some(context); + self + } + + /// Sets a custom auth manager, overriding the `rest.auth.type` configuration. + /// + /// The builder takes ownership of the manager. The loaded catalog shares it + /// across authentication sessions and requests. + pub fn with_auth_manager(mut self, auth_manager: M) -> Self + where M: AuthManager + 'static { + self.inner = self.inner.with_auth_manager(auth_manager); self } } /// Rest catalog configuration. -#[derive(Clone, Debug, TypedBuilder)] +#[derive(Clone, TypedBuilder)] pub(crate) struct RestCatalogConfig { #[builder(default, setter(strip_option))] name: Option, @@ -142,6 +197,49 @@ pub(crate) struct RestCatalogConfig { #[builder(default)] client: Option, + + /// Lazily-created default HTTP client, shared through clones of this + /// config so OAuth and catalog traffic reuse one connection pool + /// (matching the single-client behavior before the AuthManager refactor). + #[builder(default)] + default_client: Arc>, +} + +/// Property keys whose values are secrets, or may embed them (headers, +/// connection strings, keys like `adls.account-key` or `s3.sse.key`). +fn is_sensitive_prop(key: &str) -> bool { + key.contains("token") + || key.contains("credential") + || key.contains("secret") + || key.contains("password") + || key.contains("key") + || key.contains("connection-string") + || key.starts_with("header.") +} + +/// Redacts secret property values: this config is printed by the derived +/// [`Debug`] implementations of [`RestSessionCatalog`] and [`RestCatalog`]. +impl Debug for RestCatalogConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let props: HashMap<&str, &str> = self + .props + .iter() + .map(|(key, value)| { + let value = if is_sensitive_prop(key) { + "[REDACTED]" + } else { + value.as_str() + }; + (key.as_str(), value) + }) + .collect(); + f.debug_struct("RestCatalogConfig") + .field("name", &self.name) + .field("uri", &self.uri) + .field("warehouse", &self.warehouse) + .field("props", &props) + .finish_non_exhaustive() + } } impl RestCatalogConfig { @@ -158,11 +256,35 @@ impl RestCatalogConfig { } pub(crate) fn get_token_endpoint(&self) -> String { - if let Some(oauth2_uri) = self.props.get("oauth2-server-uri") { - oauth2_uri.to_string() - } else { - [&self.uri, PATH_V1, "oauth", "tokens"].join("/") - } + self.explicit_oauth2_server_uri() + .unwrap_or_else(|| default_token_endpoint(&self.uri)) + } + + /// The `oauth2-server-uri` property, only when explicitly configured. + pub(crate) fn explicit_oauth2_server_uri(&self) -> Option { + self.props.get("oauth2-server-uri").cloned() + } + + /// Parse only the effective value, after server defaults and overrides are merged. + fn page_size(&self) -> Result> { + self.props + .get(REST_CATALOG_PROP_PAGE_SIZE) + .map(|value| { + let invalid_page_size = || { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid {REST_CATALOG_PROP_PAGE_SIZE}: {value}"), + ) + }; + let page_size = value + .parse::() + .map_err(|err| invalid_page_size().with_source(err))?; + if page_size.get() > i32::MAX as u32 { + return Err(invalid_page_size()); + } + Ok(page_size) + }) + .transpose() } fn namespaces_endpoint(&self) -> String { @@ -194,9 +316,13 @@ impl RestCatalogConfig { ]) } - /// Get the client from the config. - pub(crate) fn client(&self) -> Option { - self.client.clone() + /// The HTTP client: the configured one, or a lazily-created default that + /// is shared across every user of this config (and its clones), so token + /// and catalog requests keep sharing one connection pool. + pub(crate) fn client(&self) -> Client { + self.client + .clone() + .unwrap_or_else(|| self.default_client.get_or_init(Client::default).clone()) } /// Get the token from the config. @@ -208,99 +334,80 @@ impl RestCatalogConfig { /// Get the credentials from the config. The client can use these credentials to fetch a new /// token. - /// - /// ## Output - /// - /// - `None`: No credential is set. - /// - `Some(None, client_secret)`: No client_id is set, use client_secret directly. - /// - `Some(Some(client_id), client_secret)`: Both client_id and client_secret are set. pub(crate) fn credential(&self) -> Option<(Option, String)> { - let cred = self.props.get("credential")?; - - match cred.split_once(':') { - Some((client_id, client_secret)) => { - Some((Some(client_id.to_string()), client_secret.to_string())) - } - None => Some((None, cred.to_string())), - } + credential_from_props(&self.props) } - /// Get the extra headers from config, which includes: - /// - /// - `content-type` - /// - `x-client-version` - /// - `user-agent` - /// - All headers specified by `header.xxx` in props. + /// Get the extra headers from config, see [`extra_headers_from_props`]. pub(crate) fn extra_headers(&self) -> Result { - let mut headers = HeaderMap::from_iter([ - ( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), - ), - ( - HeaderName::from_static("x-client-version"), - HeaderValue::from_static(ICEBERG_REST_SPEC_VERSION), - ), - ( - header::USER_AGENT, - HeaderValue::from_str(&format!("iceberg-rs/{}", CARGO_PKG_VERSION)).unwrap(), - ), - ]); - - for (key, value) in self - .props - .iter() - .filter_map(|(k, v)| k.strip_prefix("header.").map(|k| (k, v))) - { - headers.insert( - HeaderName::from_str(key).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid header name: {key}"), - ) - .with_source(e) - })?, - HeaderValue::from_str(value).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid header value: {value}"), - ) - .with_source(e) - })?, - ); - } - - Ok(headers) + extra_headers_from_props(&self.props) } /// Get the optional OAuth headers from the config. pub(crate) fn extra_oauth_params(&self) -> HashMap { - let mut params = HashMap::new(); + oauth_params_from_props(&self.props) + } - if let Some(scope) = self.props.get("scope") { - params.insert("scope".to_string(), scope.to_string()); - } else { - params.insert("scope".to_string(), "catalog".to_string()); - } + /// Check if header redaction is disabled in error logs. + /// + /// Returns true if the `disable-header-redaction` property is set to "true". + /// Defaults to false for security (headers are redacted by default). + pub(crate) fn disable_header_redaction(&self) -> bool { + self.props + .get(REST_CATALOG_PROP_DISABLE_HEADER_REDACTION) + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(false) + } - let optional_params = ["audience", "resource"]; - for param_name in optional_params { - if let Some(value) = self.props.get(param_name) { - params.insert(param_name.to_string(), value.to_string()); - } + /// Builds the table cache for freshness-aware loading, or `None` when + /// `rest-table-cache.max-entries` is 0. + fn table_cache(&self) -> Result> { + let parse = |key: &str, default: u64| -> Result { + self.props.get(key).map_or(Ok(default), |v| { + v.parse().map_err(|e| { + Error::new(ErrorKind::DataInvalid, format!("Invalid {key}: {v}")).with_source(e) + }) + }) + }; + let expire_after_write_ms = parse( + REST_CATALOG_PROP_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS, + TABLE_CACHE_EXPIRE_AFTER_WRITE_MS_DEFAULT, + )?; + let expire_after_write = Duration::from_millis(expire_after_write_ms); + // Zero is ambiguous: it could mean "disable" or "never expire". + if expire_after_write.is_zero() || expire_after_write > TABLE_CACHE_EXPIRE_AFTER_WRITE_MAX { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid {REST_CATALOG_PROP_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS}: {expire_after_write_ms} (must be between 1 ms and 1000 years)" + ), + )); } - - params + let max_entries = parse( + REST_CATALOG_PROP_TABLE_CACHE_MAX_ENTRIES, + TABLE_CACHE_MAX_ENTRIES_DEFAULT, + )?; + Ok((max_entries > 0).then(|| { + TableCache::builder() + .max_capacity(max_entries) + .time_to_live(expire_after_write) + .build() + })) } /// Merge the `RestCatalogConfig` with the a [`CatalogConfig`] (fetched from the REST server). pub(crate) fn merge_with_config(mut self, mut config: CatalogConfig) -> Self { - if let Some(uri) = config.overrides.remove("uri") { + if let Some(uri) = config.overrides.remove(REST_CATALOG_PROP_URI) { self.uri = uri; } let mut props = config.defaults; props.extend(self.props); + // The builder moved the client warehouse off the props; restore it + // between defaults and overrides (default < client < override). + if let Some(warehouse) = &self.warehouse { + props.insert(REST_CATALOG_PROP_WAREHOUSE.to_string(), warehouse.clone()); + } props.extend(config.overrides); self.props = props; @@ -308,188 +415,315 @@ impl RestCatalogConfig { } } -#[derive(Debug)] -struct RestContext { - client: HttpClient, +/// Parses the `credential` property. +/// +/// ## Output +/// +/// - `None`: No credential is set. +/// - `Some(None, client_secret)`: No client_id is set, use client_secret directly. +/// - `Some(Some(client_id), client_secret)`: Both client_id and client_secret are set. +pub(crate) fn credential_from_props( + props: &HashMap, +) -> Option<(Option, String)> { + let cred = props.get("credential")?; + + match cred.split_once(':') { + Some((client_id, client_secret)) => { + Some((Some(client_id.to_string()), client_secret.to_string())) + } + None => Some((None, cred.to_string())), + } +} + +/// The extra headers added to each request, which include: +/// +/// - `content-type` +/// - `x-client-version` +/// - `user-agent` +/// - All headers specified by `header.xxx` in props. +pub(crate) fn extra_headers_from_props(props: &HashMap) -> Result { + let mut headers = HeaderMap::from_iter([ + ( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ), + ( + HeaderName::from_static("x-client-version"), + HeaderValue::from_static(ICEBERG_REST_SPEC_VERSION), + ), + ( + header::USER_AGENT, + HeaderValue::from_str(&format!("iceberg-rust/{CARGO_PKG_VERSION}")).unwrap(), + ), + ]); + + headers.extend(explicit_headers_from_props(props)?); + + Ok(headers) +} + +/// The default OAuth2 token endpoint for a catalog `uri`. +pub(crate) fn default_token_endpoint(uri: &str) -> String { + [uri, PATH_V1, "oauth", "tokens"].join("/") +} + +/// Only the headers explicitly configured via `header.xxx` props (no defaults). +pub(crate) fn explicit_headers_from_props(props: &HashMap) -> Result { + let mut headers = HeaderMap::new(); + for (key, value) in props + .iter() + .filter_map(|(k, v)| k.strip_prefix("header.").map(|k| (k, v))) + { + headers.insert( + HeaderName::from_str(key).map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid header name: {key}"), + ) + .with_source(e) + })?, + HeaderValue::from_str(value).map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + // The value itself is omitted: it may be a secret. + format!("Invalid value for header: {key}"), + ) + .with_source(e) + })?, + ); + } + + Ok(headers) +} + +/// The optional OAuth parameters added to each authentication request. +pub(crate) fn oauth_params_from_props(props: &HashMap) -> HashMap { + let mut params = HashMap::new(); + + if let Some(scope) = props.get("scope") { + params.insert("scope".to_string(), scope.to_string()); + } else { + params.insert("scope".to_string(), "catalog".to_string()); + } + + let optional_params = ["audience", "resource"]; + for param_name in optional_params { + if let Some(value) = props.get(param_name) { + params.insert(param_name.to_string(), value.to_string()); + } + } + + params +} + +struct RestCatalogClient { + /// The manager that created `catalog_session`; retained so each request can + /// derive authentication for its [`SessionContext`]. + auth_manager: Arc, + /// The catalog-wide session passed to [`AuthManager::contextual_session`]. + catalog_session: Arc, + /// Shared HTTP transport and configuration. It remains unauthenticated; + /// request-time clones attach the contextual session. + http_client: HttpClient, /// Runtime config is fetched from rest server and stored here. /// - /// It's could be different from the user config. + /// It could be different from the user config. config: RestCatalogConfig, + /// Capabilities the server advertises (see [`RestSessionCatalog::supports_endpoint`]). + endpoints: HashSet, + /// Validated page size from the merged runtime configuration. + page_size: Option, + /// Tables reused when the server answers a load with `304 Not Modified`. + table_cache: Option, } -/// Rest catalog implementation. -#[derive(Debug)] -pub struct RestCatalog { - /// User config is stored as-is and never be changed. - /// - /// It could be different from the config fetched from the server and used at runtime. - user_config: RestCatalogConfig, - ctx: OnceCell, - /// Extensions for the FileIOBuilder. - file_io_extensions: io::Extensions, +impl Debug for RestCatalogClient { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // Auth managers and sessions may contain secrets, so keep them out of + // the catalog's derived Debug output just as HttpClient does. + f.debug_struct("RestCatalogClient") + .field("http_client", &self.http_client) + .field("config", &self.config) + .field("endpoints", &self.endpoints) + .finish_non_exhaustive() + } } -impl RestCatalog { - /// Creates a `RestCatalog` from a [`RestCatalogConfig`]. - fn new(config: RestCatalogConfig) -> Self { - Self { - user_config: config, - ctx: OnceCell::new(), - file_io_extensions: io::Extensions::default(), - } +impl RestCatalogClient { + /// Initializes the runtime config, advertised endpoints, auth manager, and + /// catalog authentication session shared by one REST catalog instance. + async fn init( + user_config: &RestCatalogConfig, + auth_manager: Arc, + ) -> Result { + let http_client = HttpClient::new(user_config)?; + // The init session lives only for the config handshake, so a + // manager whose session guards a one-shot resource can release + // it before deriving the catalog session. + let catalog_config = { + let init_session = auth_manager + .init_session(&http_client, &Self::auth_props(user_config)) + .await?; + Self::load_config(&http_client, init_session.as_ref(), user_config).await? + }; + // Use the advertised endpoints as-is, falling back to + // `DEFAULT_ENDPOINTS` when absent or empty. + let endpoints = match &catalog_config.endpoints { + Some(advertised) if !advertised.is_empty() => advertised.iter().cloned().collect(), + _ => crate::endpoint::DEFAULT_ENDPOINTS.clone(), + }; + let config = user_config.clone().merge_with_config(catalog_config); + let page_size = config.page_size()?; + let http_client = http_client.update_with(&config)?; + // The manager is handed an unauthenticated client: its own + // requests must not be signed by the session it is deriving. + let catalog_session = auth_manager + .catalog_session(&http_client, &Self::auth_props(&config)) + .await?; + + Ok(Self { + auth_manager, + catalog_session, + table_cache: config.table_cache()?, + config, + http_client, + endpoints, + page_size, + }) } - /// Add an extension to the file IO builder. - pub fn with_file_io_extension(mut self, ext: T) -> Self { - self.file_io_extensions.add(ext); - self + fn invalidate_table(&self, context: &SessionContext, table: &TableIdent) { + if let Some(cache) = &self.table_cache { + cache.invalidate(&(context.session_id().to_string(), table.clone())); + } } - /// Gets the [`RestContext`] from the catalog. - async fn context(&self) -> Result<&RestContext> { - self.ctx - .get_or_try_init(|| async { - let client = HttpClient::new(&self.user_config)?; - let catalog_config = RestCatalog::load_config(&client, &self.user_config).await?; - let config = self.user_config.clone().merge_with_config(catalog_config); - let client = client.update_with(&config)?; + /// Testing only: the bearer token the catalog session would attach. + #[cfg(test)] + async fn token(&self) -> Option { + self.http_client.token(self.catalog_session.as_ref()).await + } - Ok(RestContext { config, client }) - }) + /// Sends `request` with the authentication derived for `context`. + async fn query_catalog( + &self, + context: &SessionContext, + request: HttpRequest, + ) -> Result { + let session = self + .auth_manager + .contextual_session(context, Arc::clone(&self.catalog_session)) + .await?; + self.http_client + .query_catalog(session.as_ref(), request) .await } - /// Load the runtime config from the server by `user_config`. + /// The properties handed to the [`AuthManager`], with the catalog `uri` + /// and `warehouse` made explicit. + fn auth_props(config: &RestCatalogConfig) -> HashMap { + // `oauth2-server-uri` stays absent unless explicitly configured, so an + // injected manager keeps its own endpoint. The resolved `uri` and + // `warehouse` ARE passed: the builder moved them off the props, and + // the built-in manager recomputes its token endpoint from the URI. + let mut props = config.props.clone(); + props.insert(REST_CATALOG_PROP_URI.to_string(), config.uri.clone()); + if let Some(warehouse) = &config.warehouse { + // A fallback only: after the handshake the merged props hold + // the resolved warehouse, server override included. + props + .entry(REST_CATALOG_PROP_WAREHOUSE.to_string()) + .or_insert_with(|| warehouse.clone()); + } + props + } + + /// Loads the runtime config from the server using `user_config`. /// /// It's required for a REST catalog to update its config after creation. async fn load_config( - client: &HttpClient, + http_client: &HttpClient, + auth_session: &dyn AuthSession, user_config: &RestCatalogConfig, ) -> Result { - let mut request_builder = client.request(Method::GET, user_config.config_endpoint()); + let mut request_builder = http_client.request(Method::GET, user_config.config_endpoint()); if let Some(warehouse_location) = &user_config.warehouse { request_builder = request_builder.query(&[("warehouse", warehouse_location)]); } - let request = request_builder.build()?; + let request = HttpRequest::build(request_builder)?; - let http_response = client.query_catalog(request).await?; + let http_response = http_client.query_catalog(auth_session, request).await?; match http_response.status() { - StatusCode::OK => deserialize_catalog_response(http_response).await, - _ => Err(deserialize_unexpected_catalog_error(http_response).await), + StatusCode::OK => deserialize_catalog_response(http_response), + _ => Err(deserialize_unexpected_catalog_error( + http_response, + http_client.disable_header_redaction(), + )), } } +} - async fn load_file_io( - &self, - metadata_location: Option<&str>, - extra_config: Option>, - ) -> Result { - let mut props = self.context().await?.config.props.clone(); - if let Some(config) = extra_config { - props.extend(config); - } - - // If the warehouse is a logical identifier instead of a URL we don't want - // to raise an exception - let warehouse_path = match self.context().await?.config.warehouse.as_deref() { - Some(url) if Url::parse(url).is_ok() => Some(url), - Some(_) => None, - None => None, - }; - - let file_io = match metadata_location.or(warehouse_path) { - Some(url) => FileIO::from_path(url)? - .with_props(props) - .with_extensions(self.file_io_extensions.clone()) - .build()?, - None => { - return Err(Error::new( - ErrorKind::Unexpected, - "Unable to load file io, neither warehouse nor metadata location is set!", - ))?; - } - }; +/// A [`Catalog`]-compatible façade over [`RestSessionCatalog`]. +/// +/// Every operation is forwarded with the single [`SessionContext`] selected by +/// [`RestCatalogBuilder`]. Use [`RestSessionCatalog`] when the caller needs to +/// provide a context per operation. +#[derive(Debug)] +pub struct RestCatalog { + session_context: SessionContext, + inner: Arc, +} - Ok(file_io) +impl RestCatalog { + /// Creates a `RestCatalog` from a [`RestCatalogConfig`]. + #[cfg(test)] + fn new( + context: SessionContext, + config: RestCatalogConfig, + auth_manager: Option>, + storage_factory: Option>, + runtime: Runtime, + kms_client: Option>, + ) -> Self { + let session_catalog = Arc::new(RestSessionCatalog::new( + config, + auth_manager, + storage_factory, + runtime, + kms_client, + )); + + Self::from_session_catalog(context, session_catalog) } - /// Invalidate the current token without generating a new one. On the next request, the client - /// will attempt to generate a new token. - pub async fn invalidate_token(&self) -> Result<()> { - self.context().await?.client.invalidate_token().await + fn from_session_catalog(context: SessionContext, inner: Arc) -> Self { + Self { + session_context: context, + inner, + } } - /// Invalidate the current token and set a new one. Generates a new token before invalidating - /// the current token, meaning the old token will be used until this function acquires the lock - /// and overwrites the token. - /// - /// If credential is invalid, or the request fails, this method will return an error and leave - /// the current token unchanged. - pub async fn regenerate_token(&self) -> Result<()> { - self.context().await?.client.regenerate_token().await + #[cfg(test)] + async fn client(&self) -> Result<&RestCatalogClient> { + self.inner.client().await } } -/// All requests and expected responses are derived from the REST catalog API spec: -/// https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml +/// Every operation forwards to its [`RestSessionCatalog`] equivalent with the +/// bound [`SessionContext`]; see that implementation for the REST-specific +/// behavior. #[async_trait] impl Catalog for RestCatalog { async fn list_namespaces( &self, parent: Option<&NamespaceIdent>, ) -> Result> { - let context = self.context().await?; - let endpoint = context.config.namespaces_endpoint(); - let mut namespaces = Vec::new(); - let mut next_token = None; - - loop { - let mut request = context.client.request(Method::GET, endpoint.clone()); - - // Filter on `parent={namespace}` if a parent namespace exists. - if let Some(ns) = parent { - request = request.query(&[("parent", ns.to_url_string())]); - } - - if let Some(token) = next_token { - request = request.query(&[("pageToken", token)]); - } - - let http_response = context.client.query_catalog(request.build()?).await?; - - match http_response.status() { - StatusCode::OK => { - let response = - deserialize_catalog_response::(http_response) - .await?; - - let ns_identifiers = response - .namespaces - .into_iter() - .map(NamespaceIdent::from_vec) - .collect::>>()?; - - namespaces.extend(ns_identifiers); - - match response.next_page_token { - Some(token) => next_token = Some(token), - None => break, - } - } - StatusCode::NOT_FOUND => { - return Err(Error::new( - ErrorKind::Unexpected, - "The parent parameter of the namespace provided does not exist", - )); - } - _ => return Err(deserialize_unexpected_catalog_error(http_response).await), - } - } - - Ok(namespaces) + self.inner + .list_namespaces(&self.session_context, parent) + .await } async fn create_namespace( @@ -497,76 +731,457 @@ impl Catalog for RestCatalog { namespace: &NamespaceIdent, properties: HashMap, ) -> Result { - let context = self.context().await?; - - let request = context - .client - .request(Method::POST, context.config.namespaces_endpoint()) - .json(&NamespaceSerde { - namespace: namespace.as_ref().clone(), - properties: Some(properties), - }) - .build()?; - - let http_response = context.client.query_catalog(request).await?; - - match http_response.status() { - StatusCode::OK => { - let response = - deserialize_catalog_response::(http_response).await?; - Namespace::try_from(response) - } - StatusCode::CONFLICT => Err(Error::new( - ErrorKind::Unexpected, - "Tried to create a namespace that already exists", - )), - _ => Err(deserialize_unexpected_catalog_error(http_response).await), - } + self.inner + .create_namespace(&self.session_context, namespace, properties) + .await } async fn get_namespace(&self, namespace: &NamespaceIdent) -> Result { - let context = self.context().await?; + self.inner + .get_namespace(&self.session_context, namespace) + .await + } - let request = context - .client - .request(Method::GET, context.config.namespace_endpoint(namespace)) - .build()?; + async fn namespace_exists(&self, ns: &NamespaceIdent) -> Result { + self.inner.namespace_exists(&self.session_context, ns).await + } - let http_response = context.client.query_catalog(request).await?; + async fn update_namespace( + &self, + namespace: &NamespaceIdent, + properties: HashMap, + ) -> Result<()> { + self.inner + .update_namespace(&self.session_context, namespace, properties) + .await + } + + async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> { + self.inner + .drop_namespace(&self.session_context, namespace) + .await + } + + async fn list_tables(&self, namespace: &NamespaceIdent) -> Result> { + self.inner + .list_tables(&self.session_context, namespace) + .await + } + + async fn create_table( + &self, + namespace: &NamespaceIdent, + creation: TableCreation, + ) -> Result
{ + self.inner + .create_table(&self.session_context, namespace, creation) + .await + } + + async fn load_table(&self, table_ident: &TableIdent) -> Result
{ + self.inner + .load_table(&self.session_context, table_ident) + .await + } + + async fn drop_table(&self, table: &TableIdent) -> Result<()> { + self.inner.drop_table(&self.session_context, table).await + } + + async fn purge_table(&self, table: &TableIdent) -> Result<()> { + self.inner.purge_table(&self.session_context, table).await + } + + async fn table_exists(&self, table: &TableIdent) -> Result { + self.inner.table_exists(&self.session_context, table).await + } + + async fn rename_table(&self, src: &TableIdent, dest: &TableIdent) -> Result<()> { + self.inner + .rename_table(&self.session_context, src, dest) + .await + } + + async fn register_table( + &self, + table_ident: &TableIdent, + metadata_location: String, + ) -> Result
{ + self.inner + .register_table(&self.session_context, table_ident, metadata_location) + .await + } + + async fn update_table(&self, commit: TableCommit) -> Result
{ + self.inner.update_table(&self.session_context, commit).await + } +} + +/// REST catalog implementation of [`SessionCatalog`]. +/// +/// Each operation accepts a [`SessionContext`]. REST configuration, the auth manager, the catalog +/// authentication session, and the HTTP client are initialized lazily once per catalog. Each +/// REST request uses the contextual authentication session returned by the manager. +#[derive(Debug)] +pub struct RestSessionCatalog { + /// Builder-supplied override retained so lazy client initialization can + /// clone it into the runtime state. When absent, a manager is resolved from + /// `rest.auth.type` during initialization. + auth_manager_override: Option>, + /// User config is stored as-is and never changed. + /// + /// It could be different from the config fetched from the server and used at runtime. + user_config: RestCatalogConfig, + client: OnceCell, + /// Storage factory for creating FileIO instances. + storage_factory: Option>, + runtime: Runtime, + /// Optional KMS client for encrypted tables. + kms_client: Option>, +} + +impl RestSessionCatalog { + /// Creates a `RestSessionCatalog` from a [`RestCatalogConfig`]. + fn new( + config: RestCatalogConfig, + auth_manager_override: Option>, + storage_factory: Option>, + runtime: Runtime, + kms_client: Option>, + ) -> Self { + Self { + auth_manager_override: auth_manager_override.map(Arc::from), + user_config: config, + client: OnceCell::new(), + storage_factory, + runtime, + kms_client, + } + } + + /// Sends a DELETE request for the given table, optionally requesting purge. + async fn delete_table( + &self, + context: &SessionContext, + table: &TableIdent, + purge: bool, + ) -> Result<()> { + let client = self.client().await?; + + let mut request_builder = client + .http_client + .request(Method::DELETE, client.config.table_endpoint(table)); + + if purge { + request_builder = request_builder.query(&[("purgeRequested", "true")]); + } + + let request = HttpRequest::build(request_builder)?; + client.invalidate_table(context, table); + let http_response = client.query_catalog(context, request).await?; match http_response.status() { - StatusCode::OK => { - let response = - deserialize_catalog_response::(http_response).await?; - Namespace::try_from(response) - } + StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), StatusCode::NOT_FOUND => Err(Error::new( - ErrorKind::Unexpected, - "Tried to get a namespace that does not exist", + ErrorKind::TableNotFound, + "Tried to drop a table that does not exist", + )), + _ => Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), )), - _ => Err(deserialize_unexpected_catalog_error(http_response).await), } } - async fn namespace_exists(&self, ns: &NamespaceIdent) -> Result { - let context = self.context().await?; + /// The configured auth scheme: explicit `rest.auth.type` (matched + /// case-insensitively) when set; otherwise `oauth2` when a `token`, + /// `credential` or `oauth2-server-uri` is configured (preserving + /// pre-`rest.auth.type` setups), `none` when none is. + fn auth_type(config: &RestCatalogConfig) -> String { + config + .props + .get(REST_CATALOG_PROP_AUTH_TYPE) + // Matched case-insensitively, as the other flag properties are. + .map(|auth_type| auth_type.to_ascii_lowercase()) + .unwrap_or_else(|| { + if config.token().is_some() + || config.credential().is_some() + || config.explicit_oauth2_server_uri().is_some() + { + AUTH_TYPE_OAUTH2.to_string() + } else { + AUTH_TYPE_NONE.to_string() + } + }) + } + + /// Resolves the auth manager: a `with_auth_manager` override wins, + /// otherwise one is built from the `rest.auth.type` configuration. + fn resolve_auth_manager(&self) -> Result> { + if let Some(auth_manager) = &self.auth_manager_override { + return Ok(auth_manager.clone()); + } + let config = &self.user_config; + let auth_type = Self::auth_type(config); + // Java parity (`AuthManagers`): make the inference visible so users + // configure the type explicitly. + if auth_type == AUTH_TYPE_OAUTH2 && !config.props.contains_key(REST_CATALOG_PROP_AUTH_TYPE) + { + tracing::warn!( + "Inferring {REST_CATALOG_PROP_AUTH_TYPE}={AUTH_TYPE_OAUTH2} from the configured \ + OAuth properties; set it explicitly to avoid this warning" + ); + } + match auth_type.as_str() { + AUTH_TYPE_NONE => Ok(Arc::new(NoopAuthManager)), + AUTH_TYPE_OAUTH2 => Ok(Arc::new(OAuth2Manager::from_config(config)?)), + other => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "unknown '{REST_CATALOG_PROP_AUTH_TYPE}': {other}; use \ + `RestSessionCatalogBuilder::with_auth_manager` or \ + `RestCatalogBuilder::with_auth_manager` to inject a custom auth manager" + ), + )), + } + } + + /// Gets the [`RestCatalogClient`] from the catalog. + async fn client(&self) -> Result<&RestCatalogClient> { + self.client + .get_or_try_init(|| async { + RestCatalogClient::init(&self.user_config, self.resolve_auth_manager()?).await + }) + .await + } - let request = context - .client - .request(Method::HEAD, context.config.namespace_endpoint(ns)) - .build()?; + /// Returns whether the server supports `endpoint`, per the `endpoints` it + /// advertised in `GET /v1/config` (or a default base set when it advertised + /// none). + pub(crate) async fn supports_endpoint(&self, endpoint: &Endpoint) -> Result { + Ok(self.client().await?.endpoints.contains(endpoint)) + } - let http_response = context.client.query_catalog(request).await?; + /// Issue a `HEAD` request to `url` and interpret it as an existence check: + /// `2xx` means it exists, `404` means it doesn't. + async fn check_exists_via_head( + &self, + context: &SessionContext, + client: &RestCatalogClient, + url: String, + ) -> Result { + let request = HttpRequest::build(client.http_client.request(Method::HEAD, url))?; + let http_response = client.query_catalog(context, request).await?; match http_response.status() { StatusCode::NO_CONTENT | StatusCode::OK => Ok(true), StatusCode::NOT_FOUND => Ok(false), - _ => Err(deserialize_unexpected_catalog_error(http_response).await), + _ => Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )), + } + } + + async fn load_file_io( + &self, + metadata_location: Option<&str>, + extra_config: Option>, + ) -> Result { + let mut props = self.client().await?.config.props.clone(); + if let Some(config) = extra_config { + props.extend(config); + } + + // If the warehouse is a logical identifier instead of a URL we don't want + // to raise an exception + let warehouse_path = match self.client().await?.config.warehouse.as_deref() { + Some(url) if Url::parse(url).is_ok() => Some(url), + Some(_) => None, + None => None, + }; + + if metadata_location.or(warehouse_path).is_none() { + return Err(Error::new( + ErrorKind::Unexpected, + "Unable to load file io, neither warehouse nor metadata location is set!", + )); + } + + // Require a StorageFactory to be provided + let factory = self + .storage_factory + .clone() + .ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "StorageFactory must be provided for REST catalog table operations. Use `with_storage_factory` to configure it.", + ) + })?; + + let file_io = FileIOBuilder::new(factory).with_props(props).build(); + + Ok(file_io) + } +} + +/// All requests and expected responses are derived from the REST catalog API spec: +/// +#[async_trait] +impl SessionCatalog for RestSessionCatalog { + async fn list_namespaces( + &self, + context: &SessionContext, + parent: Option<&NamespaceIdent>, + ) -> Result> { + let client = self.client().await?; + let endpoint = client.config.namespaces_endpoint(); + let mut namespaces = Vec::new(); + // An empty initial token explicitly opts in to pagination. + let mut next_token = client.page_size.map(|_| String::new()); + + loop { + let mut request = client.http_client.request(Method::GET, endpoint.clone()); + + if let Some(page_size) = client.page_size { + request = request.query(&[("pageSize", page_size)]); + } + + // Filter on `parent={namespace}` if a parent namespace exists. + if let Some(ns) = parent { + request = request.query(&[("parent", ns.to_url_string())]); + } + + if let Some(token) = next_token { + request = request.query(&[("pageToken", token)]); + } + + let http_response = client + .query_catalog(context, HttpRequest::build(request)?) + .await?; + + match http_response.status() { + StatusCode::OK => { + let response = + deserialize_catalog_response::(http_response)?; + + namespaces.extend(response.namespaces); + + match response.next_page_token { + Some(token) => next_token = Some(token), + None => break, + } + } + StatusCode::NOT_FOUND => { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + "The parent parameter of the namespace provided does not exist", + )); + } + _ => { + return Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )); + } + } + } + + Ok(namespaces) + } + + async fn create_namespace( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + properties: HashMap, + ) -> Result { + let client = self.client().await?; + + let request = HttpRequest::build( + client + .http_client + .request(Method::POST, client.config.namespaces_endpoint()) + .json(&CreateNamespaceRequest { + namespace: namespace.clone(), + properties, + }), + )?; + + let http_response = client.query_catalog(context, request).await?; + + match http_response.status() { + StatusCode::OK => { + let response = deserialize_catalog_response::(http_response)?; + Ok(Namespace::from(response)) + } + StatusCode::CONFLICT => Err(Error::new( + ErrorKind::NamespaceAlreadyExists, + "Tried to create a namespace that already exists", + )), + _ => Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )), + } + } + + async fn get_namespace( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + ) -> Result { + let client = self.client().await?; + + let request = HttpRequest::build( + client + .http_client + .request(Method::GET, client.config.namespace_endpoint(namespace)), + )?; + + let http_response = client.query_catalog(context, request).await?; + + match http_response.status() { + StatusCode::OK => { + let response = deserialize_catalog_response::(http_response)?; + Ok(Namespace::from(response)) + } + StatusCode::NOT_FOUND => Err(Error::new( + ErrorKind::NamespaceNotFound, + "Tried to get a namespace that does not exist", + )), + _ => Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )), + } + } + + async fn namespace_exists( + &self, + context: &SessionContext, + ns: &NamespaceIdent, + ) -> Result { + // Prefer a cheap HEAD when the server advertises it; otherwise fall back + // to loading the namespace (GET) and treating a missing namespace as + // `false`, so this still works against servers that don't advertise the + // HEAD route. + if !self.supports_endpoint(&V1_NAMESPACE_EXISTS).await? { + return match self.get_namespace(context, ns).await { + Ok(_) => Ok(true), + Err(e) if e.kind() == ErrorKind::NamespaceNotFound => Ok(false), + Err(e) => Err(e), + }; } + + let client = self.client().await?; + self.check_exists_via_head(context, client, client.config.namespace_endpoint(ns)) + .await } async fn update_namespace( &self, + _context: &SessionContext, _namespace: &NamespaceIdent, _properties: HashMap, ) -> Result<()> { @@ -576,45 +1191,64 @@ impl Catalog for RestCatalog { )) } - async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> { - let context = self.context().await?; + async fn drop_namespace( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + ) -> Result<()> { + let client = self.client().await?; - let request = context - .client - .request(Method::DELETE, context.config.namespace_endpoint(namespace)) - .build()?; + let request = HttpRequest::build( + client + .http_client + .request(Method::DELETE, client.config.namespace_endpoint(namespace)), + )?; - let http_response = context.client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; match http_response.status() { StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), StatusCode::NOT_FOUND => Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceNotFound, "Tried to drop a namespace that does not exist", )), - _ => Err(deserialize_unexpected_catalog_error(http_response).await), + _ => Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )), } } - async fn list_tables(&self, namespace: &NamespaceIdent) -> Result> { - let context = self.context().await?; - let endpoint = context.config.tables_endpoint(namespace); + async fn list_tables( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + ) -> Result> { + let client = self.client().await?; + let endpoint = client.config.tables_endpoint(namespace); let mut identifiers = Vec::new(); - let mut next_token = None; + // An empty initial token explicitly opts in to pagination. + let mut next_token = client.page_size.map(|_| String::new()); loop { - let mut request = context.client.request(Method::GET, endpoint.clone()); + let mut request = client.http_client.request(Method::GET, endpoint.clone()); + + if let Some(page_size) = client.page_size { + request = request.query(&[("pageSize", page_size)]); + } if let Some(token) = next_token { request = request.query(&[("pageToken", token)]); } - let http_response = context.client.query_catalog(request.build()?).await?; + let http_response = client + .query_catalog(context, HttpRequest::build(request)?) + .await?; match http_response.status() { StatusCode::OK => { let response = - deserialize_catalog_response::(http_response).await?; + deserialize_catalog_response::(http_response)?; identifiers.extend(response.identifiers); @@ -625,11 +1259,16 @@ impl Catalog for RestCatalog { } StatusCode::NOT_FOUND => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceNotFound, "Tried to list tables of a namespace that does not exist", )); } - _ => return Err(deserialize_unexpected_catalog_error(http_response).await), + _ => { + return Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )); + } } } @@ -640,54 +1279,55 @@ impl Catalog for RestCatalog { /// /// In the resulting table, if there are any config properties that /// are present in both the response from the REST server and the - /// config provided when creating this `RestCatalog` instance then - /// the value provided locally to the `RestCatalog` will take precedence. + /// config provided when creating this `RestSessionCatalog` instance, then + /// the value provided locally to the `RestSessionCatalog` will take precedence. async fn create_table( &self, + context: &SessionContext, namespace: &NamespaceIdent, creation: TableCreation, ) -> Result
{ - let context = self.context().await?; + let client = self.client().await?; let table_ident = TableIdent::new(namespace.clone(), creation.name.clone()); - let request = context - .client - .request(Method::POST, context.config.tables_endpoint(namespace)) - .json(&CreateTableRequest { - name: creation.name, - location: creation.location, - schema: creation.schema, - partition_spec: creation.partition_spec, - write_order: creation.sort_order, - stage_create: Some(false), - properties: if creation.properties.is_empty() { - None - } else { - Some(creation.properties) - }, - }) - .build()?; - - let http_response = context.client.query_catalog(request).await?; + let request = HttpRequest::build( + client + .http_client + .request(Method::POST, client.config.tables_endpoint(namespace)) + .json(&CreateTableRequest { + name: creation.name, + location: creation.location, + schema: creation.schema, + partition_spec: creation.partition_spec, + write_order: creation.sort_order, + stage_create: Some(false), + properties: creation.properties, + }), + )?; + + let http_response = client.query_catalog(context, request).await?; let response = match http_response.status() { - StatusCode::OK => { - deserialize_catalog_response::(http_response).await? - } + StatusCode::OK => deserialize_catalog_response::(http_response)?, StatusCode::NOT_FOUND => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceNotFound, "Tried to create a table under a namespace that does not exist", )); } StatusCode::CONFLICT => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableAlreadyExists, "The table already exists", )); } - _ => return Err(deserialize_unexpected_catalog_error(http_response).await), + _ => { + return Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )); + } }; let metadata_location = response.metadata_location.as_ref().ok_or(Error::new( @@ -697,7 +1337,6 @@ impl Catalog for RestCatalog { let config = response .config - .unwrap_or_default() .into_iter() .chain(self.user_config.props.clone()) .collect(); @@ -706,10 +1345,14 @@ impl Catalog for RestCatalog { .load_file_io(Some(metadata_location), Some(config)) .await?; - let table_builder = Table::builder() + let mut table_builder = Table::builder() .identifier(table_ident.clone()) .file_io(file_io) - .metadata(response.metadata); + .metadata(response.metadata) + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + table_builder = table_builder.kms_client(kms_client); + } if let Some(metadata_location) = response.metadata_location { table_builder.metadata_location(metadata_location).build() @@ -721,34 +1364,76 @@ impl Catalog for RestCatalog { /// Load table from the catalog. /// /// If there are any config properties that are present in both the response from the REST - /// server and the config provided when creating this `RestCatalog` instance, then the value - /// provided locally to the `RestCatalog` will take precedence. - async fn load_table(&self, table_ident: &TableIdent) -> Result
{ - let context = self.context().await?; - - let request = context - .client - .request(Method::GET, context.config.table_endpoint(table_ident)) - .build()?; + /// server and the config provided when creating this `RestSessionCatalog` instance, then the + /// value provided locally to the `RestSessionCatalog` will take precedence. + async fn load_table( + &self, + context: &SessionContext, + table_ident: &TableIdent, + ) -> Result
{ + let client = self.client().await?; + let cache = client.table_cache.as_ref().map(|cache| { + let key = (context.session_id().to_string(), table_ident.clone()); + (cache, key) + }); + let cached = cache.as_ref().and_then(|(cache, key)| cache.get(key)); + + let mut request_builder = client + .http_client + .request(Method::GET, client.config.table_endpoint(table_ident)); + if let Some((etag, _)) = &cached { + request_builder = request_builder.header(header::IF_NONE_MATCH, etag); + } + let request = HttpRequest::build(request_builder)?; - let http_response = context.client.query_catalog(request).await?; + let http_response = client.query_catalog(context, request).await?; - let response = match http_response.status() { - StatusCode::OK | StatusCode::NOT_MODIFIED => { - deserialize_catalog_response::(http_response).await? + let (response, etag) = match http_response.status() { + StatusCode::OK => { + let etag = http_response + .headers() + .get(header::ETAG) + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + ( + deserialize_catalog_response::(http_response)?, + etag, + ) + } + StatusCode::NOT_MODIFIED => { + return cached.map(|(_, table)| table).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "Received 304 Not Modified for a table load without If-None-Match", + ) + }); } StatusCode::NOT_FOUND => { + client.invalidate_table(context, table_ident); return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableNotFound, "Tried to load a table that does not exist", )); } - _ => return Err(deserialize_unexpected_catalog_error(http_response).await), + _ => { + return Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )); + } }; + let vends_credentials = response + .storage_credentials + .as_ref() + .is_some_and(|credentials| !credentials.is_empty()) + || response + .config + .keys() + .any(|key| is_short_lived_credential(key)); + let config = response .config - .unwrap_or_default() .into_iter() .chain(self.user_config.props.clone()) .collect(); @@ -757,114 +1442,129 @@ impl Catalog for RestCatalog { .load_file_io(response.metadata_location.as_deref(), Some(config)) .await?; - let table_builder = Table::builder() + let mut table_builder = Table::builder() .identifier(table_ident.clone()) .file_io(file_io) - .metadata(response.metadata); + .metadata(response.metadata) + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + table_builder = table_builder.kms_client(kms_client); + } - if let Some(metadata_location) = response.metadata_location { + let table = if let Some(metadata_location) = response.metadata_location { table_builder.metadata_location(metadata_location).build() } else { table_builder.build() + }?; + + // A load racing a change to this table may cache the pre-change table; + // the next load's `If-None-Match` check replaces it. + if let Some((cache, key)) = cache { + match etag { + Some(etag) if !vends_credentials => cache.insert(key, (etag, table.clone())), + _ => cache.invalidate(&key), + } } + Ok(table) } /// Drop a table from the catalog. - async fn drop_table(&self, table: &TableIdent) -> Result<()> { - let context = self.context().await?; - - let request = context - .client - .request(Method::DELETE, context.config.table_endpoint(table)) - .build()?; - - let http_response = context.client.query_catalog(request).await?; + async fn drop_table(&self, context: &SessionContext, table: &TableIdent) -> Result<()> { + self.delete_table(context, table, false).await + } - match http_response.status() { - StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), - StatusCode::NOT_FOUND => Err(Error::new( - ErrorKind::Unexpected, - "Tried to drop a table that does not exist", - )), - _ => Err(deserialize_unexpected_catalog_error(http_response).await), - } + /// Drop a table from the catalog and purge its data by sending + /// `purgeRequested=true` to the REST server. + async fn purge_table(&self, context: &SessionContext, table: &TableIdent) -> Result<()> { + self.delete_table(context, table, true).await } /// Check if a table exists in the catalog. - async fn table_exists(&self, table: &TableIdent) -> Result { - let context = self.context().await?; - - let request = context - .client - .request(Method::HEAD, context.config.table_endpoint(table)) - .build()?; - - let http_response = context.client.query_catalog(request).await?; - - match http_response.status() { - StatusCode::NO_CONTENT | StatusCode::OK => Ok(true), - StatusCode::NOT_FOUND => Ok(false), - _ => Err(deserialize_unexpected_catalog_error(http_response).await), + async fn table_exists(&self, context: &SessionContext, table: &TableIdent) -> Result { + // Prefer a cheap HEAD when the server advertises it; otherwise fall back + // to loading the table (GET) and treating a missing table as `false`, so + // this still works against servers that don't advertise the HEAD route. + if !self.supports_endpoint(&V1_TABLE_EXISTS).await? { + return match self.load_table(context, table).await { + Ok(_) => Ok(true), + Err(e) if e.kind() == ErrorKind::TableNotFound => Ok(false), + Err(e) => Err(e), + }; } + + let client = self.client().await?; + self.check_exists_via_head(context, client, client.config.table_endpoint(table)) + .await } /// Rename a table in the catalog. - async fn rename_table(&self, src: &TableIdent, dest: &TableIdent) -> Result<()> { - let context = self.context().await?; - - let request = context - .client - .request(Method::POST, context.config.rename_table_endpoint()) - .json(&RenameTableRequest { - source: src.clone(), - destination: dest.clone(), - }) - .build()?; + async fn rename_table( + &self, + context: &SessionContext, + src: &TableIdent, + dest: &TableIdent, + ) -> Result<()> { + let client = self.client().await?; - let http_response = context.client.query_catalog(request).await?; + let request = HttpRequest::build( + client + .http_client + .request(Method::POST, client.config.rename_table_endpoint()) + .json(&RenameTableRequest { + source: src.clone(), + destination: dest.clone(), + }), + )?; + + client.invalidate_table(context, src); + let http_response = client.query_catalog(context, request).await?; match http_response.status() { StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), StatusCode::NOT_FOUND => Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableNotFound, "Tried to rename a table that does not exist (is the namespace correct?)", )), StatusCode::CONFLICT => Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableAlreadyExists, "Tried to rename a table to a name that already exists", )), - _ => Err(deserialize_unexpected_catalog_error(http_response).await), + _ => Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )), } } async fn register_table( &self, + context: &SessionContext, table_ident: &TableIdent, metadata_location: String, ) -> Result
{ - let context = self.context().await?; - - let request = context - .client - .request( - Method::POST, - context - .config - .register_table_endpoint(table_ident.namespace()), - ) - .json(&RegisterTableRequest { - name: table_ident.name.clone(), - metadata_location: metadata_location.clone(), - overwrite: Some(false), - }) - .build()?; - - let http_response = context.client.query_catalog(request).await?; - - let response: LoadTableResponse = match http_response.status() { - StatusCode::OK => { - deserialize_catalog_response::(http_response).await? - } + let client = self.client().await?; + + let request = HttpRequest::build( + client + .http_client + .request( + Method::POST, + client + .config + .register_table_endpoint(table_ident.namespace()), + ) + .json(&RegisterTableRequest { + name: table_ident.name.clone(), + metadata_location: metadata_location.clone(), + overwrite: Some(false), + }), + )?; + + client.invalidate_table(context, table_ident); + let http_response = client.query_catalog(context, request).await?; + + let response: LoadTableResult = match http_response.status() { + StatusCode::OK => deserialize_catalog_response::(http_response)?, StatusCode::NOT_FOUND => { return Err(Error::new( ErrorKind::NamespaceNotFound, @@ -877,7 +1577,12 @@ impl Catalog for RestCatalog { "The given table already exists.", )); } - _ => return Err(deserialize_unexpected_catalog_error(http_response).await), + _ => { + return Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )); + } }; let metadata_location = response.metadata_location.as_ref().ok_or(Error::new( @@ -887,34 +1592,44 @@ impl Catalog for RestCatalog { let file_io = self.load_file_io(Some(metadata_location), None).await?; - Table::builder() + let mut table_builder = Table::builder() .identifier(table_ident.clone()) .file_io(file_io) .metadata(response.metadata) .metadata_location(metadata_location.clone()) - .build() + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + table_builder = table_builder.kms_client(kms_client); + } + table_builder.build() } - async fn update_table(&self, mut commit: TableCommit) -> Result
{ - let context = self.context().await?; - - let request = context - .client - .request( - Method::POST, - context.config.table_endpoint(commit.identifier()), - ) - .json(&CommitTableRequest { - identifier: commit.identifier().clone(), - requirements: commit.take_requirements(), - updates: commit.take_updates(), - }) - .build()?; + async fn update_table( + &self, + context: &SessionContext, + mut commit: TableCommit, + ) -> Result
{ + let client = self.client().await?; + + let request = HttpRequest::build( + client + .http_client + .request( + Method::POST, + client.config.table_endpoint(commit.identifier()), + ) + .json(&CommitTableRequest { + identifier: Some(commit.identifier().clone()), + requirements: commit.take_requirements(), + updates: commit.take_updates(), + }), + )?; - let http_response = context.client.query_catalog(request).await?; + client.invalidate_table(context, commit.identifier()); + let http_response = client.query_catalog(context, request).await?; let response: CommitTableResponse = match http_response.status() { - StatusCode::OK => deserialize_catalog_response(http_response).await?, + StatusCode::OK => deserialize_catalog_response(http_response)?, StatusCode::NOT_FOUND => { return Err(Error::new( ErrorKind::TableNotFound, @@ -946,19 +1661,216 @@ impl Catalog for RestCatalog { "A server-side gateway timeout occurred; the commit state is unknown.", )); } - _ => return Err(deserialize_unexpected_catalog_error(http_response).await), + _ => { + return Err(deserialize_unexpected_catalog_error( + http_response, + client.http_client.disable_header_redaction(), + )); + } }; let file_io = self .load_file_io(Some(&response.metadata_location), None) .await?; - Table::builder() + let mut table_builder = Table::builder() .identifier(commit.identifier().clone()) .file_io(file_io) .metadata(response.metadata) .metadata_location(response.metadata_location) - .build() + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + table_builder = table_builder.kms_client(kms_client); + } + table_builder.build() + } +} + +/// Builder for an unbound [`RestSessionCatalog`]. +/// +/// Unlike [`RestCatalogBuilder`], the resulting catalog accepts a +/// [`SessionContext`] with each [`SessionCatalog`] operation. +#[derive(Debug)] +pub struct RestSessionCatalogBuilder { + config: RestCatalogConfig, + auth_manager: Option>, + storage_factory: Option>, + kms_client_factory: Option>, + runtime: Option, +} + +impl Default for RestSessionCatalogBuilder { + fn default() -> Self { + Self { + config: RestCatalogConfig { + name: None, + uri: "".to_string(), + warehouse: None, + props: HashMap::new(), + client: None, + default_client: Arc::new(OnceLock::new()), + }, + auth_manager: None, + storage_factory: None, + kms_client_factory: None, + runtime: None, + } + } +} + +impl RestSessionCatalogBuilder { + /// Sets the requested page size for namespace and table listings. + /// + /// List operations still return all results. Without a page size, no + /// `pageSize` is sent. Precedence, lowest to highest: server defaults, this + /// value, [`REST_CATALOG_PROP_PAGE_SIZE`] passed to [`load`](Self::load), and + /// server overrides. Catalog operations fail with [`ErrorKind::DataInvalid`] + /// if the effective value is not in `1..=i32::MAX`. + pub fn with_page_size(mut self, page_size: NonZeroU32) -> Self { + self.config.props.insert( + REST_CATALOG_PROP_PAGE_SIZE.to_string(), + page_size.to_string(), + ); + self + } + + /// Configures the catalog with a custom HTTP client. + pub fn with_client(mut self, client: Client) -> Self { + self.config.client = Some(client); + self + } + + /// Sets a custom auth manager, overriding the `rest.auth.type` configuration. + /// + /// The builder takes ownership of the manager. The loaded catalog shares it + /// across authentication sessions and requests. + pub fn with_auth_manager(mut self, auth_manager: M) -> Self + where M: AuthManager + 'static { + self.auth_manager = Some(Box::new(auth_manager)); + self + } + + /// Set a custom StorageFactory to use for storage operations. + /// + /// When a StorageFactory is provided, the catalog will use it to build FileIO + /// instances for all storage operations instead of using the default factory. + /// + /// # Arguments + /// + /// * `storage_factory` - The StorageFactory to use for creating storage instances + /// + /// # Example + /// + /// ```rust,ignore + /// use iceberg::io::StorageFactory; + /// use iceberg_catalog_rest::RestSessionCatalogBuilder; + /// use iceberg_storage_opendal::OpenDalStorageFactory; + /// use std::sync::Arc; + /// + /// let catalog = RestSessionCatalogBuilder::default() + /// .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + /// customized_credential_load: None, + /// })) + /// .load("my_catalog", props) + /// .await?; + /// ``` + pub fn with_storage_factory(mut self, storage_factory: Arc) -> Self { + self.storage_factory = Some(storage_factory); + self + } + + /// Set a [`KmsClientFactory`] to enable table encryption. + /// + /// When provided, the catalog calls the factory once during + /// [`load`](Self::load) with the catalog properties to create a shared + /// [`KeyManagementClient`]. + /// That client is then passed to each table's `TableBuilder` so tables + /// with `encryption.key-id` set can construct an `EncryptionManager`. + /// + /// # Example + /// + /// ```rust,ignore + /// use iceberg::encryption::kms::KmsClientFactory; + /// use iceberg_catalog_rest::RestSessionCatalogBuilder; + /// use std::sync::Arc; + /// + /// let catalog = RestSessionCatalogBuilder::default() + /// .with_kms_client_factory(Arc::new(MyKmsClientFactory)) + /// .load("my_catalog", props) + /// .await?; + /// ``` + pub fn with_kms_client_factory( + mut self, + kms_client_factory: Arc, + ) -> Self { + self.kms_client_factory = Some(kms_client_factory); + self + } + + /// Set a custom tokio Runtime to use for spawning async tasks. + /// + /// When a Runtime is provided, the catalog will propagate it to all tables + /// it creates. Tasks such as scan planning and delete file processing + /// will be spawned on this runtime. + pub fn with_runtime(mut self, runtime: Runtime) -> Self { + self.runtime = Some(runtime); + self + } + + /// Creates a new session catalog instance. + /// + /// The server configuration handshake, endpoint negotiation, and + /// authentication sessions are initialized lazily on the first operation. + pub fn load( + mut self, + name: impl Into, + props: HashMap, + ) -> impl Future> + Send { + self.config.name = Some(name.into()); + + if props.contains_key(REST_CATALOG_PROP_URI) { + self.config.uri = props + .get(REST_CATALOG_PROP_URI) + .cloned() + .unwrap_or_default(); + } + + if props.contains_key(REST_CATALOG_PROP_WAREHOUSE) { + self.config.warehouse = props.get(REST_CATALOG_PROP_WAREHOUSE).cloned() + } + + // Collect other remaining properties + self.config.props.extend(props); + self.config.props.remove(REST_CATALOG_PROP_URI); + self.config.props.remove(REST_CATALOG_PROP_WAREHOUSE); + + async move { + if self.config.name.is_none() { + Err(Error::new( + ErrorKind::DataInvalid, + "Catalog name is required", + )) + } else if self.config.uri.is_empty() { + Err(Error::new( + ErrorKind::DataInvalid, + "Catalog uri is required", + )) + } else { + let runtime = self.runtime.unwrap_or_else(Runtime::current); + let kms_client = match self.kms_client_factory { + Some(factory) => Some(factory.create_kms_client(&self.config.props).await?), + None => None, + }; + + Ok(RestSessionCatalog::new( + self.config, + self.auth_manager, + self.storage_factory, + runtime, + kms_client, + )) + } + } } } @@ -966,20 +1878,422 @@ impl Catalog for RestCatalog { mod tests { use std::fs::File; use std::io::BufReader; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use chrono::{TimeZone, Utc}; + use iceberg::io::LocalFsStorageFactory; use iceberg::spec::{ FormatVersion, NestedField, NullOrder, Operation, PrimitiveType, Schema, Snapshot, SnapshotLog, SortDirection, SortField, SortOrder, Summary, Transform, Type, UnboundPartitionField, UnboundPartitionSpec, }; use iceberg::transaction::{ApplyTransactionAction, Transaction}; - use mockito::{Mock, Server, ServerGuard}; + use iceberg_test_utils::test_runtime; + use mockito::{Matcher, Mock, Server, ServerGuard}; use serde_json::json; use uuid::uuid; use super::*; + use crate::auth::AuthSession; + use crate::request::HttpRequest; + + #[derive(Debug)] + struct PlainContextSession; + + #[async_trait] + impl AuthSession for PlainContextSession { + async fn authenticate(&self, _request: &mut HttpRequest) -> Result<()> { + Ok(()) + } + } + + #[derive(Debug)] + struct ContextSession(String); + + #[async_trait] + impl AuthSession for ContextSession { + async fn authenticate(&self, request: &mut HttpRequest) -> Result<()> { + request.headers_mut().insert( + "x-session-id", + HeaderValue::from_str(&self.0).expect("valid test session ID"), + ); + Ok(()) + } + } + + #[derive(Debug)] + struct ContextManager { + catalog_session: Arc, + seen_session_ids: Arc>>, + fail_contextual_session: bool, + } + + #[async_trait] + impl AuthManager for ContextManager { + async fn init_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(Box::new(PlainContextSession)) + } + + async fn catalog_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(self.catalog_session.clone()) + } + + async fn contextual_session( + &self, + context: &SessionContext, + catalog_session: Arc, + ) -> Result> { + assert!(Arc::ptr_eq(&catalog_session, &self.catalog_session)); + self.seen_session_ids + .lock() + .unwrap() + .push(context.session_id().to_string()); + if self.fail_contextual_session { + return Err(Error::new( + ErrorKind::Unexpected, + "contextual session failure", + )); + } + Ok(Arc::new(ContextSession(context.session_id().to_string()))) + } + } + + fn test_catalog(config: RestCatalogConfig) -> RestSessionCatalog { + RestSessionCatalog::new(config, None, None, Runtime::current(), None) + } + + fn test_catalog_with(config: RestCatalogConfig, auth_manager: M) -> RestSessionCatalog + where M: AuthManager + 'static { + RestSessionCatalog::new( + config, + Some(Box::new(auth_manager)), + None, + Runtime::current(), + None, + ) + } + + fn context_catalog( + config: RestCatalogConfig, + fail_contextual_session: bool, + ) -> (RestSessionCatalog, Arc>>) { + let seen_session_ids = Arc::new(Mutex::new(Vec::new())); + let catalog_session: Arc = Arc::new(PlainContextSession); + let catalog = test_catalog_with(config, ContextManager { + catalog_session, + seen_session_ids: seen_session_ids.clone(), + fail_contextual_session, + }); + (catalog, seen_session_ids) + } + + fn test_client() -> HttpClient { + HttpClient::new( + &RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .build(), + ) + .unwrap() + } + + /// Builds a [`RestSessionCatalog`] with the default test storage factory and runtime. + fn session_catalog(config: RestCatalogConfig) -> RestSessionCatalog { + RestSessionCatalog::new( + config, + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ) + } + + #[tokio::test] + async fn test_page_size_precedence() { + // Server default < builder < load properties < server override. + for (default, builder, property, override_value, expected) in [ + (Some("10"), None, None, None, 10), + (Some("10"), Some(20), None, None, 20), + (None, Some(20), None, None, 20), + (Some("10"), None, Some("30"), None, 30), + (Some("10"), Some(20), Some("30"), None, 30), + (Some("10"), Some(20), Some("30"), Some("40"), 40), + (None, None, None, Some("40"), 40), + (None, Some(1), None, None, 1), + (None, Some(i32::MAX as u32), None, None, i32::MAX as u32), + // Invalid lower-priority values must not reject a valid effective value. + (Some("invalid"), Some(20), None, None, 20), + (None, Some(20), Some("0"), Some("40"), 40), + (None, Some(u32::MAX), None, Some("40"), 40), + ] { + let mut server = Server::new_async().await; + let page_size_props = |value: Option<&str>| -> HashMap { + value + .map(|value| (REST_CATALOG_PROP_PAGE_SIZE.to_string(), value.to_string())) + .into_iter() + .collect() + }; + let config_mock = server + .mock("GET", "/v1/config") + .with_status(200) + .with_body( + json!({ + "defaults": page_size_props(default), + "overrides": page_size_props(override_value), + }) + .to_string(), + ) + .create_async() + .await; + let list_mock = server + .mock("GET", "/v1/namespaces") + .match_query(Matcher::UrlEncoded("pageSize".into(), expected.to_string())) + .with_status(200) + .with_body(r#"{"namespaces": []}"#) + .expect(1) + .create_async() + .await; + + let mut builder_config = RestCatalogBuilder::default(); + if let Some(page_size) = builder { + builder_config = builder_config.with_page_size(NonZeroU32::new(page_size).unwrap()); + } + let mut props = page_size_props(property); + props.insert(REST_CATALOG_PROP_URI.to_string(), server.url()); + let catalog = builder_config.load("test", props).await.unwrap(); + + catalog.list_namespaces(None).await.unwrap_or_else(|err| { + panic!( + "default={default:?}, builder={builder:?}, property={property:?}, \ + override={override_value:?}, expected={expected}: {err}" + ) + }); + config_mock.assert_async().await; + list_mock.assert_async().await; + } + } + + #[tokio::test] + async fn test_page_size_pagination() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let catalog = RestCatalogBuilder::default() + .with_page_size(NonZeroU32::MIN) + .load( + "test", + HashMap::from([(REST_CATALOG_PROP_URI.to_string(), server.url())]), + ) + .await + .unwrap(); + + let mut mocks = Vec::new(); + for (endpoint, parent, first_body, empty_body, last_body) in [ + ( + "/v1/namespaces", + Some("parent\u{1f}child"), + json!({"namespaces": [["ns1"]], "next-page-token": "next/+"}), + json!({"namespaces": [], "next-page-token": "last"}), + json!({"namespaces": [["ns2"]], "next-page-token": null}), + ), + ( + "/v1/namespaces/ns1/tables", + None, + json!({"identifiers": [{"namespace": ["ns1"], "name": "t1"}], "next-page-token": "next/+"}), + json!({"identifiers": [], "next-page-token": "last"}), + json!({"identifiers": [{"namespace": ["ns1"], "name": "t2"}]}), + ), + ] { + // Page 1 opts in with an empty token. Later tokens and the multipart + // parent must survive URL encoding, independent of query parameter order. + // The empty middle page must not terminate pagination. + for (token, body) in [ + ("", first_body), + ("next/+", empty_body), + ("last", last_body), + ] { + let mut query = vec![ + Matcher::UrlEncoded("pageSize".into(), "1".into()), + Matcher::UrlEncoded("pageToken".into(), token.into()), + ]; + if let Some(parent) = parent { + query.push(Matcher::UrlEncoded("parent".into(), parent.into())); + } + mocks.push( + server + .mock("GET", endpoint) + .match_query(Matcher::AllOf(query)) + .with_status(200) + .with_body(body.to_string()) + .expect(1) + .create_async() + .await, + ); + } + } + + let parent = NamespaceIdent::from_vec(vec!["parent".into(), "child".into()]).unwrap(); + assert_eq!(catalog.list_namespaces(Some(&parent)).await.unwrap(), vec![ + NamespaceIdent::new("ns1".into()), + NamespaceIdent::new("ns2".into()) + ]); + let namespace = NamespaceIdent::new("ns1".into()); + assert_eq!(catalog.list_tables(&namespace).await.unwrap(), vec![ + TableIdent::new(namespace.clone(), "t1".into()), + TableIdent::new(namespace, "t2".into()) + ]); + config_mock.assert_async().await; + for mock in mocks { + mock.assert_async().await; + } + } + + #[tokio::test] + async fn test_page_size_is_not_a_result_limit() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let catalog = RestCatalogBuilder::default() + .with_page_size(NonZeroU32::MIN) + .load( + "test", + HashMap::from([(REST_CATALOG_PROP_URI.to_string(), server.url())]), + ) + .await + .unwrap(); + + // A server may ignore pageSize and return all results without a next token. + let mut mocks = Vec::new(); + for (endpoint, body) in [ + ("/v1/namespaces", json!({"namespaces": [["ns1"], ["ns2"]]})), + ( + "/v1/namespaces/ns1/tables", + json!({"identifiers": [ + {"namespace": ["ns1"], "name": "t1"}, + {"namespace": ["ns1"], "name": "t2"}, + ]}), + ), + ] { + mocks.push( + server + .mock("GET", endpoint) + .match_query(Matcher::AllOf(vec![ + Matcher::UrlEncoded("pageSize".into(), "1".into()), + Matcher::UrlEncoded("pageToken".into(), "".into()), + ])) + .with_status(200) + .with_body(body.to_string()) + .expect(1) + .create_async() + .await, + ); + } + + assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![ + NamespaceIdent::new("ns1".into()), + NamespaceIdent::new("ns2".into()) + ]); + let namespace = NamespaceIdent::new("ns1".into()); + assert_eq!(catalog.list_tables(&namespace).await.unwrap(), vec![ + TableIdent::new(namespace.clone(), "t1".into()), + TableIdent::new(namespace, "t2".into()) + ]); + config_mock.assert_async().await; + for mock in mocks { + mock.assert_async().await; + } + } + + #[tokio::test] + async fn test_page_size_absent() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let namespaces_mock = server + .mock("GET", "/v1/namespaces") + .match_query(Matcher::Missing) + .with_status(200) + .with_body(r#"{"namespaces": []}"#) + .create_async() + .await; + let tables_mock = server + .mock("GET", "/v1/namespaces/ns1/tables") + .match_query(Matcher::Missing) + .with_status(200) + .with_body(r#"{"identifiers": []}"#) + .create_async() + .await; + let catalog = RestCatalogBuilder::default() + .load( + "test", + HashMap::from([(REST_CATALOG_PROP_URI.to_string(), server.url())]), + ) + .await + .unwrap(); + + assert!(catalog.list_namespaces(None).await.unwrap().is_empty()); + assert!( + catalog + .list_tables(&NamespaceIdent::new("ns1".into())) + .await + .unwrap() + .is_empty() + ); + config_mock.assert_async().await; + namespaces_mock.assert_async().await; + tables_mock.assert_async().await; + } + + #[tokio::test] + async fn test_invalid_page_size() { + for value in [ + "", + "0", + "-1", + "abc", + "1.5", + "2147483648", + "4294967295", + "4294967296", + ] { + for source in ["defaults", "client", "overrides"] { + let mut server = Server::new_async().await; + let mut config = json!({"defaults": {}, "overrides": {}}); + let mut props = HashMap::from([(REST_CATALOG_PROP_URI.to_string(), server.url())]); + if source == "client" { + // Invalid client values must not fall back to a valid server default. + config["defaults"][REST_CATALOG_PROP_PAGE_SIZE] = json!("1000"); + props.insert(REST_CATALOG_PROP_PAGE_SIZE.to_string(), value.to_string()); + } else { + config[source][REST_CATALOG_PROP_PAGE_SIZE] = json!(value); + } + let config_mock = server + .mock("GET", "/v1/config") + .with_status(200) + .with_body(config.to_string()) + .create_async() + .await; + let catalog = RestSessionCatalogBuilder::default() + .load("test", props) + .await + .unwrap(); + let err = catalog + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + format!("Invalid {REST_CATALOG_PROP_PAGE_SIZE}: {value}") + ); + config_mock.assert_async().await; + } + } + } #[tokio::test] async fn test_update_config() { @@ -999,11 +2313,11 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); assert_eq!( catalog - .context() + .client() .await .unwrap() .config @@ -1031,14 +2345,141 @@ mod tests { .await } - async fn create_oauth_mock(server: &mut ServerGuard) -> Mock { - create_oauth_mock_with_path(server, "/v1/oauth/tokens", "ey000000000000", 200).await - } - - async fn create_oauth_mock_with_path( - server: &mut ServerGuard, - path: &str, - token: &str, + /// Config mock that advertises the HEAD table/namespace-exists endpoints, so + /// `{table,namespace}_exists` take the HEAD path rather than the GET fallback. + async fn create_config_mock_with_exists_endpoints(server: &mut ServerGuard) -> Mock { + server + .mock("GET", "/v1/config") + .with_status(200) + .with_body( + r#"{ + "overrides": { "warehouse": "s3://iceberg-catalog" }, + "defaults": {}, + "endpoints": [ + "HEAD /v1/{prefix}/namespaces/{namespace}", + "HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}" + ] + }"#, + ) + .create_async() + .await + } + + #[tokio::test] + async fn test_config_advertised_endpoints() { + let mut server = Server::new_async().await; + + let config_mock = server + .mock("GET", "/v1/config") + .with_status(200) + .with_body( + r#"{ + "overrides": {}, + "defaults": {}, + "endpoints": [ + "GET /v1/{prefix}/namespaces", + "POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan" + ] + }"#, + ) + .create_async() + .await; + + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + + let plan = "POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan" + .parse::() + .unwrap(); + assert!(catalog.inner.supports_endpoint(&plan).await.unwrap()); + // Advertised list is present but does not include this route. + let delete_ns = "DELETE /v1/{prefix}/namespaces/{namespace}" + .parse::() + .unwrap(); + assert!(!catalog.inner.supports_endpoint(&delete_ns).await.unwrap()); + + config_mock.assert_async().await; + } + + #[tokio::test] + async fn test_config_without_endpoints_falls_back_to_default_set() { + let mut server = Server::new_async().await; + + let config_mock = server + .mock("GET", "/v1/config") + .with_status(200) + .with_body(r#"{ "overrides": {}, "defaults": {} }"#) + .create_async() + .await; + + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + + // A server that omits the `endpoints` field is assumed to support the + // standard base operations. + let load_table = "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}" + .parse::() + .unwrap(); + assert!(catalog.inner.supports_endpoint(&load_table).await.unwrap()); + // But not an optional endpoint that must be advertised. + let plan = "POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/plan" + .parse::() + .unwrap(); + assert!(!catalog.inner.supports_endpoint(&plan).await.unwrap()); + + config_mock.assert_async().await; + } + + #[tokio::test] + async fn test_config_with_empty_endpoints_falls_back_to_default_set() { + let mut server = Server::new_async().await; + + // An explicit empty list is treated the same as an absent field: fall + // back to the standard base set. + let config_mock = server + .mock("GET", "/v1/config") + .with_status(200) + .with_body(r#"{ "overrides": {}, "defaults": {}, "endpoints": [] }"#) + .create_async() + .await; + + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + + let load_table = "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}" + .parse::() + .unwrap(); + assert!(catalog.inner.supports_endpoint(&load_table).await.unwrap()); + + config_mock.assert_async().await; + } + + async fn create_oauth_mock(server: &mut ServerGuard) -> Mock { + create_oauth_mock_with_path(server, "/v1/oauth/tokens", "ey000000000000", 200).await + } + + async fn create_oauth_mock_with_path( + server: &mut ServerGuard, + path: &str, + token: &str, status: usize, ) -> Mock { let body = format!( @@ -1067,14 +2508,14 @@ mod tests { let mut props = HashMap::new(); props.insert("credential".to_string(), "client1:secret1".to_string()); - let catalog = RestCatalog::new( + let catalog = session_catalog( RestCatalogConfig::builder() .uri(server.url()) .props(props) .build(), ); - let token = catalog.context().await.unwrap().client.token().await; + let token = catalog.client().await.unwrap().token().await; oauth_mock.assert_async().await; config_mock.assert_async().await; assert_eq!(token, Some("ey000000000000".to_string())); @@ -1091,13 +2532,9 @@ mod tests { let mut server = Server::new_async().await; let oauth_mock = server .mock("POST", "/v1/oauth/tokens") - .match_body(mockito::Matcher::Regex("scope=custom_scope".to_string())) - .match_body(mockito::Matcher::Regex( - "audience=custom_audience".to_string(), - )) - .match_body(mockito::Matcher::Regex( - "resource=custom_resource".to_string(), - )) + .match_body(Matcher::Regex("scope=custom_scope".to_string())) + .match_body(Matcher::Regex("audience=custom_audience".to_string())) + .match_body(Matcher::Regex("resource=custom_resource".to_string())) .with_status(200) .with_body( r#"{ @@ -1113,14 +2550,14 @@ mod tests { let config_mock = create_config_mock(&mut server).await; - let catalog = RestCatalog::new( + let catalog = session_catalog( RestCatalogConfig::builder() .uri(server.url()) .props(props) .build(), ); - let token = catalog.context().await.unwrap().client.token().await; + let token = catalog.client().await.unwrap().token().await; oauth_mock.assert_async().await; config_mock.assert_async().await; @@ -1128,305 +2565,1091 @@ mod tests { } #[tokio::test] - async fn test_invalidate_token() { + async fn test_http_headers() { + let server = Server::new_async().await; + let mut props = HashMap::new(); + props.insert("credential".to_string(), "client1:secret1".to_string()); + + let config = RestCatalogConfig::builder() + .uri(server.url()) + .props(props) + .build(); + let headers: HeaderMap = config.extra_headers().unwrap(); + + let expected_headers = HeaderMap::from_iter([ + ( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ), + ( + HeaderName::from_static("x-client-version"), + HeaderValue::from_static(ICEBERG_REST_SPEC_VERSION), + ), + ( + header::USER_AGENT, + HeaderValue::from_str(&format!("iceberg-rust/{CARGO_PKG_VERSION}")).unwrap(), + ), + ]); + assert_eq!(headers, expected_headers); + } + + #[tokio::test] + async fn test_http_headers_with_custom_headers() { + let server = Server::new_async().await; + let mut props = HashMap::new(); + props.insert("credential".to_string(), "client1:secret1".to_string()); + props.insert( + "header.content-type".to_string(), + "application/yaml".to_string(), + ); + props.insert( + "header.customized-header".to_string(), + "some/value".to_string(), + ); + + let config = RestCatalogConfig::builder() + .uri(server.url()) + .props(props) + .build(); + let headers: HeaderMap = config.extra_headers().unwrap(); + + let expected_headers = HeaderMap::from_iter([ + ( + header::CONTENT_TYPE, + HeaderValue::from_static("application/yaml"), + ), + ( + HeaderName::from_static("x-client-version"), + HeaderValue::from_static(ICEBERG_REST_SPEC_VERSION), + ), + ( + header::USER_AGENT, + HeaderValue::from_str(&format!("iceberg-rust/{CARGO_PKG_VERSION}")).unwrap(), + ), + ( + HeaderName::from_static("customized-header"), + HeaderValue::from_static("some/value"), + ), + ]); + assert_eq!(headers, expected_headers); + } + + #[tokio::test] + async fn test_oauth_with_oauth2_server_uri() { let mut server = Server::new_async().await; - let oauth_mock = create_oauth_mock(&mut server).await; let config_mock = create_config_mock(&mut server).await; + let mut auth_server = Server::new_async().await; + let auth_server_path = "/some/path"; + let oauth_mock = + create_oauth_mock_with_path(&mut auth_server, auth_server_path, "ey000000000000", 200) + .await; + let mut props = HashMap::new(); props.insert("credential".to_string(), "client1:secret1".to_string()); + props.insert( + "oauth2-server-uri".to_string(), + format!("{}{}", auth_server.url(), auth_server_path).to_string(), + ); - let catalog = RestCatalog::new( + let catalog = session_catalog( RestCatalogConfig::builder() .uri(server.url()) .props(props) .build(), ); - let token = catalog.context().await.unwrap().client.token().await; + let token = catalog.client().await.unwrap().token().await; + oauth_mock.assert_async().await; config_mock.assert_async().await; assert_eq!(token, Some("ey000000000000".to_string())); + } - let oauth_mock = - create_oauth_mock_with_path(&mut server, "/v1/oauth/tokens", "ey000000000001", 200) - .await; - catalog.invalidate_token().await.unwrap(); - let token = catalog.context().await.unwrap().client.token().await; - oauth_mock.assert_async().await; - assert_eq!(token, Some("ey000000000001".to_string())); + #[tokio::test] + async fn test_config_override() { + let mut server = Server::new_async().await; + let mut redirect_server = Server::new_async().await; + let new_uri = redirect_server.url(); + + let config_mock = server + .mock("GET", "/v1/config") + .with_status(200) + .with_body( + json!( + { + "overrides": { + "uri": new_uri, + "warehouse": "s3://iceberg-catalog", + "prefix": "ice/warehouses/my" + }, + "defaults": {}, + } + ) + .to_string(), + ) + .create_async() + .await; + + let list_ns_mock = redirect_server + .mock("GET", "/v1/ice/warehouses/my/namespaces") + .with_body( + r#"{ + "namespaces": [] + }"#, + ) + .create_async() + .await; + + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + + let _namespaces = catalog + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap(); + + config_mock.assert_async().await; + list_ns_mock.assert_async().await; } #[tokio::test] - async fn test_invalidate_token_failing_request() { + async fn test_list_namespace() { let mut server = Server::new_async().await; - let oauth_mock = create_oauth_mock(&mut server).await; + let config_mock = create_config_mock(&mut server).await; - let mut props = HashMap::new(); - props.insert("credential".to_string(), "client1:secret1".to_string()); + let list_ns_mock = server + .mock("GET", "/v1/namespaces") + .with_body( + r#"{ + "namespaces": [ + ["ns1", "ns11"], + ["ns2"] + ] + }"#, + ) + .create_async() + .await; + + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + + let namespaces = catalog + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap(); + + let expected_ns = vec![ + NamespaceIdent::from_vec(vec!["ns1".to_string(), "ns11".to_string()]).unwrap(), + NamespaceIdent::from_vec(vec!["ns2".to_string()]).unwrap(), + ]; + + assert_eq!(expected_ns, namespaces); + + config_mock.assert_async().await; + list_ns_mock.assert_async().await; + } + + #[tokio::test] + async fn test_auth_type_none_disables_auth() { + // An explicit `rest.auth.type=none` wins over a configured token. + let props = HashMap::from([ + (REST_CATALOG_PROP_AUTH_TYPE.to_string(), "none".to_string()), + ("token".to_string(), "some-oauth-token".to_string()), + ]); + let config = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props(props) + .build(); + + let session = test_catalog(config) + .resolve_auth_manager() + .unwrap() + .init_session(&test_client(), &HashMap::new()) + .await + .unwrap(); + let mut req = HttpRequest::new( + Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + session.authenticate(&mut req).await.unwrap(); + assert!(req.headers().get("authorization").is_none()); + } + #[tokio::test] + async fn test_header_prop_overrides_token_on_the_wire() { + // Pre-AuthManager behavior, preserved: extra headers are applied after + // authentication, so a user-configured `header.authorization` wins + // over a configured token. + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let list_ns_mock = server + .mock("GET", "/v1/namespaces") + .match_header("authorization", "Basic xyz") + .with_body(r#"{"namespaces": []}"#) + .create_async() + .await; + + let props = HashMap::from([ + ("token".to_string(), "some-oauth-token".to_string()), + ("header.authorization".to_string(), "Basic xyz".to_string()), + ]); let catalog = RestCatalog::new( + SessionContext::empty(), RestCatalogConfig::builder() .uri(server.url()) .props(props) .build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, ); - let token = catalog.context().await.unwrap().client.token().await; - oauth_mock.assert_async().await; + catalog.list_namespaces(None).await.unwrap(); config_mock.assert_async().await; - assert_eq!(token, Some("ey000000000000".to_string())); + list_ns_mock.assert_async().await; + } + + #[tokio::test] + async fn test_builtin_oauth_endpoint_follows_uri_override() { + // When `/v1/config` overrides `uri` (and no explicit `oauth2-server-uri` + // is set), the built-in manager's default token endpoint must follow + // the merged URI. + let mut bootstrap = Server::new_async().await; + let overridden = Server::new_async().await; + + let config_mock = bootstrap + .mock("GET", "/v1/config") + .with_status(200) + .with_body(format!( + r#"{{"overrides": {{"uri": "{}"}}, "defaults": {{}}}}"#, + overridden.url() + )) + .create_async() + .await; + // Handshake exchange still uses the bootstrap-derived default. + let bootstrap_oauth_mock = + create_oauth_mock_with_path(&mut bootstrap, "/v1/oauth/tokens", "tok-boot", 200).await; + + let props = HashMap::from([("credential".to_string(), "client1:secret1".to_string())]); + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder() + .uri(bootstrap.url()) + .props(props) + .build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + + let client = catalog.client().await.unwrap(); + config_mock.assert_async().await; + bootstrap_oauth_mock.assert_async().await; + // The catalog session's endpoint follows the overridden URI (visible + // via the session's Debug, which prints its token endpoint). + let session_debug = format!("{:?}", client.catalog_session); + assert!(session_debug.contains(&format!("{}/v1/oauth/tokens", overridden.url()))); + } + #[tokio::test] + async fn test_concurrent_authenticate_single_token_exchange() { + // Concurrent requests that all find no cached token must trigger ONE + // credential exchange (the lock is held across it), not one each. + let mut server = Server::new_async().await; + // create_oauth_mock_with_path expects exactly 1 hit. let oauth_mock = - create_oauth_mock_with_path(&mut server, "/v1/oauth/tokens", "ey000000000001", 500) - .await; - catalog.invalidate_token().await.unwrap(); - let token = catalog.context().await.unwrap().client.token().await; + create_oauth_mock_with_path(&mut server, "/v1/oauth/tokens", "tok-once", 200).await; + + let manager = OAuth2Manager::new(format!("{}/v1/oauth/tokens", server.url())) + .with_credential(Some("client1".to_string()), "secret1".to_string()); + let session: Arc = Arc::from( + manager + .init_session(&test_client(), &HashMap::new()) + .await + .unwrap(), + ); + + let client = Client::new(); + let attempts = (0..8).map(|_| { + let session = session.clone(); + let client = client.clone(); + async move { + let mut req = HttpRequest::new( + client + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + session.authenticate(&mut req).await.unwrap(); + req.headers() + .get("authorization") + .unwrap() + .to_str() + .unwrap() + .to_string() + } + }); + let bearers = futures::future::join_all(attempts).await; + oauth_mock.assert_async().await; - assert_eq!(token, None); + assert!(bearers.iter().all(|b| b == "Bearer tok-once")); } #[tokio::test] - async fn test_regenerate_token() { + async fn test_seeded_token_takes_precedence_over_credential() { + // token + credential: the seeded token is attached without any + // credential exchange. let mut server = Server::new_async().await; - let oauth_mock = create_oauth_mock(&mut server).await; - let config_mock = create_config_mock(&mut server).await; + let oauth_mock = server + .mock("POST", "/v1/oauth/tokens") + .expect(0) + .create_async() + .await; - let mut props = HashMap::new(); - props.insert("credential".to_string(), "client1:secret1".to_string()); + let manager = OAuth2Manager::new(format!("{}/v1/oauth/tokens", server.url())) + .with_token("tok-seed") + .with_credential(Some("client1".to_string()), "secret1".to_string()); + let session = manager + .init_session(&test_client(), &HashMap::new()) + .await + .unwrap(); + + let mut req = HttpRequest::new( + Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + session.authenticate(&mut req).await.unwrap(); + assert_eq!( + req.headers().get("authorization").unwrap(), + "Bearer tok-seed" + ); + + oauth_mock.assert_async().await; + } + #[tokio::test] + async fn test_injected_oauth_manager_keeps_endpoint_and_options() { + // An injected OAuth2Manager must keep its own token endpoint, extra + // headers and OAuth params across the config handshake: only explicit + // properties may override them, never synthesized defaults. + let mut server = Server::new_async().await; + // The server vends the credential, so the exchange runs through the + // post-handshake catalog session (exercising its property merging). + let config_mock = server + .mock("GET", "/v1/config") + .with_status(200) + .with_body(r#"{"defaults": {"credential": "client1:secret1"}, "overrides": {}}"#) + .create_async() + .await; + + // The catalog-host default endpoint must never see the credential. + let default_endpoint_mock = server + .mock("POST", "/v1/oauth/tokens") + .expect(0) + .create_async() + .await; + // The exchange hits the injected endpoint, carrying the injected + // header and OAuth param. + let custom_endpoint_mock = server + .mock("POST", "/custom/oauth/tokens") + .match_header("x-tenant", "t1") + // The default catalog scope must survive alongside the injected + // audience (with_extra_oauth_params merges onto the defaults). + .match_body(Matcher::AllOf(vec![ + Matcher::Regex("scope=catalog".to_string()), + Matcher::Regex("audience=aud-1".to_string()), + ])) + .with_status(200) + .with_body( + r#"{ + "access_token": "ey000000000000", + "token_type": "Bearer", + "issued_token_type": "urn:ietf:params:oauth:token-type:access_token", + "expires_in": 86400 + }"#, + ) + .create_async() + .await; + let ns_mock = server + .mock("GET", "/v1/namespaces") + .match_header("authorization", "Bearer ey000000000000") + .with_body(r#"{"namespaces": []}"#) + .create_async() + .await; + + let manager = OAuth2Manager::new(format!("{}/custom/oauth/tokens", server.url())) + .with_extra_headers(HeaderMap::from_iter([( + HeaderName::from_static("x-tenant"), + HeaderValue::from_static("t1"), + )])) + .with_extra_oauth_params(HashMap::from([( + "audience".to_string(), + "aud-1".to_string(), + )])); let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + Some(Box::new(manager)), + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + + catalog.list_namespaces(None).await.unwrap(); + + config_mock.assert_async().await; + custom_endpoint_mock.assert_async().await; + default_endpoint_mock.assert_async().await; + ns_mock.assert_async().await; + } + + #[tokio::test] + async fn test_props_token_takes_precedence_over_props_credential() { + // Both arriving through the properties rather than the builder. + let mut server = Server::new_async().await; + let oauth_mock = server + .mock("POST", "/v1/oauth/tokens") + .expect(0) + .create_async() + .await; + + let manager = OAuth2Manager::new(format!("{}/v1/oauth/tokens", server.url())); + let session = manager + .init_session( + &test_client(), + &HashMap::from([ + ("token".to_string(), "tok-props".to_string()), + ("credential".to_string(), "client1:secret1".to_string()), + ]), + ) + .await + .unwrap(); + + let mut req = HttpRequest::new( + Client::new() + .get("https://rest.example.com/v1/config") + .build() + .unwrap(), + ); + session.authenticate(&mut req).await.unwrap(); + assert_eq!( + req.headers().get("authorization").unwrap(), + "Bearer tok-props" + ); + oauth_mock.assert_async().await; + } + + #[tokio::test] + async fn test_manager_exchanges_over_the_catalog_client() { + // The manager exchanges the credential over the client it is handed, + // sharing the catalog's connection pool. Its own extra headers go on + // the token request; the catalog's do not. + let mut server = Server::new_async().await; + let token_mock = server + .mock("POST", "/v1/oauth/tokens") + .match_header("x-from", "manager") + .match_header("x-catalog-only", Matcher::Missing) + .with_status(200) + .with_body(r#"{"access_token": "tok", "token_type": "Bearer"}"#) + .create_async() + .await; + + let catalog_client = HttpClient::new( + &RestCatalogConfig::builder() + .uri(server.url()) + .props(HashMap::from([( + "header.x-catalog-only".to_string(), + "not-on-token-requests".to_string(), + )])) + .build(), + ) + .unwrap(); + + let manager = OAuth2Manager::new(format!("{}/v1/oauth/tokens", server.url())) + .with_credential(Some("client1".to_string()), "secret1".to_string()) + .with_extra_headers(HeaderMap::from_iter([( + HeaderName::from_static("x-from"), + HeaderValue::from_static("manager"), + )])); + let session = manager + .init_session(&catalog_client, &HashMap::new()) + .await + .unwrap(); + + let mut req = HttpRequest::new( + Client::new() + .get("https://rest.example.com/v1/namespaces") + .build() + .unwrap(), + ); + session.authenticate(&mut req).await.unwrap(); + token_mock.assert_async().await; + } + + #[tokio::test] + async fn test_handshake_is_authenticated_by_the_init_session() { + // `/v1/config` goes out with the init session's authentication, not + // unauthenticated and not with a later one. + let mut server = Server::new_async().await; + let config_mock = server + .mock("GET", "/v1/config") + .match_header("authorization", "Bearer tok-init") + .with_status(200) + .with_body(r#"{"defaults": {}, "overrides": {}}"#) + .create_async() + .await; + + let catalog = RestCatalog::new( + SessionContext::empty(), RestCatalogConfig::builder() .uri(server.url()) - .props(props) + .props(HashMap::from([( + "token".to_string(), + "tok-init".to_string(), + )])) .build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, ); - let token = catalog.context().await.unwrap().client.token().await; - oauth_mock.assert_async().await; + catalog.client().await.unwrap(); config_mock.assert_async().await; - assert_eq!(token, Some("ey000000000000".to_string())); - - let oauth_mock = - create_oauth_mock_with_path(&mut server, "/v1/oauth/tokens", "ey000000000001", 200) - .await; - catalog.regenerate_token().await.unwrap(); - oauth_mock.assert_async().await; - let token = catalog.context().await.unwrap().client.token().await; - assert_eq!(token, Some("ey000000000001".to_string())); } #[tokio::test] - async fn test_regenerate_token_failing_request() { + async fn test_init_session_receives_user_props() { + use tokio::sync::Mutex as AsyncMutex; + + // A custom manager initializes from the user configuration: the + // props carry the catalog `uri` and the user's credentials. + #[derive(Debug)] + struct PlainSession; + #[async_trait] + impl AuthSession for PlainSession { + async fn authenticate(&self, _request: &mut HttpRequest) -> Result<()> { + Ok(()) + } + } + + #[derive(Debug)] + struct CapturingManager(Arc>>>); + #[async_trait] + impl AuthManager for CapturingManager { + async fn init_session( + &self, + _client: &HttpClient, + props: &HashMap, + ) -> Result> { + *self.0.lock().await = Some(props.clone()); + Ok(Box::new(PlainSession)) + } + async fn catalog_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(Arc::new(PlainSession)) + } + } + let mut server = Server::new_async().await; - let oauth_mock = create_oauth_mock(&mut server).await; let config_mock = create_config_mock(&mut server).await; + let captured = Arc::new(AsyncMutex::new(None)); + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder() + .uri(server.url()) + .props(HashMap::from([( + "token".to_string(), + "tok-user".to_string(), + )])) + .build(), + Some(Box::new(CapturingManager(captured.clone()))), + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); - let mut props = HashMap::new(); - props.insert("credential".to_string(), "client1:secret1".to_string()); + catalog.client().await.unwrap(); + config_mock.assert_async().await; + let props = captured.lock().await.clone().unwrap(); + assert_eq!(props.get("token").map(String::as_str), Some("tok-user")); + assert_eq!( + props.get(REST_CATALOG_PROP_URI).map(String::as_str), + Some(server.url().as_str()) + ); + } + #[tokio::test] + async fn test_catalog_session_receives_resolved_warehouse() { + use tokio::sync::Mutex as AsyncMutex; + + // A custom manager must receive the resolved warehouse in the props + // handed to `catalog_session`, with the standard precedence: + // server default < client-side warehouse < server override. + #[derive(Debug)] + struct PlainSession; + #[async_trait] + impl AuthSession for PlainSession { + async fn authenticate(&self, _request: &mut HttpRequest) -> Result<()> { + Ok(()) + } + } + + #[derive(Debug)] + struct CapturingManager(Arc>>>); + #[async_trait] + impl AuthManager for CapturingManager { + async fn init_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(Box::new(PlainSession)) + } + async fn catalog_session( + &self, + _client: &HttpClient, + props: &HashMap, + ) -> Result> { + *self.0.lock().await = Some(props.clone()); + Ok(Arc::new(PlainSession)) + } + } + + // Client warehouse wins over a server default. + let mut server = Server::new_async().await; + let config_mock = server + .mock("GET", "/v1/config") + .match_query(Matcher::UrlEncoded( + "warehouse".to_string(), + "client-wh".to_string(), + )) + .with_status(200) + .with_body(r#"{"defaults": {"warehouse": "default-wh"}, "overrides": {}}"#) + .create_async() + .await; + let captured = Arc::new(AsyncMutex::new(None)); let catalog = RestCatalog::new( + SessionContext::empty(), RestCatalogConfig::builder() .uri(server.url()) - .props(props) + .warehouse("client-wh".to_string()) .build(), + Some(Box::new(CapturingManager(captured.clone()))), + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + catalog.client().await.unwrap(); + config_mock.assert_async().await; + let props = captured.lock().await.clone().unwrap(); + assert_eq!( + props.get("warehouse").map(String::as_str), + Some("client-wh") + ); + + // A server override wins over the client warehouse. + let mut server = Server::new_async().await; + let config_mock = server + .mock("GET", "/v1/config") + .match_query(Matcher::UrlEncoded( + "warehouse".to_string(), + "client-wh".to_string(), + )) + .with_status(200) + .with_body(r#"{"defaults": {}, "overrides": {"warehouse": "override-wh"}}"#) + .create_async() + .await; + let captured = Arc::new(AsyncMutex::new(None)); + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder() + .uri(server.url()) + .warehouse("client-wh".to_string()) + .build(), + Some(Box::new(CapturingManager(captured.clone()))), + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + catalog.client().await.unwrap(); + config_mock.assert_async().await; + let props = captured.lock().await.clone().unwrap(); + assert_eq!( + props.get("warehouse").map(String::as_str), + Some("override-wh") + ); + } + + #[tokio::test] + async fn test_init_session_dropped_before_catalog_session() { + use std::sync::atomic::{AtomicBool, Ordering}; + + // A manager whose init session guards a one-shot resource (released on + // drop) must see it released before `catalog_session` is invoked. + #[derive(Debug)] + struct GuardSession(Arc); + impl Drop for GuardSession { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + #[async_trait] + impl AuthSession for GuardSession { + async fn authenticate(&self, _request: &mut HttpRequest) -> Result<()> { + Ok(()) + } + } + + #[derive(Debug)] + struct PlainSession; + #[async_trait] + impl AuthSession for PlainSession { + async fn authenticate(&self, _request: &mut HttpRequest) -> Result<()> { + Ok(()) + } + } + + #[derive(Debug)] + struct GuardManager(Arc); + #[async_trait] + impl AuthManager for GuardManager { + async fn init_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(Box::new(GuardSession(self.0.clone()))) + } + async fn catalog_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + if !self.0.load(Ordering::SeqCst) { + return Err(Error::new( + ErrorKind::Unexpected, + "init session must be dropped before catalog_session", + )); + } + Ok(Arc::new(PlainSession)) + } + } + + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + + let dropped = Arc::new(AtomicBool::new(false)); + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + Some(Box::new(GuardManager(dropped.clone()))), + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, ); - let token = catalog.context().await.unwrap().client.token().await; - oauth_mock.assert_async().await; + catalog.client().await.unwrap(); config_mock.assert_async().await; - assert_eq!(token, Some("ey000000000000".to_string())); - - let oauth_mock = - create_oauth_mock_with_path(&mut server, "/v1/oauth/tokens", "ey000000000001", 500) - .await; - let invalidate_result = catalog.regenerate_token().await; - assert!(invalidate_result.is_err()); - oauth_mock.assert_async().await; - let token = catalog.context().await.unwrap().client.token().await; - - // original token is left intact - assert_eq!(token, Some("ey000000000000".to_string())); + assert!(dropped.load(Ordering::SeqCst)); } #[tokio::test] - async fn test_http_headers() { - let server = Server::new_async().await; - let mut props = HashMap::new(); - props.insert("credential".to_string(), "client1:secret1".to_string()); + async fn test_contextual_session_authenticates_each_catalog_request() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let first_page = server + .mock("GET", "/v1/namespaces") + .match_header("x-session-id", "session-123") + .with_body(r#"{"namespaces": [["ns1"]], "next-page-token": "next"}"#) + .create_async() + .await; + let second_page = server + .mock("GET", "/v1/namespaces?pageToken=next") + .match_header("x-session-id", "session-123") + .with_body(r#"{"namespaces": [["ns2"]]}"#) + .create_async() + .await; - let config = RestCatalogConfig::builder() - .uri(server.url()) - .props(props) + let (catalog, seen_session_ids) = context_catalog( + RestCatalogConfig::builder().uri(server.url()).build(), + false, + ); + let context = SessionContext::builder() + .session_id("session-123".to_string()) .build(); - let headers: HeaderMap = config.extra_headers().unwrap(); - let expected_headers = HeaderMap::from_iter([ - ( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), - ), - ( - HeaderName::from_static("x-client-version"), - HeaderValue::from_static(ICEBERG_REST_SPEC_VERSION), - ), - ( - header::USER_AGENT, - HeaderValue::from_str(&format!("iceberg-rs/{}", CARGO_PKG_VERSION)).unwrap(), - ), + let namespaces = catalog.list_namespaces(&context, None).await.unwrap(); + + assert_eq!(namespaces, vec![ + NamespaceIdent::new("ns1".to_string()), + NamespaceIdent::new("ns2".to_string()), ]); - assert_eq!(headers, expected_headers); + assert_eq!(*seen_session_ids.lock().unwrap(), vec![ + "session-123", + "session-123" + ]); + config_mock.assert_async().await; + first_page.assert_async().await; + second_page.assert_async().await; } #[tokio::test] - async fn test_http_headers_with_custom_headers() { - let server = Server::new_async().await; - let mut props = HashMap::new(); - props.insert("credential".to_string(), "client1:secret1".to_string()); - props.insert( - "header.content-type".to_string(), - "application/yaml".to_string(), - ); - props.insert( - "header.customized-header".to_string(), - "some/value".to_string(), + async fn test_contextual_session_authenticates_namespace_exists_head() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock_with_exists_endpoints(&mut server).await; + let exists_mock = server + .mock("HEAD", "/v1/namespaces/ns1") + .match_header("x-session-id", "namespace-session") + .with_status(204) + .create_async() + .await; + let (catalog, seen_session_ids) = context_catalog( + RestCatalogConfig::builder().uri(server.url()).build(), + false, ); - - let config = RestCatalogConfig::builder() - .uri(server.url()) - .props(props) + let context = SessionContext::builder() + .session_id("namespace-session".to_string()) .build(); - let headers: HeaderMap = config.extra_headers().unwrap(); - let expected_headers = HeaderMap::from_iter([ - ( - header::CONTENT_TYPE, - HeaderValue::from_static("application/yaml"), - ), - ( - HeaderName::from_static("x-client-version"), - HeaderValue::from_static(ICEBERG_REST_SPEC_VERSION), - ), - ( - header::USER_AGENT, - HeaderValue::from_str(&format!("iceberg-rs/{}", CARGO_PKG_VERSION)).unwrap(), - ), - ( - HeaderName::from_static("customized-header"), - HeaderValue::from_static("some/value"), - ), - ]); - assert_eq!(headers, expected_headers); + assert!( + catalog + .namespace_exists(&context, &NamespaceIdent::new("ns1".to_string())) + .await + .unwrap() + ); + assert_eq!(*seen_session_ids.lock().unwrap(), vec!["namespace-session"]); + config_mock.assert_async().await; + exists_mock.assert_async().await; } #[tokio::test] - async fn test_oauth_with_oauth2_server_uri() { + async fn test_contextual_session_authenticates_table_exists_head() { let mut server = Server::new_async().await; - let config_mock = create_config_mock(&mut server).await; - - let mut auth_server = Server::new_async().await; - let auth_server_path = "/some/path"; - let oauth_mock = - create_oauth_mock_with_path(&mut auth_server, auth_server_path, "ey000000000000", 200) - .await; - - let mut props = HashMap::new(); - props.insert("credential".to_string(), "client1:secret1".to_string()); - props.insert( - "oauth2-server-uri".to_string(), - format!("{}{}", auth_server.url(), auth_server_path).to_string(), + let config_mock = create_config_mock_with_exists_endpoints(&mut server).await; + let exists_mock = server + .mock("HEAD", "/v1/namespaces/ns1/tables/table1") + .match_header("x-session-id", "table-session") + .with_status(204) + .create_async() + .await; + let (catalog, seen_session_ids) = context_catalog( + RestCatalogConfig::builder().uri(server.url()).build(), + false, ); + let context = SessionContext::builder() + .session_id("table-session".to_string()) + .build(); - let catalog = RestCatalog::new( - RestCatalogConfig::builder() - .uri(server.url()) - .props(props) - .build(), + assert!( + catalog + .table_exists( + &context, + &TableIdent::new(NamespaceIdent::new("ns1".to_string()), "table1".to_string(),), + ) + .await + .unwrap() ); - - let token = catalog.context().await.unwrap().client.token().await; - - oauth_mock.assert_async().await; + assert_eq!(*seen_session_ids.lock().unwrap(), vec!["table-session"]); config_mock.assert_async().await; - assert_eq!(token, Some("ey000000000000".to_string())); + exists_mock.assert_async().await; } #[tokio::test] - async fn test_config_override() { + async fn test_contextual_session_authenticates_write_operation() { let mut server = Server::new_async().await; - let mut redirect_server = Server::new_async().await; - let new_uri = redirect_server.url(); - - let config_mock = server - .mock("GET", "/v1/config") - .with_status(200) - .with_body( - json!( - { - "overrides": { - "uri": new_uri, - "warehouse": "s3://iceberg-catalog", - "prefix": "ice/warehouses/my" - }, - "defaults": {}, - } - ) - .to_string(), - ) + let config_mock = create_config_mock(&mut server).await; + let drop_mock = server + .mock("DELETE", "/v1/namespaces/ns1") + .match_header("x-session-id", "write-session") + .with_status(204) .create_async() .await; + let (catalog, seen_session_ids) = context_catalog( + RestCatalogConfig::builder().uri(server.url()).build(), + false, + ); + let context = SessionContext::builder() + .session_id("write-session".to_string()) + .build(); - let list_ns_mock = redirect_server - .mock("GET", "/v1/ice/warehouses/my/namespaces") - .with_body( - r#"{ - "namespaces": [] - }"#, - ) + catalog + .drop_namespace(&context, &NamespaceIdent::new("ns1".to_string())) + .await + .unwrap(); + assert_eq!(*seen_session_ids.lock().unwrap(), vec!["write-session"]); + config_mock.assert_async().await; + drop_mock.assert_async().await; + } + + #[tokio::test] + async fn test_contextual_session_error_prevents_operation_request() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let list_mock = server + .mock("GET", "/v1/namespaces") + .expect(0) .create_async() .await; + let (catalog, seen_session_ids) = + context_catalog(RestCatalogConfig::builder().uri(server.url()).build(), true); + let context = SessionContext::builder() + .session_id("failing-session".to_string()) + .build(); - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let error = catalog.list_namespaces(&context, None).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert_eq!(error.message(), "contextual session failure"); + assert_eq!(*seen_session_ids.lock().unwrap(), vec!["failing-session"]); + config_mock.assert_async().await; + list_mock.assert_async().await; + } - let _namespaces = catalog.list_namespaces(None).await.unwrap(); + #[test] + fn test_config_debug_redacts_secrets() { + let config = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props(HashMap::from([ + ("token".to_string(), "tok-secret".to_string()), + ("credential".to_string(), "id:cred-secret".to_string()), + ("header.authorization".to_string(), "Basic xyz".to_string()), + ("adls.account-key".to_string(), "adls-secret".to_string()), + ("s3.sse.key".to_string(), "sse-secret".to_string()), + ( + "adls.connection-string".to_string(), + "cs-secret".to_string(), + ), + ("warehouse".to_string(), "wh1".to_string()), + ])) + .build(); - config_mock.assert_async().await; - list_ns_mock.assert_async().await; + let out = format!("{config:?}"); + assert!(!out.contains("tok-secret")); + assert!(!out.contains("cred-secret")); + assert!(!out.contains("Basic xyz")); + assert!(!out.contains("adls-secret")); + assert!(!out.contains("sse-secret")); + assert!(!out.contains("cs-secret")); + assert!(out.contains("[REDACTED]")); + assert!(out.contains("wh1")); } #[tokio::test] - async fn test_list_namespace() { - let mut server = Server::new_async().await; + async fn test_auth_type_defaults() { + // Unset `rest.auth.type`: `oauth2` when any OAuth material is + // configured (existing setups keep working), `none` otherwise. + let bare = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .build(); + assert!( + format!("{:?}", test_catalog(bare).resolve_auth_manager().unwrap()) + .contains("NoopAuthManager") + ); - let config_mock = create_config_mock(&mut server).await; + let with_token = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props(HashMap::from([("token".to_string(), "tok".to_string())])) + .build(); + assert!( + format!( + "{:?}", + test_catalog(with_token).resolve_auth_manager().unwrap() + ) + .contains("OAuth2Manager") + ); - let list_ns_mock = server - .mock("GET", "/v1/namespaces") - .with_body( - r#"{ - "namespaces": [ - ["ns1", "ns11"], - ["ns2"] - ] - }"#, + // An explicit type is matched case-insensitively. + let mixed_case = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props(HashMap::from([( + REST_CATALOG_PROP_AUTH_TYPE.to_string(), + "OAuth2".to_string(), + )])) + .build(); + assert!( + format!( + "{:?}", + test_catalog(mixed_case).resolve_auth_manager().unwrap() ) - .create_async() - .await; + .contains("OAuth2Manager") + ); - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + // An explicit OAuth endpoint is oauth2 intent too: the manager can + // still pick up a server-supplied token from `/v1/config`. + let with_endpoint = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props(HashMap::from([( + "oauth2-server-uri".to_string(), + "http://auth.example.com/tokens".to_string(), + )])) + .build(); + assert!( + format!( + "{:?}", + test_catalog(with_endpoint).resolve_auth_manager().unwrap() + ) + .contains("OAuth2Manager") + ); + } - let namespaces = catalog.list_namespaces(None).await.unwrap(); + #[tokio::test] + async fn test_unknown_auth_type_is_rejected() { + let props = HashMap::from([( + REST_CATALOG_PROP_AUTH_TYPE.to_string(), + "kerberos".to_string(), + )]); + let config = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props(props) + .build(); - let expected_ns = vec![ - NamespaceIdent::from_vec(vec!["ns1".to_string(), "ns11".to_string()]).unwrap(), - NamespaceIdent::from_vec(vec!["ns2".to_string()]).unwrap(), - ]; + let err = test_catalog(config).resolve_auth_manager().unwrap_err(); + assert!(err.message().contains(REST_CATALOG_PROP_AUTH_TYPE)); + } - assert_eq!(expected_ns, namespaces); + #[tokio::test] + async fn test_with_auth_manager_overrides_config() { + // A custom auth manager takes precedence over `rest.auth.type`. + #[derive(Debug)] + struct StubAuthManager; + #[async_trait] + impl AuthManager for StubAuthManager { + async fn init_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + unimplemented!() + } + async fn catalog_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + unimplemented!() + } + } - config_mock.assert_async().await; - list_ns_mock.assert_async().await; + let config = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props(HashMap::from([( + REST_CATALOG_PROP_AUTH_TYPE.to_string(), + "kerberos".to_string(), + )])) + .build(); + + // The unknown auth type is never consulted. + let catalog = test_catalog_with(config, StubAuthManager); + assert!(catalog.resolve_auth_manager().is_ok()); } #[tokio::test] @@ -1462,9 +3685,12 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); - let namespaces = catalog.list_namespaces(None).await.unwrap(); + let namespaces = catalog + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap(); let expected_ns = vec![ NamespaceIdent::from_vec(vec!["ns1".to_string(), "ns11".to_string()]).unwrap(), @@ -1558,9 +3784,12 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); - let namespaces = catalog.list_namespaces(None).await.unwrap(); + let namespaces = catalog + .list_namespaces(&SessionContext::empty(), None) + .await + .unwrap(); let expected_ns = vec![ NamespaceIdent::from_vec(vec!["ns1".to_string(), "ns11".to_string()]).unwrap(), @@ -1608,10 +3837,11 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let namespaces = catalog .create_namespace( + &SessionContext::empty(), &NamespaceIdent::from_vec(vec!["ns1".to_string(), "ns11".to_string()]).unwrap(), HashMap::from([("key1".to_string(), "value1".to_string())]), ) @@ -1648,10 +3878,13 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let namespaces = catalog - .get_namespace(&NamespaceIdent::new("ns1".to_string())) + .get_namespace( + &SessionContext::empty(), + &NamespaceIdent::new("ns1".to_string()), + ) .await .unwrap(); @@ -1670,7 +3903,7 @@ mod tests { async fn check_namespace_exists() { let mut server = Server::new_async().await; - let config_mock = create_config_mock(&mut server).await; + let config_mock = create_config_mock_with_exists_endpoints(&mut server).await; let get_ns_mock = server .mock("HEAD", "/v1/namespaces/ns1") @@ -1678,7 +3911,49 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + + assert!( + catalog + .namespace_exists( + &SessionContext::empty(), + &NamespaceIdent::new("ns1".to_string()) + ) + .await + .unwrap() + ); + + config_mock.assert_async().await; + get_ns_mock.assert_async().await; + } + + #[tokio::test] + async fn test_namespace_exists_falls_back_to_get_when_head_not_advertised() { + let mut server = Server::new_async().await; + + // No `endpoints` advertised, and the default set has no HEAD namespace + // route, so `namespace_exists` falls back to a GET load-namespace. + let config_mock = create_config_mock(&mut server).await; + let get_ns_mock = server + .mock("GET", "/v1/namespaces/ns1") + .with_status(200) + .with_body( + r#"{ + "namespace": ["ns1"], + "properties": {} + }"#, + ) + .create_async() + .await; + + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); assert!( catalog @@ -1703,10 +3978,13 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); catalog - .drop_namespace(&NamespaceIdent::new("ns1".to_string())) + .drop_namespace( + &SessionContext::empty(), + &NamespaceIdent::new("ns1".to_string()), + ) .await .unwrap(); @@ -1740,10 +4018,13 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let tables = catalog - .list_tables(&NamespaceIdent::new("ns1".to_string())) + .list_tables( + &SessionContext::empty(), + &NamespaceIdent::new("ns1".to_string()), + ) .await .unwrap(); @@ -1805,10 +4086,13 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let tables = catalog - .list_tables(&NamespaceIdent::new("ns1".to_string())) + .list_tables( + &SessionContext::empty(), + &NamespaceIdent::new("ns1".to_string()), + ) .await .unwrap(); @@ -1933,10 +4217,13 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let tables = catalog - .list_tables(&NamespaceIdent::new("ns1".to_string())) + .list_tables( + &SessionContext::empty(), + &NamespaceIdent::new("ns1".to_string()), + ) .await .unwrap(); @@ -1974,13 +4261,13 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); catalog - .drop_table(&TableIdent::new( - NamespaceIdent::new("ns1".to_string()), - "table1".to_string(), - )) + .drop_table( + &SessionContext::empty(), + &TableIdent::new(NamespaceIdent::new("ns1".to_string()), "table1".to_string()), + ) .await .unwrap(); @@ -1992,7 +4279,7 @@ mod tests { async fn test_check_table_exists() { let mut server = Server::new_async().await; - let config_mock = create_config_mock(&mut server).await; + let config_mock = create_config_mock_with_exists_endpoints(&mut server).await; let check_table_exists_mock = server .mock("HEAD", "/v1/namespaces/ns1/tables/table1") @@ -2000,7 +4287,48 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + + assert!( + catalog + .table_exists( + &SessionContext::empty(), + &TableIdent::new(NamespaceIdent::new("ns1".to_string()), "table1".to_string(),), + ) + .await + .unwrap() + ); + + config_mock.assert_async().await; + check_table_exists_mock.assert_async().await; + } + + #[tokio::test] + async fn test_table_exists_falls_back_to_load_when_head_not_advertised() { + let mut server = Server::new_async().await; + + // No `endpoints` advertised, and the default set has no HEAD table + // route, so `table_exists` falls back to a GET load-table. + let config_mock = create_config_mock(&mut server).await; + let load_table_mock = server + .mock("GET", "/v1/namespaces/ns1/tables/table1") + .with_status(200) + .with_body_from_file(format!( + "{}/testdata/{}", + env!("CARGO_MANIFEST_DIR"), + "load_table_response.json" + )) + .create_async() + .await; + + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); assert!( catalog @@ -2013,7 +4341,7 @@ mod tests { ); config_mock.assert_async().await; - check_table_exists_mock.assert_async().await; + load_table_mock.assert_async().await; } #[tokio::test] @@ -2028,10 +4356,11 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); catalog .rename_table( + &SessionContext::empty(), &TableIdent::new(NamespaceIdent::new("ns1".to_string()), "table1".to_string()), &TableIdent::new(NamespaceIdent::new("ns1".to_string()), "table2".to_string()), ) @@ -2059,13 +4388,13 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let table = catalog - .load_table(&TableIdent::new( - NamespaceIdent::new("ns1".to_string()), - "test1".to_string(), - )) + .load_table( + &SessionContext::empty(), + &TableIdent::new(NamespaceIdent::new("ns1".to_string()), "test1".to_string()), + ) .await .unwrap(); @@ -2152,6 +4481,342 @@ mod tests { rename_table_mock.assert_async().await; } + /// A `loadTable` body for `ns1.test1` whose `metadata-location` names `tag`. + fn load_table_body(tag: &str) -> serde_json::Value { + let file = File::open(format!( + "{}/testdata/load_table_response.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let mut body: serde_json::Value = serde_json::from_reader(BufReader::new(file)).unwrap(); + body["metadata-location"] = json!(format!("s3://warehouse/{tag}.metadata.json")); + body + } + + /// Mocks one `loadTable` of `ns1.test1` that expects `if_none_match` and + /// answers `status`, carrying `etag` if given. + async fn load_table_mock( + server: &mut ServerGuard, + if_none_match: Matcher, + status: usize, + etag: Option<&str>, + ) -> Mock { + let mut mock = server + .mock("GET", "/v1/namespaces/ns1/tables/test1") + .match_header("if-none-match", if_none_match) + .with_status(status); + if let Some(etag) = etag { + mock = mock.with_header("etag", etag); + if status == 200 { + mock = mock.with_body(load_table_body(etag.trim_matches('"')).to_string()); + } + } + mock.create_async().await + } + + fn test_table_ident() -> TableIdent { + TableIdent::from_strs(["ns1", "test1"]).unwrap() + } + + #[tokio::test] + async fn test_load_table_revalidates_cached_table_with_etag() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let _v1 = load_table_mock(&mut server, Matcher::Missing, 200, Some("\"v1\"")).await; + let reuse_v1 = load_table_mock(&mut server, "\"v1\"".into(), 304, None).await; + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + let context = SessionContext::empty(); + let location = |table: Table| table.metadata_location().unwrap().to_string(); + + let v1 = "s3://warehouse/v1.metadata.json"; + let v2 = "s3://warehouse/v2.metadata.json"; + let ident = test_table_ident(); + let load = || catalog.load_table(&context, &ident); + assert_eq!(location(load().await.unwrap()), v1); + assert_eq!(location(load().await.unwrap()), v1); + + reuse_v1.remove_async().await; + let _v2 = load_table_mock(&mut server, "\"v1\"".into(), 200, Some("\"v2\"")).await; + let reuse_v2 = load_table_mock(&mut server, "\"v2\"".into(), 304, None).await; + assert_eq!(location(load().await.unwrap()), v2); + assert_eq!(location(load().await.unwrap()), v2); + reuse_v2.assert_async().await; + } + + #[tokio::test] + async fn test_load_table_not_modified_without_cached_table() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let _load = load_table_mock(&mut server, Matcher::Missing, 304, None).await; + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + + let err = catalog + .load_table(&SessionContext::empty(), &test_table_ident()) + .await + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::Unexpected); + } + + #[tokio::test] + async fn test_table_cache_is_per_session() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let miss = load_table_mock(&mut server, Matcher::Missing, 200, Some("\"v1\"")) + .await + .expect(2); + let hit = load_table_mock(&mut server, "\"v1\"".into(), 304, None).await; + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + let session = |id: &str| SessionContext::builder().session_id(id.to_string()).build(); + + for id in ["a", "b", "a"] { + catalog + .load_table(&session(id), &test_table_ident()) + .await + .unwrap(); + } + + miss.assert_async().await; + hit.assert_async().await; + } + + #[tokio::test] + async fn test_table_cache_disabled_with_zero_max_entries() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let load = load_table_mock(&mut server, Matcher::Missing, 200, Some("\"v1\"")) + .await + .expect(2); + let catalog = session_catalog( + RestCatalogConfig::builder() + .uri(server.url()) + .props(HashMap::from([( + REST_CATALOG_PROP_TABLE_CACHE_MAX_ENTRIES.to_string(), + "0".to_string(), + )])) + .build(), + ); + let context = SessionContext::empty(); + + catalog + .load_table(&context, &test_table_ident()) + .await + .unwrap(); + catalog + .load_table(&context, &test_table_ident()) + .await + .unwrap(); + + load.assert_async().await; + } + + #[tokio::test] + async fn test_table_changes_invalidate_cached_table() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let miss = load_table_mock(&mut server, Matcher::Missing, 200, Some("\"v1\"")) + .await + .expect(5); + let hit = load_table_mock(&mut server, "\"v1\"".into(), 304, None).await; + let _drop = server + .mock("DELETE", "/v1/namespaces/ns1/tables/test1") + .with_status(204) + .create_async() + .await; + let _rename = server + .mock("POST", "/v1/tables/rename") + .with_status(204) + .create_async() + .await; + let _register = server + .mock("POST", "/v1/namespaces/ns1/register") + .with_status(200) + .with_body(load_table_body("v1").to_string()) + .create_async() + .await; + let _commit = server + .mock("POST", "/v1/namespaces/ns1/tables/test1") + .with_status(200) + .with_body_from_file(format!( + "{}/testdata/update_table_response.json", + env!("CARGO_MANIFEST_DIR") + )) + .create_async() + .await; + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + let ident = test_table_ident(); + + // After each change the next load must miss the cache; only the + // commit's own refresh is answered from it. + catalog.load_table(&ident).await.unwrap(); + catalog.drop_table(&ident).await.unwrap(); + catalog.load_table(&ident).await.unwrap(); + let dest = TableIdent::from_strs(["ns1", "test2"]).unwrap(); + catalog.rename_table(&ident, &dest).await.unwrap(); + catalog.load_table(&ident).await.unwrap(); + let location = "s3://warehouse/v1.metadata.json".to_string(); + catalog.register_table(&ident, location).await.unwrap(); + let table = catalog.load_table(&ident).await.unwrap(); + let tx = Transaction::new(&table); + tx.update_table_properties() + .set("k".to_string(), "v".to_string()) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + catalog.load_table(&ident).await.unwrap(); + + miss.assert_async().await; + hit.assert_async().await; + } + + #[tokio::test] + async fn test_load_table_404_invalidates_cached_table() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let miss = load_table_mock(&mut server, Matcher::Missing, 200, Some("\"v1\"")) + .await + .expect(2); + let _gone = load_table_mock(&mut server, "\"v1\"".into(), 404, None).await; + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + let context = SessionContext::empty(); + let ident = test_table_ident(); + + catalog.load_table(&context, &ident).await.unwrap(); + let err = catalog.load_table(&context, &ident).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::TableNotFound); + catalog.load_table(&context, &ident).await.unwrap(); + + miss.assert_async().await; + } + + #[tokio::test] + async fn test_load_table_vending_credentials_evicts_cached_table() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let miss = load_table_mock(&mut server, Matcher::Missing, 200, Some("\"v1\"")) + .await + .expect(2); + let mut body = load_table_body("v2"); + body["storage-credentials"] = json!([{ + "prefix": "s3://warehouse/", + "config": { "s3.session-token": "token" }, + }]); + let _vended = server + .mock("GET", "/v1/namespaces/ns1/tables/test1") + .match_header("if-none-match", "\"v1\"") + .with_status(200) + .with_header("etag", "\"v2\"") + .with_body(body.to_string()) + .create_async() + .await; + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + let context = SessionContext::empty(); + + for _ in 0..3 { + catalog + .load_table(&context, &test_table_ident()) + .await + .unwrap(); + } + + miss.assert_async().await; + } + + #[tokio::test] + async fn test_load_table_with_vended_credentials_is_not_cached() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let mut body = load_table_body("v1"); + body["config"] = json!({ "s3.session-token": "token" }); + let load = server + .mock("GET", "/v1/namespaces/ns1/tables/test1") + .match_header("if-none-match", Matcher::Missing) + .with_status(200) + .with_header("etag", "\"v1\"") + .with_body(body.to_string()) + .expect(2) + .create_async() + .await; + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + let context = SessionContext::empty(); + + catalog + .load_table(&context, &test_table_ident()) + .await + .unwrap(); + catalog + .load_table(&context, &test_table_ident()) + .await + .unwrap(); + + load.assert_async().await; + } + + #[test] + fn test_table_cache_rejects_out_of_range_expiry() { + for value in ["0", "31536000000001"] { + let config = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props(HashMap::from([( + REST_CATALOG_PROP_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS.to_string(), + value.to_string(), + )])) + .build(); + + assert_eq!( + config.table_cache().unwrap_err().kind(), + ErrorKind::DataInvalid + ); + } + } + + #[test] + fn test_is_short_lived_credential() { + for key in [ + "S3.Session-Token", + "gcs.oauth2.token", + "adls.sas-token.account", + ] { + assert!(is_short_lived_credential(key), "{key}"); + } + for key in ["s3.sse.key", "s3.access-key-id", "encryption.key-id"] { + assert!(!is_short_lived_credential(key), "{key}"); + } + } + + #[tokio::test] + async fn test_failed_drop_still_invalidates_cached_table() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let load = load_table_mock(&mut server, Matcher::Missing, 200, Some("\"v1\"")) + .await + .expect(2); + // Cut the connection mid-response so the drop fails inside `query_catalog`. + let _drop = server + .mock("DELETE", "/v1/namespaces/ns1/tables/test1") + .with_chunked_body(|_| Err(std::io::Error::other("connection reset"))) + .create_async() + .await; + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + let context = SessionContext::empty(); + let ident = test_table_ident(); + + catalog.load_table(&context, &ident).await.unwrap(); + catalog.drop_table(&context, &ident).await.unwrap_err(); + catalog.load_table(&context, &ident).await.unwrap(); + + load.assert_async().await; + } + #[tokio::test] async fn test_load_table_404() { let mut server = Server::new_async().await; @@ -2173,13 +4838,13 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let table = catalog - .load_table(&TableIdent::new( - NamespaceIdent::new("ns1".to_string()), - "test1".to_string(), - )) + .load_table( + &SessionContext::empty(), + &TableIdent::new(NamespaceIdent::new("ns1".to_string()), "test1".to_string()), + ) .await; assert!(table.is_err()); @@ -2206,7 +4871,7 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let table_creation = TableCreation::builder() .name("test1".to_string()) @@ -2227,13 +4892,14 @@ mod tests { .properties(HashMap::from([("owner".to_string(), "testx".to_string())])) .partition_spec( UnboundPartitionSpec::builder() - .add_partition_fields(vec![ + .add_partition_field( UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) + .name("id") .transform(Transform::Truncate(3)) - .name("id".to_string()) - .build(), - ]) + .build() + .unwrap(), + ) .unwrap() .build(), ) @@ -2253,7 +4919,11 @@ mod tests { .build(); let table = catalog - .create_table(&NamespaceIdent::from_strs(["ns1"]).unwrap(), table_creation) + .create_table( + &SessionContext::empty(), + &NamespaceIdent::from_strs(["ns1"]).unwrap(), + table_creation, + ) .await .unwrap(); @@ -2352,7 +5022,7 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let table_creation = TableCreation::builder() .name("test1".to_string()) @@ -2374,7 +5044,11 @@ mod tests { .build(); let table_result = catalog - .create_table(&NamespaceIdent::from_strs(["ns1"]).unwrap(), table_creation) + .create_table( + &SessionContext::empty(), + &NamespaceIdent::from_strs(["ns1"]).unwrap(), + table_creation, + ) .await; assert!(table_result.is_err()); @@ -2418,7 +5092,14 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); let table1 = { let file = File::open(format!( @@ -2428,13 +5109,14 @@ mod tests { )) .unwrap(); let reader = BufReader::new(file); - let resp = serde_json::from_reader::<_, LoadTableResponse>(reader).unwrap(); + let resp = serde_json::from_reader::<_, LoadTableResult>(reader).unwrap(); Table::builder() .metadata(resp.metadata) .metadata_location(resp.metadata_location.unwrap()) .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .file_io(FileIO::from_path("/tmp").unwrap().build().unwrap()) + .file_io(FileIO::new_with_fs()) + .runtime(test_runtime()) .build() .unwrap() }; @@ -2558,7 +5240,14 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); let table1 = { let file = File::open(format!( @@ -2568,13 +5257,14 @@ mod tests { )) .unwrap(); let reader = BufReader::new(file); - let resp = serde_json::from_reader::<_, LoadTableResponse>(reader).unwrap(); + let resp = serde_json::from_reader::<_, LoadTableResult>(reader).unwrap(); Table::builder() .metadata(resp.metadata) .metadata_location(resp.metadata_location.unwrap()) .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .file_io(FileIO::from_path("/tmp").unwrap().build().unwrap()) + .file_io(FileIO::new_with_fs()) + .runtime(test_runtime()) .build() .unwrap() }; @@ -2619,7 +5309,7 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let table_ident = TableIdent::new(NamespaceIdent::new("ns1".to_string()), "test1".to_string()); let metadata_location = String::from( @@ -2627,7 +5317,7 @@ mod tests { ); let table = catalog - .register_table(&table_ident, metadata_location) + .register_table(&SessionContext::empty(), &table_ident, metadata_location) .await .unwrap(); @@ -2667,7 +5357,7 @@ mod tests { .create_async() .await; - let catalog = RestCatalog::new(RestCatalogConfig::builder().uri(server.url()).build()); + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); let table_ident = TableIdent::new(NamespaceIdent::new("ns1".to_string()), "test1".to_string()); @@ -2675,7 +5365,7 @@ mod tests { "s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", ); let table = catalog - .register_table(&table_ident, metadata_location) + .register_table(&SessionContext::empty(), &table_ident, metadata_location) .await; assert!(table.is_err()); @@ -2704,7 +5394,8 @@ mod tests { assert!(catalog.is_ok()); - let catalog_config = catalog.unwrap().user_config; + let catalog = catalog.unwrap(); + let catalog_config = &catalog.inner.user_config; assert_eq!(catalog_config.name.as_deref(), Some("test")); assert_eq!(catalog_config.uri, "http://localhost:8080"); assert_eq!(catalog_config.warehouse, None); @@ -2734,4 +5425,115 @@ mod tests { assert_eq!(err.message(), "Catalog uri is required"); } } + + #[tokio::test] + async fn test_create_session_catalog() { + let builder = RestSessionCatalogBuilder::default(); + + let result = builder + .load( + "test", + HashMap::from([ + ( + REST_CATALOG_PROP_URI.to_string(), + "http://localhost:8080".to_string(), + ), + ("a".to_string(), "b".to_string()), + ]), + ) + .await; + + assert!(result.is_ok()); + + let catalog = result.unwrap(); + + let catalog_config = catalog.user_config; + assert_eq!(catalog_config.name.as_deref(), Some("test")); + assert_eq!(catalog_config.uri, "http://localhost:8080"); + assert_eq!(catalog_config.warehouse, None); + // The default builder sets no client (only `with_client` does). + assert!(catalog_config.client.is_none()); + + // `uri` is consumed into its own field; other props are retained. + assert_eq!(catalog_config.props.get("a"), Some(&"b".to_string())); + assert!(!catalog_config.props.contains_key(REST_CATALOG_PROP_URI)); + } + + #[tokio::test] + async fn test_create_rest_catalog_with_session() { + let context = SessionContext::builder() + .session_id("test-id".to_string()) + .build(); + + let result = RestCatalogBuilder::default() + .with_session_context(context) + .load( + "test", + HashMap::from([( + REST_CATALOG_PROP_URI.to_string(), + "http://localhost:8080".to_string(), + )]), + ) + .await; + + assert!(result.is_ok()); + + // The context passed to `with_session_context` is the one the catalog is bound to. + let catalog = result.unwrap(); + assert_eq!(catalog.session_context.session_id(), "test-id"); + } + + #[tokio::test] + async fn test_create_rest_catalog_default_session() { + let result = RestCatalogBuilder::default() + .load( + "test", + HashMap::from([( + REST_CATALOG_PROP_URI.to_string(), + "http://localhost:8080".to_string(), + )]), + ) + .await; + + assert!(result.is_ok()); + + // Without `with_session_context`, the catalog falls back to `SessionContext::empty()`, + // which assigns a fresh v4 UUID. + let catalog = result.unwrap(); + assert!(uuid::Uuid::parse_str(catalog.session_context.session_id()).is_ok()); + } + + /// Smoke test: a [`Catalog`] trait method delegates through the façade to `RestSessionCatalog`. + #[tokio::test] + async fn test_rest_catalog_delegates() { + let mut server = Server::new_async().await; + let config_mock = create_config_mock(&mut server).await; + let list_ns_mock = server + .mock("GET", "/v1/namespaces") + .with_body( + r#"{ + "namespaces": [["ns1"]] + }"#, + ) + .create_async() + .await; + + let catalog = RestCatalog::new( + SessionContext::empty(), + RestCatalogConfig::builder().uri(server.url()).build(), + None, + Some(Arc::new(LocalFsStorageFactory)), + Runtime::current(), + None, + ); + + let namespaces = catalog.list_namespaces(None).await.unwrap(); + + assert_eq!(namespaces, vec![ + NamespaceIdent::from_vec(vec!["ns1".to_string()]).unwrap() + ]); + + config_mock.assert_async().await; + list_ns_mock.assert_async().await; + } } diff --git a/crates/catalog/rest/src/client.rs b/crates/catalog/rest/src/client.rs index 361c036bb6..f81f86574b 100644 --- a/crates/catalog/rest/src/client.rs +++ b/crates/catalog/rest/src/client.rs @@ -18,53 +18,83 @@ use std::collections::HashMap; use std::fmt::{Debug, Formatter}; -use http::StatusCode; use iceberg::{Error, ErrorKind, Result}; use reqwest::header::HeaderMap; -use reqwest::{Client, IntoUrl, Method, Request, RequestBuilder, Response}; +use reqwest::{Client, IntoUrl, Method, RequestBuilder}; use serde::de::DeserializeOwned; -use tokio::sync::Mutex; use crate::RestCatalogConfig; -use crate::types::{ErrorResponse, TokenResponse}; +use crate::auth::AuthSession; +use crate::request::HttpRequest; +use crate::response::HttpResponse; -pub(crate) struct HttpClient { +/// The catalog's HTTP client, handed to an [`AuthManager`] so its own +/// requests share the catalog's connection pool and configuration. +/// +/// [`AuthManager`]: crate::auth::AuthManager +#[derive(Clone)] +pub struct HttpClient { client: Client, - /// The token to be used for authentication. - /// - /// It's possible to fetch the token from the server while needed. - token: Mutex>, - /// The token endpoint to be used for authentication. - token_endpoint: String, - /// The credential to be used for authentication. - credential: Option<(Option, String)>, /// Extra headers to be added to each request. extra_headers: HeaderMap, - /// Extra oauth parameters to be added to each authentication request. - extra_oauth_params: HashMap, + /// Whether to disable header redaction in error logs (defaults to false for security). + disable_header_redaction: bool, } impl Debug for HttpClient { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // The inner client is omitted: an injected one may carry secrets in + // its default headers. f.debug_struct("HttpClient") - .field("client", &self.client) - .field("extra_headers", &self.extra_headers) + .field( + // `header.*` values may hold secrets (e.g. `authorization`). + "extra_headers", + &format_headers_redacted(&self.extra_headers, self.disable_header_redaction), + ) .finish_non_exhaustive() } } impl HttpClient { + /// Sends an unauthenticated form-encoded POST and returns the response, + /// which is what an [`AuthManager`] needs to exchange a credential for a + /// token. + /// + /// Only `headers` are sent: neither the catalog's extra headers nor + /// any session's authentication is applied, so a token request carries + /// exactly the credentials its caller puts into `headers` or `form`. + /// + /// [`AuthManager`]: crate::auth::AuthManager + pub async fn post_form( + &self, + url: &str, + headers: &HeaderMap, + form: &HashMap<&str, &str>, + ) -> Result { + let mut request = HttpRequest::build( + self.client + .request(Method::POST, url) + .headers(headers.clone()) + .form(form), + )?; + // `headers` may carry a `content-type: application/json` that `form` + // leaves in place. + request.headers_mut().insert( + reqwest::header::CONTENT_TYPE, + reqwest::header::HeaderValue::from_static("application/x-www-form-urlencoded"), + ); + + let response = self.client.execute(request.into_inner()).await?; + HttpResponse::read(response).await + } + /// Create a new http client. - pub fn new(cfg: &RestCatalogConfig) -> Result { - let extra_headers = cfg.extra_headers()?; + pub(crate) fn new(cfg: &RestCatalogConfig) -> Result { Ok(HttpClient { - client: cfg.client().unwrap_or_default(), - token: Mutex::new(cfg.token()), - token_endpoint: cfg.get_token_endpoint(), - credential: cfg.credential(), - extra_headers, - extra_oauth_params: cfg.extra_oauth_params(), + client: cfg.client(), + extra_headers: cfg.extra_headers()?, + disable_header_redaction: cfg.disable_header_redaction(), }) } @@ -72,228 +102,410 @@ impl HttpClient { /// /// If cfg carries new value, we will use cfg instead. /// Otherwise, we will keep the old value. - pub fn update_with(self, cfg: &RestCatalogConfig) -> Result { + pub(crate) fn update_with(self, cfg: &RestCatalogConfig) -> Result { let extra_headers = (!cfg.extra_headers()?.is_empty()) .then(|| cfg.extra_headers()) .transpose()? .unwrap_or(self.extra_headers); Ok(HttpClient { - client: cfg.client().unwrap_or(self.client), - token: Mutex::new(cfg.token().or_else(|| self.token.into_inner())), - token_endpoint: if !cfg.get_token_endpoint().is_empty() { - cfg.get_token_endpoint() - } else { - self.token_endpoint - }, - credential: cfg.credential().or(self.credential), + // `cfg.client()` returns the same shared client. + client: cfg.client(), extra_headers, - extra_oauth_params: if !cfg.extra_oauth_params().is_empty() { - cfg.extra_oauth_params() - } else { - self.extra_oauth_params - }, + disable_header_redaction: cfg.disable_header_redaction(), }) } - /// This API is testing only to assert the token. - #[cfg(test)] - pub(crate) async fn token(&self) -> Option { - let mut req = self - .request(Method::GET, &self.token_endpoint) - .build() - .unwrap(); - self.authenticate(&mut req).await.ok(); - self.token.lock().await.clone() - } - - async fn exchange_credential_for_token(&self) -> Result { - // Credential must exist here. - let (client_id, client_secret) = self.credential.as_ref().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Credential must be provided for authentication", - ) - })?; - - let mut params = HashMap::with_capacity(4); - params.insert("grant_type", "client_credentials"); - if let Some(client_id) = client_id { - params.insert("client_id", client_id); - } - params.insert("client_secret", client_secret); - params.extend( - self.extra_oauth_params - .iter() - .map(|(k, v)| (k.as_str(), v.as_str())), - ); - - let mut auth_req = self - .request(Method::POST, &self.token_endpoint) - .form(¶ms) - .build()?; - // extra headers add content-type application/json header it's necessary to override it with proper type - // note that form call doesn't add content-type header if already present - auth_req.headers_mut().insert( - http::header::CONTENT_TYPE, - http::HeaderValue::from_static("application/x-www-form-urlencoded"), - ); - let auth_url = auth_req.url().clone(); - let auth_resp = self.client.execute(auth_req).await?; - - let auth_res: TokenResponse = if auth_resp.status() == StatusCode::OK { - let text = auth_resp - .bytes() - .await - .map_err(|err| err.with_url(auth_url.clone()))?; - Ok(serde_json::from_slice(&text).map_err(|e| { - Error::new( - ErrorKind::Unexpected, - "Failed to parse response from rest catalog server!", - ) - .with_context("operation", "auth") - .with_context("url", auth_url.to_string()) - .with_context("json", String::from_utf8_lossy(&text)) - .with_source(e) - })?) - } else { - let code = auth_resp.status(); - let text = auth_resp - .bytes() - .await - .map_err(|err| err.with_url(auth_url.clone()))?; - let e: ErrorResponse = serde_json::from_slice(&text).map_err(|e| { - Error::new(ErrorKind::Unexpected, "Received unexpected response") - .with_context("code", code.to_string()) - .with_context("operation", "auth") - .with_context("url", auth_url.to_string()) - .with_context("json", String::from_utf8_lossy(&text)) - .with_source(e) - })?; - Err(Error::from(e)) - }?; - Ok(auth_res.access_token) - } - - /// Invalidate the current token without generating a new one. On the next request, the client - /// will attempt to generate a new token. - pub(crate) async fn invalidate_token(&self) -> Result<()> { - *self.token.lock().await = None; - Ok(()) - } - - /// Invalidate the current token and set a new one. Generates a new token before invalidating - /// the current token, meaning the old token will be used until this function acquires the lock - /// and overwrites the token. - /// - /// If credential is invalid, or the request fails, this method will return an error and leave - /// the current token unchanged. - pub(crate) async fn regenerate_token(&self) -> Result<()> { - let new_token = self.exchange_credential_for_token().await?; - *self.token.lock().await = Some(new_token.clone()); - Ok(()) - } - - /// Authenticates the request by adding a bearer token to the authorization header. - /// - /// This method supports three authentication modes: - /// - /// 1. **No authentication** - Skip authentication when both `credential` and `token` are missing. - /// 2. **Token authentication** - Use the provided `token` directly for authentication. - /// 3. **OAuth authentication** - Exchange `credential` for a token, cache it, then use it for authentication. + /// Testing only: the bearer token `session` would attach. /// - /// When both `credential` and `token` are present, `token` takes precedence. - /// - /// # TODO: Support automatic token refreshing. - async fn authenticate(&self, req: &mut Request) -> Result<()> { - // Clone the token from lock without holding the lock for entire function. - let token = self.token.lock().await.clone(); - - if self.credential.is_none() && token.is_none() { - return Ok(()); - } - - // Either use the provided token or exchange credential for token, cache and use that - let token = match token { - Some(token) => token, - None => { - let token = self.exchange_credential_for_token().await?; - // Update token so that we use it for next request instead of - // exchanging credential for token from the server again - *self.token.lock().await = Some(token.clone()); - token - } - }; - - // Insert token in request. - req.headers_mut().insert( - http::header::AUTHORIZATION, - format!("Bearer {token}").parse().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Invalid token received from catalog server!", - ) - .with_source(e) - })?, - ); - - Ok(()) + /// Authenticates a throwaway request (never sent) and reads the header + /// back, so it works for any [`AuthSession`]. + #[cfg(test)] + pub(crate) async fn token(&self, auth_session: &dyn AuthSession) -> Option { + let mut request = HttpRequest::build( + self.client + .request(Method::GET, "http://localhost/token-probe"), + ) + .ok()?; + auth_session.authenticate(&mut request).await.ok()?; + let request = request.into_inner(); + request + .headers() + .get(reqwest::header::AUTHORIZATION)? + .to_str() + .ok()? + .strip_prefix("Bearer ") + .map(str::to_string) } #[inline] - pub fn request(&self, method: Method, url: U) -> RequestBuilder { + pub(crate) fn request(&self, method: Method, url: U) -> RequestBuilder { self.client .request(method, url) .headers(self.extra_headers.clone()) } - /// Executes the given `Request` and returns a `Response`. - pub async fn execute(&self, mut request: Request) -> Result { + /// Sends `request` to the Iceberg REST catalog, authenticated by this + /// client's session. + pub(crate) async fn query_catalog( + &self, + auth_session: &dyn AuthSession, + mut request: HttpRequest, + ) -> Result { + // Authenticate first, then apply extra headers, so a configured + // `header.authorization` keeps overriding a token (unchanged behavior). + auth_session.authenticate(&mut request).await?; + + let mut request = request.into_inner(); request.headers_mut().extend(self.extra_headers.clone()); - Ok(self.client.execute(request).await?) + HttpResponse::read(self.client.execute(request).await?).await } - // Queries the Iceberg REST catalog after authentication with the given `Request` and - // returns a `Response`. - pub async fn query_catalog(&self, mut request: Request) -> Result { - self.authenticate(&mut request).await?; - self.execute(request).await + /// Returns whether header redaction is disabled for this client. + pub(crate) fn disable_header_redaction(&self) -> bool { + self.disable_header_redaction } } -/// Deserializes a catalog response into the given [`DeserializedOwned`] type. +/// Deserializes a catalog response into the given [`DeserializeOwned`] type. /// /// Returns an error if unable to parse the response bytes. -pub(crate) async fn deserialize_catalog_response( - response: Response, +pub(crate) fn deserialize_catalog_response( + response: HttpResponse, ) -> Result { - let bytes = response.bytes().await?; + let bytes = response.body(); - serde_json::from_slice::(&bytes).map_err(|e| { + serde_json::from_slice::(bytes).map_err(|e| { Error::new( ErrorKind::Unexpected, "Failed to parse response from rest catalog server", ) - .with_context("json", String::from_utf8_lossy(&bytes)) + .with_context("json", String::from_utf8_lossy(bytes)) .with_source(e) }) } +/// Returns true if the header may carry a secret (matched by substring, so +/// e.g. `x-client-secret` is covered along with `authorization`). +fn is_sensitive_header(name: &str) -> bool { + let name_lower = name.to_lowercase(); + [ + "auth", + "token", + "secret", + "key", + "password", + "cookie", + "credential", + ] + .iter() + .any(|pattern| name_lower.contains(pattern)) +} + +/// Redacts sensitive headers and returns a debug-formatted string. +/// +/// If `disable_redaction` is true, returns all headers without redaction. +/// Otherwise, replaces sensitive header values with `[REDACTED]`. +pub(crate) fn format_headers_redacted(headers: &HeaderMap, disable_redaction: bool) -> String { + if disable_redaction { + // Return all headers as-is without redaction + let all: HashMap<&str, &str> = headers + .iter() + .filter_map(|(name, value)| value.to_str().ok().map(|v| (name.as_str(), v))) + .collect(); + return format!("{all:?}"); + } + + // Redact sensitive headers by replacing their values with "[REDACTED]" + let redacted: HashMap<&str, &str> = headers + .iter() + .filter_map(|(name, value)| { + if is_sensitive_header(name.as_str()) { + Some((name.as_str(), "[REDACTED]")) + } else { + value.to_str().ok().map(|v| (name.as_str(), v)) + } + }) + .collect(); + format!("{redacted:?}") +} + /// Deserializes a unexpected catalog response into an error. -pub(crate) async fn deserialize_unexpected_catalog_error(response: Response) -> Error { +pub(crate) fn deserialize_unexpected_catalog_error( + response: HttpResponse, + disable_header_redaction: bool, +) -> Error { let err = Error::new( ErrorKind::Unexpected, "Received response with unexpected status code", ) .with_context("status", response.status().to_string()) - .with_context("headers", format!("{:?}", response.headers())); - - let bytes = match response.bytes().await { - Ok(bytes) => bytes, - Err(err) => return err.into(), - }; + .with_context( + "headers", + format_headers_redacted(response.headers(), disable_header_redaction), + ); + let bytes = response.body(); if bytes.is_empty() { return err; } - err.with_context("json", String::from_utf8_lossy(&bytes)) + err.with_context("json", String::from_utf8_lossy(bytes)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_a_truncated_body_error_names_the_url() { + // `bytes()` builds its error without a URL, so `read` attaches the one + // the response came from; otherwise a failure can't be attributed to a + // catalog. Needs a raw socket: the body has to be cut short. + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut buf = [0u8; 1024]; + let _ = std::io::Read::read(&mut stream, &mut buf); + // Promise more than is sent, then hang up. + let _ = std::io::Write::write_all( + &mut stream, + b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nshort", + ); + }); + + let url = format!("http://{addr}/token"); + let err = HttpClient::new(&RestCatalogConfig::builder().uri(url.clone()).build()) + .unwrap() + .post_form(&url, &HeaderMap::new(), &HashMap::new()) + .await + .unwrap_err(); + + assert!(format!("{err:?}").contains(&url), "{err:?}"); + } + + #[tokio::test] + async fn test_reading_a_response_keeps_status_headers_and_body() { + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("POST", "/token") + .with_status(418) + .with_header("x-request-id", "abc123") + .with_body("brewing") + .create_async() + .await; + + let response = HttpClient::new(&RestCatalogConfig::builder().uri(server.url()).build()) + .unwrap() + .post_form( + &format!("{}/token", server.url()), + &HeaderMap::new(), + &HashMap::new(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), 418); + assert_eq!(response.headers().get("x-request-id").unwrap(), "abc123"); + assert_eq!(response.body(), b"brewing"); + mock.assert_async().await; + } + + #[test] + fn test_unexpected_error_carries_status_headers_and_body() { + // Everything a user needs to diagnose an unexpected status, with the + // sensitive headers held back. + let mut headers = HeaderMap::new(); + headers.insert("authorization", "Bearer leaked".parse().unwrap()); + headers.insert("x-request-id", "abc123".parse().unwrap()); + let response = HttpResponse::new( + http::StatusCode::IM_A_TEAPOT, + headers, + br#"{"error": "nope"}"#.to_vec(), + ); + + let err = format!( + "{:?}", + deserialize_unexpected_catalog_error(response, false) + ); + + assert!(err.contains("418"), "{err}"); + assert!(err.contains("x-request-id"), "{err}"); + assert!(err.contains("abc123"), "{err}"); + assert!(err.contains("nope"), "{err}"); + assert!(!err.contains("leaked"), "{err}"); + } + + #[tokio::test] + async fn test_post_form_is_never_authenticated() { + // A token request carries no authentication, not even a + // user-configured `header.authorization` meant for the catalog. + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("POST", "/token") + .match_header("authorization", mockito::Matcher::Missing) + .with_status(200) + .create_async() + .await; + + HttpClient::new( + &RestCatalogConfig::builder() + .uri(server.url()) + .props(HashMap::from([( + "header.authorization".to_string(), + "Basic xyz".to_string(), + )])) + .build(), + ) + .unwrap() + .post_form( + &format!("{}/token", server.url()), + &HeaderMap::new(), + &HashMap::new(), + ) + .await + .unwrap(); + + mock.assert_async().await; + } + + #[test] + fn test_format_headers_redacted_empty() { + let headers = HeaderMap::new(); + let result = format_headers_redacted(&headers, false); + assert_eq!(result, "{}"); + } + + #[test] + fn test_format_headers_redacted_non_sensitive() { + let mut headers = HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + headers.insert("x-request-id", "abc123".parse().unwrap()); + + let result = format_headers_redacted(&headers, false); + + assert!(result.contains("content-type")); + assert!(result.contains("application/json")); + assert!(result.contains("x-request-id")); + assert!(result.contains("abc123")); + } + + #[tokio::test] + async fn test_http_client_debug_redacts_headers() { + let config = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props(HashMap::from([ + ("header.authorization".to_string(), "Basic xyz".to_string()), + ( + "header.x-client-secret".to_string(), + "shh-secret".to_string(), + ), + ( + "header.x-client-credential".to_string(), + "cred-value".to_string(), + ), + ])) + .build(); + let client = HttpClient::new(&config).unwrap(); + + let out = format!("{client:?}"); + assert!(!out.contains("Basic xyz")); + assert!(!out.contains("shh-secret")); + assert!(!out.contains("cred-value")); + assert!(out.contains("[REDACTED]")); + } + + #[test] + fn test_format_headers_redacted_filters_sensitive() { + let mut headers = HeaderMap::new(); + headers.insert("authorization", "Bearer secret-token".parse().unwrap()); + headers.insert("content-type", "application/json".parse().unwrap()); + + let result = format_headers_redacted(&headers, false); + + // Sensitive header should be present but with redacted value + assert!(result.contains("authorization")); + assert!(result.contains("[REDACTED]")); + // Sensitive value should NOT be present + assert!(!result.contains("secret-token")); + // Non-sensitive header should be present with actual value + assert!(result.contains("content-type")); + assert!(result.contains("application/json")); + } + + #[test] + fn test_format_headers_redacted_filters_set_cookie() { + let mut headers = HeaderMap::new(); + headers.insert( + "set-cookie", + "CF_Authorization=sensitive-session-token; Path=/; Secure;" + .parse() + .unwrap(), + ); + headers.insert("server", "cloudflare".parse().unwrap()); + + let result = format_headers_redacted(&headers, false); + + // Sensitive header should be present but with redacted value + assert!(result.contains("set-cookie")); + assert!(result.contains("[REDACTED]")); + // Sensitive value should NOT be present + assert!(!result.contains("sensitive-session-token")); + // Non-sensitive header should be present with actual value + assert!(result.contains("server")); + assert!(result.contains("cloudflare")); + } + + #[test] + fn test_format_headers_redacted_filters_all_sensitive() { + let mut headers = HeaderMap::new(); + headers.insert("authorization", "Bearer token".parse().unwrap()); + headers.insert("proxy-authorization", "Basic creds".parse().unwrap()); + headers.insert("set-cookie", "session=abc".parse().unwrap()); + headers.insert("cookie", "session=abc".parse().unwrap()); + headers.insert("x-api-key", "api-key-123".parse().unwrap()); + headers.insert("x-auth-token", "auth-token-456".parse().unwrap()); + headers.insert("x-request-id", "req-123".parse().unwrap()); + + let result = format_headers_redacted(&headers, false); + + // All sensitive headers should be present but with redacted values + assert!(result.contains("authorization")); + assert!(result.contains("proxy-authorization")); + assert!(result.contains("set-cookie")); + assert!(result.contains("cookie")); + assert!(result.contains("x-api-key")); + assert!(result.contains("x-auth-token")); + assert!(result.contains("[REDACTED]")); + + // Ensure no sensitive values leaked + assert!(!result.contains("Bearer token")); + assert!(!result.contains("Basic creds")); + assert!(!result.contains("session=abc")); + assert!(!result.contains("api-key-123")); + assert!(!result.contains("auth-token-456")); + + // Non-sensitive header should be present with actual value + assert!(result.contains("x-request-id")); + assert!(result.contains("req-123")); + } + + #[test] + fn test_format_headers_with_redaction_disabled() { + let mut headers = HeaderMap::new(); + headers.insert("authorization", "Bearer secret-token".parse().unwrap()); + headers.insert("x-api-key", "api-key-123".parse().unwrap()); + headers.insert("content-type", "application/json".parse().unwrap()); + + let result = format_headers_redacted(&headers, true); + + // When redaction is disabled, all headers and values should be present + assert!(result.contains("authorization")); + assert!(result.contains("Bearer secret-token")); + assert!(result.contains("x-api-key")); + assert!(result.contains("api-key-123")); + assert!(result.contains("content-type")); + assert!(result.contains("application/json")); + // [REDACTED] should NOT be present when redaction is disabled + assert!(!result.contains("[REDACTED]")); + } } diff --git a/crates/catalog/rest/src/endpoint.rs b/crates/catalog/rest/src/endpoint.rs new file mode 100644 index 0000000000..3d82071f21 --- /dev/null +++ b/crates/catalog/rest/src/endpoint.rs @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Server capability negotiation via the `endpoints` field of `GET /v1/config`. +//! +//! A REST server may advertise the set of routes it supports in the `endpoints` +//! field of its configuration response, letting clients negotiate optional +//! capabilities instead of assuming every server implements every operation. +//! Each entry is a `"{method} {path}"` string, for example +//! `"POST /v1/{prefix}/namespaces/{namespace}/tables"`; parse one through +//! [`Endpoint`]'s [`FromStr`] implementation. +//! The REST catalog uses the advertised set internally to select supported +//! operations and compatibility fallbacks. + +use std::collections::HashSet; +use std::fmt::{self, Display, Formatter}; +use std::str::FromStr; +use std::sync::LazyLock; + +use iceberg::{Error, ErrorKind}; +use reqwest::Method; +use serde::de::{Error as DeError, Visitor}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +/// A single route a REST server advertises support for, parsed from the +/// `endpoints` field of `GET /v1/config`. +/// +/// The wire form is `"{method} {path}"` — an HTTP method and a path template +/// separated by a single space, e.g. +/// `"POST /v1/{prefix}/namespaces/{namespace}/tables"`. Parse one with +/// [`str::parse`]: the method is validated and normalized, and the path is the +/// template the server advertises (with `{prefix}`, `{namespace}`, … +/// placeholders), compared verbatim. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Endpoint { + method: Method, + path: String, +} + +impl Endpoint { + /// Builds an endpoint from a known-valid method and path template. + /// + /// Intended for internal constants and tests; untrusted input (such as a + /// server's config response) is parsed through [`FromStr`], which validates + /// it. + pub(crate) fn new(method: Method, path: impl Into) -> Self { + Self { + method, + path: path.into(), + } + } + + /// The HTTP method, e.g. `GET` or `POST`. + pub fn method(&self) -> &str { + self.method.as_str() + } + + /// The path template, e.g. `/v1/{prefix}/namespaces`. + pub fn path(&self) -> &str { + &self.path + } +} + +impl FromStr for Endpoint { + type Err = Error; + + fn from_str(s: &str) -> Result { + // The wire form is exactly `" "` separated by a single + // space; paths never contain spaces, so require exactly two non-empty + // parts and a valid HTTP method. + let mut parts = s.split(' '); + match (parts.next(), parts.next(), parts.next()) { + (Some(method), Some(path), None) if !method.is_empty() && !path.is_empty() => { + let method = Method::from_str(&method.to_ascii_uppercase()).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!("invalid HTTP method in endpoint: {s:?}"), + ) + })?; + Ok(Self { + method, + path: path.to_string(), + }) + } + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + r#"invalid endpoint {s:?}: expected " " separated by a single space"# + ), + )), + } + } +} + +impl Display for Endpoint { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{} {}", self.method, self.path) + } +} + +impl Serialize for Endpoint { + fn serialize(&self, serializer: S) -> Result + where S: Serializer { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for Endpoint { + fn deserialize(deserializer: D) -> Result + where D: Deserializer<'de> { + struct EndpointVisitor; + + impl Visitor<'_> for EndpointVisitor { + type Value = Endpoint; + + fn expecting(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(r#"an endpoint string of the form " ""#) + } + + fn visit_str(self, v: &str) -> Result + where E: DeError { + Endpoint::from_str(v).map_err(E::custom) + } + } + + deserializer.deserialize_str(EndpointVisitor) + } +} + +/// Declares named [`Endpoint`] constants for routes the client may negotiate. +macro_rules! endpoints { + ($($name:ident => $method:ident $path:literal),+ $(,)?) => { + $( + pub(crate) static $name: LazyLock = + LazyLock::new(|| Endpoint::new(Method::$method, $path)); + )+ + }; +} + +endpoints! { + V1_LIST_NAMESPACES => GET "/v1/{prefix}/namespaces", + V1_CREATE_NAMESPACE => POST "/v1/{prefix}/namespaces", + V1_LOAD_NAMESPACE => GET "/v1/{prefix}/namespaces/{namespace}", + V1_DELETE_NAMESPACE => DELETE "/v1/{prefix}/namespaces/{namespace}", + V1_UPDATE_NAMESPACE => POST "/v1/{prefix}/namespaces/{namespace}/properties", + V1_NAMESPACE_EXISTS => HEAD "/v1/{prefix}/namespaces/{namespace}", + V1_LIST_TABLES => GET "/v1/{prefix}/namespaces/{namespace}/tables", + V1_CREATE_TABLE => POST "/v1/{prefix}/namespaces/{namespace}/tables", + V1_LOAD_TABLE => GET "/v1/{prefix}/namespaces/{namespace}/tables/{table}", + V1_UPDATE_TABLE => POST "/v1/{prefix}/namespaces/{namespace}/tables/{table}", + V1_DELETE_TABLE => DELETE "/v1/{prefix}/namespaces/{namespace}/tables/{table}", + V1_TABLE_EXISTS => HEAD "/v1/{prefix}/namespaces/{namespace}/tables/{table}", + V1_RENAME_TABLE => POST "/v1/{prefix}/tables/rename", + V1_REGISTER_TABLE => POST "/v1/{prefix}/namespaces/{namespace}/register", + V1_REPORT_METRICS => POST "/v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics", + V1_COMMIT_TRANSACTION => POST "/v1/{prefix}/transactions/commit", +} + +/// The standard v1 endpoints assumed to be supported when a server's +/// `GET /v1/config` response omits the `endpoints` field or sends an empty list +/// (the two are treated alike). These are the minimum a server is expected to +/// support; a server that advertises a non-empty list is taken at its word +/// instead. +/// +/// Note: existence-check `HEAD` routes are intentionally omitted — servers that +/// do not advertise them are expected to fall back to `GET` load paths. +pub(crate) static DEFAULT_ENDPOINTS: LazyLock> = LazyLock::new(|| { + [ + &*V1_LIST_NAMESPACES, + &*V1_CREATE_NAMESPACE, + &*V1_LOAD_NAMESPACE, + &*V1_DELETE_NAMESPACE, + &*V1_UPDATE_NAMESPACE, + &*V1_LIST_TABLES, + &*V1_CREATE_TABLE, + &*V1_LOAD_TABLE, + &*V1_UPDATE_TABLE, + &*V1_DELETE_TABLE, + &*V1_RENAME_TABLE, + &*V1_REGISTER_TABLE, + &*V1_REPORT_METRICS, + &*V1_COMMIT_TRANSACTION, + ] + .into_iter() + .cloned() + .collect() +}); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_and_round_trips() { + let ep: Endpoint = "POST /v1/{prefix}/namespaces/{namespace}/tables" + .parse() + .unwrap(); + assert_eq!(ep.method(), "POST"); + assert_eq!(ep.path(), "/v1/{prefix}/namespaces/{namespace}/tables"); + + let json = serde_json::to_string(&ep).unwrap(); + assert_eq!(json, r#""POST /v1/{prefix}/namespaces/{namespace}/tables""#); + assert_eq!(serde_json::from_str::(&json).unwrap(), ep); + } + + #[test] + fn rejects_malformed_endpoints() { + for invalid in ["GET", "GET /v1", " GET /v1", "GET ", " /v1", ""] { + assert!( + invalid.parse::().is_err(), + "expected {invalid:?} to be rejected" + ); + } + } + + #[test] + fn normalizes_http_method_to_uppercase() { + assert_eq!("get /v1/x".parse::().unwrap().method(), "GET"); + } +} diff --git a/crates/catalog/rest/src/lib.rs b/crates/catalog/rest/src/lib.rs index 70cdeaabd0..8390a82828 100644 --- a/crates/catalog/rest/src/lib.rs +++ b/crates/catalog/rest/src/lib.rs @@ -17,8 +17,14 @@ //! Iceberg REST API implementation. //! -//! To build a rest catalog with configurations -//! # Example +//! The crate provides two REST catalog APIs: +//! +//! - [`RestCatalog`] implements [`iceberg::Catalog`] by binding one +//! [`iceberg::SessionContext`] to every operation. +//! - [`RestSessionCatalog`] implements [`iceberg::SessionCatalog`] and accepts a +//! session context with each operation. +//! +//! # Catalog compatibility API //! //! ```rust, no_run //! use std::collections::HashMap; @@ -48,11 +54,54 @@ //! .unwrap(); //! } //! ``` +//! +//! # Pagination +//! +//! Set a page size for namespace and table listings with +//! [`RestSessionCatalogBuilder::with_page_size`] or [`REST_CATALOG_PROP_PAGE_SIZE`]. +//! +//! # Session catalog API +//! +//! ```rust, no_run +//! use std::collections::HashMap; +//! +//! use iceberg::{SessionCatalog, SessionContext}; +//! use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestSessionCatalogBuilder}; +//! +//! #[tokio::main] +//! async fn main() { +//! let catalog = RestSessionCatalogBuilder::default() +//! .load( +//! "rest", +//! HashMap::from([( +//! REST_CATALOG_PROP_URI.to_string(), +//! "http://localhost:8181".to_string(), +//! )]), +//! ) +//! .await +//! .unwrap(); +//! let context = SessionContext::builder() +//! .identity("user123".to_string()) +//! .build(); +//! +//! let namespaces = catalog.list_namespaces(&context, None).await.unwrap(); +//! } +//! ``` #![deny(missing_docs)] +mod auth; mod catalog; mod client; +pub use client::HttpClient; +mod request; +pub use request::{HttpRequest, HttpRequestBody}; +mod response; +pub use response::HttpResponse; +mod endpoint; mod types; +pub use auth::*; pub use catalog::*; +pub use endpoint::Endpoint; +pub use types::*; diff --git a/crates/catalog/rest/src/request.rs b/crates/catalog/rest/src/request.rs new file mode 100644 index 0000000000..8c6dd7f571 --- /dev/null +++ b/crates/catalog/rest/src/request.rs @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The request type REST catalog authentication works with. + +use http::{HeaderMap, Method}; +use iceberg::Result; +#[cfg(feature = "sigv4")] +use reqwest::Url; +use reqwest::{Request, RequestBuilder}; + +/// An outgoing REST request being authenticated by an +/// [`AuthSession`](crate::AuthSession). +/// +/// Wraps the request so a session mutates it through the stable +/// `http` crate types rather than the concrete request type the REST catalog +/// uses internally. +pub struct HttpRequest { + inner: Request, +} + +impl HttpRequest { + /// Wraps a request, e.g. to unit-test a custom + /// [`AuthSession`](crate::AuthSession). + pub fn new(inner: Request) -> Self { + Self { inner } + } + + /// Builds the request `builder` describes. + pub(crate) fn build(builder: RequestBuilder) -> Result { + Ok(Self::new(builder.build()?)) + } + + /// The wrapped request, for the client that sends it. + pub(crate) fn into_inner(self) -> Request { + self.inner + } + + /// The request URL. + #[cfg(feature = "sigv4")] + pub(crate) fn url(&self) -> &Url { + self.inner.url() + } + + /// The mutable request URL, for signers that rewrite it. + #[cfg(feature = "sigv4")] + pub(crate) fn url_mut(&mut self) -> &mut Url { + self.inner.url_mut() + } + + /// The request method. + pub fn method(&self) -> &Method { + self.inner.method() + } + + /// The request URL, as a string (scheme, host, path and query). + pub fn url_str(&self) -> &str { + self.inner.url().as_str() + } + + /// The request headers. + pub fn headers(&self) -> &HeaderMap { + self.inner.headers() + } + + /// The mutable request headers, e.g. to add an `Authorization` header. + pub fn headers_mut(&mut self) -> &mut HeaderMap { + self.inner.headers_mut() + } + + /// The request body, distinguishing an absent body from a streaming one: + /// signers can sign [`HttpRequestBody::Empty`] (empty-payload hash) and + /// [`HttpRequestBody::Buffered`], but not [`HttpRequestBody::Streaming`]. + pub fn body(&self) -> HttpRequestBody<'_> { + match self.inner.body() { + None => HttpRequestBody::Empty, + Some(body) => match body.as_bytes() { + Some(bytes) => HttpRequestBody::Buffered(bytes), + None => HttpRequestBody::Streaming, + }, + } + } +} + +/// The body of an [`HttpRequest`], as seen by authentication. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HttpRequestBody<'a> { + /// No body is set. + Empty, + /// An in-memory body. + Buffered(&'a [u8]), + /// A streaming body, whose bytes are not available for e.g. signing. + Streaming, +} + +impl<'a> HttpRequestBody<'a> { + /// The signable bytes: empty for [`Self::Empty`], the buffer for + /// [`Self::Buffered`], and `None` for [`Self::Streaming`]. + pub fn as_bytes(&self) -> Option<&'a [u8]> { + match self { + HttpRequestBody::Empty => Some(&[]), + HttpRequestBody::Buffered(bytes) => Some(bytes), + HttpRequestBody::Streaming => None, + } + } +} + +#[cfg(test)] +mod tests { + use reqwest::Client; + + use super::*; + + #[test] + fn test_http_request_body_states() { + let client = Client::new(); + + // No body at all. + let req = client + .get("https://rest.example.com/v1/config") + .build() + .unwrap(); + let http_req = HttpRequest::new(req); + let body = http_req.body(); + assert_eq!(body, HttpRequestBody::Empty); + assert_eq!(body.as_bytes(), Some(&[] as &[u8])); + + // An in-memory body. + let req = client + .post("https://rest.example.com/v1/namespaces") + .body("{}") + .build() + .unwrap(); + let http_req = HttpRequest::new(req); + let body = http_req.body(); + assert_eq!(body, HttpRequestBody::Buffered(b"{}")); + assert_eq!(body.as_bytes(), Some(b"{}" as &[u8])); + + // A streaming body: bytes are unavailable, so it must not sign as empty. + let req = client + .post("https://rest.example.com/v1/namespaces") + .body(reqwest::Body::wrap_stream(futures::stream::once(async { + Ok::<_, std::io::Error>(bytes::Bytes::from_static(b"chunk")) + }))) + .build() + .unwrap(); + let http_req = HttpRequest::new(req); + let body = http_req.body(); + assert_eq!(body, HttpRequestBody::Streaming); + assert_eq!(body.as_bytes(), None); + } +} diff --git a/crates/catalog/rest/src/response.rs b/crates/catalog/rest/src/response.rs new file mode 100644 index 0000000000..3158851b57 --- /dev/null +++ b/crates/catalog/rest/src/response.rs @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The response type REST catalog requests come back as. + +use std::fmt::{Debug, Formatter}; + +use http::{HeaderMap, StatusCode}; +use iceberg::Result; +use reqwest::Response; + +use crate::client::format_headers_redacted; + +/// A REST catalog response, read into memory. +/// +/// The counterpart of [`HttpRequest`](crate::HttpRequest): it keeps the +/// concrete client type inside [`HttpClient`](crate::HttpClient) so callers +/// work with the stable `http` crate types instead. Catalog responses are +/// small JSON documents that every caller reads in full, so the body is +/// buffered rather than streamed. +#[derive(Clone)] +pub struct HttpResponse { + status: StatusCode, + headers: HeaderMap, + body: Vec, +} + +impl Debug for HttpResponse { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // A token exchange answers with a credential in the body, and headers + // may carry `set-cookie`, so both are held back. + f.debug_struct("HttpResponse") + .field("status", &self.status) + // Always redacted: a response carries no config, and the + // `disable-header-redaction` escape hatch is for the request side. + .field("headers", &format_headers_redacted(&self.headers, false)) + .field("body", &format_args!("{} bytes", self.body.len())) + .finish() + } +} + +impl HttpResponse { + /// Reads `response` into memory. + pub(crate) async fn read(response: Response) -> Result { + let status = response.status(); + let headers = response.headers().clone(); + // `bytes()` builds its error without a URL, so keep the one the + // response came from. + let url = response.url().clone(); + Ok(Self { + status, + headers, + body: response + .bytes() + .await + .map_err(|err| err.with_url(url))? + .to_vec(), + }) + } + + /// Builds a response, e.g. to unit-test code that consumes one. + pub fn new(status: StatusCode, headers: HeaderMap, body: Vec) -> Self { + Self { + status, + headers, + body, + } + } + + /// The response status. + pub fn status(&self) -> StatusCode { + self.status + } + + /// The response headers. + pub fn headers(&self) -> &HeaderMap { + &self.headers + } + + /// The response body. + pub fn body(&self) -> &[u8] { + &self.body + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_debug_holds_back_secrets() { + // The body of a token exchange is a credential, and `set-cookie` is + // redacted everywhere else in this crate. + let mut headers = HeaderMap::new(); + headers.insert("set-cookie", "session=secret".parse().unwrap()); + headers.insert("content-type", "application/json".parse().unwrap()); + let debug = format!( + "{:?}", + HttpResponse::new( + StatusCode::OK, + headers, + br#"{"access_token": "tok"}"#.to_vec() + ) + ); + + assert!(!debug.contains("secret"), "{debug}"); + assert!(!debug.contains("tok"), "{debug}"); + assert!(debug.contains("content-type"), "{debug}"); + assert!(debug.contains("200"), "{debug}"); + } + + #[test] + fn test_new_exposes_what_it_was_built_from() { + let mut headers = HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + let response = HttpResponse::new(StatusCode::OK, headers, "{}".as_bytes().to_vec()); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers().get("content-type").unwrap(), + "application/json" + ); + assert_eq!(response.body(), b"{}"); + } +} diff --git a/crates/catalog/rest/src/types.rs b/crates/catalog/rest/src/types.rs index 70ed72051a..390521229e 100644 --- a/crates/catalog/rest/src/types.rs +++ b/crates/catalog/rest/src/types.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//! Request and response types for the Iceberg REST API. + use std::collections::HashMap; use iceberg::spec::{Schema, SortOrder, TableMetadata, UnboundPartitionSpec}; @@ -23,14 +25,20 @@ use iceberg::{ }; use serde_derive::{Deserialize, Serialize}; +use crate::endpoint::Endpoint; + #[derive(Clone, Debug, Serialize, Deserialize)] pub(super) struct CatalogConfig { pub(super) overrides: HashMap, pub(super) defaults: HashMap, + /// Endpoints the server advertises support for; `None` when the field is + /// absent. + pub(super) endpoints: Option>, } #[derive(Debug, Serialize, Deserialize)] -pub(super) struct ErrorResponse { +/// Wrapper for all non-2xx error responses from the REST API +pub struct ErrorResponse { error: ErrorModel, } @@ -41,11 +49,16 @@ impl From for Error { } #[derive(Debug, Serialize, Deserialize)] -pub(super) struct ErrorModel { - pub(super) message: String, - pub(super) r#type: String, - pub(super) code: u16, - pub(super) stack: Option>, +/// Error payload returned in a response with further details on the error +pub struct ErrorModel { + /// Human-readable error message + pub message: String, + /// Internal type definition of the error + pub r#type: String, + /// HTTP response code + pub code: u16, + /// Optional error stack / context + pub stack: Option>, } impl From for Error { @@ -96,106 +109,400 @@ pub(super) struct TokenResponse { pub(super) issued_token_type: Option, } -#[derive(Debug, Serialize, Deserialize)] -pub(super) struct NamespaceSerde { - pub(super) namespace: Vec, - pub(super) properties: Option>, -} - -impl TryFrom for Namespace { - type Error = Error; - fn try_from(value: NamespaceSerde) -> std::result::Result { - Ok(Namespace::with_properties( - NamespaceIdent::from_vec(value.namespace)?, - value.properties.unwrap_or_default(), - )) - } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// Namespace response +pub struct NamespaceResponse { + /// Namespace identifier + pub namespace: NamespaceIdent, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + /// Properties stored on the namespace, if supported by the server. + pub properties: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// Create namespace request +pub struct CreateNamespaceRequest { + /// Name of the namespace to create + pub namespace: NamespaceIdent, + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + /// Properties to set on the namespace + pub properties: HashMap, } -impl From<&Namespace> for NamespaceSerde { +impl From<&Namespace> for NamespaceResponse { fn from(value: &Namespace) -> Self { Self { - namespace: value.name().as_ref().clone(), - properties: Some(value.properties().clone()), + namespace: value.name().clone(), + properties: value.properties().clone(), } } } -#[derive(Debug, Serialize, Deserialize)] +impl From for Namespace { + fn from(value: NamespaceResponse) -> Self { + Namespace::with_properties(value.namespace, value.properties) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] -pub(super) struct ListNamespaceResponse { - pub(super) namespaces: Vec>, - #[serde(default)] - pub(super) next_page_token: Option, +/// Response containing a list of namespace identifiers, with optional pagination support. +pub struct ListNamespaceResponse { + /// List of namespace identifiers returned by the server + pub namespaces: Vec, + /// Opaque token for pagination. If present, indicates there are more results available. + /// Use this value in subsequent requests to retrieve the next page. + pub next_page_token: Option, } -#[allow(dead_code)] -#[derive(Debug, Serialize, Deserialize)] -pub(super) struct UpdateNamespacePropsRequest { - removals: Option>, - updates: Option>, +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// Request to update properties on a namespace. +/// +/// Properties that are not in the request are not modified or removed by this call. +/// Server implementations are not required to support namespace properties. +pub struct UpdateNamespacePropertiesRequest { + /// List of property keys to remove from the namespace + pub removals: Option>, + /// Map of property keys to values to set or update on the namespace + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub updates: HashMap, } -#[allow(dead_code)] -#[derive(Debug, Serialize, Deserialize)] -pub(super) struct UpdateNamespacePropsResponse { - updated: Vec, - removed: Vec, - missing: Option>, +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// Response from updating namespace properties, indicating which properties were changed. +pub struct UpdateNamespacePropertiesResponse { + /// List of property keys that were added or updated + pub updated: Vec, + /// List of properties that were removed + pub removed: Vec, + /// List of properties requested for removal that were not found in the namespace's properties. + /// Represents a partial success response. Servers do not need to implement this. + #[serde(skip_serializing_if = "Option::is_none")] + pub missing: Option>, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] -pub(super) struct ListTableResponse { - pub(super) identifiers: Vec, +/// Response containing a list of table identifiers, with optional pagination support. +pub struct ListTablesResponse { + /// List of table identifiers under the requested namespace + pub identifiers: Vec, + /// Opaque token for pagination. If present, indicates there are more results available. + /// Use this value in subsequent requests to retrieve the next page. #[serde(default)] - pub(super) next_page_token: Option, + pub next_page_token: Option, } -#[derive(Debug, Serialize, Deserialize)] -pub(super) struct RenameTableRequest { - pub(super) source: TableIdent, - pub(super) destination: TableIdent, +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// Request to rename a table from one identifier to another. +/// +/// It's valid to move a table across namespaces, but the server implementation +/// is not required to support it. +pub struct RenameTableRequest { + /// Current table identifier to rename + pub source: TableIdent, + /// New table identifier to rename to + pub destination: TableIdent, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] -pub(super) struct LoadTableResponse { - pub(super) metadata_location: Option, - pub(super) metadata: TableMetadata, - pub(super) config: Option>, +/// Result returned when a table is successfully loaded or created. +/// +/// The table metadata JSON is returned in the `metadata` field. The corresponding file location +/// of table metadata should be returned in the `metadata_location` field, unless the metadata +/// is not yet committed. For example, a create transaction may return metadata that is staged +/// but not committed. +/// +/// The `config` map returns table-specific configuration for the table's resources, including +/// its HTTP client and FileIO. For example, config may contain a specific FileIO implementation +/// class for the table depending on its underlying storage. +pub struct LoadTableResult { + /// May be null if the table is staged as part of a transaction + pub metadata_location: Option, + /// The table's full metadata + pub metadata: TableMetadata, + /// Table-specific configuration overriding catalog configuration + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub config: HashMap, + /// Storage credentials for accessing table data. Clients should check this field + /// before falling back to credentials in the `config` field. + #[serde(skip_serializing_if = "Option::is_none")] + pub storage_credentials: Option>, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// Storage credential for a specific location prefix. +/// +/// Indicates a storage location prefix where the credential is relevant. Clients should +/// choose the most specific prefix (by selecting the longest prefix) if several credentials +/// of the same type are available. +pub struct StorageCredential { + /// Storage location prefix where this credential is relevant + pub prefix: String, + /// Configuration map containing credential information + pub config: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] -pub(super) struct CreateTableRequest { - pub(super) name: String, - pub(super) location: Option, - pub(super) schema: Schema, - pub(super) partition_spec: Option, - pub(super) write_order: Option, - pub(super) stage_create: Option, - pub(super) properties: Option>, +/// Request to create a new table in a namespace. +/// +/// If `stage_create` is false, the table is created immediately. +/// If `stage_create` is true, the table is not created, but table metadata is initialized +/// and returned. The service should prepare as needed for a commit to the table commit +/// endpoint to complete the create transaction. +pub struct CreateTableRequest { + /// Name of the table to create + pub name: String, + /// Optional table location. If not provided, the server will choose a location. + #[serde(skip_serializing_if = "Option::is_none")] + pub location: Option, + /// Table schema + pub schema: Schema, + /// Optional partition specification. If not provided, the table will be unpartitioned. + #[serde(skip_serializing_if = "Option::is_none")] + pub partition_spec: Option, + /// Optional sort order for the table + #[serde(skip_serializing_if = "Option::is_none")] + pub write_order: Option, + /// Whether to stage the create for a transaction (true) or create immediately (false) + #[serde(skip_serializing_if = "Option::is_none")] + pub stage_create: Option, + /// Optional properties to set on the table + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub properties: HashMap, } -#[derive(Debug, Serialize, Deserialize)] -pub(super) struct CommitTableRequest { - pub(super) identifier: TableIdent, - pub(super) requirements: Vec, - pub(super) updates: Vec, +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +/// Request to commit updates to a table. +/// +/// Commits have two parts: requirements and updates. Requirements are assertions that will +/// be validated before attempting to make and commit changes. Updates are changes to make +/// to table metadata. +/// +/// Create table transactions that are started by createTable with `stage-create` set to true +/// are committed using this request. Transactions should include all changes to the table, +/// including table initialization, like AddSchemaUpdate and SetCurrentSchemaUpdate. +pub struct CommitTableRequest { + /// Table identifier to update; must be present for CommitTransactionRequest + #[serde(skip_serializing_if = "Option::is_none")] + pub identifier: Option, + /// List of requirements that must be satisfied before committing changes + pub requirements: Vec, + /// List of updates to apply to the table metadata + pub updates: Vec, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] -pub(super) struct CommitTableResponse { - pub(super) metadata_location: String, - pub(super) metadata: TableMetadata, +/// Response returned when a table is successfully updated. +/// +/// The table metadata JSON is returned in the metadata field. The corresponding file location +/// of table metadata must be returned in the metadata-location field. Clients can check whether +/// metadata has changed by comparing metadata locations. +pub struct CommitTableResponse { + /// Location of the updated table metadata file + pub metadata_location: String, + /// The table's updated metadata + pub metadata: TableMetadata, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] -pub(super) struct RegisterTableRequest { - pub(super) name: String, - pub(super) metadata_location: String, - pub(super) overwrite: Option, +/// Request to register a table using an existing metadata file location. +pub struct RegisterTableRequest { + /// Name of the table to register + pub name: String, + /// Location of the metadata file for the table + pub metadata_location: String, + /// Whether to overwrite table metadata if the table already exists + pub overwrite: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_namespace_response_serde() { + let json = serde_json::json!({ + "namespace": ["nested", "ns"], + "properties": { + "key1": "value1", + "key2": "value2" + } + }); + let ns_response: NamespaceResponse = + serde_json::from_value(json.clone()).expect("Deserialization failed"); + assert_eq!(ns_response, NamespaceResponse { + namespace: NamespaceIdent::from_vec(vec!["nested".to_string(), "ns".to_string()]) + .unwrap(), + properties: HashMap::from([ + ("key1".to_string(), "value1".to_string()), + ("key2".to_string(), "value2".to_string()), + ]), + }); + assert_eq!( + serde_json::to_value(&ns_response).expect("Serialization failed"), + json + ); + + // Without properties + let json_no_props = serde_json::json!({ + "namespace": ["db", "schema"] + }); + let ns_response_no_props: NamespaceResponse = + serde_json::from_value(json_no_props.clone()).expect("Deserialization failed"); + assert_eq!(ns_response_no_props, NamespaceResponse { + namespace: NamespaceIdent::from_vec(vec!["db".to_string(), "schema".to_string()]) + .unwrap(), + properties: HashMap::new(), + }); + assert_eq!( + serde_json::to_value(&ns_response_no_props).expect("Serialization failed"), + json_no_props + ); + } + + fn test_create_table_request_schema() -> Schema { + serde_json::from_value(serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [ + { + "id": 1, + "name": "foo", + "required": false, + "type": "string" + }, + { + "id": 2, + "name": "bar", + "required": true, + "type": "int" + } + ], + "identifier-field-ids": [2] + })) + .expect("Failed to deserialize test schema") + } + + #[test] + fn test_create_table_request_minimal_serialization() { + let request = CreateTableRequest { + name: "tbl1".to_string(), + location: None, + schema: test_create_table_request_schema(), + partition_spec: None, + write_order: None, + stage_create: None, + properties: HashMap::new(), + }; + + let serialized = serde_json::to_value(&request).expect("Serialization failed"); + let object = serialized.as_object().expect("Expected a JSON object"); + assert!(object.contains_key("name")); + assert!(object.contains_key("schema")); + assert!(!object.contains_key("location")); + assert!(!object.contains_key("partition-spec")); + assert!(!object.contains_key("write-order")); + assert!(!object.contains_key("stage-create")); + assert!(!object.contains_key("properties")); + } + + #[test] + fn test_create_table_request_full_serialization() { + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "tbl1", + "location": "s3://warehouse/tbl1", + "schema": test_create_table_request_schema(), + "partition-spec": { + "spec-id": 1, + "fields": [ + { + "source-id": 2, + "field-id": 1000, + "name": "bar", + "transform": "identity" + } + ] + }, + "write-order": { + "order-id": 1, + "fields": [ + { + "transform": "identity", + "source-id": 2, + "direction": "asc", + "null-order": "nulls-first" + } + ] + }, + "stage-create": true, + "properties": { + "owner": "test" + } + })) + .expect("Deserialization failed"); + + let serialized = serde_json::to_value(&request).expect("Serialization failed"); + let object = serialized.as_object().expect("Expected a JSON object"); + assert_eq!( + object.get("location"), + Some(&serde_json::json!("s3://warehouse/tbl1")) + ); + assert!(object.contains_key("partition-spec")); + assert!(object.contains_key("write-order")); + assert_eq!(object.get("stage-create"), Some(&serde_json::json!(true))); + assert!(object.contains_key("properties")); + } + + #[test] + fn test_create_table_request_deserialize_explicit_nulls() { + let request: CreateTableRequest = serde_json::from_value(serde_json::json!({ + "name": "tbl1", + "location": null, + "schema": test_create_table_request_schema(), + "partition-spec": null, + "write-order": null, + "stage-create": null + })) + .expect("Deserialization failed"); + + assert_eq!(request.name, "tbl1"); + assert_eq!(request.location, None); + assert_eq!(request.partition_spec, None); + assert_eq!(request.write_order, None); + assert_eq!(request.stage_create, None); + assert!(request.properties.is_empty()); + } + + #[test] + fn config_parses_advertised_endpoints() { + let json = r#"{"overrides":{},"defaults":{}, + "endpoints":["GET /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/tables"]}"#; + let config: CatalogConfig = serde_json::from_str(json).unwrap(); + let endpoints = config.endpoints.expect("endpoints should be present"); + assert_eq!(endpoints.len(), 2); + assert!(endpoints.contains(&"GET /v1/{prefix}/namespaces".parse().unwrap())); + } + + #[test] + fn config_without_endpoints_field_deserializes_to_none() { + // `Option>` defaults to `None` for a missing field without + // an explicit `#[serde(default)]`. + let config: CatalogConfig = + serde_json::from_str(r#"{"overrides":{},"defaults":{}}"#).unwrap(); + assert!(config.endpoints.is_none()); + } + + #[test] + fn malformed_endpoint_fails_config_parse() { + // A single malformed endpoint string fails the whole config parse, + // matching the Java reference (`ConfigResponseParser` rejects it rather + // than silently dropping it). + let json = r#"{"overrides":{},"defaults":{},"endpoints":["GET_v1/namespaces"]}"#; + assert!(serde_json::from_str::(json).is_err()); + } } diff --git a/crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml b/crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml deleted file mode 100644 index 29b44b2a23..0000000000 --- a/crates/catalog/rest/testdata/rest_catalog/docker-compose.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -networks: - rest_bridge: - -services: - rest: - image: apache/iceberg-rest-fixture:1.10.0 - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog - - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory - - CATALOG_WAREHOUSE=s3://icebergdata/demo - - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://minio:9000 - depends_on: - - minio - networks: - rest_bridge: - expose: - - 8181 - - minio: - image: minio/minio:RELEASE.2025-05-24T17-08-30Z - environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio - hostname: icebergdata.minio - networks: - rest_bridge: - expose: - - 9001 - - 9000 - command: ["server", "/data", "--console-address", ":9001"] - - mc: - depends_on: - - minio - image: minio/mc:RELEASE.2025-05-21T01-59-54Z - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - entrypoint: > - /bin/sh -c " until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; /usr/bin/mc rm -r --force minio/icebergdata; /usr/bin/mc mb minio/icebergdata; /usr/bin/mc policy set public minio/icebergdata; tail -f /dev/null " - networks: - rest_bridge: diff --git a/crates/catalog/rest/tests/rest_catalog_test.rs b/crates/catalog/rest/tests/rest_catalog_test.rs deleted file mode 100644 index 7ce76ec344..0000000000 --- a/crates/catalog/rest/tests/rest_catalog_test.rs +++ /dev/null @@ -1,451 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for rest catalog. - -use std::collections::HashMap; -use std::net::SocketAddr; -use std::sync::RwLock; - -use ctor::{ctor, dtor}; -use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type}; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; -use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent, TableCreation, TableIdent}; -use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalog, RestCatalogBuilder}; -use iceberg_test_utils::docker::DockerCompose; -use iceberg_test_utils::{normalize_test_name, set_up}; -use port_scanner::scan_port_addr; -use tokio::time::sleep; -use tracing::info; - -const REST_CATALOG_PORT: u16 = 8181; -static DOCKER_COMPOSE_ENV: RwLock> = RwLock::new(None); - -#[ctor] -fn before_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - let docker_compose = DockerCompose::new( - normalize_test_name(module_path!()), - format!("{}/testdata/rest_catalog", env!("CARGO_MANIFEST_DIR")), - ); - docker_compose.up(); - guard.replace(docker_compose); -} - -#[dtor] -fn after_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - guard.take(); -} - -async fn get_catalog() -> RestCatalog { - set_up(); - - let rest_catalog_ip = { - let guard = DOCKER_COMPOSE_ENV.read().unwrap(); - let docker_compose = guard.as_ref().unwrap(); - docker_compose.get_container_ip("rest") - }; - - let rest_socket_addr = SocketAddr::new(rest_catalog_ip, REST_CATALOG_PORT); - while !scan_port_addr(rest_socket_addr) { - info!("Waiting for 1s rest catalog to ready..."); - sleep(std::time::Duration::from_millis(1000)).await; - } - - RestCatalogBuilder::default() - .load( - "rest", - HashMap::from([( - REST_CATALOG_PROP_URI.to_string(), - format!("http://{}", rest_socket_addr), - )]), - ) - .await - .unwrap() -} - -#[tokio::test] -async fn test_get_non_exist_namespace() { - let catalog = get_catalog().await; - - let result = catalog - .get_namespace(&NamespaceIdent::from_strs(["test_get_non_exist_namespace"]).unwrap()) - .await; - - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("does not exist")); -} - -#[tokio::test] -async fn test_get_namespace() { - let catalog = get_catalog().await; - - let ns = Namespace::with_properties( - NamespaceIdent::from_strs(["apple", "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Verify that namespace doesn't exist - assert!(catalog.get_namespace(ns.name()).await.is_err()); - - // Create this namespace - let created_ns = catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await - .unwrap(); - - assert_eq!(ns.name(), created_ns.name()); - assert_map_contains(ns.properties(), created_ns.properties()); - - // Check that this namespace already exists - let get_ns = catalog.get_namespace(ns.name()).await.unwrap(); - assert_eq!(ns.name(), get_ns.name()); - assert_map_contains(ns.properties(), created_ns.properties()); -} - -#[tokio::test] -async fn test_list_namespace() { - let catalog = get_catalog().await; - - let ns1 = Namespace::with_properties( - NamespaceIdent::from_strs(["test_list_namespace", "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - let ns2 = Namespace::with_properties( - NamespaceIdent::from_strs(["test_list_namespace", "macos"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "xuanwo".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Currently this namespace doesn't exist, so it should return error. - assert!( - catalog - .list_namespaces(Some( - &NamespaceIdent::from_strs(["test_list_namespace"]).unwrap() - )) - .await - .is_err() - ); - - // Create namespaces - catalog - .create_namespace(ns1.name(), ns1.properties().clone()) - .await - .unwrap(); - catalog - .create_namespace(ns2.name(), ns1.properties().clone()) - .await - .unwrap(); - - // List namespace - let nss = catalog - .list_namespaces(Some( - &NamespaceIdent::from_strs(["test_list_namespace"]).unwrap(), - )) - .await - .unwrap(); - - assert!(nss.contains(ns1.name())); - assert!(nss.contains(ns2.name())); -} - -#[tokio::test] -async fn test_list_empty_namespace() { - let catalog = get_catalog().await; - - let ns_apple = Namespace::with_properties( - NamespaceIdent::from_strs(["test_list_empty_namespace", "apple"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Currently this namespace doesn't exist, so it should return error. - assert!( - catalog - .list_namespaces(Some(ns_apple.name())) - .await - .is_err() - ); - - // Create namespaces - catalog - .create_namespace(ns_apple.name(), ns_apple.properties().clone()) - .await - .unwrap(); - - // List namespace - let nss = catalog - .list_namespaces(Some(ns_apple.name())) - .await - .unwrap(); - assert!(nss.is_empty()); -} - -#[tokio::test] -async fn test_list_root_namespace() { - let catalog = get_catalog().await; - - let ns1 = Namespace::with_properties( - NamespaceIdent::from_strs(["test_list_root_namespace", "apple", "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - let ns2 = Namespace::with_properties( - NamespaceIdent::from_strs(["test_list_root_namespace", "google", "android"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "xuanwo".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Currently this namespace doesn't exist, so it should return error. - assert!( - catalog - .list_namespaces(Some( - &NamespaceIdent::from_strs(["test_list_root_namespace"]).unwrap() - )) - .await - .is_err() - ); - - // Create namespaces - catalog - .create_namespace(ns1.name(), ns1.properties().clone()) - .await - .unwrap(); - catalog - .create_namespace(ns2.name(), ns1.properties().clone()) - .await - .unwrap(); - - // List namespace - let nss = catalog.list_namespaces(None).await.unwrap(); - assert!(nss.contains(&NamespaceIdent::from_strs(["test_list_root_namespace"]).unwrap())); -} - -#[tokio::test] -async fn test_create_table() { - let catalog = get_catalog().await; - - let ns = Namespace::with_properties( - NamespaceIdent::from_strs(["test_create_table", "apple", "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Create namespaces - catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await - .unwrap(); - - let schema = Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![2]) - .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), - ]) - .build() - .unwrap(); - - let table_creation = TableCreation::builder() - .name("t1".to_string()) - .schema(schema.clone()) - .build(); - - let table = catalog - .create_table(ns.name(), table_creation) - .await - .unwrap(); - - assert_eq!( - table.identifier(), - &TableIdent::new(ns.name().clone(), "t1".to_string()) - ); - - assert_eq!( - table.metadata().current_schema().as_struct(), - schema.as_struct() - ); - assert_eq!(table.metadata().format_version(), FormatVersion::V2); - assert!(table.metadata().current_snapshot().is_none()); - assert!(table.metadata().history().is_empty()); - assert!(table.metadata().default_sort_order().is_unsorted()); - assert!(table.metadata().default_partition_spec().is_unpartitioned()); -} - -#[tokio::test] -async fn test_update_table() { - let catalog = get_catalog().await; - - let ns = Namespace::with_properties( - NamespaceIdent::from_strs(["test_update_table", "apple", "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Create namespaces - catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await - .unwrap(); - - let schema = Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![2]) - .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), - ]) - .build() - .unwrap(); - - // Now we create a table - let table_creation = TableCreation::builder() - .name("t1".to_string()) - .schema(schema.clone()) - .build(); - - let table = catalog - .create_table(ns.name(), table_creation) - .await - .unwrap(); - - assert_eq!( - table.identifier(), - &TableIdent::new(ns.name().clone(), "t1".to_string()) - ); - - let tx = Transaction::new(&table); - // Update table by committing transaction - let table2 = tx - .update_table_properties() - .set("prop1".to_string(), "v1".to_string()) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - - assert_map_contains( - &HashMap::from([("prop1".to_string(), "v1".to_string())]), - table2.metadata().properties(), - ); -} - -fn assert_map_contains(map1: &HashMap, map2: &HashMap) { - for (k, v) in map1 { - assert!(map2.contains_key(k)); - assert_eq!(map2.get(k).unwrap(), v); - } -} - -#[tokio::test] -async fn test_list_empty_multi_level_namespace() { - let catalog = get_catalog().await; - - let ns_apple = Namespace::with_properties( - NamespaceIdent::from_strs(["test_list_empty_multi_level_namespace", "a_a", "apple"]) - .unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Currently this namespace doesn't exist, so it should return error. - assert!( - catalog - .list_namespaces(Some(ns_apple.name())) - .await - .is_err() - ); - - // Create namespaces - catalog - .create_namespace(ns_apple.name(), ns_apple.properties().clone()) - .await - .unwrap(); - - // List namespace - let nss = catalog - .list_namespaces(Some( - &NamespaceIdent::from_strs(["test_list_empty_multi_level_namespace", "a_a", "apple"]) - .unwrap(), - )) - .await - .unwrap(); - assert!(nss.is_empty()); -} - -#[tokio::test] -async fn test_register_table() { - let catalog = get_catalog().await; - - // Create namespace - let ns = NamespaceIdent::from_strs(["ns"]).unwrap(); - catalog.create_namespace(&ns, HashMap::new()).await.unwrap(); - - // Create the table, store the metadata location, drop the table - let empty_schema = Schema::builder().build().unwrap(); - let table_creation = TableCreation::builder() - .name("t1".to_string()) - .schema(empty_schema) - .build(); - - let table = catalog.create_table(&ns, table_creation).await.unwrap(); - - let metadata_location = table.metadata_location().unwrap(); - catalog.drop_table(table.identifier()).await.unwrap(); - - let new_table_identifier = TableIdent::from_strs(["ns", "t2"]).unwrap(); - let table_registered = catalog - .register_table(&new_table_identifier, metadata_location.to_string()) - .await - .unwrap(); - - assert_eq!( - table.metadata_location(), - table_registered.metadata_location() - ); - assert_ne!( - table.identifier().to_string(), - table_registered.identifier().to_string() - ); -} diff --git a/crates/catalog/s3tables/Cargo.toml b/crates/catalog/s3tables/Cargo.toml index c5df489693..ef6f3e045d 100644 --- a/crates/catalog/s3tables/Cargo.toml +++ b/crates/catalog/s3tables/Cargo.toml @@ -19,6 +19,8 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg-catalog-s3tables" +publish = true +readme = "README.md" rust-version = { workspace = true } version = { workspace = true } @@ -33,9 +35,17 @@ async-trait = { workspace = true } aws-config = { workspace = true } aws-sdk-s3tables = { workspace = true } iceberg = { workspace = true } - +iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] } [dev-dependencies] +arrow-array = { workspace = true } +arrow-schema = { workspace = true } +futures = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } itertools = { workspace = true } +parquet = { workspace = true } tokio = { workspace = true } +uuid = { workspace = true } + +[lints] +workspace = true diff --git a/crates/catalog/s3tables/DEPENDENCIES.rust.tsv b/crates/catalog/s3tables/DEPENDENCIES.rust.tsv deleted file mode 100644 index 13d8eebe7a..0000000000 --- a/crates/catalog/s3tables/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,380 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayvec@0.7.6 X X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-ord@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-config@1.8.6 X -aws-credential-types@1.2.6 X -aws-runtime@1.5.10 X -aws-sdk-s3tables@1.37.0 X -aws-sdk-sso@1.83.0 X -aws-sdk-ssooidc@1.84.0 X -aws-sdk-sts@1.85.0 X -aws-sigv4@1.3.4 X -aws-smithy-async@1.2.5 X -aws-smithy-http@0.62.3 X -aws-smithy-http-client@1.1.1 X -aws-smithy-json@0.61.5 X -aws-smithy-observability@0.1.3 X -aws-smithy-query@0.60.7 X -aws-smithy-runtime@1.9.1 X -aws-smithy-runtime-api@1.9.0 X -aws-smithy-types@1.3.2 X -aws-smithy-xml@0.60.10 X -aws-types@1.3.8 X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.21.7 X X -base64@0.22.1 X X -base64-simd@0.8.0 X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -bytes-utils@0.1.4 X X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation@0.10.1 X X -core-foundation@0.9.4 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -deranged@0.5.3 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -h2@0.3.27 X -h2@0.4.12 X -half@2.6.0 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@0.2.12 X X -http@1.3.1 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hyper@0.14.32 X -hyper@1.7.0 X -hyper-rustls@0.24.2 X X X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-catalog-s3tables@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-conv@0.1.0 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -openssl-probe@0.1.6 X X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustls@0.21.12 X X X -rustls@0.23.31 X X X -rustls-native-certs@0.6.3 X X X -rustls-native-certs@0.8.1 X X X -rustls-pemfile@1.0.4 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.101.7 X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -schannel@0.1.27 X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -sct@0.7.1 X X X -security-framework@2.11.1 X X -security-framework@3.4.0 X X -security-framework-sys@2.15.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.6 X X -simdutf8@0.1.5 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.5.10 X X -socket2@0.6.0 X X -stable_deref_trait@1.2.0 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -time@0.3.43 X X -time-core@0.1.6 X X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.24.1 X X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.7 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -vsimd@0.8.0 X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -webpki-roots@1.0.2 X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -xmlparser@0.13.6 X X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/s3tables/LICENSE b/crates/catalog/s3tables/LICENSE new file mode 120000 index 0000000000..5853aaea53 --- /dev/null +++ b/crates/catalog/s3tables/LICENSE @@ -0,0 +1 @@ +../../../LICENSE \ No newline at end of file diff --git a/crates/catalog/s3tables/NOTICE b/crates/catalog/s3tables/NOTICE new file mode 120000 index 0000000000..295f6bdb3a --- /dev/null +++ b/crates/catalog/s3tables/NOTICE @@ -0,0 +1 @@ +../../../NOTICE \ No newline at end of file diff --git a/crates/catalog/s3tables/README.md b/crates/catalog/s3tables/README.md new file mode 100644 index 0000000000..d9784d5231 --- /dev/null +++ b/crates/catalog/s3tables/README.md @@ -0,0 +1,56 @@ + + +# Apache Iceberg S3Tables Catalog (Rust) + +[![crates.io](https://img.shields.io/crates/v/iceberg.svg)](https://crates.io/crates/iceberg-catalog-s3tables) +[![docs.rs](https://img.shields.io/docsrs/iceberg.svg)](https://docs.rs/iceberg/latest/iceberg_catalog_s3tables/) + +Official Native Rust implementation of the Apache Iceberg S3Tables catalog. + +## Quick start + +```rust,no_run +use std::collections::HashMap; + +use iceberg::CatalogBuilder; +use iceberg_catalog_s3tables::{ + S3TABLES_CATALOG_PROP_ENDPOINT_URL, S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN, + S3TablesCatalogBuilder, +}; + +#[tokio::main] +async fn main() { + let catalog = S3TablesCatalogBuilder::default() + .with_endpoint_url("http://localhost:4566") + .load( + "s3tables", + HashMap::from([( + S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN.to_string(), + "arn:aws:s3tables:us-east-1:123456789012:bucket/my-bucket".to_string(), + )]), + ) + .await + .unwrap(); + + // use `catalog` as any Iceberg Catalog +} +``` + +See the [API documentation](https://docs.rs/iceberg_catalog_s3tables/latest) for the full API surface. diff --git a/crates/catalog/s3tables/public-api.txt b/crates/catalog/s3tables/public-api.txt new file mode 100644 index 0000000000..6d1b089e12 --- /dev/null +++ b/crates/catalog/s3tables/public-api.txt @@ -0,0 +1,37 @@ +pub mod iceberg_catalog_s3tables +pub struct iceberg_catalog_s3tables::S3TablesCatalog +impl core::fmt::Debug for iceberg_catalog_s3tables::S3TablesCatalog +pub fn iceberg_catalog_s3tables::S3TablesCatalog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::Catalog for iceberg_catalog_s3tables::S3TablesCatalog +pub fn iceberg_catalog_s3tables::S3TablesCatalog::create_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, _properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::create_table<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, creation: iceberg::catalog::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::drop_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::drop_table<'life0, 'life1, 'async_trait>(&'life0 self, _table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::get_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::list_namespaces<'life0, 'life1, 'async_trait>(&'life0 self, parent: core::option::Option<&'life1 iceberg::catalog::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::list_tables<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::load_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::namespace_exists<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::purge_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::register_table<'life0, 'life1, 'async_trait>(&'life0 self, _table_ident: &'life1 iceberg::catalog::TableIdent, _metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::rename_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, src: &'life1 iceberg::catalog::TableIdent, dest: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::table_exists<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::update_namespace<'life0, 'life1, 'async_trait>(&'life0 self, _namespace: &'life1 iceberg::catalog::NamespaceIdent, _properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_s3tables::S3TablesCatalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::catalog::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub struct iceberg_catalog_s3tables::S3TablesCatalogBuilder +impl iceberg_catalog_s3tables::S3TablesCatalogBuilder +pub fn iceberg_catalog_s3tables::S3TablesCatalogBuilder::with_client(self, client: aws_sdk_s3tables::client::Client) -> Self +pub fn iceberg_catalog_s3tables::S3TablesCatalogBuilder::with_endpoint_url(self, endpoint_url: impl core::convert::Into) -> Self +pub fn iceberg_catalog_s3tables::S3TablesCatalogBuilder::with_table_bucket_arn(self, table_bucket_arn: impl core::convert::Into) -> Self +impl core::default::Default for iceberg_catalog_s3tables::S3TablesCatalogBuilder +pub fn iceberg_catalog_s3tables::S3TablesCatalogBuilder::default() -> Self +impl core::fmt::Debug for iceberg_catalog_s3tables::S3TablesCatalogBuilder +pub fn iceberg_catalog_s3tables::S3TablesCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::CatalogBuilder for iceberg_catalog_s3tables::S3TablesCatalogBuilder +pub type iceberg_catalog_s3tables::S3TablesCatalogBuilder::C = iceberg_catalog_s3tables::S3TablesCatalog +pub fn iceberg_catalog_s3tables::S3TablesCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg_catalog_s3tables::S3TablesCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg_catalog_s3tables::S3TablesCatalogBuilder::with_runtime(self, runtime: iceberg::runtime::Runtime) -> Self +pub fn iceberg_catalog_s3tables::S3TablesCatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +pub const iceberg_catalog_s3tables::S3TABLES_CATALOG_PROP_ENDPOINT_URL: &str +pub const iceberg_catalog_s3tables::S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN: &str diff --git a/crates/catalog/s3tables/src/catalog.rs b/crates/catalog/s3tables/src/catalog.rs index 858bb92896..c96d06abd4 100644 --- a/crates/catalog/s3tables/src/catalog.rs +++ b/crates/catalog/s3tables/src/catalog.rs @@ -17,20 +17,25 @@ use std::collections::HashMap; use std::future::Future; +use std::str::FromStr; +use std::sync::Arc; use async_trait::async_trait; use aws_sdk_s3tables::operation::create_table::CreateTableOutput; use aws_sdk_s3tables::operation::get_namespace::GetNamespaceOutput; -use aws_sdk_s3tables::operation::get_table::GetTableOutput; +use aws_sdk_s3tables::operation::get_table::{GetTableError, GetTableOutput}; use aws_sdk_s3tables::operation::list_tables::ListTablesOutput; +use aws_sdk_s3tables::operation::update_table_metadata_location::UpdateTableMetadataLocationError; use aws_sdk_s3tables::types::OpenTableFormat; -use iceberg::io::{FileIO, FileIOBuilder}; +use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use iceberg::io::{FileIO, FileIOBuilder, StorageFactory}; use iceberg::spec::{TableMetadata, TableMetadataBuilder}; use iceberg::table::Table; use iceberg::{ Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, - TableCommit, TableCreation, TableIdent, + Runtime, TableCommit, TableCreation, TableIdent, }; +use iceberg_storage_opendal::OpenDalStorageFactory; use crate::utils::create_sdk_config; @@ -64,18 +69,28 @@ struct S3TablesCatalogConfig { /// Builder for [`S3TablesCatalog`]. #[derive(Debug)] -pub struct S3TablesCatalogBuilder(S3TablesCatalogConfig); +pub struct S3TablesCatalogBuilder { + config: S3TablesCatalogConfig, + storage_factory: Option>, + kms_client_factory: Option>, + runtime: Option, +} /// Default builder for [`S3TablesCatalog`]. impl Default for S3TablesCatalogBuilder { fn default() -> Self { - Self(S3TablesCatalogConfig { - name: None, - table_bucket_arn: "".to_string(), - endpoint_url: None, - client: None, - props: HashMap::new(), - }) + Self { + config: S3TablesCatalogConfig { + name: None, + table_bucket_arn: "".to_string(), + endpoint_url: None, + client: None, + props: HashMap::new(), + }, + storage_factory: None, + kms_client_factory: None, + runtime: None, + } } } @@ -90,13 +105,13 @@ impl S3TablesCatalogBuilder { /// This follows the general pattern where properties specified in the `load()` method /// have higher priority than builder method configurations. pub fn with_endpoint_url(mut self, endpoint_url: impl Into) -> Self { - self.0.endpoint_url = Some(endpoint_url.into()); + self.config.endpoint_url = Some(endpoint_url.into()); self } /// Configure the catalog with a pre-built AWS SDK client. pub fn with_client(mut self, client: aws_sdk_s3tables::Client) -> Self { - self.0.client = Some(client); + self.config.client = Some(client); self } @@ -109,7 +124,7 @@ impl S3TablesCatalogBuilder { /// This follows the general pattern where properties specified in the `load()` method /// have higher priority than builder method configurations. pub fn with_table_bucket_arn(mut self, table_bucket_arn: impl Into) -> Self { - self.0.table_bucket_arn = table_bucket_arn.into(); + self.config.table_bucket_arn = table_bucket_arn.into(); self } } @@ -117,27 +132,42 @@ impl S3TablesCatalogBuilder { impl CatalogBuilder for S3TablesCatalogBuilder { type C = S3TablesCatalog; + fn with_storage_factory(mut self, storage_factory: Arc) -> Self { + self.storage_factory = Some(storage_factory); + self + } + + fn with_kms_client_factory(mut self, kms_client_factory: Arc) -> Self { + self.kms_client_factory = Some(kms_client_factory); + self + } + + fn with_runtime(mut self, runtime: Runtime) -> Self { + self.runtime = Some(runtime); + self + } + fn load( mut self, name: impl Into, props: HashMap, ) -> impl Future> + Send { let catalog_name = name.into(); - self.0.name = Some(catalog_name.clone()); + self.config.name = Some(catalog_name.clone()); if props.contains_key(S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN) { - self.0.table_bucket_arn = props + self.config.table_bucket_arn = props .get(S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN) .cloned() .unwrap_or_default(); } if props.contains_key(S3TABLES_CATALOG_PROP_ENDPOINT_URL) { - self.0.endpoint_url = props.get(S3TABLES_CATALOG_PROP_ENDPOINT_URL).cloned(); + self.config.endpoint_url = props.get(S3TABLES_CATALOG_PROP_ENDPOINT_URL).cloned(); } // Collect other remaining properties - self.0.props = props + self.config.props = props .into_iter() .filter(|(k, _)| { k != S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN @@ -151,13 +181,21 @@ impl CatalogBuilder for S3TablesCatalogBuilder { ErrorKind::DataInvalid, "Catalog name cannot be empty", )) - } else if self.0.table_bucket_arn.is_empty() { + } else if self.config.table_bucket_arn.is_empty() { Err(Error::new( ErrorKind::DataInvalid, "Table bucket ARN is required", )) } else { - S3TablesCatalog::new(self.0).await + let runtime = match self.runtime { + Some(rt) => rt, + None => Runtime::try_current()?, + }; + let kms_client = match self.kms_client_factory { + Some(factory) => Some(factory.create_kms_client(&self.config.props).await?), + None => None, + }; + S3TablesCatalog::new(self.config, self.storage_factory, runtime, kms_client).await } } } @@ -169,11 +207,18 @@ pub struct S3TablesCatalog { config: S3TablesCatalogConfig, s3tables_client: aws_sdk_s3tables::Client, file_io: FileIO, + runtime: Runtime, + kms_client: Option>, } impl S3TablesCatalog { /// Creates a new S3Tables catalog. - async fn new(config: S3TablesCatalogConfig) -> Result { + async fn new( + config: S3TablesCatalogConfig, + storage_factory: Option>, + runtime: Runtime, + kms_client: Option>, + ) -> Result { let s3tables_client = if let Some(client) = config.client.clone() { client } else { @@ -181,14 +226,77 @@ impl S3TablesCatalog { aws_sdk_s3tables::Client::new(&aws_config) }; - let file_io = FileIOBuilder::new("s3").with_props(&config.props).build()?; + // Use provided factory or default to OpenDalStorageFactory::S3 + let factory = storage_factory.unwrap_or_else(|| { + Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + }) + }); + let file_io = FileIOBuilder::new(factory) + .with_props(&config.props) + .build(); Ok(Self { config, s3tables_client, file_io, + runtime, + kms_client, }) } + + async fn load_table_with_version_token( + &self, + table_ident: &TableIdent, + ) -> Result<(Table, String)> { + let req = self + .s3tables_client + .get_table() + .table_bucket_arn(self.config.table_bucket_arn.clone()) + .namespace(table_ident.namespace().to_url_string()) + .name(table_ident.name()); + + let resp = req.send().await.map_err(|err| { + if err + .as_service_error() + .is_some_and(GetTableError::is_not_found_exception) + { + // S3 Tables GetTable API only reports that the resource was not found, and does not distinguish between namespaces and tables. + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_s3Buckets_GetTable.html#API_s3Buckets_GetTable_Errors + Error::new( + ErrorKind::TableNotFound, + format!("Table {table_ident} is not found, either because the namespace or table did not exist"), + ) + .with_source(err) + } else { + from_aws_sdk_error(err) + } + })?; + + // when a table is created, it's possible that the metadata location is not set. + let metadata_location = resp.metadata_location().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!( + "Table {} does not have metadata location", + table_ident.name() + ), + ) + })?; + let metadata = TableMetadata::read_from(&self.file_io, metadata_location).await?; + + let mut builder = Table::builder() + .identifier(table_ident.clone()) + .metadata(metadata) + .metadata_location(metadata_location) + .file_io(self.file_io.clone()) + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + let table = builder.build()?; + Ok((table, resp.version_token)) + } } #[async_trait] @@ -248,6 +356,13 @@ impl Catalog for S3TablesCatalog { namespace: &NamespaceIdent, _properties: HashMap, ) -> Result { + if self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceAlreadyExists, + format!("Namespace {namespace:?} already exists"), + )); + } + let req = self .s3tables_client .create_namespace() @@ -270,6 +385,13 @@ impl Catalog for S3TablesCatalog { /// - If there is an error querying the database, returned by /// `from_aws_sdk_error`. async fn get_namespace(&self, namespace: &NamespaceIdent) -> Result { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + let req = self .s3tables_client .get_namespace() @@ -337,6 +459,13 @@ impl Catalog for S3TablesCatalog { /// - Errors from the underlying database deletion process, converted using /// `from_aws_sdk_error`. async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + let req = self .s3tables_client .delete_namespace() @@ -414,9 +543,9 @@ impl Catalog for S3TablesCatalog { .await .map_err(from_aws_sdk_error)?; - // prepare metadata location. the warehouse location is generated by s3tables catalog, + // prepare table location. the warehouse location is generated by s3tables catalog, // which looks like: s3://e6c9bf20-991a-46fb-kni5xs1q2yxi3xxdyxzjzigdeop1quse2b--table-s3 - let metadata_location = match &creation.location { + let table_location = match &creation.location { Some(_) => { return Err(Error::new( ErrorKind::DataInvalid, @@ -433,36 +562,41 @@ impl Catalog for S3TablesCatalog { .send() .await .map_err(from_aws_sdk_error)?; - let warehouse_location = get_resp.warehouse_location().to_string(); - MetadataLocation::new_with_table_location(warehouse_location).to_string() + get_resp.warehouse_location().to_string() } }; // write metadata to file - creation.location = Some(metadata_location.clone()); + creation.location = Some(table_location.clone()); let metadata = TableMetadataBuilder::from_table_creation(creation)? .build()? .metadata; + let metadata_location = MetadataLocation::try_new_with_metadata(&metadata)?; metadata.write_to(&self.file_io, &metadata_location).await?; // update metadata location + let metadata_location_str = metadata_location.to_string(); self.s3tables_client .update_table_metadata_location() .table_bucket_arn(self.config.table_bucket_arn.clone()) .namespace(namespace.to_url_string()) .name(table_ident.name()) - .metadata_location(metadata_location.clone()) + .metadata_location(metadata_location_str.clone()) .version_token(create_resp.version_token()) .send() .await .map_err(from_aws_sdk_error)?; - let table = Table::builder() + let mut builder = Table::builder() .identifier(table_ident) - .metadata_location(metadata_location) + .metadata_location(metadata_location_str) .metadata(metadata) .file_io(self.file_io.clone()) - .build()?; + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + let table = builder.build()?; Ok(table) } @@ -477,44 +611,21 @@ impl Catalog for S3TablesCatalog { /// - Errors from the underlying database query process, converted using /// `from_aws_sdk_error`. async fn load_table(&self, table_ident: &TableIdent) -> Result
{ - let req = self - .s3tables_client - .get_table() - .table_bucket_arn(self.config.table_bucket_arn.clone()) - .namespace(table_ident.namespace().to_url_string()) - .name(table_ident.name()); - let resp: GetTableOutput = req.send().await.map_err(from_aws_sdk_error)?; - - // when a table is created, it's possible that the metadata location is not set. - let metadata_location = resp.metadata_location().ok_or_else(|| { - Error::new( - ErrorKind::Unexpected, - format!( - "Table {} does not have metadata location", - table_ident.name() - ), - ) - })?; - let metadata = TableMetadata::read_from(&self.file_io, metadata_location).await?; - - let table = Table::builder() - .identifier(table_ident.clone()) - .metadata(metadata) - .metadata_location(metadata_location) - .file_io(self.file_io.clone()) - .build()?; - Ok(table) + Ok(self.load_table_with_version_token(table_ident).await?.0) } - /// Drops an existing table from the s3tables catalog. - /// - /// Validates the table identifier and then deletes the corresponding - /// table from the s3tables catalog. + /// Not supported for S3Tables. Use `purge_table` instead. /// - /// This function can return an error in the following situations: - /// - Errors from the underlying database deletion process, converted using - /// `from_aws_sdk_error`. - async fn drop_table(&self, table: &TableIdent) -> Result<()> { + /// S3 Tables doesn't support soft delete, so dropping a table will permanently remove it from the catalog. + async fn drop_table(&self, _table: &TableIdent) -> Result<()> { + Err(Error::new( + ErrorKind::FeatureUnsupported, + "drop_table is not supported for S3Tables; use purge_table instead", + )) + } + + /// Purge a table from the S3 Tables catalog. + async fn purge_table(&self, table: &TableIdent) -> Result<()> { let req = self .s3tables_client .delete_table() @@ -589,31 +700,87 @@ impl Catalog for S3TablesCatalog { } /// Updates an existing table within the s3tables catalog. - /// - /// This function is still in development and will always return an error. - async fn update_table(&self, _commit: TableCommit) -> Result
{ - Err(Error::new( - ErrorKind::FeatureUnsupported, - "Updating a table is not supported yet", - )) + async fn update_table(&self, commit: TableCommit) -> Result
{ + let table_ident = commit.identifier().clone(); + let table_namespace = table_ident.namespace(); + let (current_table, version_token) = + self.load_table_with_version_token(&table_ident).await?; + + let staged_table = commit.apply(current_table)?; + let staged_metadata_location_str = staged_table.metadata_location_result()?; + let staged_metadata_location = MetadataLocation::from_str(staged_metadata_location_str)?; + + staged_table + .metadata() + .write_to(staged_table.file_io(), &staged_metadata_location) + .await?; + + let builder = self + .s3tables_client + .update_table_metadata_location() + .table_bucket_arn(&self.config.table_bucket_arn) + .namespace(table_namespace.to_url_string()) + .name(table_ident.name()) + .version_token(version_token) + .metadata_location(staged_metadata_location_str); + + let _ = builder + .send() + .await + .map_err(|e| match e.as_service_error() { + Some(UpdateTableMetadataLocationError::ConflictException(_)) => Error::new( + ErrorKind::CatalogCommitConflicts, + format!("Commit conflicted for table: {table_ident}"), + ) + .with_retryable(true) + .with_source(e), + Some(UpdateTableMetadataLocationError::NotFoundException(_)) => Error::new( + ErrorKind::TableNotFound, + format!("Table {table_ident} is not found"), + ) + .with_source(e), + _ => from_aws_sdk_error(e), + })?; + + Ok(staged_table) } } /// Format AWS SDK error into iceberg error pub(crate) fn from_aws_sdk_error(error: aws_sdk_s3tables::error::SdkError) -> Error -where T: std::fmt::Debug { +where T: std::error::Error + Send + Sync + 'static { Error::new( ErrorKind::Unexpected, - format!("Operation failed for hitting aws sdk error: {:?}", error), + format!("Operation failed for hitting aws sdk error: {error:?}"), ) + .with_source(error) } #[cfg(test)] mod tests { + use std::error::Error as _; + + use futures::TryStreamExt; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; + use iceberg::transaction::{ApplyTransactionAction, Transaction}; use super::*; + #[test] + fn test_from_aws_sdk_error_keeps_sdk_error_as_source() { + let err = from_aws_sdk_error( + aws_sdk_s3tables::error::SdkError::::construction_failure("boom"), + ); + + assert_eq!(err.kind(), ErrorKind::Unexpected); + assert!( + err.source() + .and_then(|source| source + .downcast_ref::>()) + .is_some() + ); + } + async fn load_s3tables_catalog_from_env() -> Result> { let table_bucket_arn = match std::env::var("TABLE_BUCKET_ARN").ok() { Some(table_bucket_arn) => table_bucket_arn, @@ -628,7 +795,9 @@ mod tests { props: HashMap::new(), }; - Ok(Some(S3TablesCatalog::new(config).await?)) + Ok(Some( + S3TablesCatalog::new(config, None, Runtime::current(), None).await?, + )) } #[tokio::test] @@ -636,7 +805,7 @@ mod tests { let catalog = match load_s3tables_catalog_from_env().await { Ok(Some(catalog)) => catalog, Ok(None) => return, - Err(e) => panic!("Error loading catalog: {}", e), + Err(e) => panic!("Error loading catalog: {e}"), }; let namespaces = catalog.list_namespaces(None).await.unwrap(); @@ -648,7 +817,7 @@ mod tests { let catalog = match load_s3tables_catalog_from_env().await { Ok(Some(catalog)) => catalog, Ok(None) => return, - Err(e) => panic!("Error loading catalog: {}", e), + Err(e) => panic!("Error loading catalog: {e}"), }; let tables = catalog @@ -663,17 +832,29 @@ mod tests { let catalog = match load_s3tables_catalog_from_env().await { Ok(Some(catalog)) => catalog, Ok(None) => return, - Err(e) => panic!("Error loading catalog: {}", e), + Err(e) => panic!("Error loading catalog: {e}"), }; - let table = catalog + let _table = catalog .load_table(&TableIdent::new( NamespaceIdent::new("aws_s3_metadata".to_string()), "query_storage_metadata".to_string(), )) .await - .unwrap(); - println!("{:?}", table); + .expect("table that exists should be loaded"); + + let load_table_err = catalog + .load_table(&TableIdent::new( + NamespaceIdent::new("not_a_namespace".to_string()), + "not_a_table_name".to_string(), + )) + .await + .expect_err("loading a table that does not exist should fail"); + assert_eq!( + load_table_err.kind(), + ErrorKind::TableNotFound, + "must return table not found error for non-existent table" + ); } #[tokio::test] @@ -681,7 +862,7 @@ mod tests { let catalog = match load_s3tables_catalog_from_env().await { Ok(Some(catalog)) => catalog, Ok(None) => return, - Err(e) => panic!("Error loading catalog: {}", e), + Err(e) => panic!("Error loading catalog: {e}"), }; let namespace = NamespaceIdent::new("test_s3tables_create_delete_namespace".to_string()); @@ -699,7 +880,7 @@ mod tests { let catalog = match load_s3tables_catalog_from_env().await { Ok(Some(catalog)) => catalog, Ok(None) => return, - Err(e) => panic!("Error loading catalog: {}", e), + Err(e) => panic!("Error loading catalog: {e}"), }; let creation = { @@ -737,6 +918,91 @@ mod tests { catalog.drop_namespace(&namespace).await.unwrap(); } + #[tokio::test] + async fn test_s3tables_update_table() { + let catalog = match load_s3tables_catalog_from_env().await { + Ok(Some(catalog)) => catalog, + Ok(None) => return, + Err(e) => panic!("Error loading catalog: {e}"), + }; + + // Create a test namespace and table + let namespace = NamespaceIdent::new("test_s3tables_update_table".to_string()); + let table_ident = + TableIdent::new(namespace.clone(), "test_s3tables_update_table".to_string()); + + // Clean up any existing resources from previous test runs + catalog.drop_table(&table_ident).await.ok(); + catalog.drop_namespace(&namespace).await.ok(); + + // Create namespace and table + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap(); + + let creation = { + let schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "bar", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + TableCreation::builder() + .name(table_ident.name().to_string()) + .properties(HashMap::new()) + .schema(schema) + .build() + }; + + let table = catalog.create_table(&namespace, creation).await.unwrap(); + + // Create a transaction to update the table + let tx = Transaction::new(&table); + + // Store the original metadata location for comparison + let original_metadata_location = table.metadata_location(); + + // Update table properties using the transaction + let tx = tx + .update_table_properties() + .set("test_property".to_string(), "test_value".to_string()) + .apply(tx) + .unwrap(); + + // Commit the transaction to the catalog + let updated_table = tx.commit(&catalog).await.unwrap(); + + // Verify the update was successful + assert_eq!( + updated_table.metadata().properties().get("test_property"), + Some(&"test_value".to_string()) + ); + + // Verify the metadata location has been updated + assert_ne!( + updated_table.metadata_location(), + original_metadata_location, + "Metadata location should be updated after commit" + ); + + // Load the table again from the catalog to verify changes were persisted + let reloaded_table = catalog.load_table(&table_ident).await.unwrap(); + + // Verify the reloaded table matches the updated table + assert_eq!( + reloaded_table.metadata().properties().get("test_property"), + Some(&"test_value".to_string()) + ); + assert_eq!( + reloaded_table.metadata_location(), + updated_table.metadata_location(), + "Reloaded table should have the same metadata location as the updated table" + ); + } + #[tokio::test] async fn test_builder_load_missing_bucket_arn() { let builder = S3TablesCatalogBuilder::default(); @@ -992,4 +1258,108 @@ mod tests { assert_eq!(err.message(), "Catalog name cannot be empty"); } } + + /// Verify that an S3 Table catalog can create a table, write data, load the same table, and read from it. + #[tokio::test] + async fn test_s3tables_create_table_write_load_table_read() { + use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; + use iceberg::writer::file_writer::ParquetWriterBuilder; + use iceberg::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; + + let catalog = match load_s3tables_catalog_from_env().await { + Ok(Some(c)) => c, + Ok(None) => return, + Err(e) => panic!("Error loading catalog: {e}"), + }; + + let ns = NamespaceIdent::new(format!("test_rw_{}", uuid::Uuid::new_v4().simple())); + catalog.create_namespace(&ns, HashMap::new()).await.unwrap(); + + let table_name = String::from("table"); + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let creation = TableCreation::builder() + .name(table_name.clone()) + .schema(schema) + .build(); + + let table = catalog.create_table(&ns, creation).await.unwrap(); + + // Write one row. + let arrow_schema: Arc = Arc::new( + table + .metadata() + .current_schema() + .as_ref() + .try_into() + .unwrap(), + ); + let batch = arrow_array::RecordBatch::try_new(arrow_schema, vec![Arc::new( + arrow_array::Int32Array::from(vec![42]), + )]) + .unwrap(); + + // Locations will be generated based on the table metadata, which will be using `s3://` for Amazon S3 Tables. + let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); + let file_name_generator = DefaultFileNameGenerator::new( + "test".to_string(), + None, + iceberg::spec::DataFileFormat::Parquet, + ); + let parquet_writer_builder = ParquetWriterBuilder::new( + parquet::file::properties::WriterProperties::default(), + table.metadata().current_schema().clone(), + ); + let rw = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + table.file_io().clone(), + location_generator, + file_name_generator, + ); + let mut writer = DataFileWriterBuilder::new(rw).build(None).await.unwrap(); + writer.write(batch.clone()).await.unwrap(); + let data_files = writer.close().await.unwrap(); + + let tx = Transaction::new(&table); + let tx = tx + .fast_append() + .add_data_files(data_files) + .apply(tx) + .unwrap(); + tx.commit(&catalog).await.unwrap(); + + // Reload from catalog and read back. + let table_ident = TableIdent::new(ns.clone(), table_name.clone()); + let reloaded = catalog.load_table(&table_ident).await.unwrap(); + let batches: Vec = reloaded + .scan() + .select_all() + .build() + .expect("scan to be valid (snapshot exists, schema is OK)") + .to_arrow(Default::default()) + .await + .expect("scan tasks should be OK") + .try_collect() + .await + .expect("scan should complete successfully"); + + assert_eq!(batches.len(), 1); + assert_eq!( + batches[0], batch, + "read records should match records written earlier" + ); + + // Clean up. + catalog.purge_table(&table_ident).await.ok(); + catalog.drop_namespace(&ns).await.ok(); + } } diff --git a/crates/catalog/s3tables/src/utils.rs b/crates/catalog/s3tables/src/utils.rs index 2f3e16a937..0f7e3663aa 100644 --- a/crates/catalog/s3tables/src/utils.rs +++ b/crates/catalog/s3tables/src/utils.rs @@ -39,10 +39,6 @@ pub(crate) async fn create_sdk_config( ) -> SdkConfig { let mut config = aws_config::defaults(BehaviorVersion::latest()); - if properties.is_empty() { - return config.load().await; - } - if let Some(endpoint_url) = endpoint_url { config = config.endpoint_url(endpoint_url); } @@ -69,3 +65,20 @@ pub(crate) async fn create_sdk_config( config.load().await } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_config_with_custom_endpoint() { + let properties = HashMap::new(); + let endpoint_url = "http://localhost:5001"; + + let sdk_config = create_sdk_config(&properties, Some(endpoint_url.to_string())).await; + + let result = sdk_config.endpoint_url().unwrap(); + + assert_eq!(result, endpoint_url); + } +} diff --git a/crates/catalog/sql/Cargo.toml b/crates/catalog/sql/Cargo.toml index 33ca700bf7..e78e4e0ffc 100644 --- a/crates/catalog/sql/Cargo.toml +++ b/crates/catalog/sql/Cargo.toml @@ -19,6 +19,7 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg-catalog-sql" +publish = true rust-version = { workspace = true } version = { workspace = true } @@ -31,13 +32,14 @@ repository = { workspace = true } [dependencies] async-trait = { workspace = true } iceberg = { workspace = true } +iceberg-property-macro = { workspace = true } sqlx = { version = "0.8.1", features = ["any"], default-features = false } -typed-builder = { workspace = true } +strum = { workspace = true } +tracing = { workspace = true } [dev-dependencies] -iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } itertools = { workspace = true } -regex = "1.10.5" +regex = { workspace = true } sqlx = { version = "0.8.1", features = [ "tls-rustls", "runtime-tokio", @@ -47,3 +49,6 @@ sqlx = { version = "0.8.1", features = [ ], default-features = false } tempfile = { workspace = true } tokio = { workspace = true } + +[lints] +workspace = true diff --git a/crates/catalog/sql/DEPENDENCIES.rust.tsv b/crates/catalog/sql/DEPENDENCIES.rust.tsv deleted file mode 100644 index 335c980967..0000000000 --- a/crates/catalog/sql/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,342 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayvec@0.7.6 X X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-ord@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc@3.3.0 X X -crc-catalog@2.4.0 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-queue@0.3.12 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -flume@0.11.1 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-intrusive@0.5.0 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -half@2.6.0 X X -hashbrown@0.15.5 X X -hashlink@0.10.0 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-catalog-sql@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libsqlite3-sys@0.30.1 X -libz-rs-sys@0.5.2 X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustls@0.23.31 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simdutf8@0.1.5 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.0 X X -spin@0.9.8 X -sqlx@0.8.6 X X -sqlx-core@0.8.6 X X -sqlx-sqlite@0.8.6 X X -stable_deref_trait@1.2.0 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-stream@0.1.17 X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -vcpkg@0.2.15 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -webpki-roots@0.26.11 X -webpki-roots@1.0.2 X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/catalog/sql/public-api.txt b/crates/catalog/sql/public-api.txt new file mode 100644 index 0000000000..964a1cae5e --- /dev/null +++ b/crates/catalog/sql/public-api.txt @@ -0,0 +1,76 @@ +pub mod iceberg_catalog_sql +pub enum iceberg_catalog_sql::SchemaVersion +pub iceberg_catalog_sql::SchemaVersion::V0 +pub iceberg_catalog_sql::SchemaVersion::V1 +impl core::clone::Clone for iceberg_catalog_sql::SchemaVersion +pub fn iceberg_catalog_sql::SchemaVersion::clone(&self) -> iceberg_catalog_sql::SchemaVersion +impl core::cmp::PartialEq for iceberg_catalog_sql::SchemaVersion +pub fn iceberg_catalog_sql::SchemaVersion::eq(&self, other: &iceberg_catalog_sql::SchemaVersion) -> bool +impl core::convert::TryFrom<&str> for iceberg_catalog_sql::SchemaVersion +pub type iceberg_catalog_sql::SchemaVersion::Error = strum::ParseError +pub fn iceberg_catalog_sql::SchemaVersion::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg_catalog_sql::SchemaVersion +pub fn iceberg_catalog_sql::SchemaVersion::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg_catalog_sql::SchemaVersion +pub fn iceberg_catalog_sql::SchemaVersion::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::Copy for iceberg_catalog_sql::SchemaVersion +impl core::marker::StructuralPartialEq for iceberg_catalog_sql::SchemaVersion +impl core::str::traits::FromStr for iceberg_catalog_sql::SchemaVersion +pub type iceberg_catalog_sql::SchemaVersion::Err = strum::ParseError +pub fn iceberg_catalog_sql::SchemaVersion::from_str(s: &str) -> core::result::Result::Err> +pub enum iceberg_catalog_sql::SqlBindStyle +pub iceberg_catalog_sql::SqlBindStyle::DollarNumeric +pub iceberg_catalog_sql::SqlBindStyle::QMark +impl core::cmp::PartialEq for iceberg_catalog_sql::SqlBindStyle +pub fn iceberg_catalog_sql::SqlBindStyle::eq(&self, other: &iceberg_catalog_sql::SqlBindStyle) -> bool +impl core::convert::TryFrom<&str> for iceberg_catalog_sql::SqlBindStyle +pub type iceberg_catalog_sql::SqlBindStyle::Error = strum::ParseError +pub fn iceberg_catalog_sql::SqlBindStyle::try_from(s: &str) -> core::result::Result>::Error> +impl core::fmt::Debug for iceberg_catalog_sql::SqlBindStyle +pub fn iceberg_catalog_sql::SqlBindStyle::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg_catalog_sql::SqlBindStyle +pub fn iceberg_catalog_sql::SqlBindStyle::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::result::Result<(), core::fmt::Error> +impl core::marker::StructuralPartialEq for iceberg_catalog_sql::SqlBindStyle +impl core::str::traits::FromStr for iceberg_catalog_sql::SqlBindStyle +pub type iceberg_catalog_sql::SqlBindStyle::Err = strum::ParseError +pub fn iceberg_catalog_sql::SqlBindStyle::from_str(s: &str) -> core::result::Result::Err> +pub struct iceberg_catalog_sql::SqlCatalog +impl core::fmt::Debug for iceberg_catalog_sql::SqlCatalog +pub fn iceberg_catalog_sql::SqlCatalog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::Catalog for iceberg_catalog_sql::SqlCatalog +pub fn iceberg_catalog_sql::SqlCatalog::create_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::create_table<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, creation: iceberg::catalog::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::drop_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::drop_table<'life0, 'life1, 'async_trait>(&'life0 self, identifier: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::get_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::list_namespaces<'life0, 'life1, 'async_trait>(&'life0 self, parent: core::option::Option<&'life1 iceberg::catalog::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::list_tables<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::load_table<'life0, 'life1, 'async_trait>(&'life0 self, identifier: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::namespace_exists<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::purge_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::register_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::catalog::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::rename_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, src: &'life1 iceberg::catalog::TableIdent, dest: &'life2 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::table_exists<'life0, 'life1, 'async_trait>(&'life0 self, identifier: &'life1 iceberg::catalog::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::update_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::catalog::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_catalog_sql::SqlCatalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::catalog::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub struct iceberg_catalog_sql::SqlCatalogBuilder +impl iceberg_catalog_sql::SqlCatalogBuilder +pub fn iceberg_catalog_sql::SqlCatalogBuilder::prop(self, key: impl core::convert::Into, value: impl core::convert::Into) -> Self +pub fn iceberg_catalog_sql::SqlCatalogBuilder::props(self, props: std::collections::hash::map::HashMap) -> Self +pub fn iceberg_catalog_sql::SqlCatalogBuilder::sql_bind_style(self, sql_bind_style: iceberg_catalog_sql::SqlBindStyle) -> Self +pub fn iceberg_catalog_sql::SqlCatalogBuilder::uri(self, uri: impl core::convert::Into) -> Self +pub fn iceberg_catalog_sql::SqlCatalogBuilder::warehouse_location(self, location: impl core::convert::Into) -> Self +impl core::default::Default for iceberg_catalog_sql::SqlCatalogBuilder +pub fn iceberg_catalog_sql::SqlCatalogBuilder::default() -> iceberg_catalog_sql::SqlCatalogBuilder +impl core::fmt::Debug for iceberg_catalog_sql::SqlCatalogBuilder +pub fn iceberg_catalog_sql::SqlCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::catalog::CatalogBuilder for iceberg_catalog_sql::SqlCatalogBuilder +pub type iceberg_catalog_sql::SqlCatalogBuilder::C = iceberg_catalog_sql::SqlCatalog +pub fn iceberg_catalog_sql::SqlCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg_catalog_sql::SqlCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg_catalog_sql::SqlCatalogBuilder::with_runtime(self, runtime: iceberg::runtime::Runtime) -> Self +pub fn iceberg_catalog_sql::SqlCatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +pub const iceberg_catalog_sql::SQL_CATALOG_PROP_BIND_STYLE: &str +pub const iceberg_catalog_sql::SQL_CATALOG_PROP_SCHEMA_VERSION: &str +pub const iceberg_catalog_sql::SQL_CATALOG_PROP_URI: &str +pub const iceberg_catalog_sql::SQL_CATALOG_PROP_WAREHOUSE: &str diff --git a/crates/catalog/sql/src/catalog.rs b/crates/catalog/sql/src/catalog.rs index 35889d451a..7aeba57c89 100644 --- a/crates/catalog/sql/src/catalog.rs +++ b/crates/catalog/sql/src/catalog.rs @@ -16,24 +16,48 @@ // under the License. use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; -use iceberg::io::FileIO; +use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use iceberg::io::{FileIO, FileIOBuilder, StorageFactory}; use iceberg::spec::{TableMetadata, TableMetadataBuilder}; use iceberg::table::Table; use iceberg::{ - Catalog, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, TableCommit, - TableCreation, TableIdent, + Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, + Runtime, TableCommit, TableCreation, TableIdent, delete_removed_metadata_files, }; +use iceberg_property_macro::Properties; use sqlx::any::{AnyPoolOptions, AnyQueryResult, AnyRow, install_default_drivers}; -use sqlx::{Any, AnyPool, Row, Transaction}; -use typed_builder::TypedBuilder; +use sqlx::{Any, AnyPool, Column, Executor, Row, Transaction}; use crate::error::{ from_sqlx_error, no_such_namespace_err, no_such_table_err, table_already_exists_err, }; +/// catalog URI +pub const SQL_CATALOG_PROP_URI: &str = "uri"; +/// catalog warehouse location +pub const SQL_CATALOG_PROP_WAREHOUSE: &str = "warehouse"; +/// catalog sql bind style +pub const SQL_CATALOG_PROP_BIND_STYLE: &str = "sql.bind-style"; +/// Legacy (pre-`sql.bind-style`) key for [`SQL_CATALOG_PROP_BIND_STYLE`], still accepted for +/// backward compatibility. +const SQL_CATALOG_PROP_BIND_STYLE_LEGACY: &str = "sql_bind_style"; +/// Expected catalog schema version. +/// +/// If this property is set and it is newer than the detected schema version, +/// a migration will be attempted. +/// If it is older, it is ignored with a warning. +/// If the catalog table didn't already exist, this value is ignored and it will be created with `V1`. +/// +/// `V0` is a compatibility mode for catalog tables created before the `iceberg_type` column +/// existed; it cannot be requested for a new catalog table, since table creation and +/// registration are unsupported on `V0`. +pub const SQL_CATALOG_PROP_SCHEMA_VERSION: &str = "sql.schema-version"; + static CATALOG_TABLE_NAME: &str = "iceberg_tables"; static CATALOG_FIELD_CATALOG_NAME: &str = "catalog_name"; static CATALOG_FIELD_TABLE_NAME: &str = "table_name"; @@ -54,36 +78,313 @@ static MAX_CONNECTIONS: u32 = 10; // Default the SQL pool to 10 connections if n static IDLE_TIMEOUT: u64 = 10; // Default the maximum idle timeout per connection to 10s before it is closed static TEST_BEFORE_ACQUIRE: bool = true; // Default the health-check of each connection to enabled prior to returning -/// A struct representing the SQL catalog configuration. -/// -/// This struct contains various parameters that are used to configure a SQL catalog, -/// such as the database URI, warehouse location, and file I/O settings. -/// You are required to provide a `SqlBindStyle`, which determines how SQL statements will be bound to values in the catalog. -/// The options available for this parameter include: -/// - `SqlBindStyle::DollarNumeric`: Binds SQL statements using `$1`, `$2`, etc., as placeholders. This is for PostgreSQL databases. -/// - `SqlBindStyle::QuestionMark`: Binds SQL statements using `?` as a placeholder. This is for MySQL and SQLite databases. -#[derive(Debug, TypedBuilder)] -pub struct SqlCatalogConfig { +/// Parses one pool-property value; the `Properties` derive adds the property-key context. +fn parse_pool_property(value: &str) -> Result +where + T: FromStr, + T::Err: std::error::Error + Send + Sync + 'static, +{ + value.parse().map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + "Failed to parse SQL catalog pool property", + ) + .with_context("value", value) + .with_source(error) + }) +} + +/// Builder for [`SqlCatalog`] +#[derive(Debug, Default)] +pub struct SqlCatalogBuilder { + props: HashMap, + storage_factory: Option>, + kms_client_factory: Option>, + runtime: Option, +} + +impl SqlCatalogBuilder { + /// Configure the database URI + /// + /// If `SQL_CATALOG_PROP_URI` has a value set in `props` during `SqlCatalogBuilder::load`, + /// that value takes precedence, and the value specified by this method will not be used. + pub fn uri(mut self, uri: impl Into) -> Self { + self.props + .insert(SQL_CATALOG_PROP_URI.to_string(), uri.into()); + self + } + + /// Configure the warehouse location + /// + /// If `SQL_CATALOG_PROP_WAREHOUSE` has a value set in `props` during `SqlCatalogBuilder::load`, + /// that value takes precedence, and the value specified by this method will not be used. + pub fn warehouse_location(mut self, location: impl Into) -> Self { + self.props + .insert(SQL_CATALOG_PROP_WAREHOUSE.to_string(), location.into()); + self + } + + /// Configure the bound SQL Statement + /// + /// If either `SQL_CATALOG_PROP_BIND_STYLE` or the legacy `sql_bind_style` property has a value + /// set in `props` during `SqlCatalogBuilder::load`, that load-time value takes precedence. + /// When both load-time keys are present, `SQL_CATALOG_PROP_BIND_STYLE` takes precedence. + pub fn sql_bind_style(mut self, sql_bind_style: SqlBindStyle) -> Self { + self.props.insert( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + sql_bind_style.to_string(), + ); + self + } + + /// Configure the any properties + /// + /// If the same key has values set in `props` during `SqlCatalogBuilder::load`, + /// those values will take precedence. + pub fn props(mut self, props: HashMap) -> Self { + for (k, v) in props { + self.props.insert(k, v); + } + self + } + + /// Set a new property on the property to be configured. + /// When multiple methods are executed with the same key, + /// the later-set value takes precedence. + /// + /// If the same key has values set in `props` during `SqlCatalogBuilder::load`, + /// those values will take precedence. + pub fn prop(mut self, key: impl Into, value: impl Into) -> Self { + self.props.insert(key.into(), value.into()); + self + } +} + +impl CatalogBuilder for SqlCatalogBuilder { + type C = SqlCatalog; + + fn with_storage_factory(mut self, storage_factory: Arc) -> Self { + self.storage_factory = Some(storage_factory); + self + } + + fn with_kms_client_factory(mut self, kms_client_factory: Arc) -> Self { + self.kms_client_factory = Some(kms_client_factory); + self + } + + fn with_runtime(mut self, runtime: Runtime) -> Self { + self.runtime = Some(runtime); + self + } + + fn load( + self, + name: impl Into, + props: HashMap, + ) -> impl Future> + Send { + let name = name.into(); + + async move { + if name.trim().is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + "Catalog name cannot be empty", + )); + } + + let load_overrides_bind_style = props.contains_key(SQL_CATALOG_PROP_BIND_STYLE) + || props.contains_key(SQL_CATALOG_PROP_BIND_STYLE_LEGACY); + let mut merged_props = self.props; + if load_overrides_bind_style { + // Treat the preferred and legacy names as aliases so either load-time spelling + // overrides a value configured on the builder. + merged_props.remove(SQL_CATALOG_PROP_BIND_STYLE); + merged_props.remove(SQL_CATALOG_PROP_BIND_STYLE_LEGACY); + } + merged_props.extend(props); + let catalog_properties = SqlCatalogProperties::from_properties(&merged_props)?; + + // The catalog URI is consumed above and may contain database credentials. Do not + // expose this security-sensitive value to FileIO or KMS extension points. + merged_props.remove(SQL_CATALOG_PROP_URI); + + let runtime = match self.runtime { + Some(rt) => rt, + None => Runtime::try_current()?, + }; + let kms_client = match self.kms_client_factory { + Some(factory) => Some(factory.create_kms_client(&merged_props).await?), + None => None, + }; + SqlCatalog::new( + name, + catalog_properties, + merged_props, + self.storage_factory, + runtime, + kms_client, + ) + .await + } + } +} + +fn parse_sql_bind_style( + properties: &HashMap, + key: &str, + additional_keys: &[&str], + default: SqlBindStyle, +) -> Result { + let configured_value = properties.get(key).map(|value| (key, value)).or_else(|| { + additional_keys + .iter() + .find_map(|alt_key| properties.get(*alt_key).map(|value| (*alt_key, value))) + }); + + configured_value.map_or(Ok(default), |(configured_key, value)| { + SqlBindStyle::from_str(value).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "`{}` values are valid only if they're `{}` or `{}`", + configured_key, + SqlBindStyle::DollarNumeric, + SqlBindStyle::QMark + ), + ) + }) + }) +} + +fn parse_schema_version(value: &str) -> Result { + SchemaVersion::from_str(value).map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "`{}` values are valid only if they're `{}` or `{}`", + SQL_CATALOG_PROP_SCHEMA_VERSION, + SchemaVersion::V0, + SchemaVersion::V1 + ), + ) + }) +} + +#[derive(Debug, Properties)] +pub(crate) struct SqlCatalogProperties { + #[property(key = SQL_CATALOG_PROP_URI, default = "")] uri: String, - name: String, + #[property(key = SQL_CATALOG_PROP_WAREHOUSE, default = "")] warehouse_location: String, - file_io: FileIO, + #[property( + key = SQL_CATALOG_PROP_BIND_STYLE, + additional_keys = [SQL_CATALOG_PROP_BIND_STYLE_LEGACY], + default = SqlBindStyle::DollarNumeric, + parse_properties_with = parse_sql_bind_style + )] sql_bind_style: SqlBindStyle, - #[builder(default)] - props: HashMap, + #[property( + key = SQL_CATALOG_PROP_SCHEMA_VERSION, + default = None, + parse_with = parse_schema_version + )] + schema_version: Option, + #[property( + key = "pool.max-connections", + default = MAX_CONNECTIONS, + parse_with = parse_pool_property + )] + max_connections: u32, + #[property( + key = "pool.idle-timeout", + default = IDLE_TIMEOUT, + parse_with = parse_pool_property + )] + idle_timeout: u64, + #[property( + key = "pool.test-before-acquire", + default = TEST_BEFORE_ACQUIRE, + parse_with = parse_pool_property + )] + test_before_acquire: bool, } #[derive(Debug)] -/// Sql catalog implementation. +/// SQL catalog implementation. +/// +/// The catalog supports SQL catalog schema V1, as well as limited support for V0. +/// Catalogs can opt-in to automatic migration by configuring the `sql.schema-version` catalog property. pub struct SqlCatalog { name: String, + properties: SqlCatalogProperties, connection: AnyPool, - warehouse_location: String, fileio: FileIO, - sql_bind_style: SqlBindStyle, + runtime: Runtime, + kms_client: Option>, + schema_version: SchemaVersion, +} + +#[derive(Debug, Clone, Copy, PartialEq, strum::EnumString, strum::Display)] +#[strum(ascii_case_insensitive)] +/// Schema version of the `iceberg_tables` catalog table. +pub enum SchemaVersion { + /// Original schema without the `iceberg_type` column. + V0, + /// Extended schema with the `iceberg_type` column for view support. + V1, +} + +impl SchemaVersion { + /// Detect the schema version of an existing catalog table by introspecting its columns. + async fn detect(pool: &AnyPool) -> Result { + let catalog_table_description = pool + .describe(&format!("SELECT * FROM {CATALOG_TABLE_NAME}")) + .await + .map_err(from_sqlx_error)?; + + let has_type_column = catalog_table_description.columns().iter().any(|column| { + column + .name() + .eq_ignore_ascii_case(CATALOG_FIELD_RECORD_TYPE) + }); + + Ok(if has_type_column { + SchemaVersion::V1 + } else { + SchemaVersion::V0 + }) + } + + /// The trailing SQL `AND` clause used to exclude view rows when querying for tables. + /// + /// `V1` schemas carry an `iceberg_type` column, so table rows are those tagged `TABLE` + /// (or `NULL`, for rows written before the column existed). `V0` schemas have no such + /// column, so no filter is applied. + fn record_type_filter(self) -> String { + match self { + SchemaVersion::V1 => format!( + "AND ({CATALOG_FIELD_RECORD_TYPE} = '{CATALOG_FIELD_TABLE_RECORD_TYPE}' \ + OR {CATALOG_FIELD_RECORD_TYPE} IS NULL)" + ), + SchemaVersion::V0 => String::new(), + } + } + + /// The SQL needed to migrate a `V0` catalog table up to this schema version. + /// + /// Returns `None` when the target version requires no migration (i.e. `V0`). + fn migration_sql(self) -> Option { + match self { + SchemaVersion::V1 => Some(format!( + "ALTER TABLE {CATALOG_TABLE_NAME} ADD COLUMN {CATALOG_FIELD_RECORD_TYPE} VARCHAR(5)" + )), + SchemaVersion::V0 => None, + } + } } -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, strum::EnumString, strum::Display)] /// Set the SQL parameter bind style to either $1..$N (Postgres style) or ? (SQLite/MySQL/MariaDB) pub enum SqlBindStyle { /// DollarNumeric uses parameters of the form `$1..$N``, which is the Postgres style @@ -94,29 +395,30 @@ pub enum SqlBindStyle { impl SqlCatalog { /// Create new sql catalog instance - pub async fn new(config: SqlCatalogConfig) -> Result { + async fn new( + name: String, + properties: SqlCatalogProperties, + props: HashMap, + storage_factory: Option>, + runtime: Runtime, + kms_client: Option>, + ) -> Result { + let factory = storage_factory.ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "StorageFactory must be provided for SqlCatalog. Use `with_storage_factory` to configure it.", + ) + })?; install_default_drivers(); - let max_connections: u32 = config - .props - .get("pool.max-connections") - .map(|v| v.parse().unwrap()) - .unwrap_or(MAX_CONNECTIONS); - let idle_timeout: u64 = config - .props - .get("pool.idle-timeout") - .map(|v| v.parse().unwrap()) - .unwrap_or(IDLE_TIMEOUT); - let test_before_acquire: bool = config - .props - .get("pool.test-before-acquire") - .map(|v| v.parse().unwrap()) - .unwrap_or(TEST_BEFORE_ACQUIRE); + // Forward the complete property map so storage-backend keys reach FileIO. + // Unrecognized keys are ignored by backends. + let fileio = FileIOBuilder::new(factory).with_props(props).build(); let pool = AnyPoolOptions::new() - .max_connections(max_connections) - .idle_timeout(Duration::from_secs(idle_timeout)) - .test_before_acquire(test_before_acquire) - .connect(&config.uri) + .max_connections(properties.max_connections) + .idle_timeout(Duration::from_secs(properties.idle_timeout)) + .test_before_acquire(properties.test_before_acquire) + .connect(&properties.uri) .await .map_err(from_sqlx_error)?; @@ -146,18 +448,66 @@ impl SqlCatalog { .await .map_err(from_sqlx_error)?; + let detected_schema_version = SchemaVersion::detect(&pool).await?; + let expected_schema_version = properties.schema_version; + + // Detect schema by describing columns. If expected is configured then automigrate, otherwise gracefully support older schemas. + let schema_version = match (detected_schema_version, expected_schema_version) { + (SchemaVersion::V1, Some(SchemaVersion::V1) | None) => { + tracing::debug!( + "detected {CATALOG_TABLE_NAME} schema {} which already supports views", + detected_schema_version, + ); + SchemaVersion::V1 + } + (SchemaVersion::V0, Some(expected_schema_version @ SchemaVersion::V1)) => { + tracing::warn!( + "table {CATALOG_TABLE_NAME} has inferred schema {} but expected schema {}, performing migration", + detected_schema_version, + expected_schema_version, + ); + if let Some(migration_sql) = SchemaVersion::V1.migration_sql() { + sqlx::query(&migration_sql) + .execute(&pool) + .await + .map_err(from_sqlx_error)?; + } + SchemaVersion::V1 + } + (SchemaVersion::V0, Some(SchemaVersion::V0) | None) => { + tracing::warn!( + "table {CATALOG_TABLE_NAME} has inferred schema {}; SQL catalog is initialized without view support, table creation, and table registration. \ + To auto-migrate the database schema, set {}=V1", + detected_schema_version, + SQL_CATALOG_PROP_SCHEMA_VERSION, + ); + SchemaVersion::V0 + } + (SchemaVersion::V1, Some(expected_schema_version @ SchemaVersion::V0)) => { + tracing::warn!( + "ignoring expected schema {} for table {CATALOG_TABLE_NAME}: the table is \ + already at schema {}, and downgrade migration is not supported", + expected_schema_version, + detected_schema_version, + ); + SchemaVersion::V1 + } + }; + Ok(SqlCatalog { - name: config.name.to_owned(), + name, + properties, connection: pool, - warehouse_location: config.warehouse_location, - fileio: config.file_io, - sql_bind_style: config.sql_bind_style, + fileio, + runtime, + kms_client, + schema_version, }) } /// SQLX Any does not implement PostgresSQL bindings, so we have to do this. fn replace_placeholders(&self, query: &str) -> String { - match self.sql_bind_style { + match self.properties.sql_bind_style { SqlBindStyle::DollarNumeric => { let mut count = 1; query @@ -210,9 +560,12 @@ impl SqlCatalog { Some(t) => sqlx_query.execute(&mut **t).await.map_err(from_sqlx_error), None => { let mut tx = self.connection.begin().await.map_err(from_sqlx_error)?; - let result = sqlx_query.execute(&mut *tx).await.map_err(from_sqlx_error); - let _ = tx.commit().await.map_err(from_sqlx_error); - result + let result = sqlx_query + .execute(&mut *tx) + .await + .map_err(from_sqlx_error)?; + tx.commit().await.map_err(from_sqlx_error)?; + Ok(result) } } } @@ -283,8 +636,8 @@ impl Catalog for SqlCatalog { if exists { return Err(Error::new( - iceberg::ErrorKind::Unexpected, - format!("Namespace {:?} already exists", namespace), + ErrorKind::NamespaceAlreadyExists, + format!("Namespace {namespace:?} already exists"), )); } @@ -433,7 +786,7 @@ impl Catalog for SqlCatalog { let mut tx = self.connection.begin().await.map_err(from_sqlx_error)?; let update_stmt = format!( "UPDATE {NAMESPACE_TABLE_NAME} SET {NAMESPACE_FIELD_PROPERTY_VALUE} = ? - WHERE {CATALOG_FIELD_CATALOG_NAME} = ? + WHERE {CATALOG_FIELD_CATALOG_NAME} = ? AND {NAMESPACE_FIELD_NAME} = ? AND {NAMESPACE_FIELD_PROPERTY_KEY} = ?" ); @@ -486,7 +839,7 @@ impl Catalog for SqlCatalog { let tables = self.list_tables(namespace).await?; if !tables.is_empty() { return Err(Error::new( - iceberg::ErrorKind::Unexpected, + ErrorKind::Unexpected, format!( "Namespace {:?} is not empty. {} tables exist.", namespace, @@ -523,10 +876,8 @@ impl Catalog for SqlCatalog { FROM {CATALOG_TABLE_NAME} WHERE {CATALOG_FIELD_TABLE_NAMESPACE} = ? AND {CATALOG_FIELD_CATALOG_NAME} = ? - AND ( - {CATALOG_FIELD_RECORD_TYPE} = '{CATALOG_FIELD_TABLE_RECORD_TYPE}' - OR {CATALOG_FIELD_RECORD_TYPE} IS NULL - )", + {}", + self.schema_version.record_type_filter() ), vec![Some(&namespace.join(".")), Some(&self.name)], ) @@ -562,10 +913,8 @@ impl Catalog for SqlCatalog { WHERE {CATALOG_FIELD_TABLE_NAMESPACE} = ? AND {CATALOG_FIELD_CATALOG_NAME} = ? AND {CATALOG_FIELD_TABLE_NAME} = ? - AND ( - {CATALOG_FIELD_RECORD_TYPE} = '{CATALOG_FIELD_TABLE_RECORD_TYPE}' - OR {CATALOG_FIELD_RECORD_TYPE} IS NULL - )" + {}", + self.schema_version.record_type_filter() ), vec![Some(&namespace), Some(&self.name), Some(table_name)], ) @@ -589,10 +938,8 @@ impl Catalog for SqlCatalog { WHERE {CATALOG_FIELD_CATALOG_NAME} = ? AND {CATALOG_FIELD_TABLE_NAME} = ? AND {CATALOG_FIELD_TABLE_NAMESPACE} = ? - AND ( - {CATALOG_FIELD_RECORD_TYPE} = '{CATALOG_FIELD_TABLE_RECORD_TYPE}' - OR {CATALOG_FIELD_RECORD_TYPE} IS NULL - )" + {}", + self.schema_version.record_type_filter() ), vec![ Some(&self.name), @@ -606,6 +953,12 @@ impl Catalog for SqlCatalog { Ok(()) } + async fn purge_table(&self, table: &TableIdent) -> Result<()> { + let table_info = self.load_table(table).await?; + self.drop_table(table).await?; + iceberg::drop_table_data(&table_info).await + } + async fn load_table(&self, identifier: &TableIdent) -> Result
{ if !self.table_exists(identifier).await? { return no_such_table_err(identifier); @@ -619,10 +972,8 @@ impl Catalog for SqlCatalog { WHERE {CATALOG_FIELD_CATALOG_NAME} = ? AND {CATALOG_FIELD_TABLE_NAME} = ? AND {CATALOG_FIELD_TABLE_NAMESPACE} = ? - AND ( - {CATALOG_FIELD_RECORD_TYPE} = '{CATALOG_FIELD_TABLE_RECORD_TYPE}' - OR {CATALOG_FIELD_RECORD_TYPE} IS NULL - )" + {}", + self.schema_version.record_type_filter() ), vec![ Some(&self.name), @@ -643,12 +994,16 @@ impl Catalog for SqlCatalog { let metadata = TableMetadata::read_from(&self.fileio, &tbl_metadata_location).await?; - Ok(Table::builder() + let mut builder = Table::builder() .file_io(self.fileio.clone()) .identifier(identifier.clone()) .metadata_location(tbl_metadata_location) .metadata(metadata) - .build()?) + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + Ok(builder.build()?) } async fn create_table( @@ -656,6 +1011,16 @@ impl Catalog for SqlCatalog { namespace: &NamespaceIdent, creation: TableCreation, ) -> Result
{ + if self.schema_version != SchemaVersion::V1 { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Table creation is not supported for SQL catalog schema version {}", + self.schema_version + ), + )); + } + if !self.namespace_exists(namespace).await? { return no_such_namespace_err(namespace); } @@ -667,57 +1032,57 @@ impl Catalog for SqlCatalog { return table_already_exists_err(&tbl_ident); } - let (tbl_creation, location) = match creation.location.clone() { - Some(location) => (creation, location), - None => { - // fall back to namespace-specific location - // and then to warehouse location - let nsp_properties = self.get_namespace(namespace).await?.properties().clone(); - let nsp_location = match nsp_properties.get(NAMESPACE_LOCATION_PROPERTY_KEY) { - Some(location) => location.clone(), - None => { - format!( - "{}/{}", - self.warehouse_location.clone(), - namespace.join("/") - ) - } - }; + let tbl_creation = if creation.location.is_some() { + creation + } else { + // fall back to namespace-specific location + // and then to warehouse location + let nsp_properties = self.get_namespace(namespace).await?.properties().clone(); + let nsp_location = match nsp_properties.get(NAMESPACE_LOCATION_PROPERTY_KEY) { + Some(location) => location.clone(), + None => { + format!( + "{}/{}", + &self.properties.warehouse_location, + namespace.join("/") + ) + } + }; - let tbl_location = format!("{}/{}", nsp_location, tbl_ident.name()); + let tbl_location = format!("{}/{}", nsp_location, tbl_ident.name()); - ( - TableCreation { - location: Some(tbl_location.clone()), - ..creation - }, - tbl_location, - ) + TableCreation { + location: Some(tbl_location), + ..creation } }; let tbl_metadata = TableMetadataBuilder::from_table_creation(tbl_creation)? .build()? .metadata; - let tbl_metadata_location = - MetadataLocation::new_with_table_location(location.clone()).to_string(); + let tbl_metadata_location = MetadataLocation::try_new_with_metadata(&tbl_metadata)?; tbl_metadata .write_to(&self.fileio, &tbl_metadata_location) .await?; + let tbl_metadata_location_str = tbl_metadata_location.to_string(); self.execute(&format!( "INSERT INTO {CATALOG_TABLE_NAME} ({CATALOG_FIELD_CATALOG_NAME}, {CATALOG_FIELD_TABLE_NAMESPACE}, {CATALOG_FIELD_TABLE_NAME}, {CATALOG_FIELD_METADATA_LOCATION_PROP}, {CATALOG_FIELD_RECORD_TYPE}) VALUES (?, ?, ?, ?, ?) - "), vec![Some(&self.name), Some(&namespace.join(".")), Some(&tbl_name.clone()), Some(&tbl_metadata_location), Some(CATALOG_FIELD_TABLE_RECORD_TYPE)], None).await?; + "), vec![Some(&self.name), Some(&namespace.join(".")), Some(&tbl_name.clone()), Some(&tbl_metadata_location_str), Some(CATALOG_FIELD_TABLE_RECORD_TYPE)], None).await?; - Ok(Table::builder() + let mut builder = Table::builder() .file_io(self.fileio.clone()) - .metadata_location(tbl_metadata_location) + .metadata_location(tbl_metadata_location_str) .identifier(tbl_ident) .metadata(tbl_metadata) - .build()?) + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + Ok(builder.build()?) } async fn rename_table(&self, src: &TableIdent, dest: &TableIdent) -> Result<()> { @@ -744,10 +1109,8 @@ impl Catalog for SqlCatalog { WHERE {CATALOG_FIELD_CATALOG_NAME} = ? AND {CATALOG_FIELD_TABLE_NAME} = ? AND {CATALOG_FIELD_TABLE_NAMESPACE} = ? - AND ( - {CATALOG_FIELD_RECORD_TYPE} = '{CATALOG_FIELD_TABLE_RECORD_TYPE}' - OR {CATALOG_FIELD_RECORD_TYPE} IS NULL - )" + {}", + self.schema_version.record_type_filter() ), vec![ Some(dest.name()), @@ -765,20 +1128,103 @@ impl Catalog for SqlCatalog { async fn register_table( &self, - _table_ident: &TableIdent, - _metadata_location: String, + table_ident: &TableIdent, + metadata_location: String, ) -> Result
{ - Err(Error::new( - ErrorKind::FeatureUnsupported, - "Registering a table is not supported yet", - )) + if self.schema_version != SchemaVersion::V1 { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Table registration is not supported for SQL catalog schema version {}", + self.schema_version + ), + )); + } + + if self.table_exists(table_ident).await? { + return table_already_exists_err(table_ident); + } + + let metadata = TableMetadata::read_from(&self.fileio, &metadata_location).await?; + + let namespace = table_ident.namespace(); + let tbl_name = table_ident.name().to_string(); + + self.execute(&format!( + "INSERT INTO {CATALOG_TABLE_NAME} + ({CATALOG_FIELD_CATALOG_NAME}, {CATALOG_FIELD_TABLE_NAMESPACE}, {CATALOG_FIELD_TABLE_NAME}, {CATALOG_FIELD_METADATA_LOCATION_PROP}, {CATALOG_FIELD_RECORD_TYPE}) + VALUES (?, ?, ?, ?, ?) + "), vec![Some(&self.name), Some(&namespace.join(".")), Some(&tbl_name), Some(&metadata_location), Some(CATALOG_FIELD_TABLE_RECORD_TYPE)], None).await?; + + let mut builder = Table::builder() + .identifier(table_ident.clone()) + .metadata_location(metadata_location) + .metadata(metadata) + .file_io(self.fileio.clone()) + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + Ok(builder.build()?) } - async fn update_table(&self, _commit: TableCommit) -> Result
{ - Err(Error::new( - ErrorKind::FeatureUnsupported, - "Updating a table is not supported yet", - )) + /// Updates an existing table within the SQL catalog. + async fn update_table(&self, commit: TableCommit) -> Result
{ + let table_ident = commit.identifier().clone(); + let current_table = self.load_table(&table_ident).await?; + let current_metadata_location = current_table.metadata_location_result()?.to_string(); + let base_metadata = current_table.metadata_ref(); + + let staged_table = commit.apply(current_table)?; + let staged_metadata_location_str = staged_table.metadata_location_result()?; + let staged_metadata_location = MetadataLocation::from_str(staged_metadata_location_str)?; + + staged_table + .metadata() + .write_to(staged_table.file_io(), &staged_metadata_location) + .await?; + + let staged_metadata_location_str = staged_metadata_location.to_string(); + let update_result = self + .execute( + &format!( + "UPDATE {CATALOG_TABLE_NAME} + SET {CATALOG_FIELD_METADATA_LOCATION_PROP} = ?, {CATALOG_FIELD_PREVIOUS_METADATA_LOCATION_PROP} = ? + WHERE {CATALOG_FIELD_CATALOG_NAME} = ? + AND {CATALOG_FIELD_TABLE_NAME} = ? + AND {CATALOG_FIELD_TABLE_NAMESPACE} = ? + {} + AND {CATALOG_FIELD_METADATA_LOCATION_PROP} = ?", + self.schema_version.record_type_filter() + ), + vec![ + Some(&staged_metadata_location_str), + Some(current_metadata_location.as_str()), + Some(&self.name), + Some(table_ident.name()), + Some(&table_ident.namespace().join(".")), + Some(current_metadata_location.as_str()), + ], + None, + ) + .await?; + + if update_result.rows_affected() == 0 { + return Err(Error::new( + ErrorKind::CatalogCommitConflicts, + format!("Commit conflicted for table: {table_ident}"), + ) + .with_retryable(true)); + } + + delete_removed_metadata_files( + staged_table.file_io(), + &base_metadata, + staged_table.metadata(), + ) + .await; + + Ok(staged_table) } } @@ -786,18 +1232,28 @@ impl Catalog for SqlCatalog { mod tests { use std::collections::{HashMap, HashSet}; use std::hash::Hash; + use std::sync::Arc; - use iceberg::io::FileIOBuilder; + use iceberg::io::LocalFsStorageFactory; use iceberg::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, Type}; use iceberg::table::Table; - use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation, TableIdent}; + use iceberg::{ + Catalog, CatalogBuilder, ErrorKind, Namespace, NamespaceIdent, TableCreation, TableIdent, + }; use itertools::Itertools; use regex::Regex; + use sqlx::any::install_default_drivers; use sqlx::migrate::MigrateDatabase; + use sqlx::{Column, Executor}; use tempfile::TempDir; - use crate::catalog::NAMESPACE_LOCATION_PROPERTY_KEY; - use crate::{SqlBindStyle, SqlCatalog, SqlCatalogConfig}; + use crate::catalog::{ + CATALOG_FIELD_RECORD_TYPE, CATALOG_TABLE_NAME, IDLE_TIMEOUT, MAX_CONNECTIONS, + NAMESPACE_LOCATION_PROPERTY_KEY, NAMESPACE_TABLE_NAME, SQL_CATALOG_PROP_BIND_STYLE, + SQL_CATALOG_PROP_BIND_STYLE_LEGACY, SQL_CATALOG_PROP_SCHEMA_VERSION, SQL_CATALOG_PROP_URI, + SQL_CATALOG_PROP_WAREHOUSE, SqlCatalogProperties, TEST_BEFORE_ACQUIRE, + }; + use crate::{SchemaVersion, SqlBindStyle, SqlCatalog, SqlCatalogBuilder}; const UUID_REGEX_STR: &str = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; @@ -806,7 +1262,86 @@ mod tests { temp_dir.path().to_str().unwrap().to_string() } - fn to_set(vec: Vec) -> HashSet { + #[test] + fn test_catalog_properties() { + let properties = SqlCatalogProperties::from_properties(&HashMap::from([ + ( + SQL_CATALOG_PROP_URI.to_string(), + "sqlite://catalog".to_string(), + ), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + "/warehouse".to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ( + SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(), + "V1".to_string(), + ), + ("pool.max-connections".to_string(), "5".to_string()), + ("pool.idle-timeout".to_string(), "20".to_string()), + ("pool.test-before-acquire".to_string(), "false".to_string()), + ])) + .unwrap(); + + assert_eq!(properties.uri, "sqlite://catalog"); + assert_eq!(properties.warehouse_location, "/warehouse"); + assert_eq!(properties.sql_bind_style, SqlBindStyle::QMark); + assert_eq!(properties.schema_version, Some(SchemaVersion::V1)); + assert_eq!(properties.max_connections, 5); + assert_eq!(properties.idle_timeout, 20); + assert!(!properties.test_before_acquire); + } + + #[test] + fn test_catalog_properties_defaults() { + let properties = SqlCatalogProperties::from_properties(&HashMap::new()).unwrap(); + + assert_eq!(properties.uri, ""); + assert_eq!(properties.warehouse_location, ""); + assert_eq!(properties.sql_bind_style, SqlBindStyle::DollarNumeric); + assert_eq!(properties.schema_version, None); + assert_eq!(properties.max_connections, MAX_CONNECTIONS); + assert_eq!(properties.idle_timeout, IDLE_TIMEOUT); + assert_eq!(properties.test_before_acquire, TEST_BEFORE_ACQUIRE); + } + + #[test] + fn test_preferred_bind_style_key_takes_precedence_over_legacy_key() { + let properties = SqlCatalogProperties::from_properties(&HashMap::from([ + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(), + SqlBindStyle::DollarNumeric.to_string(), + ), + ])) + .unwrap(); + + assert_eq!(properties.sql_bind_style, SqlBindStyle::QMark); + } + + #[test] + fn test_invalid_legacy_bind_style_error_names_configured_key() { + let error = SqlCatalogProperties::from_properties(&HashMap::from([( + SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(), + "invalid".to_string(), + )])) + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert_eq!( + error.message(), + "`sql_bind_style` values are valid only if they're `DollarNumeric` or `QMark`" + ); + } + + fn to_set(vec: Vec) -> HashSet { HashSet::from_iter(vec) } @@ -814,19 +1349,32 @@ mod tests { HashMap::from([("exists".to_string(), "true".to_string())]) } - async fn new_sql_catalog(warehouse_location: String) -> impl Catalog { + /// Create a new SQLite catalog for testing. If name is not specified it defaults to "iceberg". + async fn new_sql_catalog( + warehouse_location: String, + name: Option, + ) -> impl Catalog { + let name = if let Some(name) = name { + name.to_string() + } else { + "iceberg".to_string() + }; let sql_lite_uri = format!("sqlite:{}", temp_path()); sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); - let config = SqlCatalogConfig::builder() - .uri(sql_lite_uri.to_string()) - .name("iceberg".to_string()) - .warehouse_location(warehouse_location) - .file_io(FileIOBuilder::new_fs_io().build().unwrap()) - .sql_bind_style(SqlBindStyle::QMark) - .build(); - - SqlCatalog::new(config).await.unwrap() + let props = HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.to_string()), + (SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::DollarNumeric.to_string(), + ), + ]); + SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load(&name, props) + .await + .unwrap() } async fn create_namespace(catalog: &C, namespace_ident: &NamespaceIdent) { @@ -919,84 +1467,449 @@ mod tests { #[tokio::test] async fn test_initialized() { let warehouse_loc = temp_path(); - new_sql_catalog(warehouse_loc.clone()).await; + new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; // catalog instantiation should not fail even if tables exist - new_sql_catalog(warehouse_loc.clone()).await; - new_sql_catalog(warehouse_loc.clone()).await; + new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; + new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; } - #[tokio::test] - async fn test_list_namespaces_returns_empty_vector() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + async fn new_commit_error_catalog() -> SqlCatalog { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .prop("pool.max-connections", "1") + .load( + "iceberg", + HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri), + (SQL_CATALOG_PROP_WAREHOUSE.to_string(), temp_path()), + ]), + ) + .await + .unwrap(); - assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![]); + catalog + .connection + .execute("PRAGMA foreign_keys = ON") + .await + .unwrap(); + // This deferred constraint lets an INSERT succeed while COMMIT fails. + catalog + .connection + .execute("CREATE TABLE parent(id INTEGER PRIMARY KEY)") + .await + .unwrap(); + catalog + .connection + .execute( + "CREATE TABLE child(parent_id INTEGER REFERENCES parent(id) \ + DEFERRABLE INITIALLY DEFERRED)", + ) + .await + .unwrap(); + + catalog } #[tokio::test] - async fn test_list_namespaces_returns_multiple_namespaces() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident_1 = NamespaceIdent::new("a".into()); - let namespace_ident_2 = NamespaceIdent::new("b".into()); - create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await; + async fn test_execute_returns_commit_error() { + let catalog = new_commit_error_catalog().await; + + // Make the public namespace operation insert a child row whose deferred + // foreign-key constraint succeeds during execution but fails at commit. + let trigger = format!( + "CREATE TRIGGER fail_namespace_commit + AFTER INSERT ON {NAMESPACE_TABLE_NAME} + BEGIN INSERT INTO child VALUES (1); END" + ); + catalog.connection.execute(trigger.as_str()).await.unwrap(); - assert_eq!( - to_set(catalog.list_namespaces(None).await.unwrap()), - to_set(vec![namespace_ident_1, namespace_ident_2]) + let failed_namespace = NamespaceIdent::new("failed".into()); + let error = catalog + .create_namespace(&failed_namespace, HashMap::new()) + .await + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(!catalog.namespace_exists(&failed_namespace).await.unwrap()); + + // A valid relationship confirms that successful transactions still commit. + catalog + .connection + .execute("INSERT INTO parent VALUES (1)") + .await + .unwrap(); + let committed_namespace = NamespaceIdent::new("committed".into()); + catalog + .create_namespace(&committed_namespace, HashMap::new()) + .await + .unwrap(); + assert!( + catalog + .namespace_exists(&committed_namespace) + .await + .unwrap() ); } + // Regression test: storage-backend props set on the catalog must reach + // the FileIO; otherwise authenticated backends fail with 401s on writes. #[tokio::test] - async fn test_list_namespaces_returns_only_top_level_namespaces() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident_1 = NamespaceIdent::new("a".into()); - let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); - let namespace_ident_3 = NamespaceIdent::new("b".into()); - create_namespaces(&catalog, &vec![ - &namespace_ident_1, - &namespace_ident_2, - &namespace_ident_3, - ]) - .await; + async fn test_storage_props_propagate_to_file_io() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let warehouse_location = temp_path(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "iceberg", + HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + warehouse_location.clone(), + ), + ("s3.region".to_string(), "us-east-1".to_string()), + ("hf.token".to_string(), "hf_test_token".to_string()), + ]), + ) + .await + .unwrap(); + let props = catalog.fileio.config().props(); + assert_eq!(props.get(SQL_CATALOG_PROP_URI), None); assert_eq!( - to_set(catalog.list_namespaces(None).await.unwrap()), - to_set(vec![namespace_ident_1, namespace_ident_3]) + props.get(SQL_CATALOG_PROP_WAREHOUSE), + Some(&warehouse_location) ); + assert_eq!(props.get("s3.region"), Some(&"us-east-1".to_string())); + assert_eq!(props.get("hf.token"), Some(&"hf_test_token".to_string())); } #[tokio::test] - async fn test_list_namespaces_returns_no_namespaces_under_parent() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident_1 = NamespaceIdent::new("a".into()); - let namespace_ident_2 = NamespaceIdent::new("b".into()); - create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await; + async fn test_builder_method() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let warehouse_location = temp_path(); - assert_eq!( - catalog - .list_namespaces(Some(&namespace_ident_1)) - .await - .unwrap(), - vec![] - ); + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .uri(sql_lite_uri.to_string()) + .warehouse_location(warehouse_location.clone()) + .sql_bind_style(SqlBindStyle::QMark) + .load("iceberg", HashMap::default()) + .await; + assert!(catalog.is_ok()); + + let catalog = catalog.unwrap(); + assert!(catalog.properties.warehouse_location == warehouse_location); + assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark); } + /// Overwriting an sqlite database with a non-existent path causes + /// catalog generation to fail #[tokio::test] - async fn test_list_namespaces_returns_namespace_under_parent() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident_1 = NamespaceIdent::new("a".into()); - let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); - let namespace_ident_3 = NamespaceIdent::new("c".into()); - create_namespaces(&catalog, &vec![ - &namespace_ident_1, - &namespace_ident_2, - &namespace_ident_3, - ]) - .await; + async fn test_builder_props_non_existent_path_fails() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + let sql_lite_uri2 = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let warehouse_location = temp_path(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .uri(sql_lite_uri) + .warehouse_location(warehouse_location) + .load( + "iceberg", + HashMap::from_iter([(SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri2)]), + ) + .await; + assert!(catalog.is_err()); + } + + /// Even when an invalid URI is specified in a builder method, + /// it can be successfully overridden with a valid URI in props + /// for catalog generation to succeed. + #[tokio::test] + async fn test_builder_props_set_valid_uri() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + let sql_lite_uri2 = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let warehouse_location = temp_path(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .uri(sql_lite_uri2) + .warehouse_location(warehouse_location) + .load( + "iceberg", + HashMap::from_iter([(SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.clone())]), + ) + .await; + assert!(catalog.is_ok()); + } + + /// values assigned via props take precedence + #[tokio::test] + async fn test_builder_props_take_precedence() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let warehouse_location = temp_path(); + let warehouse_location2 = temp_path(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .warehouse_location(warehouse_location2) + .sql_bind_style(SqlBindStyle::DollarNumeric) + .load( + "iceberg", + HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + warehouse_location.clone(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]), + ) + .await; + + assert!(catalog.is_ok()); + + let catalog = catalog.unwrap(); + assert!(catalog.properties.warehouse_location == warehouse_location); + assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark); + } + + #[tokio::test] + async fn test_load_legacy_bind_style_overrides_builder_method() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .sql_bind_style(SqlBindStyle::DollarNumeric) + .load( + "iceberg", + HashMap::from([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri), + (SQL_CATALOG_PROP_WAREHOUSE.to_string(), temp_path()), + ( + SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]), + ) + .await + .unwrap(); + + assert_eq!(catalog.properties.sql_bind_style, SqlBindStyle::QMark); + } + + /// values assigned via props take precedence + #[tokio::test] + async fn test_builder_props_take_precedence_props() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + let sql_lite_uri2 = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let warehouse_location = temp_path(); + let warehouse_location2 = temp_path(); + + let props = HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.clone()), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + warehouse_location.clone(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]); + let props2 = HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri2.clone()), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + warehouse_location2.clone(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::DollarNumeric.to_string(), + ), + ]); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .props(props2) + .load("iceberg", props) + .await; + + assert!(catalog.is_ok()); + + let catalog = catalog.unwrap(); + assert!(catalog.properties.warehouse_location == warehouse_location); + assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark); + } + + /// values assigned via props take precedence + #[tokio::test] + async fn test_builder_props_take_precedence_prop() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + let sql_lite_uri2 = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let warehouse_location = temp_path(); + let warehouse_location2 = temp_path(); + + let props = HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.clone()), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + warehouse_location.clone(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .prop(SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri2) + .prop(SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location2) + .prop( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::DollarNumeric.to_string(), + ) + .load("iceberg", props) + .await; + + assert!(catalog.is_ok()); + + let catalog = catalog.unwrap(); + assert!(catalog.properties.warehouse_location == warehouse_location); + assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark); + } + + /// invalid value for `SqlBindStyle` causes catalog creation to fail + #[tokio::test] + async fn test_builder_props_invalid_bind_style_fails() { + let sql_lite_uri = format!("sqlite:{}", temp_path()); + sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap(); + let warehouse_location = temp_path(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "iceberg", + HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri), + (SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location), + (SQL_CATALOG_PROP_BIND_STYLE.to_string(), "AAA".to_string()), + ]), + ) + .await; + + assert!(catalog.is_err()); + } + + #[tokio::test] + async fn test_builder_props_invalid_pool_property_fails() { + for property in [ + "pool.max-connections", + "pool.idle-timeout", + "pool.test-before-acquire", + ] { + let error = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .prop(property, "invalid") + .load("iceberg", HashMap::new()) + .await + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.to_string().contains(property)); + assert!(error.to_string().contains("invalid")); + } + } + + #[tokio::test] + async fn test_list_namespaces_returns_empty_vector() { + let warehouse_loc = temp_path(); + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; + + assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![]); + } + + #[tokio::test] + async fn test_list_namespaces_returns_empty_different_name() { + let warehouse_loc = temp_path(); + let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; + let namespace_ident_1 = NamespaceIdent::new("a".into()); + let namespace_ident_2 = NamespaceIdent::new("b".into()); + create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await; + assert_eq!( + to_set(catalog.list_namespaces(None).await.unwrap()), + to_set(vec![namespace_ident_1, namespace_ident_2]) + ); + + let catalog2 = new_sql_catalog(warehouse_loc, Some("test")).await; + assert_eq!(catalog2.list_namespaces(None).await.unwrap(), vec![]); + } + + #[tokio::test] + async fn test_list_namespaces_returns_only_top_level_namespaces() { + let warehouse_loc = temp_path(); + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; + let namespace_ident_1 = NamespaceIdent::new("a".into()); + let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); + let namespace_ident_3 = NamespaceIdent::new("b".into()); + create_namespaces(&catalog, &vec![ + &namespace_ident_1, + &namespace_ident_2, + &namespace_ident_3, + ]) + .await; + + assert_eq!( + to_set(catalog.list_namespaces(None).await.unwrap()), + to_set(vec![namespace_ident_1, namespace_ident_3]) + ); + } + + #[tokio::test] + async fn test_list_namespaces_returns_no_namespaces_under_parent() { + let warehouse_loc = temp_path(); + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; + let namespace_ident_1 = NamespaceIdent::new("a".into()); + let namespace_ident_2 = NamespaceIdent::new("b".into()); + create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await; + + assert_eq!( + catalog + .list_namespaces(Some(&namespace_ident_1)) + .await + .unwrap(), + vec![] + ); + } + + #[tokio::test] + async fn test_list_namespaces_returns_namespace_under_parent() { + let warehouse_loc = temp_path(); + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; + let namespace_ident_1 = NamespaceIdent::new("a".into()); + let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); + let namespace_ident_3 = NamespaceIdent::new("c".into()); + create_namespaces(&catalog, &vec![ + &namespace_ident_1, + &namespace_ident_2, + &namespace_ident_3, + ]) + .await; assert_eq!( to_set(catalog.list_namespaces(None).await.unwrap()), @@ -1015,7 +1928,7 @@ mod tests { #[tokio::test] async fn test_list_namespaces_returns_multiple_namespaces_under_parent() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident_1 = NamespaceIdent::new("a".to_string()); let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "a"]).unwrap(); let namespace_ident_3 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); @@ -1048,7 +1961,7 @@ mod tests { #[tokio::test] async fn test_namespace_exists_returns_false() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident = NamespaceIdent::new("a".into()); create_namespace(&catalog, &namespace_ident).await; @@ -1063,7 +1976,7 @@ mod tests { #[tokio::test] async fn test_namespace_exists_returns_true() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident = NamespaceIdent::new("a".into()); create_namespace(&catalog, &namespace_ident).await; @@ -1073,7 +1986,7 @@ mod tests { #[tokio::test] async fn test_create_namespace_with_properties() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident = NamespaceIdent::new("abc".into()); let mut properties = default_properties(); @@ -1093,35 +2006,10 @@ mod tests { ); } - #[tokio::test] - async fn test_create_namespace_throws_error_if_namespace_already_exists() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - - assert_eq!( - catalog - .create_namespace(&namespace_ident, HashMap::new()) - .await - .unwrap_err() - .to_string(), - format!( - "Unexpected => Namespace {:?} already exists", - &namespace_ident - ) - ); - - assert_eq!( - catalog.get_namespace(&namespace_ident).await.unwrap(), - Namespace::with_properties(namespace_ident, default_properties()) - ); - } - #[tokio::test] async fn test_create_nested_namespace() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let parent_namespace_ident = NamespaceIdent::new("a".into()); create_namespace(&catalog, &parent_namespace_ident).await; @@ -1144,7 +2032,7 @@ mod tests { #[tokio::test] async fn test_create_deeply_nested_namespace() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident_a = NamespaceIdent::new("a".into()); let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await; @@ -1168,7 +2056,7 @@ mod tests { #[tokio::test] async fn test_update_namespace_noop() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident = NamespaceIdent::new("a".into()); create_namespace(&catalog, &namespace_ident).await; @@ -1187,39 +2075,10 @@ mod tests { ) } - #[tokio::test] - async fn test_update_namespace() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - - let mut props = HashMap::from_iter([ - ("prop1".to_string(), "val1".to_string()), - ("prop2".into(), "val2".into()), - ]); - - catalog - .update_namespace(&namespace_ident, props.clone()) - .await - .unwrap(); - - props.insert("exists".into(), "true".into()); - - assert_eq!( - *catalog - .get_namespace(&namespace_ident) - .await - .unwrap() - .properties(), - props - ) - } - #[tokio::test] async fn test_update_nested_namespace() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident = NamespaceIdent::from_strs(["a", "b"]).unwrap(); create_namespace(&catalog, &namespace_ident).await; @@ -1245,32 +2104,10 @@ mod tests { ) } - #[tokio::test] - async fn test_update_namespace_errors_if_namespace_doesnt_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident = NamespaceIdent::new("a".into()); - - let props = HashMap::from_iter([ - ("prop1".to_string(), "val1".to_string()), - ("prop2".into(), "val2".into()), - ]); - - let err = catalog - .update_namespace(&namespace_ident, props) - .await - .unwrap_err(); - - assert_eq!( - err.message(), - format!("No such namespace: {:?}", namespace_ident) - ); - } - #[tokio::test] async fn test_update_namespace_errors_if_nested_namespace_doesnt_exist() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident = NamespaceIdent::from_strs(["a", "b"]).unwrap(); let props = HashMap::from_iter([ @@ -1285,26 +2122,14 @@ mod tests { assert_eq!( err.message(), - format!("No such namespace: {:?}", namespace_ident) + format!("No such namespace: {namespace_ident:?}") ); } - #[tokio::test] - async fn test_drop_namespace() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident = NamespaceIdent::new("abc".into()); - create_namespace(&catalog, &namespace_ident).await; - - catalog.drop_namespace(&namespace_ident).await.unwrap(); - - assert!(!catalog.namespace_exists(&namespace_ident).await.unwrap()) - } - #[tokio::test] async fn test_drop_nested_namespace() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident_a = NamespaceIdent::new("a".into()); let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await; @@ -1324,7 +2149,7 @@ mod tests { #[tokio::test] async fn test_drop_deeply_nested_namespace() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident_a = NamespaceIdent::new("a".into()); let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap(); @@ -1357,29 +2182,10 @@ mod tests { assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap()); } - #[tokio::test] - async fn test_drop_namespace_throws_error_if_namespace_doesnt_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - - let non_existent_namespace_ident = NamespaceIdent::new("abc".into()); - assert_eq!( - catalog - .drop_namespace(&non_existent_namespace_ident) - .await - .unwrap_err() - .to_string(), - format!( - "Unexpected => No such namespace: {:?}", - non_existent_namespace_ident - ) - ) - } - #[tokio::test] async fn test_drop_namespace_throws_error_if_nested_namespace_doesnt_exist() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; create_namespace(&catalog, &NamespaceIdent::new("a".into())).await; let non_existent_namespace_ident = @@ -1390,17 +2196,14 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "Unexpected => No such namespace: {:?}", - non_existent_namespace_ident - ) + format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}") ) } #[tokio::test] async fn test_dropping_a_namespace_does_not_drop_namespaces_nested_under_that_one() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident_a = NamespaceIdent::new("a".into()); let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await; @@ -1418,97 +2221,26 @@ mod tests { } #[tokio::test] - async fn test_list_tables_returns_empty_vector() { + async fn test_create_table_falls_back_to_namespace_location_if_table_location_is_missing() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - - assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![]); - } + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - #[tokio::test] - async fn test_list_tables_throws_error_if_namespace_doesnt_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let namespace_ident = NamespaceIdent::new("a".into()); + let mut namespace_properties = HashMap::new(); + let namespace_location = temp_path(); + namespace_properties.insert( + NAMESPACE_LOCATION_PROPERTY_KEY.to_string(), + namespace_location.to_string(), + ); + catalog + .create_namespace(&namespace_ident, namespace_properties) + .await + .unwrap(); - let non_existent_namespace_ident = NamespaceIdent::new("n1".into()); - - assert_eq!( - catalog - .list_tables(&non_existent_namespace_ident) - .await - .unwrap_err() - .to_string(), - format!( - "Unexpected => No such namespace: {:?}", - non_existent_namespace_ident - ), - ); - } - - #[tokio::test] - async fn test_create_table_with_location() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone()).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - - let table_name = "abc"; - let location = warehouse_loc.clone(); - let table_creation = TableCreation::builder() - .name(table_name.into()) - .location(location.clone()) - .schema(simple_table_schema()) - .build(); - - let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - - assert_table_eq( - &catalog - .create_table(&namespace_ident, table_creation) - .await - .unwrap(), - &expected_table_ident, - &simple_table_schema(), - ); - - let table = catalog.load_table(&expected_table_ident).await.unwrap(); - - assert_table_eq(&table, &expected_table_ident, &simple_table_schema()); - - assert!( - table - .metadata_location() - .unwrap() - .to_string() - .starts_with(&location) - ) - } - - #[tokio::test] - async fn test_create_table_falls_back_to_namespace_location_if_table_location_is_missing() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - - let namespace_ident = NamespaceIdent::new("a".into()); - let mut namespace_properties = HashMap::new(); - let namespace_location = temp_path(); - namespace_properties.insert( - NAMESPACE_LOCATION_PROPERTY_KEY.to_string(), - namespace_location.to_string(), - ); - catalog - .create_namespace(&namespace_ident, namespace_properties) - .await - .unwrap(); - - let table_name = "tbl1"; - let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - let expected_table_metadata_location_regex = format!( - "^{}/tbl1/metadata/00000-{}.metadata.json$", - namespace_location, UUID_REGEX_STR, - ); + let table_name = "tbl1"; + let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); + let expected_table_metadata_location_regex = + format!("^{namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",); let table = catalog .create_table( @@ -1533,7 +2265,7 @@ mod tests { async fn test_create_table_in_nested_namespace_falls_back_to_nested_namespace_location_if_table_location_is_missing() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident = NamespaceIdent::new("a".into()); let mut namespace_properties = HashMap::new(); @@ -1563,8 +2295,7 @@ mod tests { let expected_table_ident = TableIdent::new(nested_namespace_ident.clone(), table_name.into()); let expected_table_metadata_location_regex = format!( - "^{}/tbl1/metadata/00000-{}.metadata.json$", - nested_namespace_location, UUID_REGEX_STR, + "^{nested_namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$", ); let table = catalog @@ -1590,7 +2321,7 @@ mod tests { async fn test_create_table_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone()).await; + let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; let namespace_ident = NamespaceIdent::new("a".into()); // note: no location specified in namespace_properties @@ -1602,10 +2333,8 @@ mod tests { let table_name = "tbl1"; let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - let expected_table_metadata_location_regex = format!( - "^{}/a/tbl1/metadata/00000-{}.metadata.json$", - warehouse_loc, UUID_REGEX_STR - ); + let expected_table_metadata_location_regex = + format!("^{warehouse_loc}/a/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$"); let table = catalog .create_table( @@ -1630,7 +2359,7 @@ mod tests { async fn test_create_table_in_nested_namespace_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone()).await; + let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; let namespace_ident = NamespaceIdent::new("a".into()); create_namespace(&catalog, &namespace_ident).await; @@ -1641,10 +2370,8 @@ mod tests { let table_name = "tbl1"; let expected_table_ident = TableIdent::new(nested_namespace_ident.clone(), table_name.into()); - let expected_table_metadata_location_regex = format!( - "^{}/a/b/tbl1/metadata/00000-{}.metadata.json$", - warehouse_loc, UUID_REGEX_STR - ); + let expected_table_metadata_location_regex = + format!("^{warehouse_loc}/a/b/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$"); let table = catalog .create_table( @@ -1668,7 +2395,7 @@ mod tests { #[tokio::test] async fn test_create_table_throws_error_if_table_with_same_name_already_exists() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone()).await; + let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; let namespace_ident = NamespaceIdent::new("a".into()); create_namespace(&catalog, &namespace_ident).await; let table_name = "tbl1"; @@ -1691,61 +2418,17 @@ mod tests { .await .unwrap_err() .to_string(), - format!("Unexpected => Table {:?} already exists.", &table_ident) - ); - } - - #[tokio::test] - async fn test_rename_table_in_same_namespace() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident = NamespaceIdent::new("n1".into()); - create_namespace(&catalog, &namespace_ident).await; - let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into()); - let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into()); - create_table(&catalog, &src_table_ident).await; - - catalog - .rename_table(&src_table_ident, &dst_table_ident) - .await - .unwrap(); - - assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![ - dst_table_ident - ],); - } - - #[tokio::test] - async fn test_rename_table_across_namespaces() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let src_namespace_ident = NamespaceIdent::new("a".into()); - let dst_namespace_ident = NamespaceIdent::new("b".into()); - create_namespaces(&catalog, &vec![&src_namespace_ident, &dst_namespace_ident]).await; - let src_table_ident = TableIdent::new(src_namespace_ident.clone(), "tbl1".into()); - let dst_table_ident = TableIdent::new(dst_namespace_ident.clone(), "tbl2".into()); - create_table(&catalog, &src_table_ident).await; - - catalog - .rename_table(&src_table_ident, &dst_table_ident) - .await - .unwrap(); - - assert_eq!( - catalog.list_tables(&src_namespace_ident).await.unwrap(), - vec![], - ); - - assert_eq!( - catalog.list_tables(&dst_namespace_ident).await.unwrap(), - vec![dst_table_ident], + format!( + "TableAlreadyExists => Table {:?} already exists.", + &table_ident + ) ); } #[tokio::test] async fn test_rename_table_src_table_is_same_as_dst_table() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident = NamespaceIdent::new("n1".into()); create_namespace(&catalog, &namespace_ident).await; let table_ident = TableIdent::new(namespace_ident.clone(), "tbl".into()); @@ -1764,7 +2447,7 @@ mod tests { #[tokio::test] async fn test_rename_table_across_nested_namespaces() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let namespace_ident_a = NamespaceIdent::new("a".into()); let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(); let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap(); @@ -1792,7 +2475,7 @@ mod tests { #[tokio::test] async fn test_rename_table_throws_error_if_dst_namespace_doesnt_exist() { let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; + let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; let src_namespace_ident = NamespaceIdent::new("n1".into()); let src_table_ident = TableIdent::new(src_namespace_ident.clone(), "tbl1".into()); create_namespace(&catalog, &src_namespace_ident).await; @@ -1807,105 +2490,516 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "Unexpected => No such namespace: {:?}", - non_existent_dst_namespace_ident - ), + format!("NamespaceNotFound => No such namespace: {non_existent_dst_namespace_ident:?}"), + ); + } + + /// Creates a V0 SQLite database (no `iceberg_type` column) with one pre-inserted table row. + /// Returns the SQLite URI and the temp dir that owns the database file. + async fn create_v0_sqlite_db() -> (String, TempDir) { + let temp_dir = TempDir::new().unwrap(); + let uri = format!( + "sqlite:{}", + temp_dir.path().join("catalog.db").to_str().unwrap() + ); + sqlx::Sqlite::create_database(&uri).await.unwrap(); + let pool = sqlx::AnyPool::connect(&uri).await.unwrap(); + sqlx::query( + "CREATE TABLE iceberg_tables ( + catalog_name VARCHAR(255) NOT NULL, + table_namespace VARCHAR(255) NOT NULL, + table_name VARCHAR(255) NOT NULL, + metadata_location VARCHAR(1000), + previous_metadata_location VARCHAR(1000), + PRIMARY KEY (catalog_name, table_namespace, table_name) + )", + ) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO iceberg_tables + (catalog_name, table_namespace, table_name, metadata_location) + VALUES ('iceberg', 'test_namespace', 'existing_test_table', '/tmp/fake-location')", + ) + .execute(&pool) + .await + .unwrap(); + pool.close().await; + (uri, temp_dir) + } + + /// Creates a V1 SQLite database (with an `iceberg_type` column) with one pre-inserted table row. + /// Returns the SQLite URI and the temp dir that owns the database file. + async fn create_v1_sqlite_db() -> (String, TempDir) { + let temp_dir = TempDir::new().unwrap(); + let uri = format!( + "sqlite:{}", + temp_dir.path().join("catalog.db").to_str().unwrap() ); + sqlx::Sqlite::create_database(&uri).await.unwrap(); + let pool = sqlx::AnyPool::connect(&uri).await.unwrap(); + sqlx::query( + "CREATE TABLE iceberg_tables ( + catalog_name VARCHAR(255) NOT NULL, + table_namespace VARCHAR(255) NOT NULL, + table_name VARCHAR(255) NOT NULL, + metadata_location VARCHAR(1000), + previous_metadata_location VARCHAR(1000), + iceberg_type VARCHAR(5), + PRIMARY KEY (catalog_name, table_namespace, table_name) + )", + ) + .execute(&pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO iceberg_tables + (catalog_name, table_namespace, table_name, metadata_location, iceberg_type) + VALUES ('iceberg', 'test_namespace', 'existing_test_table', '/tmp/fake-location', 'TABLE')", + ) + .execute(&pool) + .await + .unwrap(); + pool.close().await; + (uri, temp_dir) + } + + /// Catalog properties for opening `uri` with `warehouse` as the warehouse location, + /// optionally requesting a specific `sql.schema-version`. + fn catalog_props( + uri: &str, + warehouse: &TempDir, + schema_version: Option, + ) -> HashMap { + let mut props = HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), uri.to_string()), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + warehouse.path().to_str().unwrap().to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]); + if let Some(schema_version) = schema_version { + props.insert( + SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(), + schema_version.to_string(), + ); + } + props + } + + /// Whether the catalog table in the database at `uri` has the `iceberg_type` column. + /// + /// Connects independently of any catalog under test, so it reports what is actually on disk. + async fn record_type_column_exists(uri: &str) -> bool { + let probe_pool = sqlx::AnyPool::connect(uri).await.unwrap(); + let column_exists = probe_pool + .describe(&format!("SELECT * FROM {CATALOG_TABLE_NAME}")) + .await + .expect("connection and query should succeed") + .columns() + .iter() + .any(|column| { + column + .name() + .eq_ignore_ascii_case(CATALOG_FIELD_RECORD_TYPE) + }); + probe_pool.close().await; + column_exists } #[tokio::test] - async fn test_rename_table_throws_error_if_src_table_doesnt_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident = NamespaceIdent::new("n1".into()); - create_namespace(&catalog, &namespace_ident).await; - let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into()); - let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into()); + async fn test_detect_schema_version() { + install_default_drivers(); + let detected_schema = { + let (uri, _temp_dir) = create_v0_sqlite_db().await; + let pool = sqlx::AnyPool::connect(&uri).await.unwrap(); + let detected_schema = SchemaVersion::detect(&pool).await.unwrap(); + pool.close().await; + detected_schema + }; assert_eq!( - catalog - .rename_table(&src_table_ident, &dst_table_ident) - .await - .unwrap_err() - .to_string(), - format!("Unexpected => No such table: {:?}", src_table_ident), + detected_schema, + SchemaVersion::V0, + "a catalog table without an iceberg_type column should be V0", + ); + + let detected_schema = { + let (uri, _temp_dir) = create_v1_sqlite_db().await; + let pool = sqlx::AnyPool::connect(&uri).await.unwrap(); + let detected_schema = SchemaVersion::detect(&pool).await.unwrap(); + pool.close().await; + detected_schema + }; + assert_eq!( + detected_schema, + SchemaVersion::V1, + "a catalog table with an iceberg_type column should be V1", ); } #[tokio::test] - async fn test_rename_table_throws_error_if_dst_table_already_exists() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc).await; - let namespace_ident = NamespaceIdent::new("n1".into()); - create_namespace(&catalog, &namespace_ident).await; - let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into()); - let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into()); - create_tables(&catalog, vec![&src_table_ident, &dst_table_ident]).await; + async fn test_detect_schema_version_surfaces_errors() { + install_default_drivers(); + + let temp_dir = TempDir::new().unwrap(); + let uri = format!( + "sqlite:{}", + temp_dir.path().join("catalog.db").to_str().unwrap() + ); + sqlx::Sqlite::create_database(&uri).await.unwrap(); + let pool = sqlx::AnyPool::connect(&uri).await.unwrap(); + + // No `iceberg_tables` table at all, so the schema version is unknowable. + let err = SchemaVersion::detect(&pool) + .await + .expect_err("detection should fail rather than report V0"); + pool.close().await; + + assert!( + err.to_string().contains("iceberg_tables"), + "error should name the table it failed to introspect, got: {err}" + ); + } + + #[tokio::test] + async fn test_v0_schema_migration() { + install_default_drivers(); + + let (uri, temp_dir) = create_v0_sqlite_db().await; + + // Opening the catalog with sql.schema-version=V1 should migrate the V0 schema. + let props = HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), uri.clone()), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ( + SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(), + SchemaVersion::V1.to_string(), + ), + ]); + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load("iceberg", props) + .await + .expect("should open V0 catalog and migrate schema when sql.schema-version=V1"); + + // The V0 row (no "iceberg_type" column) should be treated as a TABLE after migration. + let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap(); + let tables = catalog.list_tables(&namespace).await.unwrap(); + assert_eq!(tables.len(), 1); + assert_eq!(tables[0].name(), "existing_test_table"); + + assert!( + record_type_column_exists(&uri).await, + "iceberg_type column should exist when sql.schema-version=V1 was set", + ); + } + + #[tokio::test] + async fn test_v0_schema_no_migration_without_property() { + install_default_drivers(); + + let (uri, temp_dir) = create_v0_sqlite_db().await; + // Opening without sql.schema-version=V1 should NOT migrate — but should still work. + let props = HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), uri.clone()), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]); + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load("iceberg", props) + .await + .expect("should open V0 catalog without migrating"); + + assert_eq!(catalog.schema_version, SchemaVersion::V0); + + // The table should still be visible via V0 queries (no iceberg_type filter). + let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap(); + let tables = catalog.list_tables(&namespace).await.unwrap(); + assert_eq!(tables.len(), 1); + assert_eq!(tables[0].name(), "existing_test_table"); + + assert!( + !record_type_column_exists(&uri).await, + "iceberg_type column should not exist when sql.schema-version=V1 was not set" + ); + } + + #[tokio::test] + async fn test_explicit_v0_schema_version_is_honored() { + install_default_drivers(); + + let (uri, temp_dir) = create_v0_sqlite_db().await; + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "iceberg", + catalog_props(&uri, &temp_dir, Some(SchemaVersion::V0)), + ) + .await + .expect("requesting V0 against a V0 catalog table should succeed"); + + assert_eq!(catalog.schema_version, SchemaVersion::V0); + + let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap(); + let tables = catalog.list_tables(&namespace).await.unwrap(); + assert_eq!(tables.len(), 1); + assert_eq!(tables[0].name(), "existing_test_table"); + + assert!( + !record_type_column_exists(&uri).await, + "iceberg_type column should not be added when V0 was requested" + ); + } + + #[tokio::test] + async fn test_v0_schema_version_is_ignored_on_v1_table() { + install_default_drivers(); + + let (uri, temp_dir) = create_v1_sqlite_db().await; + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "iceberg", + catalog_props(&uri, &temp_dir, Some(SchemaVersion::V0)), + ) + .await + .expect("requesting V0 against a V1 catalog table should succeed"); + + assert_eq!(catalog.schema_version, SchemaVersion::V1); + + let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap(); + let tables = catalog.list_tables(&namespace).await.unwrap(); + assert_eq!(tables.len(), 1); + assert_eq!(tables[0].name(), "existing_test_table"); + } + + #[tokio::test] + async fn test_v0_schema_version_on_empty_database_yields_v1() { + install_default_drivers(); + + let temp_dir = TempDir::new().unwrap(); + let uri = format!( + "sqlite:{}", + temp_dir.path().join("catalog.db").to_str().unwrap() + ); + sqlx::Sqlite::create_database(&uri).await.unwrap(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "iceberg", + catalog_props(&uri, &temp_dir, Some(SchemaVersion::V0)), + ) + .await + .expect("requesting V0 against an empty database should succeed"); + + assert_eq!(catalog.schema_version, SchemaVersion::V1); + } + + #[tokio::test] + async fn test_create_table_unsupported_on_v0_schema() { + install_default_drivers(); + + let (uri, temp_dir) = create_v0_sqlite_db().await; + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load("iceberg", catalog_props(&uri, &temp_dir, None)) + .await + .expect("should open V0 catalog without migrating"); + assert_eq!(catalog.schema_version, SchemaVersion::V0); + + // Inserts always populate `iceberg_type`, which a V0 catalog table does not have, so + // creation is refused up front rather than failing in the database. + let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap(); + let err = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("created_test_table".to_string()) + .schema(simple_table_schema()) + .location(temp_path()) + .build(), + ) + .await + .expect_err("table creation should be rejected on a V0 catalog table"); + + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert!( + err.to_string().contains("Table creation is not supported"), + "error should explain that table creation is unsupported, got: {err}" + ); + + let tables = catalog.list_tables(&namespace).await.unwrap(); assert_eq!( - catalog - .rename_table(&src_table_ident, &dst_table_ident) - .await - .unwrap_err() - .to_string(), - format!("Unexpected => Table {:?} already exists.", &dst_table_ident), + tables.len(), + 1, + "only the pre-existing row should be present, got: {tables:?}" ); } #[tokio::test] - async fn test_drop_table_throws_error_if_table_not_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone()).await; - let namespace_ident = NamespaceIdent::new("a".into()); - let table_name = "tbl1"; - let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - create_namespace(&catalog, &namespace_ident).await; + async fn test_register_table_unsupported_on_v0_schema() { + install_default_drivers(); + let (uri, temp_dir) = create_v0_sqlite_db().await; + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load("iceberg", catalog_props(&uri, &temp_dir, None)) + .await + .expect("should open V0 catalog without migrating"); + assert_eq!(catalog.schema_version, SchemaVersion::V0); + + let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap(); + let table_ident = TableIdent::new(namespace.clone(), "registered_test_table".to_string()); + // Register with non-existent table to ensure test is verifying that schema version is checked first. let err = catalog - .drop_table(&table_ident) + .register_table( + &table_ident, + "/tmp/does-not-exist/metadata.json".to_string(), + ) .await - .unwrap_err() - .to_string(); + .expect_err("table registration should be rejected on a V0 catalog table"); + + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert!( + err.to_string() + .contains("Table registration is not supported"), + "error should explain that table registration is unsupported, got: {err}" + ); + + let tables = catalog.list_tables(&namespace).await.unwrap(); assert_eq!( - err, - "Unexpected => No such table: TableIdent { namespace: NamespaceIdent([\"a\"]), name: \"tbl1\" }" + tables.len(), + 1, + "only the pre-existing row should be present, got: {tables:?}" ); } #[tokio::test] - async fn test_drop_table() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone()).await; - let namespace_ident = NamespaceIdent::new("a".into()); - let table_name = "tbl1"; - let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - create_namespace(&catalog, &namespace_ident).await; + async fn test_create_table_supported_after_v0_migration() { + install_default_drivers(); - let location = warehouse_loc.clone(); - let table_creation = TableCreation::builder() - .name(table_name.into()) - .location(location.clone()) - .schema(simple_table_schema()) - .build(); + let (uri, temp_dir) = create_v0_sqlite_db().await; - catalog - .create_table(&namespace_ident, table_creation) + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "iceberg", + catalog_props(&uri, &temp_dir, Some(SchemaVersion::V1)), + ) .await - .unwrap(); + .expect("should open V0 catalog and migrate schema when sql.schema-version=V1"); + assert_eq!(catalog.schema_version, SchemaVersion::V1); - let table = catalog.load_table(&table_ident).await.unwrap(); - assert_table_eq(&table, &table_ident, &simple_table_schema()); + let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap(); + catalog + .create_table( + &namespace, + TableCreation::builder() + .name("created_test_table".to_string()) + .schema(simple_table_schema()) + .location(temp_path()) + .build(), + ) + .await + .expect("table creation should succeed once the schema is migrated to V1"); - catalog.drop_table(&table_ident).await.unwrap(); - let err = catalog - .load_table(&table_ident) + let table_names = catalog + .list_tables(&namespace) .await - .unwrap_err() - .to_string(); + .unwrap() + .iter() + .map(|table_ident| table_ident.name().to_string()) + .sorted() + .collect_vec(); assert_eq!( - err, - "Unexpected => No such table: TableIdent { namespace: NamespaceIdent([\"a\"]), name: \"tbl1\" }" + table_names, + vec!["created_test_table", "existing_test_table"], + "the migrated pre-existing row and the new table should both be listed" ); } + + #[tokio::test] + async fn test_invalid_schema_version_is_rejected() { + install_default_drivers(); + + let (uri, temp_dir) = create_v0_sqlite_db().await; + + // An unrecognized sql.schema-version value must fail fast rather than silently + // falling back to V0. + let props = HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), uri), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ( + SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(), + "v2".to_string(), + ), + ]); + let result = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load("iceberg", props) + .await; + + let err = result.expect_err("an invalid sql.schema-version should be rejected"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_legacy_bind_style_key_is_accepted() { + install_default_drivers(); + + let (uri, temp_dir) = create_v0_sqlite_db().await; + + // The legacy `sql_bind_style` key must keep working alongside the new `sql.bind-style`. + let props = HashMap::from_iter([ + (SQL_CATALOG_PROP_URI.to_string(), uri), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + temp_dir.path().to_str().unwrap().to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]); + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load("iceberg", props) + .await + .expect("legacy sql_bind_style key should still be accepted"); + + assert_eq!(catalog.properties.sql_bind_style, SqlBindStyle::QMark); + } } diff --git a/crates/catalog/sql/src/error.rs b/crates/catalog/sql/src/error.rs index 15b56e8e23..55e0e0a368 100644 --- a/crates/catalog/sql/src/error.rs +++ b/crates/catalog/sql/src/error.rs @@ -28,21 +28,21 @@ pub fn from_sqlx_error(error: sqlx::Error) -> Error { pub fn no_such_namespace_err(namespace: &NamespaceIdent) -> Result { Err(Error::new( - ErrorKind::Unexpected, - format!("No such namespace: {:?}", namespace), + ErrorKind::NamespaceNotFound, + format!("No such namespace: {namespace:?}"), )) } pub fn no_such_table_err(table_ident: &TableIdent) -> Result { Err(Error::new( - ErrorKind::Unexpected, - format!("No such table: {:?}", table_ident), + ErrorKind::TableNotFound, + format!("No such table: {table_ident:?}"), )) } pub fn table_already_exists_err(table_ident: &TableIdent) -> Result { Err(Error::new( - ErrorKind::Unexpected, - format!("Table {:?} already exists.", table_ident), + ErrorKind::TableAlreadyExists, + format!("Table {table_ident:?} already exists."), )) } diff --git a/crates/catalog/sql/src/lib.rs b/crates/catalog/sql/src/lib.rs index 6861dab3f8..b76006ed3b 100644 --- a/crates/catalog/sql/src/lib.rs +++ b/crates/catalog/sql/src/lib.rs @@ -16,6 +16,43 @@ // under the License. //! Iceberg sql catalog implementation. +//! +//! To build a sql catalog with configurations +//! # Example +//! +//! ```rust, no_run +//! use std::collections::HashMap; +//! +//! use iceberg::CatalogBuilder; +//! use iceberg_catalog_sql::{ +//! SQL_CATALOG_PROP_BIND_STYLE, SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE, +//! SqlBindStyle, SqlCatalogBuilder, +//! }; +//! +//! #[tokio::main] +//! async fn main() { +//! let catalog = SqlCatalogBuilder::default() +//! .load( +//! "sql", +//! HashMap::from_iter([ +//! ( +//! SQL_CATALOG_PROP_URI.to_string(), +//! "http://localhost:8181".to_string(), +//! ), +//! ( +//! SQL_CATALOG_PROP_WAREHOUSE.to_string(), +//! "s3://warehouse".to_string(), +//! ), +//! ( +//! SQL_CATALOG_PROP_BIND_STYLE.to_string(), +//! SqlBindStyle::QMark.to_string(), +//! ), +//! ]), +//! ) +//! .await +//! .unwrap(); +//! } +//! ``` #![deny(missing_docs)] diff --git a/crates/examples/Cargo.toml b/crates/examples/Cargo.toml index c7874d9a17..27e7c9969d 100644 --- a/crates/examples/Cargo.toml +++ b/crates/examples/Cargo.toml @@ -20,6 +20,7 @@ edition = { workspace = true } homepage = { workspace = true } license = { workspace = true } name = "iceberg-examples" +publish = false repository = { workspace = true } rust-version = { workspace = true } version = { workspace = true } @@ -28,6 +29,7 @@ version = { workspace = true } futures = { workspace = true } iceberg = { workspace = true } iceberg-catalog-rest = { workspace = true } +iceberg-storage-opendal = { workspace = true, optional = true } tokio = { workspace = true, features = ["full"] } [[example]] @@ -38,6 +40,10 @@ path = "src/rest_catalog_namespace.rs" name = "rest-catalog-table" path = "src/rest_catalog_table.rs" +[[example]] +name = "rest-session-catalog-namespace" +path = "src/rest_session_catalog_namespace.rs" + [[example]] name = "oss-backend" path = "src/oss_backend.rs" @@ -45,4 +51,7 @@ required-features = ["storage-oss"] [features] default = [] -storage-oss = ["iceberg/storage-oss"] +storage-oss = ["iceberg-storage-opendal/opendal-oss"] + +[lints] +workspace = true diff --git a/crates/examples/DEPENDENCIES.rust.tsv b/crates/examples/DEPENDENCIES.rust.tsv deleted file mode 100644 index 20702597a7..0000000000 --- a/crates/examples/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,330 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayvec@0.7.6 X X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-ord@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -h2@0.4.12 X -half@2.6.0 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-catalog-rest@0.7.0 X -iceberg-examples@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustls@0.23.31 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.6 X X -simdutf8@0.1.5 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.0 X X -stable_deref_trait@1.2.0 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -webpki-roots@1.0.2 X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/examples/src/oss_backend.rs b/crates/examples/src/oss_backend.rs index 8f0b866d23..5f56e4a301 100644 --- a/crates/examples/src/oss_backend.rs +++ b/crates/examples/src/oss_backend.rs @@ -16,10 +16,12 @@ // under the License. use std::collections::HashMap; +use std::sync::Arc; use futures::stream::StreamExt; use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableIdent}; use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalogBuilder}; +use iceberg_storage_opendal::OpenDalStorageFactory; // Configure these values according to your environment @@ -41,8 +43,9 @@ static OSS_ACCESS_KEY_SECRET: &str = "99999999999999999999999999999999"; /// The example also requires valid OSS credentials and endpoint to be configured. #[tokio::main] async fn main() { - // Create the REST iceberg catalog. + // Create the REST iceberg catalog with explicit OSS storage factory. let catalog = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::Oss)) .load( "rest", HashMap::from([ @@ -78,7 +81,7 @@ async fn main() { println!("Table {TABLE_NAME} loaded!"); let scan = table.scan().select_all().build().unwrap(); - let reader = scan.to_arrow().await.unwrap(); + let reader = scan.to_arrow(Default::default()).await.unwrap(); let buf = reader.collect::>().await; println!("Table {TABLE_NAME} has {} batches.", buf.len()); diff --git a/crates/examples/src/rest_catalog_namespace.rs b/crates/examples/src/rest_catalog_namespace.rs index 27196a2696..d871de4387 100644 --- a/crates/examples/src/rest_catalog_namespace.rs +++ b/crates/examples/src/rest_catalog_namespace.rs @@ -45,10 +45,7 @@ async fn main() { // ANCHOR: list_all_namespace // List all namespaces already in the catalog. let existing_namespaces = catalog.list_namespaces(None).await.unwrap(); - println!( - "Namespaces alreading in the existing catalog: {:?}", - existing_namespaces - ); + println!("Namespaces alreading in the existing catalog: {existing_namespaces:?}"); // ANCHOR_END: list_all_namespace // ANCHOR: create_namespace @@ -70,9 +67,9 @@ async fn main() { ) .await .unwrap(); - println!("Namespace {:?} created!", namespace_ident); + println!("Namespace {namespace_ident:?} created!"); let loaded_namespace = catalog.get_namespace(&namespace_ident).await.unwrap(); - println!("Namespace loaded!\n\nNamespace: {:#?}", loaded_namespace,); + println!("Namespace loaded!\n\nNamespace: {loaded_namespace:#?}",); // ANCHOR_END: create_namespace } diff --git a/crates/examples/src/rest_catalog_table.rs b/crates/examples/src/rest_catalog_table.rs index 306d886ae6..64acb263f2 100644 --- a/crates/examples/src/rest_catalog_table.rs +++ b/crates/examples/src/rest_catalog_table.rs @@ -96,6 +96,6 @@ async fn main() { // Load the table back from the catalog. It should be identical to the created table. let loaded_table = catalog.load_table(&table_ident).await.unwrap(); - println!("Table {TABLE_NAME} loaded!\n\nTable: {:?}", loaded_table); + println!("Table {TABLE_NAME} loaded!\n\nTable: {loaded_table:?}"); // ANCHOR_END: load_table } diff --git a/crates/examples/src/rest_session_catalog_namespace.rs b/crates/examples/src/rest_session_catalog_namespace.rs new file mode 100644 index 0000000000..60f5d33780 --- /dev/null +++ b/crates/examples/src/rest_session_catalog_namespace.rs @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg::{NamespaceIdent, SessionCatalog, SessionContext}; +use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestSessionCatalogBuilder}; + +static REST_URI: &str = "http://localhost:8181"; + +/// This is a simple example that demonstrates how to use [`RestSessionCatalog`] to create +/// namespaces. +/// +/// Unlike [`RestCatalog`](iceberg_catalog_rest::RestCatalog), every operation +/// takes the [`SessionContext`] it should run under, so a single catalog instance +/// can serve requests on behalf of different sessions. +/// +/// A running instance of the iceberg-rest catalog on port 8181 is required. +/// You can find how to run the iceberg-rest catalog with `docker compose` in +/// the official [quickstart documentation](https://iceberg.apache.org/spark-quickstart/). +#[tokio::main] +async fn main() { + // ANCHOR: create_catalog + // Create the REST iceberg session catalog. + let catalog = RestSessionCatalogBuilder::default() + .load( + "rest", + HashMap::from([(REST_CATALOG_PROP_URI.to_string(), REST_URI.to_string())]), + ) + .await + .unwrap(); + + // Build the session context passed to each operation. + let context = SessionContext::builder() + .identity("user123".to_string()) + .build(); + // ANCHOR_END: create_catalog + + // ANCHOR: list_all_namespace + // List all namespaces already in the catalog. + let existing_namespaces = catalog.list_namespaces(&context, None).await.unwrap(); + println!("Namespaces alreading in the existing catalog: {existing_namespaces:?}"); + // ANCHOR_END: list_all_namespace + + // ANCHOR: create_namespace + // Create a new namespace identifier. + let namespace_ident = + NamespaceIdent::from_vec(vec!["ns1".to_string(), "ns11".to_string()]).unwrap(); + + // Drop the namespace if it already exists. + if catalog + .namespace_exists(&context, &namespace_ident) + .await + .unwrap() + { + println!("Namespace already exists, dropping now.",); + catalog + .drop_namespace(&context, &namespace_ident) + .await + .unwrap(); + } + + // Create the new namespace in the catalog. + let _created_namespace = catalog + .create_namespace( + &context, + &namespace_ident, + HashMap::from([("key1".to_string(), "value1".to_string())]), + ) + .await + .unwrap(); + println!("Namespace {namespace_ident:?} created!"); + + let loaded_namespace = catalog + .get_namespace(&context, &namespace_ident) + .await + .unwrap(); + println!("Namespace loaded!\n\nNamespace: {loaded_namespace:#?}",); + // ANCHOR_END: create_namespace +} diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index d592700b73..45e0cae412 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -19,6 +19,7 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg" +publish = true rust-version = { workspace = true } version = { workspace = true } @@ -29,29 +30,20 @@ license = { workspace = true } repository = { workspace = true } [features] -default = ["storage-memory", "storage-fs", "storage-s3", "tokio"] -storage-all = ["storage-memory", "storage-fs", "storage-s3", "storage-gcs"] - -storage-azdls = ["opendal/services-azdls"] -storage-fs = ["opendal/services-fs"] -storage-gcs = ["opendal/services-gcs"] -storage-memory = ["opendal/services-memory"] -storage-oss = ["opendal/services-oss"] -storage-s3 = ["opendal/services-s3", "reqsign"] - -smol = ["dep:smol"] -tokio = ["tokio/rt-multi-thread"] +default = [] [dependencies] +aes-gcm = { workspace = true } anyhow = { workspace = true } apache-avro = { workspace = true } array-init = { workspace = true } arrow-arith = { workspace = true } arrow-array = { workspace = true } +arrow-avro = { workspace = true } arrow-buffer = { workspace = true } arrow-cast = { workspace = true } arrow-ord = { workspace = true } -arrow-schema = { workspace = true } +arrow-schema = { workspace = true, features = ["canonical_extension_types"] } arrow-select = { workspace = true } arrow-string = { workspace = true } as-any = { workspace = true } @@ -61,49 +53,61 @@ base64 = { workspace = true } bimap = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } +crc32fast = { workspace = true } derive_builder = { workspace = true } -expect-test = { workspace = true } +fastnum = { workspace = true } +flate2 = { workspace = true } fnv = { workspace = true } +form_urlencoded = { workspace = true } futures = { workspace = true } +iceberg-property-macro = { workspace = true } itertools = { workspace = true } -moka = { version = "0.12.10", features = ["future"] } +moka = { workspace = true, features = ["future"] } murmur3 = { workspace = true } -num-bigint = { workspace = true } once_cell = { workspace = true } -opendal = { workspace = true } ordered-float = { workspace = true } -parquet = { workspace = true, features = ["async"] } +parquet = { workspace = true, features = [ + "async", + "encryption", + "arrow_canonical_extension_types", + "geospatial", +] } +parquet-geospatial = { workspace = true } rand = { workspace = true } reqwest = { workspace = true } -reqsign = { version = "0.16.3", optional = true, default-features = false } roaring = { workspace = true } -rust_decimal = { workspace = true } serde = { workspace = true } serde_bytes = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } serde_repr = { workspace = true } serde_with = { workspace = true } -smol = { workspace = true, optional = true } strum = { workspace = true, features = ["derive"] } -thrift = { workspace = true } -tokio = { workspace = true, optional = false, features = ["sync"] } +tokio = { workspace = true, optional = false } +tracing = { workspace = true } typed-builder = { workspace = true } +typetag = { workspace = true } url = { workspace = true } uuid = { workspace = true } +zeroize = { workspace = true } zstd = { workspace = true } [dev-dependencies] -ctor = { workspace = true } expect-test = { workspace = true } iceberg_test_utils = { path = "../test_utils", features = ["tests"] } +minijinja = { workspace = true } mockall = { workspace = true } pretty_assertions = { workspace = true } rand = { workspace = true } regex = { workspace = true } +rstest = { workspace = true } +serde_arrow = { version = "0.15", features = ["arrow-60"] } tempfile = { workspace = true } -tera = { workspace = true } +tracing-subscriber = { workspace = true } [package.metadata.cargo-machete] # These dependencies are added to ensure minimal dependency version ignored = ["tap"] + +[lints] +workspace = true diff --git a/crates/iceberg/DEPENDENCIES.rust.tsv b/crates/iceberg/DEPENDENCIES.rust.tsv deleted file mode 100644 index 460a531644..0000000000 --- a/crates/iceberg/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,323 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayvec@0.7.6 X X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-ord@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -half@2.6.0 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustls@0.23.31 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simdutf8@0.1.5 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.0 X X -stable_deref_trait@1.2.0 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -webpki-roots@1.0.2 X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/iceberg/README.md b/crates/iceberg/README.md index 14acaa2d22..bb22750519 100644 --- a/crates/iceberg/README.md +++ b/crates/iceberg/README.md @@ -29,17 +29,24 @@ See the [API documentation](https://docs.rs/iceberg/latest) for examples and the ## Usage ```rust +use std::collections::HashMap; +use std::sync::Arc; + use futures::TryStreamExt; -use iceberg::io::{FileIO, FileIOBuilder}; -use iceberg::{Catalog, Result, TableIdent}; -use iceberg_catalog_memory::MemoryCatalog; +use iceberg::io::MemoryStorageFactory; +use iceberg::memory::{MemoryCatalogBuilder, MEMORY_CATALOG_WAREHOUSE}; +use iceberg::{Catalog, CatalogBuilder, Result, TableIdent}; #[tokio::main] async fn main() -> Result<()> { - // Build your file IO. - let file_io = FileIOBuilder::new("memory").build()?; - // Connect to a catalog. - let catalog = MemoryCatalog::new(file_io, None); + // Connect to a catalog with a memory storage factory. + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(MemoryStorageFactory)) + .load( + "my_catalog", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "/tmp/warehouse".to_string())]), + ) + .await?; // Load table from catalog. let table = catalog .load_table(&TableIdent::from_strs(["hello", "world"])?) @@ -49,7 +56,7 @@ async fn main() -> Result<()> { .scan() .select(["name", "id"]) .build()? - .to_arrow() + .to_arrow(Default::default()) .await?; // Consume this stream like arrow record batch stream. @@ -58,26 +65,6 @@ async fn main() -> Result<()> { } ``` -## IO Support - -Iceberg Rust provides various storage backends through feature flags. Here are the currently supported storage backends: - -| Storage Backend | Feature Flag | Status | Description | -| -------------------- | ---------------- | -------------- | --------------------------------------------- | -| Memory | `storage-memory` | ✅ Stable | In-memory storage for testing and development | -| Local Filesystem | `storage-fs` | ✅ Stable | Local filesystem storage | -| Amazon S3 | `storage-s3` | ✅ Stable | Amazon S3 storage | -| Google Cloud Storage | `storage-gcs` | ✅ Stable | Google Cloud Storage | -| Alibaba Cloud OSS | `storage-oss` | 🧪 Experimental | Alibaba Cloud Object Storage Service | -| Azure Datalake | `storage-azdls` | 🧪 Experimental | Azure Datalake Storage v2 | +## Storage Backends -You can enable all stable storage backends at once using the `storage-all` feature flag. - -> Note that `storage-oss` and `storage-azdls` are currently experimental and not included in `storage-all`. - -Example usage in `Cargo.toml`: - -```toml -[dependencies] -iceberg = { version = "x.y.z", features = ["storage-s3", "storage-fs"] } -``` +For storage backend support (S3, GCS, local filesystem, etc.), use the [`iceberg-storage-opendal`](https://crates.io/crates/iceberg-storage-opendal) crate. See its [README](../storage/opendal/README.md) for available backends and feature flags. diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt new file mode 100644 index 0000000000..4311f99101 --- /dev/null +++ b/crates/iceberg/public-api.txt @@ -0,0 +1,4253 @@ +pub mod iceberg +pub mod iceberg::arrow +pub mod iceberg::arrow::delete_file_loader +pub trait iceberg::arrow::delete_file_loader::DeleteFileLoader +pub fn iceberg::arrow::delete_file_loader::DeleteFileLoader::read_delete_file<'life0, 'life1, 'async_trait>(&'life0 self, task: &'life1 iceberg::scan::FileScanTaskDeleteFile, schema: iceberg::spec::SchemaRef) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub mod iceberg::arrow::partition_value_calculator +pub struct iceberg::arrow::partition_value_calculator::PartitionValueCalculator +impl iceberg::arrow::PartitionValueCalculator +pub fn iceberg::arrow::PartitionValueCalculator::calculate(&self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result +pub fn iceberg::arrow::PartitionValueCalculator::partition_arrow_type(&self) -> &arrow_schema::datatype::DataType +pub fn iceberg::arrow::PartitionValueCalculator::partition_type(&self) -> &iceberg::spec::StructType +pub fn iceberg::arrow::PartitionValueCalculator::try_new(partition_spec: &iceberg::spec::PartitionSpec, table_schema: &iceberg::spec::Schema) -> iceberg::Result +impl core::fmt::Debug for iceberg::arrow::PartitionValueCalculator +pub fn iceberg::arrow::PartitionValueCalculator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub mod iceberg::arrow::record_batch_partition_splitter +pub struct iceberg::arrow::record_batch_partition_splitter::RecordBatchPartitionSplitter +impl iceberg::arrow::RecordBatchPartitionSplitter +pub fn iceberg::arrow::RecordBatchPartitionSplitter::split(&self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result> +pub fn iceberg::arrow::RecordBatchPartitionSplitter::try_new(iceberg_schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpecRef, calculator: core::option::Option) -> iceberg::Result +pub fn iceberg::arrow::RecordBatchPartitionSplitter::try_new_with_computed_values(iceberg_schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpecRef) -> iceberg::Result +pub fn iceberg::arrow::RecordBatchPartitionSplitter::try_new_with_precomputed_values(iceberg_schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpecRef) -> iceberg::Result +pub const iceberg::arrow::record_batch_partition_splitter::PROJECTED_PARTITION_VALUE_COLUMN: &str +pub mod iceberg::arrow::record_batch_projector +pub struct iceberg::arrow::record_batch_projector::RecordBatchProjector +impl iceberg::arrow::record_batch_projector::RecordBatchProjector +pub fn iceberg::arrow::record_batch_projector::RecordBatchProjector::from_iceberg_schema(iceberg_schema: alloc::sync::Arc, target_field_ids: &[i32]) -> iceberg::Result +pub fn iceberg::arrow::record_batch_projector::RecordBatchProjector::project_column(&self, batch: &[arrow_array::array::ArrayRef]) -> iceberg::Result> +impl core::clone::Clone for iceberg::arrow::record_batch_projector::RecordBatchProjector +pub fn iceberg::arrow::record_batch_projector::RecordBatchProjector::clone(&self) -> iceberg::arrow::record_batch_projector::RecordBatchProjector +impl core::cmp::Eq for iceberg::arrow::record_batch_projector::RecordBatchProjector +impl core::cmp::PartialEq for iceberg::arrow::record_batch_projector::RecordBatchProjector +pub fn iceberg::arrow::record_batch_projector::RecordBatchProjector::eq(&self, other: &iceberg::arrow::record_batch_projector::RecordBatchProjector) -> bool +impl core::fmt::Debug for iceberg::arrow::record_batch_projector::RecordBatchProjector +pub fn iceberg::arrow::record_batch_projector::RecordBatchProjector::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::arrow::record_batch_projector::RecordBatchProjector +pub enum iceberg::arrow::FieldMatchMode +pub iceberg::arrow::FieldMatchMode::Id +pub iceberg::arrow::FieldMatchMode::Name +impl iceberg::arrow::FieldMatchMode +pub fn iceberg::arrow::FieldMatchMode::match_field(&self, arrow_field: &arrow_schema::field::FieldRef, iceberg_field: &iceberg::spec::NestedField) -> bool +impl core::clone::Clone for iceberg::arrow::FieldMatchMode +pub fn iceberg::arrow::FieldMatchMode::clone(&self) -> iceberg::arrow::FieldMatchMode +impl core::fmt::Debug for iceberg::arrow::FieldMatchMode +pub fn iceberg::arrow::FieldMatchMode::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::arrow::FieldMatchMode +pub struct iceberg::arrow::ArrowArrayAccessor +impl iceberg::arrow::ArrowArrayAccessor +pub fn iceberg::arrow::ArrowArrayAccessor::new() -> Self +pub fn iceberg::arrow::ArrowArrayAccessor::new_with_match_mode(match_mode: iceberg::arrow::FieldMatchMode) -> Self +impl core::default::Default for iceberg::arrow::ArrowArrayAccessor +pub fn iceberg::arrow::ArrowArrayAccessor::default() -> Self +impl iceberg::spec::PartnerAccessor> for iceberg::arrow::ArrowArrayAccessor +pub fn iceberg::arrow::ArrowArrayAccessor::field_partner<'a>(&self, struct_partner: &'a arrow_array::array::ArrayRef, field: &iceberg::spec::NestedField) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub fn iceberg::arrow::ArrowArrayAccessor::list_element_partner<'a>(&self, list_partner: &'a arrow_array::array::ArrayRef) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub fn iceberg::arrow::ArrowArrayAccessor::map_key_partner<'a>(&self, map_partner: &'a arrow_array::array::ArrayRef) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub fn iceberg::arrow::ArrowArrayAccessor::map_value_partner<'a>(&self, map_partner: &'a arrow_array::array::ArrayRef) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub fn iceberg::arrow::ArrowArrayAccessor::struct_partner<'a>(&self, schema_partner: &'a arrow_array::array::ArrayRef) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub struct iceberg::arrow::ArrowFileReader +impl iceberg::arrow::ArrowFileReader +pub fn iceberg::arrow::ArrowFileReader::new(meta: iceberg::io::FileMetadata, r: alloc::boxed::Box) -> Self +impl parquet::arrow::async_reader::AsyncFileReader for iceberg::arrow::ArrowFileReader +pub fn iceberg::arrow::ArrowFileReader::get_byte_ranges(&mut self, ranges: alloc::vec::Vec>) -> futures_core::future::BoxFuture<'_, parquet::errors::Result>> +pub fn iceberg::arrow::ArrowFileReader::get_bytes(&mut self, range: core::ops::range::Range) -> futures_core::future::BoxFuture<'_, parquet::errors::Result> +pub fn iceberg::arrow::ArrowFileReader::get_metadata(&mut self, options: core::option::Option<&parquet::arrow::arrow_reader::ArrowReaderOptions>) -> futures_core::future::BoxFuture<'_, parquet::errors::Result>> +pub struct iceberg::arrow::ArrowReader +impl iceberg::arrow::ArrowReader +pub fn iceberg::arrow::ArrowReader::read(self, tasks: iceberg::scan::FileScanTaskStream) -> iceberg::Result +impl core::clone::Clone for iceberg::arrow::ArrowReader +pub fn iceberg::arrow::ArrowReader::clone(&self) -> iceberg::arrow::ArrowReader +pub struct iceberg::arrow::ArrowReaderBuilder +impl iceberg::arrow::ArrowReaderBuilder +pub fn iceberg::arrow::ArrowReaderBuilder::build(self) -> iceberg::arrow::ArrowReader +pub fn iceberg::arrow::ArrowReaderBuilder::new(file_io: iceberg::io::FileIO, runtime: iceberg::Runtime) -> Self +pub fn iceberg::arrow::ArrowReaderBuilder::with_options(self, options: iceberg::arrow::ArrowReaderOptions) -> Self +pub fn iceberg::arrow::ArrowReaderBuilder::with_parquet_options(self, options: iceberg::arrow::ParquetReadOptions) -> Self +pub struct iceberg::arrow::ArrowReaderOptions +impl iceberg::arrow::ArrowReaderOptions +pub fn iceberg::arrow::ArrowReaderOptions::batch_size(&self) -> core::option::Option +pub fn iceberg::arrow::ArrowReaderOptions::bloom_filter_enabled(&self) -> bool +pub fn iceberg::arrow::ArrowReaderOptions::data_file_concurrency_limit(&self) -> core::option::Option +pub fn iceberg::arrow::ArrowReaderOptions::parquet_options(&self) -> &iceberg::arrow::ParquetReadOptions +pub fn iceberg::arrow::ArrowReaderOptions::row_group_filtering_enabled(&self) -> bool +pub fn iceberg::arrow::ArrowReaderOptions::row_selection_enabled(&self) -> bool +impl core::clone::Clone for iceberg::arrow::ArrowReaderOptions +pub fn iceberg::arrow::ArrowReaderOptions::clone(&self) -> iceberg::arrow::ArrowReaderOptions +impl core::cmp::PartialEq for iceberg::arrow::ArrowReaderOptions +pub fn iceberg::arrow::ArrowReaderOptions::eq(&self, other: &iceberg::arrow::ArrowReaderOptions) -> bool +impl core::default::Default for iceberg::arrow::ArrowReaderOptions +pub fn iceberg::arrow::ArrowReaderOptions::default() -> Self +impl core::fmt::Debug for iceberg::arrow::ArrowReaderOptions +pub fn iceberg::arrow::ArrowReaderOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::arrow::ArrowReaderOptions +impl iceberg::arrow::ArrowReaderOptions +pub fn iceberg::arrow::ArrowReaderOptions::builder() -> ArrowReaderOptionsBuilder<((), (), (), (), (), ())> +impl serde_core::ser::Serialize for iceberg::arrow::ArrowReaderOptions +pub fn iceberg::arrow::ArrowReaderOptions::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::arrow::ArrowReaderOptions where iceberg::arrow::ArrowReaderOptions: core::default::Default +pub fn iceberg::arrow::ArrowReaderOptions::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::arrow::ParquetReadOptions +impl iceberg::arrow::ParquetReadOptions +pub fn iceberg::arrow::ParquetReadOptions::metadata_size_hint(&self) -> usize +pub fn iceberg::arrow::ParquetReadOptions::preload_column_index(&self) -> bool +pub fn iceberg::arrow::ParquetReadOptions::preload_offset_index(&self) -> bool +pub fn iceberg::arrow::ParquetReadOptions::preload_page_index(&self) -> bool +pub fn iceberg::arrow::ParquetReadOptions::range_coalesce_bytes(&self) -> u64 +pub fn iceberg::arrow::ParquetReadOptions::range_fetch_concurrency(&self) -> usize +impl core::clone::Clone for iceberg::arrow::ParquetReadOptions +pub fn iceberg::arrow::ParquetReadOptions::clone(&self) -> iceberg::arrow::ParquetReadOptions +impl core::cmp::PartialEq for iceberg::arrow::ParquetReadOptions +pub fn iceberg::arrow::ParquetReadOptions::eq(&self, other: &iceberg::arrow::ParquetReadOptions) -> bool +impl core::default::Default for iceberg::arrow::ParquetReadOptions +pub fn iceberg::arrow::ParquetReadOptions::default() -> Self +impl core::fmt::Debug for iceberg::arrow::ParquetReadOptions +pub fn iceberg::arrow::ParquetReadOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::arrow::ParquetReadOptions +impl iceberg::arrow::ParquetReadOptions +pub fn iceberg::arrow::ParquetReadOptions::builder() -> ParquetReadOptionsBuilder<((), (), (), (), (), ())> +impl serde_core::ser::Serialize for iceberg::arrow::ParquetReadOptions +pub fn iceberg::arrow::ParquetReadOptions::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::arrow::ParquetReadOptions where iceberg::arrow::ParquetReadOptions: core::default::Default +pub fn iceberg::arrow::ParquetReadOptions::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::arrow::PartitionValueCalculator +impl iceberg::arrow::PartitionValueCalculator +pub fn iceberg::arrow::PartitionValueCalculator::calculate(&self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result +pub fn iceberg::arrow::PartitionValueCalculator::partition_arrow_type(&self) -> &arrow_schema::datatype::DataType +pub fn iceberg::arrow::PartitionValueCalculator::partition_type(&self) -> &iceberg::spec::StructType +pub fn iceberg::arrow::PartitionValueCalculator::try_new(partition_spec: &iceberg::spec::PartitionSpec, table_schema: &iceberg::spec::Schema) -> iceberg::Result +impl core::fmt::Debug for iceberg::arrow::PartitionValueCalculator +pub fn iceberg::arrow::PartitionValueCalculator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::arrow::RecordBatchPartitionSplitter +impl iceberg::arrow::RecordBatchPartitionSplitter +pub fn iceberg::arrow::RecordBatchPartitionSplitter::split(&self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result> +pub fn iceberg::arrow::RecordBatchPartitionSplitter::try_new(iceberg_schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpecRef, calculator: core::option::Option) -> iceberg::Result +pub fn iceberg::arrow::RecordBatchPartitionSplitter::try_new_with_computed_values(iceberg_schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpecRef) -> iceberg::Result +pub fn iceberg::arrow::RecordBatchPartitionSplitter::try_new_with_precomputed_values(iceberg_schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpecRef) -> iceberg::Result +pub struct iceberg::arrow::ScanMetrics +impl iceberg::scan::ScanMetrics +pub fn iceberg::scan::ScanMetrics::bytes_read(&self) -> u64 +impl core::clone::Clone for iceberg::scan::ScanMetrics +pub fn iceberg::scan::ScanMetrics::clone(&self) -> iceberg::scan::ScanMetrics +impl core::fmt::Debug for iceberg::scan::ScanMetrics +pub fn iceberg::scan::ScanMetrics::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::arrow::ScanResult +impl iceberg::scan::ScanResult +pub fn iceberg::scan::ScanResult::metrics(&self) -> &iceberg::scan::ScanMetrics +pub fn iceberg::scan::ScanResult::stream(self) -> iceberg::scan::ArrowRecordBatchStream +pub const iceberg::arrow::DEFAULT_MAP_FIELD_NAME: &str +pub const iceberg::arrow::PROJECTED_PARTITION_VALUE_COLUMN: &str +pub const iceberg::arrow::UTC_TIME_ZONE: &str +pub trait iceberg::arrow::ArrowSchemaVisitor +pub type iceberg::arrow::ArrowSchemaVisitor::T +pub type iceberg::arrow::ArrowSchemaVisitor::U +pub fn iceberg::arrow::ArrowSchemaVisitor::after_field(&mut self, _field: &arrow_schema::field::FieldRef) -> iceberg::Result<()> +pub fn iceberg::arrow::ArrowSchemaVisitor::after_list_element(&mut self, _field: &arrow_schema::field::FieldRef) -> iceberg::Result<()> +pub fn iceberg::arrow::ArrowSchemaVisitor::after_map_key(&mut self, _field: &arrow_schema::field::FieldRef) -> iceberg::Result<()> +pub fn iceberg::arrow::ArrowSchemaVisitor::after_map_value(&mut self, _field: &arrow_schema::field::FieldRef) -> iceberg::Result<()> +pub fn iceberg::arrow::ArrowSchemaVisitor::before_field(&mut self, _field: &arrow_schema::field::FieldRef) -> iceberg::Result<()> +pub fn iceberg::arrow::ArrowSchemaVisitor::before_list_element(&mut self, _field: &arrow_schema::field::FieldRef) -> iceberg::Result<()> +pub fn iceberg::arrow::ArrowSchemaVisitor::before_map_key(&mut self, _field: &arrow_schema::field::FieldRef) -> iceberg::Result<()> +pub fn iceberg::arrow::ArrowSchemaVisitor::before_map_value(&mut self, _field: &arrow_schema::field::FieldRef) -> iceberg::Result<()> +pub fn iceberg::arrow::ArrowSchemaVisitor::list(&mut self, list: &arrow_schema::datatype::DataType, value: Self::T) -> iceberg::Result +pub fn iceberg::arrow::ArrowSchemaVisitor::map(&mut self, map: &arrow_schema::datatype::DataType, key_value: Self::T, value: Self::T) -> iceberg::Result +pub fn iceberg::arrow::ArrowSchemaVisitor::primitive(&mut self, p: &arrow_schema::datatype::DataType) -> iceberg::Result +pub fn iceberg::arrow::ArrowSchemaVisitor::schema(&mut self, schema: &arrow_schema::schema::Schema, values: alloc::vec::Vec) -> iceberg::Result +pub fn iceberg::arrow::ArrowSchemaVisitor::struct(&mut self, fields: &arrow_schema::fields::Fields, results: alloc::vec::Vec) -> iceberg::Result +pub fn iceberg::arrow::ArrowSchemaVisitor::uuid(&mut self, field: &arrow_schema::field::FieldRef) -> iceberg::Result where Self: core::marker::Sized +pub fn iceberg::arrow::ArrowSchemaVisitor::variant(&mut self, field: &arrow_schema::field::FieldRef) -> iceberg::Result where Self: core::marker::Sized +pub fn iceberg::arrow::arrow_primitive_to_literal(primitive_array: &arrow_array::array::ArrayRef, ty: &iceberg::spec::Type) -> iceberg::Result>> +pub fn iceberg::arrow::arrow_schema_to_schema(schema: &arrow_schema::schema::Schema) -> iceberg::Result +pub fn iceberg::arrow::arrow_schema_to_schema_auto_assign_ids(schema: &arrow_schema::schema::Schema) -> iceberg::Result +pub fn iceberg::arrow::arrow_struct_to_literal(struct_array: &arrow_array::array::ArrayRef, ty: &iceberg::spec::StructType) -> iceberg::Result>> +pub fn iceberg::arrow::arrow_type_to_type(ty: &arrow_schema::datatype::DataType) -> iceberg::Result +pub fn iceberg::arrow::datum_to_arrow_type_with_ree(datum: &iceberg::spec::Datum) -> arrow_schema::datatype::DataType +pub fn iceberg::arrow::read_manifest_entries(bytes: &[u8], manifest_file: &iceberg::spec::ManifestFile) -> iceberg::Result +pub fn iceberg::arrow::read_manifest_list(bytes: &[u8]) -> iceberg::Result +pub fn iceberg::arrow::schema_to_arrow_schema(schema: &iceberg::spec::Schema) -> iceberg::Result +pub fn iceberg::arrow::strip_metadata_from_schema(schema: &arrow_schema::schema::Schema) -> iceberg::Result +pub fn iceberg::arrow::type_to_arrow_type(ty: &iceberg::spec::Type) -> iceberg::Result +pub mod iceberg::cache +pub trait iceberg::cache::ObjectCache: core::marker::Send + core::marker::Sync +pub fn iceberg::cache::ObjectCache::get(&self, key: &K) -> core::option::Option +pub fn iceberg::cache::ObjectCache::set(&self, key: K, value: V) +pub trait iceberg::cache::ObjectCacheProvide: core::marker::Send + core::marker::Sync +pub fn iceberg::cache::ObjectCacheProvide::manifest_cache(&self) -> &dyn iceberg::cache::ObjectCache> +pub fn iceberg::cache::ObjectCacheProvide::manifest_list_cache(&self) -> &dyn iceberg::cache::ObjectCache> +pub type iceberg::cache::ObjectCacheProvider = alloc::sync::Arc +pub mod iceberg::compression +pub enum iceberg::compression::CompressionCodec +pub iceberg::compression::CompressionCodec::Brotli(u8) +pub iceberg::compression::CompressionCodec::Gzip(u8) +pub iceberg::compression::CompressionCodec::Lz4 +pub iceberg::compression::CompressionCodec::Lz4Raw +pub iceberg::compression::CompressionCodec::Lzo +pub iceberg::compression::CompressionCodec::None +pub iceberg::compression::CompressionCodec::Snappy +pub iceberg::compression::CompressionCodec::Zstd(u8) +impl iceberg::compression::CompressionCodec +pub const fn iceberg::compression::CompressionCodec::brotli_default() -> Self +pub const fn iceberg::compression::CompressionCodec::gzip_default() -> Self +pub fn iceberg::compression::CompressionCodec::name(&self) -> &'static str +pub const fn iceberg::compression::CompressionCodec::zstd_default() -> Self +impl iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::suffix(&self) -> iceberg::Result<&'static str> +impl core::clone::Clone for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::clone(&self) -> iceberg::compression::CompressionCodec +impl core::cmp::Eq for iceberg::compression::CompressionCodec +impl core::cmp::PartialEq for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::eq(&self, other: &iceberg::compression::CompressionCodec) -> bool +impl core::default::Default for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::default() -> iceberg::compression::CompressionCodec +impl core::fmt::Debug for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::compression::CompressionCodec +impl core::marker::StructuralPartialEq for iceberg::compression::CompressionCodec +impl serde_core::ser::Serialize for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> +impl<'de> serde_core::de::Deserialize<'de> for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::deserialize>(deserializer: D) -> core::result::Result::Error> +pub mod iceberg::cow_rewrite +pub struct iceberg::cow_rewrite::CowBatchRewrite +pub iceberg::cow_rewrite::CowBatchRewrite::changed: bool +pub iceberg::cow_rewrite::CowBatchRewrite::output: core::option::Option +impl core::fmt::Debug for iceberg::cow_rewrite::CowBatchRewrite +pub fn iceberg::cow_rewrite::CowBatchRewrite::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::cow_rewrite::CowRewriteBuilder<'a> +impl<'a> iceberg::cow_rewrite::CowRewriteBuilder<'a> +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::new(table: &'a iceberg::table::Table) -> Self +pub async fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::rewrite(self) -> iceberg::Result +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_batch_size(self, batch_size: usize) -> Self +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_case_sensitive(self, case_sensitive: bool) -> Self +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_predicate(self, predicate: iceberg::expr::Predicate) -> Self +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_rewriter(self, rewriter: alloc::sync::Arc) -> Self +pub fn iceberg::cow_rewrite::CowRewriteBuilder<'a>::with_snapshot_id(self, snapshot_id: i64) -> Self +pub struct iceberg::cow_rewrite::CowRewriteFile +impl iceberg::cow_rewrite::CowRewriteFile +pub fn iceberg::cow_rewrite::CowRewriteFile::old_data_file(&self) -> &iceberg::spec::DataFile +pub fn iceberg::cow_rewrite::CowRewriteFile::scan_task(&self) -> &iceberg::scan::FileScanTask +impl core::clone::Clone for iceberg::cow_rewrite::CowRewriteFile +pub fn iceberg::cow_rewrite::CowRewriteFile::clone(&self) -> iceberg::cow_rewrite::CowRewriteFile +impl core::fmt::Debug for iceberg::cow_rewrite::CowRewriteFile +pub fn iceberg::cow_rewrite::CowRewriteFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::cow_rewrite::CowRewriteResult +pub iceberg::cow_rewrite::CowRewriteResult::added_data_files: alloc::vec::Vec +pub iceberg::cow_rewrite::CowRewriteResult::removed_data_files: alloc::vec::Vec +pub iceberg::cow_rewrite::CowRewriteResult::stats: iceberg::cow_rewrite::CowRewriteStats +pub iceberg::cow_rewrite::CowRewriteResult::unchanged_data_files: alloc::vec::Vec +impl iceberg::cow_rewrite::CowRewriteResult +pub fn iceberg::cow_rewrite::CowRewriteResult::has_changes(&self) -> bool +impl core::default::Default for iceberg::cow_rewrite::CowRewriteResult +pub fn iceberg::cow_rewrite::CowRewriteResult::default() -> iceberg::cow_rewrite::CowRewriteResult +impl core::fmt::Debug for iceberg::cow_rewrite::CowRewriteResult +pub fn iceberg::cow_rewrite::CowRewriteResult::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::cow_rewrite::CowRewriteStats +pub iceberg::cow_rewrite::CowRewriteStats::candidate_files: usize +pub iceberg::cow_rewrite::CowRewriteStats::changed_batches: u64 +pub iceberg::cow_rewrite::CowRewriteStats::input_rows: u64 +pub iceberg::cow_rewrite::CowRewriteStats::output_rows: u64 +pub iceberg::cow_rewrite::CowRewriteStats::rewritten_files: usize +pub iceberg::cow_rewrite::CowRewriteStats::unchanged_files: usize +impl core::clone::Clone for iceberg::cow_rewrite::CowRewriteStats +pub fn iceberg::cow_rewrite::CowRewriteStats::clone(&self) -> iceberg::cow_rewrite::CowRewriteStats +impl core::cmp::Eq for iceberg::cow_rewrite::CowRewriteStats +impl core::cmp::PartialEq for iceberg::cow_rewrite::CowRewriteStats +pub fn iceberg::cow_rewrite::CowRewriteStats::eq(&self, other: &iceberg::cow_rewrite::CowRewriteStats) -> bool +impl core::default::Default for iceberg::cow_rewrite::CowRewriteStats +pub fn iceberg::cow_rewrite::CowRewriteStats::default() -> iceberg::cow_rewrite::CowRewriteStats +impl core::fmt::Debug for iceberg::cow_rewrite::CowRewriteStats +pub fn iceberg::cow_rewrite::CowRewriteStats::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::cow_rewrite::CowRewriteStats +impl core::marker::StructuralPartialEq for iceberg::cow_rewrite::CowRewriteStats +pub trait iceberg::cow_rewrite::CowBatchRewriter: core::marker::Send + core::marker::Sync +pub fn iceberg::cow_rewrite::CowBatchRewriter::rewrite_batch(&self, batch: arrow_array::record_batch::RecordBatch) -> iceberg::Result +pub mod iceberg::delete_vector +pub struct iceberg::delete_vector::DeleteVector +impl iceberg::delete_vector::DeleteVector +pub fn iceberg::delete_vector::DeleteVector::deserialize(blob: &[u8]) -> iceberg::Result +pub fn iceberg::delete_vector::DeleteVector::insert(&mut self, pos: u64) -> bool +pub fn iceberg::delete_vector::DeleteVector::insert_positions(&mut self, positions: &[u64]) -> iceberg::Result +pub fn iceberg::delete_vector::DeleteVector::is_empty(&self) -> bool +pub fn iceberg::delete_vector::DeleteVector::iter(&self) -> iceberg::delete_vector::DeleteVectorIterator<'_> +pub fn iceberg::delete_vector::DeleteVector::len(&self) -> u64 +pub fn iceberg::delete_vector::DeleteVector::new(roaring_treemap: roaring::treemap::RoaringTreemap) -> iceberg::delete_vector::DeleteVector +impl core::default::Default for iceberg::delete_vector::DeleteVector +pub fn iceberg::delete_vector::DeleteVector::default() -> iceberg::delete_vector::DeleteVector +impl core::fmt::Debug for iceberg::delete_vector::DeleteVector +pub fn iceberg::delete_vector::DeleteVector::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::ops::bit::BitOrAssign for iceberg::delete_vector::DeleteVector +pub fn iceberg::delete_vector::DeleteVector::bitor_assign(&mut self, other: Self) +pub struct iceberg::delete_vector::DeleteVectorIterator<'a> +impl iceberg::delete_vector::DeleteVectorIterator<'_> +pub fn iceberg::delete_vector::DeleteVectorIterator<'_>::advance_to(&mut self, pos: u64) +impl core::iter::traits::iterator::Iterator for iceberg::delete_vector::DeleteVectorIterator<'_> +pub type iceberg::delete_vector::DeleteVectorIterator<'_>::Item = u64 +pub fn iceberg::delete_vector::DeleteVectorIterator<'_>::next(&mut self) -> core::option::Option +pub mod iceberg::encryption +pub mod iceberg::encryption::kms +pub struct iceberg::encryption::kms::GeneratedKey +impl iceberg::encryption::GeneratedKey +pub fn iceberg::encryption::GeneratedKey::key(&self) -> &iceberg::sensitive::SensitiveBytes +pub fn iceberg::encryption::GeneratedKey::new(key: iceberg::sensitive::SensitiveBytes, wrapped_key: alloc::vec::Vec) -> Self +pub fn iceberg::encryption::GeneratedKey::wrapped_key(&self) -> &[u8] +pub struct iceberg::encryption::kms::MemoryKeyManagementClient +impl iceberg::encryption::kms::MemoryKeyManagementClient +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::add_master_key(&self, key_id: impl core::convert::Into) -> iceberg::Result<()> +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::add_master_key_bytes(&self, key_id: impl core::convert::Into, key_bytes: iceberg::sensitive::SensitiveBytes) -> iceberg::Result<()> +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::has_key(&self, key_id: &str) -> bool +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::key_count(&self) -> usize +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::new() -> Self +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::with_master_key_size(master_key_size: iceberg::encryption::AesKeySize) -> Self +impl core::clone::Clone for iceberg::encryption::kms::MemoryKeyManagementClient +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::clone(&self) -> iceberg::encryption::kms::MemoryKeyManagementClient +impl core::default::Default for iceberg::encryption::kms::MemoryKeyManagementClient +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::default() -> iceberg::encryption::kms::MemoryKeyManagementClient +impl core::fmt::Debug for iceberg::encryption::kms::MemoryKeyManagementClient +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::encryption::KeyManagementClient for iceberg::encryption::kms::MemoryKeyManagementClient +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::generate_key<'life0, 'life1, 'async_trait>(&'life0 self, _wrapping_key_id: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::supports_key_generation(&self) -> bool +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::unwrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, wrapped_key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::wrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub struct iceberg::encryption::kms::MemoryKmsClientFactory +impl iceberg::encryption::kms::MemoryKmsClientFactory +pub fn iceberg::encryption::kms::MemoryKmsClientFactory::add_master_key(&self, key_id: impl core::convert::Into) -> iceberg::Result<()> +pub fn iceberg::encryption::kms::MemoryKmsClientFactory::add_master_key_bytes(&self, key_id: impl core::convert::Into, key_bytes: iceberg::sensitive::SensitiveBytes) -> iceberg::Result<()> +pub fn iceberg::encryption::kms::MemoryKmsClientFactory::new() -> Self +pub fn iceberg::encryption::kms::MemoryKmsClientFactory::with_master_key_size(master_key_size: iceberg::encryption::AesKeySize) -> Self +impl core::clone::Clone for iceberg::encryption::kms::MemoryKmsClientFactory +pub fn iceberg::encryption::kms::MemoryKmsClientFactory::clone(&self) -> iceberg::encryption::kms::MemoryKmsClientFactory +impl core::default::Default for iceberg::encryption::kms::MemoryKmsClientFactory +pub fn iceberg::encryption::kms::MemoryKmsClientFactory::default() -> iceberg::encryption::kms::MemoryKmsClientFactory +impl core::fmt::Debug for iceberg::encryption::kms::MemoryKmsClientFactory +pub fn iceberg::encryption::kms::MemoryKmsClientFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::encryption::kms::KmsClientFactory for iceberg::encryption::kms::MemoryKmsClientFactory +pub fn iceberg::encryption::kms::MemoryKmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, _properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub trait iceberg::encryption::kms::KeyManagementClient: core::marker::Send + core::marker::Sync + core::fmt::Debug +pub fn iceberg::encryption::kms::KeyManagementClient::generate_key<'life0, 'life1, 'async_trait>(&'life0 self, wrapping_key_id: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::encryption::kms::KeyManagementClient::supports_key_generation(&self) -> bool +pub fn iceberg::encryption::kms::KeyManagementClient::unwrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, wrapped_key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::encryption::kms::KeyManagementClient::wrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +impl iceberg::encryption::KeyManagementClient for iceberg::encryption::kms::MemoryKeyManagementClient +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::generate_key<'life0, 'life1, 'async_trait>(&'life0 self, _wrapping_key_id: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::supports_key_generation(&self) -> bool +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::unwrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, wrapped_key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::wrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +impl + core::marker::Send + core::marker::Sync + core::fmt::Debug> iceberg::encryption::KeyManagementClient for T +pub fn T::generate_key<'life0, 'life1, 'async_trait>(&'life0 self, wrapping_key_id: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn T::supports_key_generation(&self) -> bool +pub fn T::unwrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, wrapped_key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn T::wrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub trait iceberg::encryption::kms::KmsClientFactory: core::fmt::Debug + core::marker::Send + core::marker::Sync +pub fn iceberg::encryption::kms::KmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +impl iceberg::encryption::kms::KmsClientFactory for iceberg::encryption::kms::MemoryKmsClientFactory +pub fn iceberg::encryption::kms::MemoryKmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, _properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub enum iceberg::encryption::AesKeySize +pub iceberg::encryption::AesKeySize::Bits128 = 128 +pub iceberg::encryption::AesKeySize::Bits192 = 192 +pub iceberg::encryption::AesKeySize::Bits256 = 256 +impl iceberg::encryption::AesKeySize +pub fn iceberg::encryption::AesKeySize::from_key_length(len: usize) -> iceberg::Result +pub fn iceberg::encryption::AesKeySize::key_length(&self) -> usize +impl core::clone::Clone for iceberg::encryption::AesKeySize +pub fn iceberg::encryption::AesKeySize::clone(&self) -> iceberg::encryption::AesKeySize +impl core::cmp::Eq for iceberg::encryption::AesKeySize +impl core::cmp::PartialEq for iceberg::encryption::AesKeySize +pub fn iceberg::encryption::AesKeySize::eq(&self, other: &iceberg::encryption::AesKeySize) -> bool +impl core::default::Default for iceberg::encryption::AesKeySize +pub fn iceberg::encryption::AesKeySize::default() -> iceberg::encryption::AesKeySize +impl core::fmt::Debug for iceberg::encryption::AesKeySize +pub fn iceberg::encryption::AesKeySize::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::encryption::AesKeySize +impl core::marker::StructuralPartialEq for iceberg::encryption::AesKeySize +impl core::str::traits::FromStr for iceberg::encryption::AesKeySize +pub type iceberg::encryption::AesKeySize::Err = iceberg::Error +pub fn iceberg::encryption::AesKeySize::from_str(s: &str) -> iceberg::Result +pub struct iceberg::encryption::AesGcmCipher +impl iceberg::encryption::AesGcmCipher +pub const iceberg::encryption::AesGcmCipher::NONCE_LEN: usize +pub const iceberg::encryption::AesGcmCipher::TAG_LEN: usize +pub fn iceberg::encryption::AesGcmCipher::decrypt(&self, ciphertext: &[u8], aad: core::option::Option<&[u8]>) -> iceberg::Result> +pub fn iceberg::encryption::AesGcmCipher::encrypt(&self, plaintext: &[u8], aad: core::option::Option<&[u8]>) -> iceberg::Result> +pub fn iceberg::encryption::AesGcmCipher::new(key: iceberg::encryption::SecureKey) -> Self +pub struct iceberg::encryption::AesGcmFileRead +impl iceberg::encryption::AesGcmFileRead +pub fn iceberg::encryption::AesGcmFileRead::calculate_plaintext_length(encrypted_file_length: u64) -> iceberg::Result +pub fn iceberg::encryption::AesGcmFileRead::new(inner: alloc::boxed::Box, cipher: alloc::sync::Arc, aad_prefix: alloc::boxed::Box<[u8]>, encrypted_file_length: u64) -> iceberg::Result +pub fn iceberg::encryption::AesGcmFileRead::plaintext_length(&self) -> u64 +impl iceberg::io::FileRead for iceberg::encryption::AesGcmFileRead +pub fn iceberg::encryption::AesGcmFileRead::read<'life0, 'async_trait>(&'life0 self, range: core::ops::range::Range) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub struct iceberg::encryption::AesGcmFileWrite +impl iceberg::encryption::AesGcmFileWrite +pub fn iceberg::encryption::AesGcmFileWrite::new(inner: alloc::boxed::Box, cipher: alloc::sync::Arc, aad_prefix: impl core::convert::Into>) -> Self +impl iceberg::io::FileWrite for iceberg::encryption::AesGcmFileWrite +pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::encryption::AesGcmFileWrite::write<'life0, 'async_trait>(&'life0 mut self, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub struct iceberg::encryption::EncryptedInputFile +impl iceberg::encryption::EncryptedInputFile +pub async fn iceberg::encryption::EncryptedInputFile::exists(&self) -> iceberg::Result +pub fn iceberg::encryption::EncryptedInputFile::into_inner(self) -> iceberg::io::InputFile +pub fn iceberg::encryption::EncryptedInputFile::key_metadata(&self) -> &iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::EncryptedInputFile::location(&self) -> &str +pub fn iceberg::encryption::EncryptedInputFile::metadata(&self) -> iceberg::Result +pub fn iceberg::encryption::EncryptedInputFile::new(inner: iceberg::io::InputFile, key_metadata: iceberg::encryption::StandardKeyMetadata) -> Self +pub async fn iceberg::encryption::EncryptedInputFile::read(&self) -> iceberg::Result +pub async fn iceberg::encryption::EncryptedInputFile::reader(&self) -> iceberg::Result> +impl core::fmt::Debug for iceberg::encryption::EncryptedInputFile +pub fn iceberg::encryption::EncryptedInputFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::encryption::EncryptedOutputFile +impl iceberg::encryption::EncryptedOutputFile +pub async fn iceberg::encryption::EncryptedOutputFile::delete(&self) -> iceberg::Result<()> +pub fn iceberg::encryption::EncryptedOutputFile::into_inner(self) -> iceberg::io::OutputFile +pub fn iceberg::encryption::EncryptedOutputFile::key_metadata_with_saved_file_metadata(&self, file_metadata: &iceberg::io::FileMetadata) -> iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::EncryptedOutputFile::location(&self) -> &str +pub fn iceberg::encryption::EncryptedOutputFile::new(inner: iceberg::io::OutputFile, key_metadata: iceberg::encryption::StandardKeyMetadata) -> Self +pub async fn iceberg::encryption::EncryptedOutputFile::write(&self, bs: bytes::bytes::Bytes) -> iceberg::Result +pub async fn iceberg::encryption::EncryptedOutputFile::writer(&self) -> iceberg::Result> +impl core::fmt::Debug for iceberg::encryption::EncryptedOutputFile +pub fn iceberg::encryption::EncryptedOutputFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::encryption::EncryptionManager +impl iceberg::encryption::EncryptionManager +pub async fn iceberg::encryption::EncryptionManager::decrypt_manifest_list_key_metadata(&self, encryption_key_id: &str) -> iceberg::Result +pub fn iceberg::encryption::EncryptionManager::encrypt(&self, raw_output: iceberg::io::OutputFile) -> iceberg::encryption::EncryptedOutputFile +pub async fn iceberg::encryption::EncryptionManager::encrypt_manifest_list_key_metadata(&self, key_metadata: &iceberg::encryption::StandardKeyMetadata) -> iceberg::Result +pub fn iceberg::encryption::EncryptionManager::generate_key_metadata(&self) -> iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::EncryptionManager::with_encryption_keys(&self, f: F) -> R where F: core::ops::function::FnOnce(&std::collections::hash::map::HashMap) -> R +impl core::fmt::Debug for iceberg::encryption::EncryptionManager +pub fn iceberg::encryption::EncryptionManager::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::encryption::EncryptionManager +pub fn iceberg::encryption::EncryptionManager::builder() -> EncryptionManagerBuilder<((), (), (), (std::sync::poison::rwlock::RwLock>))> +pub struct iceberg::encryption::GeneratedKey +impl iceberg::encryption::GeneratedKey +pub fn iceberg::encryption::GeneratedKey::key(&self) -> &iceberg::sensitive::SensitiveBytes +pub fn iceberg::encryption::GeneratedKey::new(key: iceberg::sensitive::SensitiveBytes, wrapped_key: alloc::vec::Vec) -> Self +pub fn iceberg::encryption::GeneratedKey::wrapped_key(&self) -> &[u8] +pub struct iceberg::encryption::SecureKey +impl iceberg::encryption::SecureKey +pub fn iceberg::encryption::SecureKey::as_bytes(&self) -> &[u8] +pub fn iceberg::encryption::SecureKey::generate(key_size: iceberg::encryption::AesKeySize) -> Self +pub fn iceberg::encryption::SecureKey::key_size(&self) -> iceberg::encryption::AesKeySize +pub fn iceberg::encryption::SecureKey::new(key: &[u8]) -> iceberg::Result +impl core::clone::Clone for iceberg::encryption::SecureKey +pub fn iceberg::encryption::SecureKey::clone(&self) -> iceberg::encryption::SecureKey +impl core::cmp::Eq for iceberg::encryption::SecureKey +impl core::cmp::PartialEq for iceberg::encryption::SecureKey +pub fn iceberg::encryption::SecureKey::eq(&self, other: &iceberg::encryption::SecureKey) -> bool +impl core::convert::From for iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::StandardKeyMetadata::from(encryption_key: iceberg::encryption::SecureKey) -> Self +impl core::convert::TryFrom for iceberg::encryption::SecureKey +pub type iceberg::encryption::SecureKey::Error = iceberg::Error +pub fn iceberg::encryption::SecureKey::try_from(key: iceberg::sensitive::SensitiveBytes) -> iceberg::Result +impl core::fmt::Debug for iceberg::encryption::SecureKey +pub fn iceberg::encryption::SecureKey::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::encryption::SecureKey +pub struct iceberg::encryption::SensitiveBytes(_) +impl iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::as_bytes(&self) -> &[u8] +pub fn iceberg::sensitive::SensitiveBytes::is_empty(&self) -> bool +pub fn iceberg::sensitive::SensitiveBytes::len(&self) -> usize +pub fn iceberg::sensitive::SensitiveBytes::new(bytes: impl core::convert::Into>) -> Self +impl core::clone::Clone for iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::clone(&self) -> iceberg::sensitive::SensitiveBytes +impl core::cmp::Eq for iceberg::sensitive::SensitiveBytes +impl core::cmp::PartialEq for iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::eq(&self, other: &iceberg::sensitive::SensitiveBytes) -> bool +impl core::convert::TryFrom for iceberg::encryption::SecureKey +pub type iceberg::encryption::SecureKey::Error = iceberg::Error +pub fn iceberg::encryption::SecureKey::try_from(key: iceberg::sensitive::SensitiveBytes) -> iceberg::Result +impl core::fmt::Debug for iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::sensitive::SensitiveBytes +pub struct iceberg::encryption::StandardKeyMetadata +impl iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::StandardKeyMetadata::aad_prefix(&self) -> core::option::Option<&[u8]> +pub fn iceberg::encryption::StandardKeyMetadata::decode(bytes: &[u8]) -> iceberg::Result +pub fn iceberg::encryption::StandardKeyMetadata::encode(&self) -> iceberg::Result> +pub fn iceberg::encryption::StandardKeyMetadata::encryption_key(&self) -> &iceberg::encryption::SecureKey +pub fn iceberg::encryption::StandardKeyMetadata::file_length(&self) -> core::option::Option +pub fn iceberg::encryption::StandardKeyMetadata::try_new(encryption_key: &[u8]) -> iceberg::Result +pub fn iceberg::encryption::StandardKeyMetadata::with_aad_prefix(self, aad_prefix: &[u8]) -> Self +pub fn iceberg::encryption::StandardKeyMetadata::with_file_length(self, length: u64) -> Self +impl core::clone::Clone for iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::StandardKeyMetadata::clone(&self) -> iceberg::encryption::StandardKeyMetadata +impl core::cmp::Eq for iceberg::encryption::StandardKeyMetadata +impl core::cmp::PartialEq for iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::StandardKeyMetadata::eq(&self, other: &iceberg::encryption::StandardKeyMetadata) -> bool +impl core::convert::From for iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::StandardKeyMetadata::from(encryption_key: iceberg::encryption::SecureKey) -> Self +impl core::fmt::Debug for iceberg::encryption::StandardKeyMetadata +pub fn iceberg::encryption::StandardKeyMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::encryption::StandardKeyMetadata +pub trait iceberg::encryption::KeyManagementClient: core::marker::Send + core::marker::Sync + core::fmt::Debug +pub fn iceberg::encryption::KeyManagementClient::generate_key<'life0, 'life1, 'async_trait>(&'life0 self, wrapping_key_id: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::encryption::KeyManagementClient::supports_key_generation(&self) -> bool +pub fn iceberg::encryption::KeyManagementClient::unwrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, wrapped_key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::encryption::KeyManagementClient::wrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +impl iceberg::encryption::KeyManagementClient for iceberg::encryption::kms::MemoryKeyManagementClient +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::generate_key<'life0, 'life1, 'async_trait>(&'life0 self, _wrapping_key_id: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::supports_key_generation(&self) -> bool +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::unwrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, wrapped_key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::encryption::kms::MemoryKeyManagementClient::wrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +impl + core::marker::Send + core::marker::Sync + core::fmt::Debug> iceberg::encryption::KeyManagementClient for T +pub fn T::generate_key<'life0, 'life1, 'async_trait>(&'life0 self, wrapping_key_id: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn T::supports_key_generation(&self) -> bool +pub fn T::unwrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, wrapped_key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn T::wrap_key<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, key: &'life1 [u8], wrapping_key_id: &'life2 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub mod iceberg::expr +pub enum iceberg::expr::BoundPredicate +pub iceberg::expr::BoundPredicate::AlwaysFalse +pub iceberg::expr::BoundPredicate::AlwaysTrue +pub iceberg::expr::BoundPredicate::And(iceberg::expr::LogicalExpression) +pub iceberg::expr::BoundPredicate::Binary(iceberg::expr::BinaryExpression) +pub iceberg::expr::BoundPredicate::Not(iceberg::expr::LogicalExpression) +pub iceberg::expr::BoundPredicate::Or(iceberg::expr::LogicalExpression) +pub iceberg::expr::BoundPredicate::Set(iceberg::expr::SetExpression) +pub iceberg::expr::BoundPredicate::Unary(iceberg::expr::UnaryExpression) +impl iceberg::expr::BoundPredicate +pub fn iceberg::expr::BoundPredicate::rewrite_not(self) -> iceberg::expr::BoundPredicate +impl core::clone::Clone for iceberg::expr::BoundPredicate +pub fn iceberg::expr::BoundPredicate::clone(&self) -> iceberg::expr::BoundPredicate +impl core::cmp::PartialEq for iceberg::expr::BoundPredicate +pub fn iceberg::expr::BoundPredicate::eq(&self, other: &iceberg::expr::BoundPredicate) -> bool +impl core::fmt::Debug for iceberg::expr::BoundPredicate +pub fn iceberg::expr::BoundPredicate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::expr::BoundPredicate +pub fn iceberg::expr::BoundPredicate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::expr::BoundPredicate +impl serde_core::ser::Serialize for iceberg::expr::BoundPredicate +pub fn iceberg::expr::BoundPredicate::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::expr::BoundPredicate +pub fn iceberg::expr::BoundPredicate::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::expr::Predicate +pub iceberg::expr::Predicate::AlwaysFalse +pub iceberg::expr::Predicate::AlwaysTrue +pub iceberg::expr::Predicate::And(iceberg::expr::LogicalExpression) +pub iceberg::expr::Predicate::Binary(iceberg::expr::BinaryExpression) +pub iceberg::expr::Predicate::Not(iceberg::expr::LogicalExpression) +pub iceberg::expr::Predicate::Or(iceberg::expr::LogicalExpression) +pub iceberg::expr::Predicate::Set(iceberg::expr::SetExpression) +pub iceberg::expr::Predicate::Unary(iceberg::expr::UnaryExpression) +impl iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::and(self, other: iceberg::expr::Predicate) -> iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::negate(self) -> iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::or(self, other: iceberg::expr::Predicate) -> iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::rewrite_not(self) -> iceberg::expr::Predicate +impl core::clone::Clone for iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::clone(&self) -> iceberg::expr::Predicate +impl core::cmp::PartialEq for iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::eq(&self, other: &iceberg::expr::Predicate) -> bool +impl core::fmt::Debug for iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::expr::Predicate +impl core::ops::bit::Not for iceberg::expr::Predicate +pub type iceberg::expr::Predicate::Output = iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::not(self) -> Self::Output +impl iceberg::expr::Bind for iceberg::expr::Predicate +pub type iceberg::expr::Predicate::Bound = iceberg::expr::BoundPredicate +pub fn iceberg::expr::Predicate::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl serde_core::ser::Serialize for iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::expr::Predicate +pub fn iceberg::expr::Predicate::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +#[non_exhaustive] #[repr(u16)] pub enum iceberg::expr::PredicateOperator +pub iceberg::expr::PredicateOperator::Eq = 205 +pub iceberg::expr::PredicateOperator::GreaterThan = 203 +pub iceberg::expr::PredicateOperator::GreaterThanOrEq = 204 +pub iceberg::expr::PredicateOperator::In = 301 +pub iceberg::expr::PredicateOperator::IsNan = 103 +pub iceberg::expr::PredicateOperator::IsNull = 101 +pub iceberg::expr::PredicateOperator::LessThan = 201 +pub iceberg::expr::PredicateOperator::LessThanOrEq = 202 +pub iceberg::expr::PredicateOperator::NotEq = 206 +pub iceberg::expr::PredicateOperator::NotIn = 302 +pub iceberg::expr::PredicateOperator::NotNan = 104 +pub iceberg::expr::PredicateOperator::NotNull = 102 +pub iceberg::expr::PredicateOperator::NotStartsWith = 208 +pub iceberg::expr::PredicateOperator::StartsWith = 207 +impl iceberg::expr::PredicateOperator +pub fn iceberg::expr::PredicateOperator::is_binary(self) -> bool +pub fn iceberg::expr::PredicateOperator::is_set(self) -> bool +pub fn iceberg::expr::PredicateOperator::is_unary(self) -> bool +pub fn iceberg::expr::PredicateOperator::negate(self) -> iceberg::expr::PredicateOperator +impl core::clone::Clone for iceberg::expr::PredicateOperator +pub fn iceberg::expr::PredicateOperator::clone(&self) -> iceberg::expr::PredicateOperator +impl core::cmp::PartialEq for iceberg::expr::PredicateOperator +pub fn iceberg::expr::PredicateOperator::eq(&self, other: &iceberg::expr::PredicateOperator) -> bool +impl core::fmt::Debug for iceberg::expr::PredicateOperator +pub fn iceberg::expr::PredicateOperator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::expr::PredicateOperator +pub fn iceberg::expr::PredicateOperator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::expr::PredicateOperator +impl core::marker::StructuralPartialEq for iceberg::expr::PredicateOperator +impl serde_core::ser::Serialize for iceberg::expr::PredicateOperator +pub fn iceberg::expr::PredicateOperator::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::expr::PredicateOperator +pub fn iceberg::expr::PredicateOperator::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::expr::BinaryExpression +impl iceberg::expr::BinaryExpression +pub fn iceberg::expr::BinaryExpression::literal(&self) -> &iceberg::spec::Datum +pub fn iceberg::expr::BinaryExpression::new(op: iceberg::expr::PredicateOperator, term: T, literal: iceberg::spec::Datum) -> Self +pub fn iceberg::expr::BinaryExpression::op(&self) -> iceberg::expr::PredicateOperator +pub fn iceberg::expr::BinaryExpression::term(&self) -> &T +impl<'de, T> serde_core::de::Deserialize<'de> for iceberg::expr::BinaryExpression where T: serde_core::de::Deserialize<'de> +pub fn iceberg::expr::BinaryExpression::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +impl core::clone::Clone for iceberg::expr::BinaryExpression +pub fn iceberg::expr::BinaryExpression::clone(&self) -> iceberg::expr::BinaryExpression +impl core::cmp::PartialEq for iceberg::expr::BinaryExpression +pub fn iceberg::expr::BinaryExpression::eq(&self, other: &iceberg::expr::BinaryExpression) -> bool +impl core::fmt::Debug for iceberg::expr::BinaryExpression +pub fn iceberg::expr::BinaryExpression::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::expr::BinaryExpression +pub fn iceberg::expr::BinaryExpression::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::expr::Bind for iceberg::expr::BinaryExpression +pub type iceberg::expr::BinaryExpression::Bound = iceberg::expr::BinaryExpression<::Bound> +pub fn iceberg::expr::BinaryExpression::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl core::marker::StructuralPartialEq for iceberg::expr::BinaryExpression +impl serde_core::ser::Serialize for iceberg::expr::BinaryExpression where T: serde_core::ser::Serialize +pub fn iceberg::expr::BinaryExpression::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +pub struct iceberg::expr::BoundReference +impl iceberg::expr::BoundReference +pub fn iceberg::expr::BoundReference::accessor(&self) -> &iceberg::expr::accessor::StructAccessor +pub fn iceberg::expr::BoundReference::field(&self) -> &iceberg::spec::NestedField +pub fn iceberg::expr::BoundReference::new(name: impl core::convert::Into, field: iceberg::spec::NestedFieldRef, accessor: alloc::sync::Arc) -> Self +impl core::clone::Clone for iceberg::expr::BoundReference +pub fn iceberg::expr::BoundReference::clone(&self) -> iceberg::expr::BoundReference +impl core::cmp::Eq for iceberg::expr::BoundReference +impl core::cmp::PartialEq for iceberg::expr::BoundReference +pub fn iceberg::expr::BoundReference::eq(&self, other: &iceberg::expr::BoundReference) -> bool +impl core::fmt::Debug for iceberg::expr::BoundReference +pub fn iceberg::expr::BoundReference::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::expr::BoundReference +pub fn iceberg::expr::BoundReference::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::expr::BoundReference +impl serde_core::ser::Serialize for iceberg::expr::BoundReference +pub fn iceberg::expr::BoundReference::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::expr::BoundReference +pub fn iceberg::expr::BoundReference::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::expr::LogicalExpression +impl iceberg::expr::LogicalExpression +pub fn iceberg::expr::LogicalExpression::inputs(&self) -> [&T; N] +impl<'de, T: serde_core::de::Deserialize<'de>, const N: usize> serde_core::de::Deserialize<'de> for iceberg::expr::LogicalExpression +pub fn iceberg::expr::LogicalExpression::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +impl core::marker::StructuralPartialEq for iceberg::expr::LogicalExpression +impl core::clone::Clone for iceberg::expr::LogicalExpression +pub fn iceberg::expr::LogicalExpression::clone(&self) -> iceberg::expr::LogicalExpression +impl core::cmp::PartialEq for iceberg::expr::LogicalExpression +pub fn iceberg::expr::LogicalExpression::eq(&self, other: &iceberg::expr::LogicalExpression) -> bool +impl core::fmt::Debug for iceberg::expr::LogicalExpression +pub fn iceberg::expr::LogicalExpression::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::expr::Bind for iceberg::expr::LogicalExpression where ::Bound: core::marker::Sized +pub type iceberg::expr::LogicalExpression::Bound = iceberg::expr::LogicalExpression<::Bound, N> +pub fn iceberg::expr::LogicalExpression::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl serde_core::ser::Serialize for iceberg::expr::LogicalExpression +pub fn iceberg::expr::LogicalExpression::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +pub struct iceberg::expr::Reference +impl iceberg::expr::Reference +pub fn iceberg::expr::Reference::equal_to(self, datum: iceberg::spec::Datum) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::greater_than(self, datum: iceberg::spec::Datum) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::greater_than_or_equal_to(self, datum: iceberg::spec::Datum) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::is_in(self, literals: impl core::iter::traits::collect::IntoIterator) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::is_nan(self) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::is_not_in(self, literals: impl core::iter::traits::collect::IntoIterator) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::is_not_nan(self) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::is_not_null(self) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::is_null(self) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::less_than(self, datum: iceberg::spec::Datum) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::less_than_or_equal_to(self, datum: iceberg::spec::Datum) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::not_equal_to(self, datum: iceberg::spec::Datum) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::not_starts_with(self, datum: iceberg::spec::Datum) -> iceberg::expr::Predicate +pub fn iceberg::expr::Reference::starts_with(self, datum: iceberg::spec::Datum) -> iceberg::expr::Predicate +impl iceberg::expr::Reference +pub fn iceberg::expr::Reference::name(&self) -> &str +pub fn iceberg::expr::Reference::new(name: impl core::convert::Into) -> Self +impl core::clone::Clone for iceberg::expr::Reference +pub fn iceberg::expr::Reference::clone(&self) -> iceberg::expr::Reference +impl core::cmp::PartialEq for iceberg::expr::Reference +pub fn iceberg::expr::Reference::eq(&self, other: &iceberg::expr::Reference) -> bool +impl core::fmt::Debug for iceberg::expr::Reference +pub fn iceberg::expr::Reference::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::expr::Reference +pub fn iceberg::expr::Reference::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::expr::Reference +impl iceberg::expr::Bind for iceberg::expr::Reference +pub type iceberg::expr::Reference::Bound = iceberg::expr::BoundReference +pub fn iceberg::expr::Reference::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl serde_core::ser::Serialize for iceberg::expr::Reference +pub fn iceberg::expr::Reference::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::expr::Reference +pub fn iceberg::expr::Reference::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::expr::SetExpression +impl iceberg::expr::SetExpression +pub fn iceberg::expr::SetExpression::literals(&self) -> &fnv::FnvHashSet +pub fn iceberg::expr::SetExpression::new(op: iceberg::expr::PredicateOperator, term: T, literals: fnv::FnvHashSet) -> Self +pub fn iceberg::expr::SetExpression::op(&self) -> iceberg::expr::PredicateOperator +pub fn iceberg::expr::SetExpression::term(&self) -> &T +impl<'de, T> serde_core::de::Deserialize<'de> for iceberg::expr::SetExpression where T: serde_core::de::Deserialize<'de> +pub fn iceberg::expr::SetExpression::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +impl core::clone::Clone for iceberg::expr::SetExpression +pub fn iceberg::expr::SetExpression::clone(&self) -> iceberg::expr::SetExpression +impl core::cmp::PartialEq for iceberg::expr::SetExpression +pub fn iceberg::expr::SetExpression::eq(&self, other: &iceberg::expr::SetExpression) -> bool +impl core::fmt::Debug for iceberg::expr::SetExpression +pub fn iceberg::expr::SetExpression::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::expr::SetExpression +pub fn iceberg::expr::SetExpression::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::expr::Bind for iceberg::expr::SetExpression +pub type iceberg::expr::SetExpression::Bound = iceberg::expr::SetExpression<::Bound> +pub fn iceberg::expr::SetExpression::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl core::marker::StructuralPartialEq for iceberg::expr::SetExpression +impl serde_core::ser::Serialize for iceberg::expr::SetExpression where T: serde_core::ser::Serialize +pub fn iceberg::expr::SetExpression::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +pub struct iceberg::expr::UnaryExpression +impl iceberg::expr::UnaryExpression +pub fn iceberg::expr::UnaryExpression::new(op: iceberg::expr::PredicateOperator, term: T) -> Self +pub fn iceberg::expr::UnaryExpression::op(&self) -> iceberg::expr::PredicateOperator +pub fn iceberg::expr::UnaryExpression::term(&self) -> &T +impl<'de, T> serde_core::de::Deserialize<'de> for iceberg::expr::UnaryExpression where T: serde_core::de::Deserialize<'de> +pub fn iceberg::expr::UnaryExpression::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +impl core::clone::Clone for iceberg::expr::UnaryExpression +pub fn iceberg::expr::UnaryExpression::clone(&self) -> iceberg::expr::UnaryExpression +impl core::cmp::PartialEq for iceberg::expr::UnaryExpression +pub fn iceberg::expr::UnaryExpression::eq(&self, other: &iceberg::expr::UnaryExpression) -> bool +impl core::fmt::Debug for iceberg::expr::UnaryExpression +pub fn iceberg::expr::UnaryExpression::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::expr::UnaryExpression +pub fn iceberg::expr::UnaryExpression::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::expr::Bind for iceberg::expr::UnaryExpression +pub type iceberg::expr::UnaryExpression::Bound = iceberg::expr::UnaryExpression<::Bound> +pub fn iceberg::expr::UnaryExpression::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl core::marker::StructuralPartialEq for iceberg::expr::UnaryExpression +impl serde_core::ser::Serialize for iceberg::expr::UnaryExpression where T: serde_core::ser::Serialize +pub fn iceberg::expr::UnaryExpression::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +pub trait iceberg::expr::Bind +pub type iceberg::expr::Bind::Bound +pub fn iceberg::expr::Bind::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl iceberg::expr::Bind for iceberg::expr::Predicate +pub type iceberg::expr::Predicate::Bound = iceberg::expr::BoundPredicate +pub fn iceberg::expr::Predicate::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl iceberg::expr::Bind for iceberg::expr::Reference +pub type iceberg::expr::Reference::Bound = iceberg::expr::BoundReference +pub fn iceberg::expr::Reference::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl iceberg::expr::Bind for iceberg::expr::LogicalExpression where ::Bound: core::marker::Sized +pub type iceberg::expr::LogicalExpression::Bound = iceberg::expr::LogicalExpression<::Bound, N> +pub fn iceberg::expr::LogicalExpression::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl iceberg::expr::Bind for iceberg::expr::BinaryExpression +pub type iceberg::expr::BinaryExpression::Bound = iceberg::expr::BinaryExpression<::Bound> +pub fn iceberg::expr::BinaryExpression::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl iceberg::expr::Bind for iceberg::expr::SetExpression +pub type iceberg::expr::SetExpression::Bound = iceberg::expr::SetExpression<::Bound> +pub fn iceberg::expr::SetExpression::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +impl iceberg::expr::Bind for iceberg::expr::UnaryExpression +pub type iceberg::expr::UnaryExpression::Bound = iceberg::expr::UnaryExpression<::Bound> +pub fn iceberg::expr::UnaryExpression::bind(&self, schema: iceberg::spec::SchemaRef, case_sensitive: bool) -> iceberg::Result +pub type iceberg::expr::BoundTerm = iceberg::expr::BoundReference +pub type iceberg::expr::Term = iceberg::expr::Reference +pub mod iceberg::inspect +pub enum iceberg::inspect::MetadataTableType +pub iceberg::inspect::MetadataTableType::History +pub iceberg::inspect::MetadataTableType::Manifests +pub iceberg::inspect::MetadataTableType::Snapshots +impl iceberg::inspect::MetadataTableType +pub fn iceberg::inspect::MetadataTableType::all_types() -> impl core::iter::traits::iterator::Iterator +pub fn iceberg::inspect::MetadataTableType::as_str(&self) -> &str +impl core::clone::Clone for iceberg::inspect::MetadataTableType +pub fn iceberg::inspect::MetadataTableType::clone(&self) -> iceberg::inspect::MetadataTableType +impl core::convert::TryFrom<&str> for iceberg::inspect::MetadataTableType +pub type iceberg::inspect::MetadataTableType::Error = alloc::string::String +pub fn iceberg::inspect::MetadataTableType::try_from(value: &str) -> core::result::Result +impl core::fmt::Debug for iceberg::inspect::MetadataTableType +pub fn iceberg::inspect::MetadataTableType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl strum::IntoEnumIterator for iceberg::inspect::MetadataTableType +pub type iceberg::inspect::MetadataTableType::Iterator = iceberg::inspect::MetadataTableTypeIter +pub fn iceberg::inspect::MetadataTableType::iter() -> iceberg::inspect::MetadataTableTypeIter +pub struct iceberg::inspect::HistoryTable<'a> +impl<'a> iceberg::inspect::HistoryTable<'a> +pub fn iceberg::inspect::HistoryTable<'a>::new(table: &'a iceberg::table::Table) -> Self +pub async fn iceberg::inspect::HistoryTable<'a>::scan(&self) -> iceberg::Result +pub fn iceberg::inspect::HistoryTable<'a>::schema(&self) -> iceberg::spec::Schema +pub struct iceberg::inspect::ManifestsTable<'a> +impl<'a> iceberg::inspect::ManifestsTable<'a> +pub fn iceberg::inspect::ManifestsTable<'a>::new(table: &'a iceberg::table::Table) -> Self +pub async fn iceberg::inspect::ManifestsTable<'a>::scan(&self) -> iceberg::Result +pub fn iceberg::inspect::ManifestsTable<'a>::schema(&self) -> iceberg::spec::Schema +pub struct iceberg::inspect::MetadataTable<'a>(_) +impl<'a> iceberg::inspect::MetadataTable<'a> +pub fn iceberg::inspect::MetadataTable<'a>::history(&self) -> iceberg::inspect::HistoryTable<'_> +pub fn iceberg::inspect::MetadataTable<'a>::manifests(&self) -> iceberg::inspect::ManifestsTable<'_> +pub fn iceberg::inspect::MetadataTable<'a>::new(table: &'a iceberg::table::Table) -> Self +pub fn iceberg::inspect::MetadataTable<'a>::snapshots(&self) -> iceberg::inspect::SnapshotsTable<'_> +impl<'a> core::fmt::Debug for iceberg::inspect::MetadataTable<'a> +pub fn iceberg::inspect::MetadataTable<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::inspect::MetadataTableTypeIter +impl core::clone::Clone for iceberg::inspect::MetadataTableTypeIter +pub fn iceberg::inspect::MetadataTableTypeIter::clone(&self) -> iceberg::inspect::MetadataTableTypeIter +impl core::fmt::Debug for iceberg::inspect::MetadataTableTypeIter +pub fn iceberg::inspect::MetadataTableTypeIter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::iter::traits::double_ended::DoubleEndedIterator for iceberg::inspect::MetadataTableTypeIter +pub fn iceberg::inspect::MetadataTableTypeIter::next_back(&mut self) -> core::option::Option<::Item> +impl core::iter::traits::exact_size::ExactSizeIterator for iceberg::inspect::MetadataTableTypeIter +pub fn iceberg::inspect::MetadataTableTypeIter::len(&self) -> usize +impl core::iter::traits::iterator::Iterator for iceberg::inspect::MetadataTableTypeIter +pub type iceberg::inspect::MetadataTableTypeIter::Item = iceberg::inspect::MetadataTableType +pub fn iceberg::inspect::MetadataTableTypeIter::next(&mut self) -> core::option::Option<::Item> +pub fn iceberg::inspect::MetadataTableTypeIter::nth(&mut self, n: usize) -> core::option::Option<::Item> +pub fn iceberg::inspect::MetadataTableTypeIter::size_hint(&self) -> (usize, core::option::Option) +impl core::iter::traits::marker::FusedIterator for iceberg::inspect::MetadataTableTypeIter +pub struct iceberg::inspect::SnapshotsTable<'a> +impl<'a> iceberg::inspect::SnapshotsTable<'a> +pub fn iceberg::inspect::SnapshotsTable<'a>::new(table: &'a iceberg::table::Table) -> Self +pub async fn iceberg::inspect::SnapshotsTable<'a>::scan(&self) -> iceberg::Result +pub fn iceberg::inspect::SnapshotsTable<'a>::schema(&self) -> iceberg::spec::Schema +pub mod iceberg::io +pub struct iceberg::io::AzdlsConfig +pub iceberg::io::AzdlsConfig::account_key: core::option::Option +pub iceberg::io::AzdlsConfig::account_name: core::option::Option +pub iceberg::io::AzdlsConfig::authority_host: core::option::Option +pub iceberg::io::AzdlsConfig::client_id: core::option::Option +pub iceberg::io::AzdlsConfig::client_secret: core::option::Option +pub iceberg::io::AzdlsConfig::connection_string: core::option::Option +pub iceberg::io::AzdlsConfig::endpoint: core::option::Option +pub iceberg::io::AzdlsConfig::filesystem: alloc::string::String +pub iceberg::io::AzdlsConfig::sas_token: core::option::Option +pub iceberg::io::AzdlsConfig::tenant_id: core::option::Option +impl core::clone::Clone for iceberg::io::AzdlsConfig +pub fn iceberg::io::AzdlsConfig::clone(&self) -> iceberg::io::AzdlsConfig +impl core::cmp::Eq for iceberg::io::AzdlsConfig +impl core::cmp::PartialEq for iceberg::io::AzdlsConfig +pub fn iceberg::io::AzdlsConfig::eq(&self, other: &iceberg::io::AzdlsConfig) -> bool +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::AzdlsConfig +pub type iceberg::io::AzdlsConfig::Error = iceberg::Error +pub fn iceberg::io::AzdlsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::default::Default for iceberg::io::AzdlsConfig +pub fn iceberg::io::AzdlsConfig::default() -> iceberg::io::AzdlsConfig +impl core::fmt::Debug for iceberg::io::AzdlsConfig +pub fn iceberg::io::AzdlsConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::AzdlsConfig +impl iceberg::io::AzdlsConfig +pub fn iceberg::io::AzdlsConfig::builder() -> AzdlsConfigBuilder<((), (), (), (), (), (), (), (), (), ())> +impl serde_core::ser::Serialize for iceberg::io::AzdlsConfig +pub fn iceberg::io::AzdlsConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::AzdlsConfig +pub fn iceberg::io::AzdlsConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::FileIO +impl iceberg::io::FileIO +pub fn iceberg::io::FileIO::config(&self) -> &iceberg::io::StorageConfig +pub async fn iceberg::io::FileIO::delete(&self, path: impl core::convert::AsRef) -> iceberg::Result<()> +pub async fn iceberg::io::FileIO::delete_prefix(&self, path: impl core::convert::AsRef) -> iceberg::Result<()> +pub async fn iceberg::io::FileIO::delete_stream(&self, paths: impl futures_core::stream::Stream + core::marker::Send + 'static) -> iceberg::Result<()> +pub fn iceberg::io::FileIO::deserialize_all(bytes: &[u8]) -> iceberg::Result +pub async fn iceberg::io::FileIO::exists(&self, path: impl core::convert::AsRef) -> iceberg::Result +pub fn iceberg::io::FileIO::new_input(&self, path: impl core::convert::AsRef) -> iceberg::Result +pub fn iceberg::io::FileIO::new_output(&self, path: impl core::convert::AsRef) -> iceberg::Result +pub fn iceberg::io::FileIO::new_with_fs() -> Self +pub fn iceberg::io::FileIO::new_with_memory() -> Self +pub fn iceberg::io::FileIO::serialize_all(&self) -> iceberg::Result> +impl core::clone::Clone for iceberg::io::FileIO +pub fn iceberg::io::FileIO::clone(&self) -> iceberg::io::FileIO +impl core::fmt::Debug for iceberg::io::FileIO +pub fn iceberg::io::FileIO::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::io::FileIOBuilder +impl iceberg::io::FileIOBuilder +pub fn iceberg::io::FileIOBuilder::build(self) -> iceberg::io::FileIO +pub fn iceberg::io::FileIOBuilder::config(&self) -> &iceberg::io::StorageConfig +pub fn iceberg::io::FileIOBuilder::new(factory: alloc::sync::Arc) -> Self +pub fn iceberg::io::FileIOBuilder::with_prop(self, key: impl alloc::string::ToString, value: impl alloc::string::ToString) -> Self +pub fn iceberg::io::FileIOBuilder::with_props(self, args: impl core::iter::traits::collect::IntoIterator) -> Self +impl core::clone::Clone for iceberg::io::FileIOBuilder +pub fn iceberg::io::FileIOBuilder::clone(&self) -> iceberg::io::FileIOBuilder +impl core::fmt::Debug for iceberg::io::FileIOBuilder +pub fn iceberg::io::FileIOBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::io::FileMetadata +pub iceberg::io::FileMetadata::size: u64 +pub struct iceberg::io::GcsConfig +pub iceberg::io::GcsConfig::allow_anonymous: bool +pub iceberg::io::GcsConfig::credential: core::option::Option +pub iceberg::io::GcsConfig::disable_config_load: bool +pub iceberg::io::GcsConfig::disable_vm_metadata: bool +pub iceberg::io::GcsConfig::endpoint: core::option::Option +pub iceberg::io::GcsConfig::project_id: core::option::Option +pub iceberg::io::GcsConfig::token: core::option::Option +pub iceberg::io::GcsConfig::user_project: core::option::Option +impl core::clone::Clone for iceberg::io::GcsConfig +pub fn iceberg::io::GcsConfig::clone(&self) -> iceberg::io::GcsConfig +impl core::cmp::Eq for iceberg::io::GcsConfig +impl core::cmp::PartialEq for iceberg::io::GcsConfig +pub fn iceberg::io::GcsConfig::eq(&self, other: &iceberg::io::GcsConfig) -> bool +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::GcsConfig +pub type iceberg::io::GcsConfig::Error = iceberg::Error +pub fn iceberg::io::GcsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::default::Default for iceberg::io::GcsConfig +pub fn iceberg::io::GcsConfig::default() -> iceberg::io::GcsConfig +impl core::fmt::Debug for iceberg::io::GcsConfig +pub fn iceberg::io::GcsConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::GcsConfig +impl iceberg::io::GcsConfig +pub fn iceberg::io::GcsConfig::builder() -> GcsConfigBuilder<((), (), (), (), (), (), (), ())> +impl serde_core::ser::Serialize for iceberg::io::GcsConfig +pub fn iceberg::io::GcsConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::GcsConfig +pub fn iceberg::io::GcsConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::HfConfig +pub iceberg::io::HfConfig::endpoint: core::option::Option +pub iceberg::io::HfConfig::revision: core::option::Option +pub iceberg::io::HfConfig::token: core::option::Option +impl core::clone::Clone for iceberg::io::HfConfig +pub fn iceberg::io::HfConfig::clone(&self) -> iceberg::io::HfConfig +impl core::cmp::Eq for iceberg::io::HfConfig +impl core::cmp::PartialEq for iceberg::io::HfConfig +pub fn iceberg::io::HfConfig::eq(&self, other: &iceberg::io::HfConfig) -> bool +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::HfConfig +pub type iceberg::io::HfConfig::Error = iceberg::Error +pub fn iceberg::io::HfConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::default::Default for iceberg::io::HfConfig +pub fn iceberg::io::HfConfig::default() -> iceberg::io::HfConfig +impl core::fmt::Debug for iceberg::io::HfConfig +pub fn iceberg::io::HfConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::HfConfig +impl iceberg::io::HfConfig +pub fn iceberg::io::HfConfig::builder() -> HfConfigBuilder<((), (), ())> +impl serde_core::ser::Serialize for iceberg::io::HfConfig +pub fn iceberg::io::HfConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::HfConfig +pub fn iceberg::io::HfConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::InputFile +impl iceberg::io::InputFile +pub async fn iceberg::io::InputFile::exists(&self) -> iceberg::Result +pub fn iceberg::io::InputFile::location(&self) -> &str +pub async fn iceberg::io::InputFile::metadata(&self) -> iceberg::Result +pub fn iceberg::io::InputFile::new(storage: alloc::sync::Arc, path: alloc::string::String) -> Self +pub async fn iceberg::io::InputFile::read(&self) -> iceberg::Result +pub async fn iceberg::io::InputFile::reader(&self) -> iceberg::Result> +impl core::fmt::Debug for iceberg::io::InputFile +pub fn iceberg::io::InputFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::io::LocalFsStorage +impl iceberg::io::LocalFsStorage +pub fn iceberg::io::LocalFsStorage::new() -> Self +impl core::clone::Clone for iceberg::io::LocalFsStorage +pub fn iceberg::io::LocalFsStorage::clone(&self) -> iceberg::io::LocalFsStorage +impl core::default::Default for iceberg::io::LocalFsStorage +pub fn iceberg::io::LocalFsStorage::default() -> iceberg::io::LocalFsStorage +impl core::fmt::Debug for iceberg::io::LocalFsStorage +pub fn iceberg::io::LocalFsStorage::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::io::Storage for iceberg::io::LocalFsStorage +pub fn iceberg::io::LocalFsStorage::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::delete_prefix<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::delete_stream<'life0, 'async_trait>(&'life0 self, paths: futures_core::stream::BoxStream<'static, alloc::string::String>) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::io::LocalFsStorage::exists<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::metadata<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::new_input(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::LocalFsStorage::new_output(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::LocalFsStorage::read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::reader<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::write<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +impl serde_core::ser::Serialize for iceberg::io::LocalFsStorage +pub fn iceberg::io::LocalFsStorage::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::LocalFsStorage +pub fn iceberg::io::LocalFsStorage::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::LocalFsStorageFactory +impl core::clone::Clone for iceberg::io::LocalFsStorageFactory +pub fn iceberg::io::LocalFsStorageFactory::clone(&self) -> iceberg::io::LocalFsStorageFactory +impl core::default::Default for iceberg::io::LocalFsStorageFactory +pub fn iceberg::io::LocalFsStorageFactory::default() -> iceberg::io::LocalFsStorageFactory +impl core::fmt::Debug for iceberg::io::LocalFsStorageFactory +pub fn iceberg::io::LocalFsStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::io::StorageFactory for iceberg::io::LocalFsStorageFactory +pub fn iceberg::io::LocalFsStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +impl serde_core::ser::Serialize for iceberg::io::LocalFsStorageFactory +pub fn iceberg::io::LocalFsStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::LocalFsStorageFactory +pub fn iceberg::io::LocalFsStorageFactory::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::MemoryStorage +impl iceberg::io::MemoryStorage +pub fn iceberg::io::MemoryStorage::new() -> Self +impl core::clone::Clone for iceberg::io::MemoryStorage +pub fn iceberg::io::MemoryStorage::clone(&self) -> iceberg::io::MemoryStorage +impl core::default::Default for iceberg::io::MemoryStorage +pub fn iceberg::io::MemoryStorage::default() -> iceberg::io::MemoryStorage +impl core::fmt::Debug for iceberg::io::MemoryStorage +pub fn iceberg::io::MemoryStorage::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::io::Storage for iceberg::io::MemoryStorage +pub fn iceberg::io::MemoryStorage::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::delete_prefix<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::delete_stream<'life0, 'async_trait>(&'life0 self, paths: futures_core::stream::BoxStream<'static, alloc::string::String>) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::io::MemoryStorage::exists<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::metadata<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::new_input(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::MemoryStorage::new_output(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::MemoryStorage::read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::reader<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::write<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +impl serde_core::ser::Serialize for iceberg::io::MemoryStorage +pub fn iceberg::io::MemoryStorage::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::MemoryStorage +pub fn iceberg::io::MemoryStorage::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::MemoryStorageFactory +impl core::clone::Clone for iceberg::io::MemoryStorageFactory +pub fn iceberg::io::MemoryStorageFactory::clone(&self) -> iceberg::io::MemoryStorageFactory +impl core::default::Default for iceberg::io::MemoryStorageFactory +pub fn iceberg::io::MemoryStorageFactory::default() -> iceberg::io::MemoryStorageFactory +impl core::fmt::Debug for iceberg::io::MemoryStorageFactory +pub fn iceberg::io::MemoryStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::io::StorageFactory for iceberg::io::MemoryStorageFactory +pub fn iceberg::io::MemoryStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +impl serde_core::ser::Serialize for iceberg::io::MemoryStorageFactory +pub fn iceberg::io::MemoryStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::MemoryStorageFactory +pub fn iceberg::io::MemoryStorageFactory::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::OssConfig +pub iceberg::io::OssConfig::access_key_id: core::option::Option +pub iceberg::io::OssConfig::access_key_secret: core::option::Option +pub iceberg::io::OssConfig::endpoint: core::option::Option +impl core::clone::Clone for iceberg::io::OssConfig +pub fn iceberg::io::OssConfig::clone(&self) -> iceberg::io::OssConfig +impl core::cmp::Eq for iceberg::io::OssConfig +impl core::cmp::PartialEq for iceberg::io::OssConfig +pub fn iceberg::io::OssConfig::eq(&self, other: &iceberg::io::OssConfig) -> bool +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::OssConfig +pub type iceberg::io::OssConfig::Error = iceberg::Error +pub fn iceberg::io::OssConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::default::Default for iceberg::io::OssConfig +pub fn iceberg::io::OssConfig::default() -> iceberg::io::OssConfig +impl core::fmt::Debug for iceberg::io::OssConfig +pub fn iceberg::io::OssConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::OssConfig +impl iceberg::io::OssConfig +pub fn iceberg::io::OssConfig::builder() -> OssConfigBuilder<((), (), ())> +impl serde_core::ser::Serialize for iceberg::io::OssConfig +pub fn iceberg::io::OssConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::OssConfig +pub fn iceberg::io::OssConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::OutputFile +impl iceberg::io::OutputFile +pub async fn iceberg::io::OutputFile::delete(&self) -> iceberg::Result<()> +pub async fn iceberg::io::OutputFile::exists(&self) -> iceberg::Result +pub fn iceberg::io::OutputFile::location(&self) -> &str +pub fn iceberg::io::OutputFile::new(storage: alloc::sync::Arc, path: alloc::string::String) -> Self +pub fn iceberg::io::OutputFile::to_input_file(self) -> iceberg::io::InputFile +pub async fn iceberg::io::OutputFile::write(&self, bs: bytes::bytes::Bytes) -> iceberg::Result<()> +pub async fn iceberg::io::OutputFile::writer(&self) -> iceberg::Result> +impl core::fmt::Debug for iceberg::io::OutputFile +pub fn iceberg::io::OutputFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::io::S3Config +pub iceberg::io::S3Config::access_key_id: core::option::Option +pub iceberg::io::S3Config::allow_anonymous: bool +pub iceberg::io::S3Config::disable_config_load: bool +pub iceberg::io::S3Config::disable_ec2_metadata: bool +pub iceberg::io::S3Config::enable_virtual_host_style: bool +pub iceberg::io::S3Config::endpoint: core::option::Option +pub iceberg::io::S3Config::external_id: core::option::Option +pub iceberg::io::S3Config::region: core::option::Option +pub iceberg::io::S3Config::role_arn: core::option::Option +pub iceberg::io::S3Config::role_session_name: core::option::Option +pub iceberg::io::S3Config::secret_access_key: core::option::Option +pub iceberg::io::S3Config::server_side_encryption: core::option::Option +pub iceberg::io::S3Config::server_side_encryption_aws_kms_key_id: core::option::Option +pub iceberg::io::S3Config::server_side_encryption_customer_algorithm: core::option::Option +pub iceberg::io::S3Config::server_side_encryption_customer_key: core::option::Option +pub iceberg::io::S3Config::server_side_encryption_customer_key_md5: core::option::Option +pub iceberg::io::S3Config::session_token: core::option::Option +impl core::clone::Clone for iceberg::io::S3Config +pub fn iceberg::io::S3Config::clone(&self) -> iceberg::io::S3Config +impl core::cmp::Eq for iceberg::io::S3Config +impl core::cmp::PartialEq for iceberg::io::S3Config +pub fn iceberg::io::S3Config::eq(&self, other: &iceberg::io::S3Config) -> bool +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::S3Config +pub type iceberg::io::S3Config::Error = iceberg::Error +pub fn iceberg::io::S3Config::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::default::Default for iceberg::io::S3Config +pub fn iceberg::io::S3Config::default() -> Self +impl core::fmt::Debug for iceberg::io::S3Config +pub fn iceberg::io::S3Config::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::S3Config +impl iceberg::io::S3Config +pub fn iceberg::io::S3Config::builder() -> S3ConfigBuilder<((), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), ())> +impl serde_core::ser::Serialize for iceberg::io::S3Config +pub fn iceberg::io::S3Config::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::S3Config +pub fn iceberg::io::S3Config::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::StorageConfig +impl iceberg::io::StorageConfig +pub fn iceberg::io::StorageConfig::from_props(props: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::io::StorageConfig::get(&self, key: &str) -> core::option::Option<&alloc::string::String> +pub fn iceberg::io::StorageConfig::new() -> Self +pub fn iceberg::io::StorageConfig::props(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::io::StorageConfig::with_prop(self, key: impl core::convert::Into, value: impl core::convert::Into) -> Self +pub fn iceberg::io::StorageConfig::with_props(self, props: impl core::iter::traits::collect::IntoIterator, impl core::convert::Into)>) -> Self +impl core::clone::Clone for iceberg::io::StorageConfig +pub fn iceberg::io::StorageConfig::clone(&self) -> iceberg::io::StorageConfig +impl core::cmp::Eq for iceberg::io::StorageConfig +impl core::cmp::PartialEq for iceberg::io::StorageConfig +pub fn iceberg::io::StorageConfig::eq(&self, other: &iceberg::io::StorageConfig) -> bool +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::AzdlsConfig +pub type iceberg::io::AzdlsConfig::Error = iceberg::Error +pub fn iceberg::io::AzdlsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::GcsConfig +pub type iceberg::io::GcsConfig::Error = iceberg::Error +pub fn iceberg::io::GcsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::HfConfig +pub type iceberg::io::HfConfig::Error = iceberg::Error +pub fn iceberg::io::HfConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::OssConfig +pub type iceberg::io::OssConfig::Error = iceberg::Error +pub fn iceberg::io::OssConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::S3Config +pub type iceberg::io::S3Config::Error = iceberg::Error +pub fn iceberg::io::S3Config::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::default::Default for iceberg::io::StorageConfig +pub fn iceberg::io::StorageConfig::default() -> iceberg::io::StorageConfig +impl core::fmt::Debug for iceberg::io::StorageConfig +pub fn iceberg::io::StorageConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::StorageConfig +impl serde_core::ser::Serialize for iceberg::io::StorageConfig +pub fn iceberg::io::StorageConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageConfig +pub fn iceberg::io::StorageConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub const iceberg::io::ADLS_ACCOUNT_KEY: &str +pub const iceberg::io::ADLS_ACCOUNT_NAME: &str +pub const iceberg::io::ADLS_AUTHORITY_HOST: &str +pub const iceberg::io::ADLS_CLIENT_ID: &str +pub const iceberg::io::ADLS_CLIENT_SECRET: &str +pub const iceberg::io::ADLS_CONNECTION_STRING: &str +pub const iceberg::io::ADLS_SAS_TOKEN: &str +pub const iceberg::io::ADLS_TENANT_ID: &str +pub const iceberg::io::CLIENT_REGION: &str +pub const iceberg::io::GCS_ALLOW_ANONYMOUS: &str +pub const iceberg::io::GCS_CREDENTIALS_JSON: &str +pub const iceberg::io::GCS_DISABLE_CONFIG_LOAD: &str +pub const iceberg::io::GCS_DISABLE_VM_METADATA: &str +pub const iceberg::io::GCS_NO_AUTH: &str +pub const iceberg::io::GCS_PROJECT_ID: &str +pub const iceberg::io::GCS_SERVICE_HOST: &str +pub const iceberg::io::GCS_TOKEN: &str +pub const iceberg::io::GCS_USER_PROJECT: &str +pub const iceberg::io::HDFS_HADOOP_CONF_PREFIX: &str +pub const iceberg::io::HDFS_HOST: &str +pub const iceberg::io::HDFS_NAME_NODE: &str +pub const iceberg::io::HDFS_PORT: &str +pub const iceberg::io::HF_ENDPOINT: &str +pub const iceberg::io::HF_REVISION: &str +pub const iceberg::io::HF_TOKEN: &str +pub const iceberg::io::OSS_ACCESS_KEY_ID: &str +pub const iceberg::io::OSS_ACCESS_KEY_SECRET: &str +pub const iceberg::io::OSS_ENDPOINT: &str +pub const iceberg::io::S3_ACCESS_KEY_ID: &str +pub const iceberg::io::S3_ALLOW_ANONYMOUS: &str +pub const iceberg::io::S3_ASSUME_ROLE_ARN: &str +pub const iceberg::io::S3_ASSUME_ROLE_EXTERNAL_ID: &str +pub const iceberg::io::S3_ASSUME_ROLE_SESSION_NAME: &str +pub const iceberg::io::S3_DISABLE_CONFIG_LOAD: &str +pub const iceberg::io::S3_DISABLE_EC2_METADATA: &str +pub const iceberg::io::S3_ENDPOINT: &str +pub const iceberg::io::S3_MULTIPART_PART_SIZE_BYTES: &str +pub const iceberg::io::S3_PATH_STYLE_ACCESS: &str +pub const iceberg::io::S3_REGION: &str +pub const iceberg::io::S3_RETRY_MAX_WAIT_MS: &str +pub const iceberg::io::S3_RETRY_MIN_WAIT_MS: &str +pub const iceberg::io::S3_RETRY_NUM_RETRIES: &str +pub const iceberg::io::S3_SECRET_ACCESS_KEY: &str +pub const iceberg::io::S3_SESSION_TOKEN: &str +pub const iceberg::io::S3_SSE_KEY: &str +pub const iceberg::io::S3_SSE_MD5: &str +pub const iceberg::io::S3_SSE_TYPE: &str +pub trait iceberg::io::FileRead: core::marker::Send + core::marker::Sync + core::marker::Unpin + 'static +pub fn iceberg::io::FileRead::read<'life0, 'async_trait>(&'life0 self, range: core::ops::range::Range) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::io::FileRead for iceberg::encryption::AesGcmFileRead +pub fn iceberg::encryption::AesGcmFileRead::read<'life0, 'async_trait>(&'life0 self, range: core::ops::range::Range) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl + core::marker::Send + core::marker::Sync + core::marker::Unpin + 'static> iceberg::io::FileRead for T +pub fn T::read<'life0, 'async_trait>(&'life0 self, range: core::ops::range::Range) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub trait iceberg::io::FileWrite: core::marker::Send + core::marker::Unpin + 'static +pub fn iceberg::io::FileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::io::FileWrite::write<'life0, 'async_trait>(&'life0 mut self, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::io::FileWrite for iceberg::encryption::AesGcmFileWrite +pub fn iceberg::encryption::AesGcmFileWrite::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::encryption::AesGcmFileWrite::write<'life0, 'async_trait>(&'life0 mut self, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub trait iceberg::io::Storage: core::fmt::Debug + core::marker::Send + core::marker::Sync + typetag::Serialize + typetag::Deserialize +pub fn iceberg::io::Storage::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::Storage::delete_prefix<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::Storage::delete_stream<'life0, 'async_trait>(&'life0 self, paths: futures_core::stream::BoxStream<'static, alloc::string::String>) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::io::Storage::exists<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::Storage::metadata<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::Storage::new_input(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::Storage::new_output(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::Storage::read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::Storage::reader<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::Storage::write<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::Storage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +impl iceberg::io::Storage for iceberg::io::LocalFsStorage +pub fn iceberg::io::LocalFsStorage::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::delete_prefix<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::delete_stream<'life0, 'async_trait>(&'life0 self, paths: futures_core::stream::BoxStream<'static, alloc::string::String>) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::io::LocalFsStorage::exists<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::metadata<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::new_input(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::LocalFsStorage::new_output(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::LocalFsStorage::read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::reader<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::write<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::LocalFsStorage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +impl iceberg::io::Storage for iceberg::io::MemoryStorage +pub fn iceberg::io::MemoryStorage::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::delete_prefix<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::delete_stream<'life0, 'async_trait>(&'life0 self, paths: futures_core::stream::BoxStream<'static, alloc::string::String>) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::io::MemoryStorage::exists<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::metadata<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::new_input(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::MemoryStorage::new_output(&self, path: &str) -> iceberg::Result +pub fn iceberg::io::MemoryStorage::read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::reader<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::write<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::io::MemoryStorage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub trait iceberg::io::StorageFactory: core::fmt::Debug + core::marker::Send + core::marker::Sync + typetag::Serialize + typetag::Deserialize +pub fn iceberg::io::StorageFactory::build(&self, config: &iceberg::io::StorageConfig) -> iceberg::Result> +impl iceberg::io::StorageFactory for iceberg::io::LocalFsStorageFactory +pub fn iceberg::io::LocalFsStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +impl iceberg::io::StorageFactory for iceberg::io::MemoryStorageFactory +pub fn iceberg::io::MemoryStorageFactory::build(&self, _config: &iceberg::io::StorageConfig) -> iceberg::Result> +pub mod iceberg::memory +pub struct iceberg::memory::MemoryCatalog +impl core::fmt::Debug for iceberg::memory::MemoryCatalog +pub fn iceberg::memory::MemoryCatalog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::Catalog for iceberg::memory::MemoryCatalog +pub fn iceberg::memory::MemoryCatalog::create_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::create_table<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent, table_creation: iceberg::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::drop_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::drop_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::get_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::list_namespaces<'life0, 'life1, 'async_trait>(&'life0 self, maybe_parent: core::option::Option<&'life1 iceberg::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::list_tables<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::load_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::namespace_exists<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::purge_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::register_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::rename_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, src_table_ident: &'life1 iceberg::TableIdent, dst_table_ident: &'life2 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::memory::MemoryCatalog::table_exists<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::update_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub struct iceberg::memory::MemoryCatalogBuilder +impl core::default::Default for iceberg::memory::MemoryCatalogBuilder +pub fn iceberg::memory::MemoryCatalogBuilder::default() -> iceberg::memory::MemoryCatalogBuilder +impl core::fmt::Debug for iceberg::memory::MemoryCatalogBuilder +pub fn iceberg::memory::MemoryCatalogBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::CatalogBuilder for iceberg::memory::MemoryCatalogBuilder +pub type iceberg::memory::MemoryCatalogBuilder::C = iceberg::memory::MemoryCatalog +pub fn iceberg::memory::MemoryCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg::memory::MemoryCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg::memory::MemoryCatalogBuilder::with_runtime(self, runtime: iceberg::Runtime) -> Self +pub fn iceberg::memory::MemoryCatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +pub const iceberg::memory::MEMORY_CATALOG_WAREHOUSE: &str +pub mod iceberg::metadata_columns +pub const iceberg::metadata_columns::RESERVED_COL_NAME_CHANGE_ORDINAL: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_CHANGE_TYPE: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_COMMIT_SNAPSHOT_ID: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_DELETED: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_DELETE_FILE_PATH: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_DELETE_FILE_POS: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_FILE: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_PARTITION: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_POS: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_ROW_ID: &str +pub const iceberg::metadata_columns::RESERVED_COL_NAME_SPEC_ID: &str +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_CHANGE_ORDINAL: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_CHANGE_TYPE: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_DELETED: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_DELETE_FILE_PATH: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_DELETE_FILE_POS: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_FILE: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_PARTITION: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_POS: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_ROW_ID: i32 +pub const iceberg::metadata_columns::RESERVED_FIELD_ID_SPEC_ID: i32 +pub fn iceberg::metadata_columns::change_ordinal_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::change_type_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::commit_snapshot_id_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::delete_file_path_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::delete_file_pos_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::deleted_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::file_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::get_metadata_field(field_id: i32) -> iceberg::Result<&'static iceberg::spec::NestedFieldRef> +pub fn iceberg::metadata_columns::get_metadata_field_id(column_name: &str) -> iceberg::Result +pub fn iceberg::metadata_columns::is_metadata_column_name(column_name: &str) -> bool +pub fn iceberg::metadata_columns::is_metadata_field(field_id: i32) -> bool +pub fn iceberg::metadata_columns::last_updated_sequence_number_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::partition_field(partition_fields: alloc::vec::Vec) -> iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::pos_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::row_id_field() -> &'static iceberg::spec::NestedFieldRef +pub fn iceberg::metadata_columns::spec_id_field() -> &'static iceberg::spec::NestedFieldRef +pub mod iceberg::partitioning +pub fn iceberg::partitioning::compute_unified_partition_type<'a>(partition_specs: impl core::iter::traits::iterator::Iterator, schema: &iceberg::spec::Schema) -> iceberg::Result +pub mod iceberg::puffin +pub enum iceberg::puffin::CompressionCodec +pub iceberg::puffin::CompressionCodec::Brotli(u8) +pub iceberg::puffin::CompressionCodec::Gzip(u8) +pub iceberg::puffin::CompressionCodec::Lz4 +pub iceberg::puffin::CompressionCodec::Lz4Raw +pub iceberg::puffin::CompressionCodec::Lzo +pub iceberg::puffin::CompressionCodec::None +pub iceberg::puffin::CompressionCodec::Snappy +pub iceberg::puffin::CompressionCodec::Zstd(u8) +impl iceberg::compression::CompressionCodec +pub const fn iceberg::compression::CompressionCodec::brotli_default() -> Self +pub const fn iceberg::compression::CompressionCodec::gzip_default() -> Self +pub fn iceberg::compression::CompressionCodec::name(&self) -> &'static str +pub const fn iceberg::compression::CompressionCodec::zstd_default() -> Self +impl iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::suffix(&self) -> iceberg::Result<&'static str> +impl core::clone::Clone for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::clone(&self) -> iceberg::compression::CompressionCodec +impl core::cmp::Eq for iceberg::compression::CompressionCodec +impl core::cmp::PartialEq for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::eq(&self, other: &iceberg::compression::CompressionCodec) -> bool +impl core::default::Default for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::default() -> iceberg::compression::CompressionCodec +impl core::fmt::Debug for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::compression::CompressionCodec +impl core::marker::StructuralPartialEq for iceberg::compression::CompressionCodec +impl serde_core::ser::Serialize for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> +impl<'de> serde_core::de::Deserialize<'de> for iceberg::compression::CompressionCodec +pub fn iceberg::compression::CompressionCodec::deserialize>(deserializer: D) -> core::result::Result::Error> +pub struct iceberg::puffin::Blob +impl iceberg::puffin::Blob +pub fn iceberg::puffin::Blob::blob_type(&self) -> &str +pub fn iceberg::puffin::Blob::data(&self) -> &[u8] +pub fn iceberg::puffin::Blob::fields(&self) -> &[i32] +pub fn iceberg::puffin::Blob::properties(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::puffin::Blob::sequence_number(&self) -> i64 +pub fn iceberg::puffin::Blob::snapshot_id(&self) -> i64 +impl core::clone::Clone for iceberg::puffin::Blob +pub fn iceberg::puffin::Blob::clone(&self) -> iceberg::puffin::Blob +impl core::cmp::PartialEq for iceberg::puffin::Blob +pub fn iceberg::puffin::Blob::eq(&self, other: &iceberg::puffin::Blob) -> bool +impl core::fmt::Debug for iceberg::puffin::Blob +pub fn iceberg::puffin::Blob::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::puffin::Blob +impl iceberg::puffin::Blob +pub fn iceberg::puffin::Blob::builder() -> BlobBuilder<((), (), (), (), (), ())> +pub struct iceberg::puffin::BlobMetadata +impl iceberg::puffin::BlobMetadata +pub fn iceberg::puffin::BlobMetadata::blob_type(&self) -> &str +pub fn iceberg::puffin::BlobMetadata::compression_codec(&self) -> iceberg::compression::CompressionCodec +pub fn iceberg::puffin::BlobMetadata::fields(&self) -> &[i32] +pub fn iceberg::puffin::BlobMetadata::length(&self) -> u64 +pub fn iceberg::puffin::BlobMetadata::offset(&self) -> u64 +pub fn iceberg::puffin::BlobMetadata::properties(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::puffin::BlobMetadata::sequence_number(&self) -> i64 +pub fn iceberg::puffin::BlobMetadata::snapshot_id(&self) -> i64 +impl core::clone::Clone for iceberg::puffin::BlobMetadata +pub fn iceberg::puffin::BlobMetadata::clone(&self) -> iceberg::puffin::BlobMetadata +impl core::cmp::Eq for iceberg::puffin::BlobMetadata +impl core::cmp::PartialEq for iceberg::puffin::BlobMetadata +pub fn iceberg::puffin::BlobMetadata::eq(&self, other: &iceberg::puffin::BlobMetadata) -> bool +impl core::fmt::Debug for iceberg::puffin::BlobMetadata +pub fn iceberg::puffin::BlobMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::puffin::BlobMetadata +impl serde_core::ser::Serialize for iceberg::puffin::BlobMetadata +pub fn iceberg::puffin::BlobMetadata::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::puffin::BlobMetadata +pub fn iceberg::puffin::BlobMetadata::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::puffin::FileMetadata +impl iceberg::puffin::FileMetadata +pub fn iceberg::puffin::FileMetadata::blobs(&self) -> &[iceberg::puffin::BlobMetadata] +pub fn iceberg::puffin::FileMetadata::new(blobs: alloc::vec::Vec, properties: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::puffin::FileMetadata::properties(&self) -> &std::collections::hash::map::HashMap +impl core::clone::Clone for iceberg::puffin::FileMetadata +pub fn iceberg::puffin::FileMetadata::clone(&self) -> iceberg::puffin::FileMetadata +impl core::cmp::Eq for iceberg::puffin::FileMetadata +impl core::cmp::PartialEq for iceberg::puffin::FileMetadata +pub fn iceberg::puffin::FileMetadata::eq(&self, other: &iceberg::puffin::FileMetadata) -> bool +impl core::fmt::Debug for iceberg::puffin::FileMetadata +pub fn iceberg::puffin::FileMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::puffin::FileMetadata +impl serde_core::ser::Serialize for iceberg::puffin::FileMetadata +pub fn iceberg::puffin::FileMetadata::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::puffin::FileMetadata +pub fn iceberg::puffin::FileMetadata::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::puffin::PuffinReader +impl iceberg::puffin::PuffinReader +pub async fn iceberg::puffin::PuffinReader::blob(&self, blob_metadata: &iceberg::puffin::BlobMetadata) -> iceberg::Result +pub async fn iceberg::puffin::PuffinReader::file_metadata(&self) -> iceberg::Result<&iceberg::puffin::FileMetadata> +pub async fn iceberg::puffin::PuffinReader::new(input_file: iceberg::io::InputFile) -> iceberg::Result +pub async fn iceberg::puffin::PuffinReader::new_from_encrypted(encrypted_input: iceberg::encryption::EncryptedInputFile) -> iceberg::Result +pub struct iceberg::puffin::PuffinWriter +impl iceberg::puffin::PuffinWriter +pub async fn iceberg::puffin::PuffinWriter::add(&mut self, blob: iceberg::puffin::Blob, compression_codec: iceberg::compression::CompressionCodec) -> iceberg::Result<()> +pub async fn iceberg::puffin::PuffinWriter::close(self) -> iceberg::Result +pub async fn iceberg::puffin::PuffinWriter::new(output_file: &iceberg::io::OutputFile, properties: std::collections::hash::map::HashMap, compress_footer: bool) -> iceberg::Result +pub async fn iceberg::puffin::PuffinWriter::new_from_encrypted(encrypted_output: &iceberg::encryption::EncryptedOutputFile, properties: std::collections::hash::map::HashMap, compress_footer: bool) -> iceberg::Result +pub const iceberg::puffin::APACHE_DATASKETCHES_THETA_V1: &str +pub const iceberg::puffin::CREATED_BY_PROPERTY: &str +pub const iceberg::puffin::DELETION_VECTOR_V1: &str +pub mod iceberg::scan +pub struct iceberg::scan::FileScanTask +impl iceberg::scan::FileScanTask +pub fn iceberg::scan::FileScanTask::case_sensitive(&self) -> bool +pub fn iceberg::scan::FileScanTask::column_stats(&self) -> core::option::Option<&iceberg::scan::FileScanTaskColumnStats> +pub fn iceberg::scan::FileScanTask::data_file_format(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::scan::FileScanTask::data_file_path(&self) -> &str +pub fn iceberg::scan::FileScanTask::data_sequence_number(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTask::deletes(&self) -> &[iceberg::scan::FileScanTaskDeleteFile] +pub fn iceberg::scan::FileScanTask::file_size_in_bytes(&self) -> u64 +pub fn iceberg::scan::FileScanTask::first_row_id(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTask::key_metadata(&self) -> core::option::Option<&[u8]> +pub fn iceberg::scan::FileScanTask::length(&self) -> u64 +pub fn iceberg::scan::FileScanTask::name_mapping(&self) -> core::option::Option<&alloc::sync::Arc> +pub fn iceberg::scan::FileScanTask::partition(&self) -> core::option::Option<&iceberg::spec::Struct> +pub fn iceberg::scan::FileScanTask::partition_spec(&self) -> core::option::Option<&alloc::sync::Arc> +pub fn iceberg::scan::FileScanTask::predicate(&self) -> core::option::Option<&iceberg::expr::BoundPredicate> +pub fn iceberg::scan::FileScanTask::project_field_ids(&self) -> &[i32] +pub fn iceberg::scan::FileScanTask::record_count(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTask::schema(&self) -> &iceberg::spec::Schema +pub fn iceberg::scan::FileScanTask::schema_ref(&self) -> iceberg::spec::SchemaRef +pub fn iceberg::scan::FileScanTask::sort_order(&self) -> core::option::Option<&iceberg::spec::SortOrderRef> +pub fn iceberg::scan::FileScanTask::sort_order_id(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTask::start(&self) -> u64 +pub fn iceberg::scan::FileScanTask::unified_partition_type(&self) -> core::option::Option<&alloc::sync::Arc> +impl core::clone::Clone for iceberg::scan::FileScanTask +pub fn iceberg::scan::FileScanTask::clone(&self) -> iceberg::scan::FileScanTask +impl core::cmp::PartialEq for iceberg::scan::FileScanTask +pub fn iceberg::scan::FileScanTask::eq(&self, other: &iceberg::scan::FileScanTask) -> bool +impl core::convert::From for iceberg::Result +pub fn iceberg::Result::from(task: iceberg::scan::FileScanTask) -> Self +impl core::fmt::Debug for iceberg::scan::FileScanTask +pub fn iceberg::scan::FileScanTask::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::scan::FileScanTask +impl iceberg::scan::FileScanTask +pub fn iceberg::scan::FileScanTask::builder() -> FileScanTaskBuilder<((), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), (), ())> +impl serde_core::ser::Serialize for iceberg::scan::FileScanTask +pub fn iceberg::scan::FileScanTask::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::scan::FileScanTask +pub fn iceberg::scan::FileScanTask::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::scan::FileScanTaskColumnStats +impl iceberg::scan::FileScanTaskColumnStats +pub fn iceberg::scan::FileScanTaskColumnStats::column_sizes(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::scan::FileScanTaskColumnStats::lower_bounds(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::scan::FileScanTaskColumnStats::nan_value_counts(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::scan::FileScanTaskColumnStats::null_value_counts(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::scan::FileScanTaskColumnStats::upper_bounds(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::scan::FileScanTaskColumnStats::value_counts(&self) -> &std::collections::hash::map::HashMap +impl core::clone::Clone for iceberg::scan::FileScanTaskColumnStats +pub fn iceberg::scan::FileScanTaskColumnStats::clone(&self) -> iceberg::scan::FileScanTaskColumnStats +impl core::cmp::PartialEq for iceberg::scan::FileScanTaskColumnStats +pub fn iceberg::scan::FileScanTaskColumnStats::eq(&self, other: &iceberg::scan::FileScanTaskColumnStats) -> bool +impl core::default::Default for iceberg::scan::FileScanTaskColumnStats +pub fn iceberg::scan::FileScanTaskColumnStats::default() -> iceberg::scan::FileScanTaskColumnStats +impl core::fmt::Debug for iceberg::scan::FileScanTaskColumnStats +pub fn iceberg::scan::FileScanTaskColumnStats::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::scan::FileScanTaskColumnStats +impl iceberg::scan::FileScanTaskColumnStats +pub fn iceberg::scan::FileScanTaskColumnStats::builder() -> FileScanTaskColumnStatsBuilder<((), (), (), (), (), ())> +impl serde_core::ser::Serialize for iceberg::scan::FileScanTaskColumnStats +pub fn iceberg::scan::FileScanTaskColumnStats::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::scan::FileScanTaskColumnStats +pub fn iceberg::scan::FileScanTaskColumnStats::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::scan::FileScanTaskDeleteFile +impl iceberg::scan::FileScanTaskDeleteFile +pub fn iceberg::scan::FileScanTaskDeleteFile::content_offset(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTaskDeleteFile::content_size_in_bytes(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTaskDeleteFile::equality_ids(&self) -> core::option::Option<&[i32]> +pub fn iceberg::scan::FileScanTaskDeleteFile::file_format(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::scan::FileScanTaskDeleteFile::file_path(&self) -> &str +pub fn iceberg::scan::FileScanTaskDeleteFile::file_size_in_bytes(&self) -> u64 +pub fn iceberg::scan::FileScanTaskDeleteFile::file_type(&self) -> iceberg::spec::DataContentType +pub fn iceberg::scan::FileScanTaskDeleteFile::key_metadata(&self) -> core::option::Option<&[u8]> +pub fn iceberg::scan::FileScanTaskDeleteFile::partition_spec_id(&self) -> i32 +pub fn iceberg::scan::FileScanTaskDeleteFile::record_count(&self) -> core::option::Option +pub fn iceberg::scan::FileScanTaskDeleteFile::referenced_data_file(&self) -> core::option::Option<&str> +impl core::clone::Clone for iceberg::scan::FileScanTaskDeleteFile +pub fn iceberg::scan::FileScanTaskDeleteFile::clone(&self) -> iceberg::scan::FileScanTaskDeleteFile +impl core::cmp::PartialEq for iceberg::scan::FileScanTaskDeleteFile +pub fn iceberg::scan::FileScanTaskDeleteFile::eq(&self, other: &iceberg::scan::FileScanTaskDeleteFile) -> bool +impl core::convert::From for iceberg::Result +pub fn iceberg::Result::from(task: iceberg::scan::FileScanTaskDeleteFile) -> Self +impl core::fmt::Debug for iceberg::scan::FileScanTaskDeleteFile +pub fn iceberg::scan::FileScanTaskDeleteFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::scan::FileScanTaskDeleteFile +impl iceberg::scan::FileScanTaskDeleteFile +pub fn iceberg::scan::FileScanTaskDeleteFile::builder() -> FileScanTaskDeleteFileBuilder<((), (), (), (), (), (), (), (), (), (), ())> +impl serde_core::ser::Serialize for iceberg::scan::FileScanTaskDeleteFile +pub fn iceberg::scan::FileScanTaskDeleteFile::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::scan::FileScanTaskDeleteFile +pub fn iceberg::scan::FileScanTaskDeleteFile::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::scan::ScanMetrics +impl iceberg::scan::ScanMetrics +pub fn iceberg::scan::ScanMetrics::bytes_read(&self) -> u64 +impl core::clone::Clone for iceberg::scan::ScanMetrics +pub fn iceberg::scan::ScanMetrics::clone(&self) -> iceberg::scan::ScanMetrics +impl core::fmt::Debug for iceberg::scan::ScanMetrics +pub fn iceberg::scan::ScanMetrics::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::scan::ScanResult +impl iceberg::scan::ScanResult +pub fn iceberg::scan::ScanResult::metrics(&self) -> &iceberg::scan::ScanMetrics +pub fn iceberg::scan::ScanResult::stream(self) -> iceberg::scan::ArrowRecordBatchStream +pub struct iceberg::scan::TableScan +impl iceberg::scan::TableScan +pub fn iceberg::scan::TableScan::column_names(&self) -> core::option::Option<&[alloc::string::String]> +pub async fn iceberg::scan::TableScan::plan_files(&self) -> iceberg::Result +pub fn iceberg::scan::TableScan::snapshot(&self) -> core::option::Option<&iceberg::spec::SnapshotRef> +pub async fn iceberg::scan::TableScan::to_arrow(&self, options: iceberg::arrow::ArrowReaderOptions) -> iceberg::Result +impl core::fmt::Debug for iceberg::scan::TableScan +pub fn iceberg::scan::TableScan::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::scan::TableScanBuilder<'a> +impl<'a> iceberg::scan::TableScanBuilder<'a> +pub fn iceberg::scan::TableScanBuilder<'a>::as_of_time(self, timestamp_ms: i64) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::build(self) -> iceberg::Result +pub fn iceberg::scan::TableScanBuilder<'a>::select(self, column_names: impl core::iter::traits::collect::IntoIterator) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::select_all(self) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::select_empty(self) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::snapshot_id(self, snapshot_id: i64) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::with_case_sensitive(self, case_sensitive: bool) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::with_column_stats(self, column_names: impl core::iter::traits::collect::IntoIterator) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::with_concurrency_limit(self, limit: usize) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::with_filter(self, predicate: iceberg::expr::Predicate) -> Self +pub fn iceberg::scan::TableScanBuilder<'a>::with_manifest_entry_concurrency_limit(self, limit: usize) -> Self +pub type iceberg::scan::ArrowRecordBatchStream = futures_core::stream::BoxStream<'static, iceberg::Result> +pub type iceberg::scan::FileScanTaskStream = futures_core::stream::BoxStream<'static, iceberg::Result> +pub mod iceberg::sensitive +pub struct iceberg::sensitive::SensitiveBytes(_) +impl iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::as_bytes(&self) -> &[u8] +pub fn iceberg::sensitive::SensitiveBytes::is_empty(&self) -> bool +pub fn iceberg::sensitive::SensitiveBytes::len(&self) -> usize +pub fn iceberg::sensitive::SensitiveBytes::new(bytes: impl core::convert::Into>) -> Self +impl core::clone::Clone for iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::clone(&self) -> iceberg::sensitive::SensitiveBytes +impl core::cmp::Eq for iceberg::sensitive::SensitiveBytes +impl core::cmp::PartialEq for iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::eq(&self, other: &iceberg::sensitive::SensitiveBytes) -> bool +impl core::convert::TryFrom for iceberg::encryption::SecureKey +pub type iceberg::encryption::SecureKey::Error = iceberg::Error +pub fn iceberg::encryption::SecureKey::try_from(key: iceberg::sensitive::SensitiveBytes) -> iceberg::Result +impl core::fmt::Debug for iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::sensitive::SensitiveBytes +pub fn iceberg::sensitive::SensitiveBytes::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::sensitive::SensitiveBytes +pub struct iceberg::sensitive::SensitiveString(_) +impl iceberg::sensitive::SensitiveString +pub fn iceberg::sensitive::SensitiveString::expose(&self) -> &str +pub fn iceberg::sensitive::SensitiveString::is_empty(&self) -> bool +impl core::clone::Clone for iceberg::sensitive::SensitiveString +pub fn iceberg::sensitive::SensitiveString::clone(&self) -> iceberg::sensitive::SensitiveString +impl core::cmp::Eq for iceberg::sensitive::SensitiveString +impl core::cmp::PartialEq for iceberg::sensitive::SensitiveString +pub fn iceberg::sensitive::SensitiveString::eq(&self, other: &iceberg::sensitive::SensitiveString) -> bool +impl core::convert::From for iceberg::sensitive::SensitiveString +pub fn iceberg::sensitive::SensitiveString::from(value: alloc::string::String) -> Self +impl core::fmt::Debug for iceberg::sensitive::SensitiveString +pub fn iceberg::sensitive::SensitiveString::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::sensitive::SensitiveString +pub mod iceberg::spec +pub use iceberg::spec::ByteBuf +pub mod iceberg::spec::geospatial +pub enum iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Andoyer +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Karney +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Spherical +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Thomas +pub iceberg::spec::geospatial::EdgeInterpolationAlgorithm::Vincenty +impl core::clone::Clone for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::clone(&self) -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl core::cmp::Eq for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl core::cmp::PartialEq for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::eq(&self, other: &iceberg::spec::geospatial::EdgeInterpolationAlgorithm) -> bool +impl core::convert::From for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::from(edges: parquet_geospatial::types::Edges) -> Self +impl core::default::Default for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::default() -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl core::fmt::Debug for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::Copy for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +impl serde_core::ser::Serialize for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::EdgeInterpolationAlgorithm::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::geospatial::GeographyType +impl iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::algorithm(&self) -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::GeographyType::crs(&self) -> &str +pub fn iceberg::spec::geospatial::GeographyType::new(crs: core::option::Option, algorithm: iceberg::spec::geospatial::EdgeInterpolationAlgorithm) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::clone(&self) -> iceberg::spec::geospatial::GeographyType +impl core::cmp::Eq for iceberg::spec::geospatial::GeographyType +impl core::cmp::PartialEq for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::eq(&self, other: &iceberg::spec::geospatial::GeographyType) -> bool +impl core::default::Default for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::default() -> Self +impl core::fmt::Debug for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeographyType +impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::geospatial::GeometryType +impl iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::crs(&self) -> &str +pub fn iceberg::spec::geospatial::GeometryType::new(crs: core::option::Option) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::clone(&self) -> iceberg::spec::geospatial::GeometryType +impl core::cmp::Eq for iceberg::spec::geospatial::GeometryType +impl core::cmp::PartialEq for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::eq(&self, other: &iceberg::spec::geospatial::GeometryType) -> bool +impl core::default::Default for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::default() -> iceberg::spec::geospatial::GeometryType +impl core::fmt::Debug for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeometryType +impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::DataContentType +pub iceberg::spec::DataContentType::Data = 0 +pub iceberg::spec::DataContentType::EqualityDeletes = 2 +pub iceberg::spec::DataContentType::PositionDeletes = 1 +impl core::clone::Clone for iceberg::spec::DataContentType +pub fn iceberg::spec::DataContentType::clone(&self) -> iceberg::spec::DataContentType +impl core::cmp::Eq for iceberg::spec::DataContentType +impl core::cmp::PartialEq for iceberg::spec::DataContentType +pub fn iceberg::spec::DataContentType::eq(&self, other: &iceberg::spec::DataContentType) -> bool +impl core::convert::TryFrom for iceberg::spec::DataContentType +pub type iceberg::spec::DataContentType::Error = iceberg::Error +pub fn iceberg::spec::DataContentType::try_from(v: i32) -> iceberg::Result +impl core::default::Default for iceberg::spec::DataContentType +pub fn iceberg::spec::DataContentType::default() -> iceberg::spec::DataContentType +impl core::fmt::Debug for iceberg::spec::DataContentType +pub fn iceberg::spec::DataContentType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::spec::DataContentType +impl core::marker::StructuralPartialEq for iceberg::spec::DataContentType +impl serde_core::ser::Serialize for iceberg::spec::DataContentType +pub fn iceberg::spec::DataContentType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::DataContentType +pub fn iceberg::spec::DataContentType::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +#[non_exhaustive] pub enum iceberg::spec::DataFileBuilderError +pub iceberg::spec::DataFileBuilderError::UninitializedField(&'static str) +pub iceberg::spec::DataFileBuilderError::ValidationError(alloc::string::String) +impl core::convert::From for iceberg::spec::DataFileBuilderError +pub fn iceberg::spec::DataFileBuilderError::from(s: alloc::string::String) -> Self +impl core::convert::From for iceberg::spec::DataFileBuilderError +pub fn iceberg::spec::DataFileBuilderError::from(s: derive_builder::error::UninitializedFieldError) -> Self +impl core::error::Error for iceberg::spec::DataFileBuilderError +impl core::fmt::Debug for iceberg::spec::DataFileBuilderError +pub fn iceberg::spec::DataFileBuilderError::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::DataFileBuilderError +pub fn iceberg::spec::DataFileBuilderError::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub enum iceberg::spec::DataFileFormat +pub iceberg::spec::DataFileFormat::Avro +pub iceberg::spec::DataFileFormat::Orc +pub iceberg::spec::DataFileFormat::Parquet +pub iceberg::spec::DataFileFormat::Puffin +impl core::clone::Clone for iceberg::spec::DataFileFormat +pub fn iceberg::spec::DataFileFormat::clone(&self) -> iceberg::spec::DataFileFormat +impl core::cmp::Eq for iceberg::spec::DataFileFormat +impl core::cmp::PartialEq for iceberg::spec::DataFileFormat +pub fn iceberg::spec::DataFileFormat::eq(&self, other: &iceberg::spec::DataFileFormat) -> bool +impl core::fmt::Debug for iceberg::spec::DataFileFormat +pub fn iceberg::spec::DataFileFormat::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::DataFileFormat +pub fn iceberg::spec::DataFileFormat::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::spec::DataFileFormat +impl core::marker::StructuralPartialEq for iceberg::spec::DataFileFormat +impl core::str::traits::FromStr for iceberg::spec::DataFileFormat +pub type iceberg::spec::DataFileFormat::Err = iceberg::Error +pub fn iceberg::spec::DataFileFormat::from_str(s: &str) -> iceberg::Result +impl serde_core::ser::Serialize for iceberg::spec::DataFileFormat where Self: core::fmt::Display +pub fn iceberg::spec::DataFileFormat::serialize<__S>(&self, serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::DataFileFormat where Self: core::str::traits::FromStr, ::Err: core::fmt::Display +pub fn iceberg::spec::DataFileFormat::deserialize<__D>(deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +#[repr(u8)] pub enum iceberg::spec::FormatVersion +pub iceberg::spec::FormatVersion::V1 = 1 +pub iceberg::spec::FormatVersion::V2 = 2 +pub iceberg::spec::FormatVersion::V3 = 3 +impl core::clone::Clone for iceberg::spec::FormatVersion +pub fn iceberg::spec::FormatVersion::clone(&self) -> iceberg::spec::FormatVersion +impl core::cmp::Eq for iceberg::spec::FormatVersion +impl core::cmp::Ord for iceberg::spec::FormatVersion +pub fn iceberg::spec::FormatVersion::cmp(&self, other: &Self) -> core::cmp::Ordering +impl core::cmp::PartialEq for iceberg::spec::FormatVersion +pub fn iceberg::spec::FormatVersion::eq(&self, other: &iceberg::spec::FormatVersion) -> bool +impl core::cmp::PartialOrd for iceberg::spec::FormatVersion +pub fn iceberg::spec::FormatVersion::partial_cmp(&self, other: &Self) -> core::option::Option +impl core::fmt::Debug for iceberg::spec::FormatVersion +pub fn iceberg::spec::FormatVersion::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::FormatVersion +pub fn iceberg::spec::FormatVersion::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::FormatVersion +pub fn iceberg::spec::FormatVersion::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::Copy for iceberg::spec::FormatVersion +impl core::marker::StructuralPartialEq for iceberg::spec::FormatVersion +impl serde_core::ser::Serialize for iceberg::spec::FormatVersion +pub fn iceberg::spec::FormatVersion::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::FormatVersion +pub fn iceberg::spec::FormatVersion::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::Literal +pub iceberg::spec::Literal::List(alloc::vec::Vec>) +pub iceberg::spec::Literal::Map(iceberg::spec::Map) +pub iceberg::spec::Literal::Primitive(iceberg::spec::PrimitiveLiteral) +pub iceberg::spec::Literal::Struct(iceberg::spec::Struct) +impl iceberg::spec::Literal +pub fn iceberg::spec::Literal::as_primitive_literal(&self) -> core::option::Option +pub fn iceberg::spec::Literal::binary>(input: I) -> Self +pub fn iceberg::spec::Literal::bool>(t: T) -> Self +pub fn iceberg::spec::Literal::bool_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Literal::date(days: i32) -> Self +pub fn iceberg::spec::Literal::date_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Literal::date_from_ymd(year: i32, month: u32, day: u32) -> iceberg::Result +pub fn iceberg::spec::Literal::decimal(decimal: i128) -> Self +pub fn iceberg::spec::Literal::decimal_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Literal::double>(t: T) -> Self +pub fn iceberg::spec::Literal::fixed>(input: I) -> Self +pub fn iceberg::spec::Literal::float>(t: T) -> Self +pub fn iceberg::spec::Literal::int>(t: T) -> Self +pub fn iceberg::spec::Literal::into_any(self) -> alloc::boxed::Box +pub fn iceberg::spec::Literal::long>(t: T) -> Self +pub fn iceberg::spec::Literal::string(s: S) -> Self +pub fn iceberg::spec::Literal::time(value: i64) -> Self +pub fn iceberg::spec::Literal::time_from_hms_micro(hour: u32, min: u32, sec: u32, micro: u32) -> iceberg::Result +pub fn iceberg::spec::Literal::time_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Literal::timestamp(value: i64) -> Self +pub fn iceberg::spec::Literal::timestamp_from_datetime(dt: chrono::datetime::DateTime) -> Self +pub fn iceberg::spec::Literal::timestamp_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Literal::timestamptz(value: i64) -> Self +pub fn iceberg::spec::Literal::timestamptz_from_datetime(dt: chrono::datetime::DateTime) -> Self +pub fn iceberg::spec::Literal::timestamptz_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Literal::try_from_json(value: serde_json::value::Value, data_type: &iceberg::spec::Type) -> iceberg::Result> +pub fn iceberg::spec::Literal::try_into_json(self, type: &iceberg::spec::Type) -> iceberg::Result +pub fn iceberg::spec::Literal::uuid(uuid: uuid::Uuid) -> Self +pub fn iceberg::spec::Literal::uuid_from_str>(s: S) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::Literal +pub fn iceberg::spec::Literal::clone(&self) -> iceberg::spec::Literal +impl core::cmp::Eq for iceberg::spec::Literal +impl core::cmp::PartialEq for iceberg::spec::Literal +pub fn iceberg::spec::Literal::eq(&self, other: &iceberg::spec::Literal) -> bool +impl core::convert::From for iceberg::spec::Literal +pub fn iceberg::spec::Literal::from(value: iceberg::spec::Datum) -> Self +impl core::fmt::Debug for iceberg::spec::Literal +pub fn iceberg::spec::Literal::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::Literal +pub fn iceberg::spec::Literal::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::Literal +pub enum iceberg::spec::ManifestContentType +pub iceberg::spec::ManifestContentType::Data = 0 +pub iceberg::spec::ManifestContentType::Deletes = 1 +impl core::clone::Clone for iceberg::spec::ManifestContentType +pub fn iceberg::spec::ManifestContentType::clone(&self) -> iceberg::spec::ManifestContentType +impl core::cmp::Eq for iceberg::spec::ManifestContentType +impl core::cmp::PartialEq for iceberg::spec::ManifestContentType +pub fn iceberg::spec::ManifestContentType::eq(&self, other: &iceberg::spec::ManifestContentType) -> bool +impl core::convert::TryFrom for iceberg::spec::ManifestContentType +pub type iceberg::spec::ManifestContentType::Error = iceberg::Error +pub fn iceberg::spec::ManifestContentType::try_from(value: i32) -> core::result::Result +impl core::default::Default for iceberg::spec::ManifestContentType +pub fn iceberg::spec::ManifestContentType::default() -> iceberg::spec::ManifestContentType +impl core::fmt::Debug for iceberg::spec::ManifestContentType +pub fn iceberg::spec::ManifestContentType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::ManifestContentType +pub fn iceberg::spec::ManifestContentType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::ManifestContentType +pub fn iceberg::spec::ManifestContentType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::Copy for iceberg::spec::ManifestContentType +impl core::marker::StructuralPartialEq for iceberg::spec::ManifestContentType +impl core::str::traits::FromStr for iceberg::spec::ManifestContentType +pub type iceberg::spec::ManifestContentType::Err = iceberg::Error +pub fn iceberg::spec::ManifestContentType::from_str(s: &str) -> iceberg::Result +pub enum iceberg::spec::ManifestStatus +pub iceberg::spec::ManifestStatus::Added = 1 +pub iceberg::spec::ManifestStatus::Deleted = 2 +pub iceberg::spec::ManifestStatus::Existing = 0 +impl core::clone::Clone for iceberg::spec::ManifestStatus +pub fn iceberg::spec::ManifestStatus::clone(&self) -> iceberg::spec::ManifestStatus +impl core::cmp::Eq for iceberg::spec::ManifestStatus +impl core::cmp::PartialEq for iceberg::spec::ManifestStatus +pub fn iceberg::spec::ManifestStatus::eq(&self, other: &iceberg::spec::ManifestStatus) -> bool +impl core::convert::TryFrom for iceberg::spec::ManifestStatus +pub type iceberg::spec::ManifestStatus::Error = iceberg::Error +pub fn iceberg::spec::ManifestStatus::try_from(v: i32) -> iceberg::Result +impl core::fmt::Debug for iceberg::spec::ManifestStatus +pub fn iceberg::spec::ManifestStatus::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::spec::ManifestStatus +impl core::marker::StructuralPartialEq for iceberg::spec::ManifestStatus +pub enum iceberg::spec::NullOrder +pub iceberg::spec::NullOrder::First +pub iceberg::spec::NullOrder::Last +impl core::clone::Clone for iceberg::spec::NullOrder +pub fn iceberg::spec::NullOrder::clone(&self) -> iceberg::spec::NullOrder +impl core::cmp::Eq for iceberg::spec::NullOrder +impl core::cmp::PartialEq for iceberg::spec::NullOrder +pub fn iceberg::spec::NullOrder::eq(&self, other: &iceberg::spec::NullOrder) -> bool +impl core::fmt::Debug for iceberg::spec::NullOrder +pub fn iceberg::spec::NullOrder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::NullOrder +pub fn iceberg::spec::NullOrder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::spec::NullOrder +impl core::marker::StructuralPartialEq for iceberg::spec::NullOrder +impl serde_core::ser::Serialize for iceberg::spec::NullOrder +pub fn iceberg::spec::NullOrder::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NullOrder +pub fn iceberg::spec::NullOrder::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::Operation +pub iceberg::spec::Operation::Append +pub iceberg::spec::Operation::Delete +pub iceberg::spec::Operation::Overwrite +pub iceberg::spec::Operation::Replace +impl iceberg::spec::Operation +pub fn iceberg::spec::Operation::as_str(&self) -> &str +impl core::clone::Clone for iceberg::spec::Operation +pub fn iceberg::spec::Operation::clone(&self) -> iceberg::spec::Operation +impl core::cmp::Eq for iceberg::spec::Operation +impl core::cmp::PartialEq for iceberg::spec::Operation +pub fn iceberg::spec::Operation::eq(&self, other: &iceberg::spec::Operation) -> bool +impl core::default::Default for iceberg::spec::Operation +pub fn iceberg::spec::Operation::default() -> iceberg::spec::Operation +impl core::fmt::Debug for iceberg::spec::Operation +pub fn iceberg::spec::Operation::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::Operation +impl serde_core::ser::Serialize for iceberg::spec::Operation +pub fn iceberg::spec::Operation::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::Operation +pub fn iceberg::spec::Operation::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::PrimitiveLiteral +pub iceberg::spec::PrimitiveLiteral::AboveMax +pub iceberg::spec::PrimitiveLiteral::BelowMin +pub iceberg::spec::PrimitiveLiteral::Binary(alloc::vec::Vec) +pub iceberg::spec::PrimitiveLiteral::Boolean(bool) +pub iceberg::spec::PrimitiveLiteral::Double(ordered_float::OrderedFloat) +pub iceberg::spec::PrimitiveLiteral::Float(ordered_float::OrderedFloat) +pub iceberg::spec::PrimitiveLiteral::Int(i32) +pub iceberg::spec::PrimitiveLiteral::Int128(i128) +pub iceberg::spec::PrimitiveLiteral::Long(i64) +pub iceberg::spec::PrimitiveLiteral::String(alloc::string::String) +pub iceberg::spec::PrimitiveLiteral::UInt128(u128) +impl iceberg::spec::PrimitiveLiteral +pub fn iceberg::spec::PrimitiveLiteral::is_nan(&self) -> bool +impl core::clone::Clone for iceberg::spec::PrimitiveLiteral +pub fn iceberg::spec::PrimitiveLiteral::clone(&self) -> iceberg::spec::PrimitiveLiteral +impl core::cmp::Eq for iceberg::spec::PrimitiveLiteral +impl core::cmp::PartialEq for iceberg::spec::PrimitiveLiteral +pub fn iceberg::spec::PrimitiveLiteral::eq(&self, other: &Self) -> bool +impl core::cmp::PartialOrd for iceberg::spec::PrimitiveLiteral +pub fn iceberg::spec::PrimitiveLiteral::partial_cmp(&self, other: &Self) -> core::option::Option +impl core::convert::From for iceberg::spec::PrimitiveLiteral +pub fn iceberg::spec::PrimitiveLiteral::from(value: iceberg::spec::Datum) -> Self +impl core::fmt::Debug for iceberg::spec::PrimitiveLiteral +pub fn iceberg::spec::PrimitiveLiteral::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::PrimitiveLiteral +pub fn iceberg::spec::PrimitiveLiteral::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +pub enum iceberg::spec::PrimitiveType +pub iceberg::spec::PrimitiveType::Binary +pub iceberg::spec::PrimitiveType::Boolean +pub iceberg::spec::PrimitiveType::Date +pub iceberg::spec::PrimitiveType::Decimal +pub iceberg::spec::PrimitiveType::Decimal::precision: u32 +pub iceberg::spec::PrimitiveType::Decimal::scale: u32 +pub iceberg::spec::PrimitiveType::Double +pub iceberg::spec::PrimitiveType::Fixed(u64) +pub iceberg::spec::PrimitiveType::Float +pub iceberg::spec::PrimitiveType::Geography(iceberg::spec::geospatial::GeographyType) +pub iceberg::spec::PrimitiveType::Geometry(iceberg::spec::geospatial::GeometryType) +pub iceberg::spec::PrimitiveType::Int +pub iceberg::spec::PrimitiveType::Long +pub iceberg::spec::PrimitiveType::String +pub iceberg::spec::PrimitiveType::Time +pub iceberg::spec::PrimitiveType::Timestamp +pub iceberg::spec::PrimitiveType::TimestampNs +pub iceberg::spec::PrimitiveType::Timestamptz +pub iceberg::spec::PrimitiveType::TimestamptzNs +pub iceberg::spec::PrimitiveType::Unknown +pub iceberg::spec::PrimitiveType::Uuid +impl iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::compatible(&self, literal: &iceberg::spec::PrimitiveLiteral) -> bool +impl core::clone::Clone for iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::clone(&self) -> iceberg::spec::PrimitiveType +impl core::cmp::Eq for iceberg::spec::PrimitiveType +impl core::cmp::PartialEq for iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::eq(&self, other: &iceberg::spec::PrimitiveType) -> bool +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(value: iceberg::spec::PrimitiveType) -> Self +impl core::fmt::Debug for iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::PrimitiveType +impl iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::serialize<__S>(__self: &iceberg::spec::PrimitiveType, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl serde_core::ser::Serialize for iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::PrimitiveType +pub fn iceberg::spec::PrimitiveType::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::SnapshotRetention +pub iceberg::spec::SnapshotRetention::Branch +pub iceberg::spec::SnapshotRetention::Branch::max_ref_age_ms: core::option::Option +pub iceberg::spec::SnapshotRetention::Branch::max_snapshot_age_ms: core::option::Option +pub iceberg::spec::SnapshotRetention::Branch::min_snapshots_to_keep: core::option::Option +pub iceberg::spec::SnapshotRetention::Tag +pub iceberg::spec::SnapshotRetention::Tag::max_ref_age_ms: core::option::Option +impl iceberg::spec::SnapshotRetention +pub fn iceberg::spec::SnapshotRetention::branch(min_snapshots_to_keep: core::option::Option, max_snapshot_age_ms: core::option::Option, max_ref_age_ms: core::option::Option) -> Self +impl core::clone::Clone for iceberg::spec::SnapshotRetention +pub fn iceberg::spec::SnapshotRetention::clone(&self) -> iceberg::spec::SnapshotRetention +impl core::cmp::Eq for iceberg::spec::SnapshotRetention +impl core::cmp::PartialEq for iceberg::spec::SnapshotRetention +pub fn iceberg::spec::SnapshotRetention::eq(&self, other: &iceberg::spec::SnapshotRetention) -> bool +impl core::fmt::Debug for iceberg::spec::SnapshotRetention +pub fn iceberg::spec::SnapshotRetention::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::SnapshotRetention +impl serde_core::ser::Serialize for iceberg::spec::SnapshotRetention +pub fn iceberg::spec::SnapshotRetention::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::SnapshotRetention +pub fn iceberg::spec::SnapshotRetention::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::SortDirection +pub iceberg::spec::SortDirection::Ascending +pub iceberg::spec::SortDirection::Descending +impl core::clone::Clone for iceberg::spec::SortDirection +pub fn iceberg::spec::SortDirection::clone(&self) -> iceberg::spec::SortDirection +impl core::cmp::Eq for iceberg::spec::SortDirection +impl core::cmp::PartialEq for iceberg::spec::SortDirection +pub fn iceberg::spec::SortDirection::eq(&self, other: &iceberg::spec::SortDirection) -> bool +impl core::fmt::Debug for iceberg::spec::SortDirection +pub fn iceberg::spec::SortDirection::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::SortDirection +pub fn iceberg::spec::SortDirection::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::spec::SortDirection +impl core::marker::StructuralPartialEq for iceberg::spec::SortDirection +impl serde_core::ser::Serialize for iceberg::spec::SortDirection +pub fn iceberg::spec::SortDirection::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::SortDirection +pub fn iceberg::spec::SortDirection::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +#[non_exhaustive] pub enum iceberg::spec::SortOrderBuilderError +pub iceberg::spec::SortOrderBuilderError::UninitializedField(&'static str) +pub iceberg::spec::SortOrderBuilderError::ValidationError(alloc::string::String) +impl core::convert::From for iceberg::spec::SortOrderBuilderError +pub fn iceberg::spec::SortOrderBuilderError::from(s: alloc::string::String) -> Self +impl core::convert::From for iceberg::spec::SortOrderBuilderError +pub fn iceberg::spec::SortOrderBuilderError::from(s: derive_builder::error::UninitializedFieldError) -> Self +impl core::error::Error for iceberg::spec::SortOrderBuilderError +impl core::fmt::Debug for iceberg::spec::SortOrderBuilderError +pub fn iceberg::spec::SortOrderBuilderError::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::SortOrderBuilderError +pub fn iceberg::spec::SortOrderBuilderError::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub enum iceberg::spec::Transform +pub iceberg::spec::Transform::Bucket(u32) +pub iceberg::spec::Transform::Day +pub iceberg::spec::Transform::Hour +pub iceberg::spec::Transform::Identity +pub iceberg::spec::Transform::Month +pub iceberg::spec::Transform::Truncate(u32) +pub iceberg::spec::Transform::Unknown +pub iceberg::spec::Transform::Void +pub iceberg::spec::Transform::Year +impl iceberg::spec::Transform +pub fn iceberg::spec::Transform::dedup_name(&self) -> alloc::string::String +pub fn iceberg::spec::Transform::preserves_order(&self) -> bool +pub fn iceberg::spec::Transform::project(&self, name: &str, predicate: &iceberg::expr::BoundPredicate) -> iceberg::Result> +pub fn iceberg::spec::Transform::result_type(&self, input_type: &iceberg::spec::Type) -> iceberg::Result +pub fn iceberg::spec::Transform::satisfies_order_of(&self, other: &Self) -> bool +pub fn iceberg::spec::Transform::strict_project(&self, name: &str, predicate: &iceberg::expr::BoundPredicate) -> iceberg::Result> +pub fn iceberg::spec::Transform::to_human_string(&self, field_type: &iceberg::spec::Type, value: core::option::Option<&iceberg::spec::Literal>) -> alloc::string::String +impl core::clone::Clone for iceberg::spec::Transform +pub fn iceberg::spec::Transform::clone(&self) -> iceberg::spec::Transform +impl core::cmp::Eq for iceberg::spec::Transform +impl core::cmp::PartialEq for iceberg::spec::Transform +pub fn iceberg::spec::Transform::eq(&self, other: &iceberg::spec::Transform) -> bool +impl core::fmt::Debug for iceberg::spec::Transform +pub fn iceberg::spec::Transform::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::Transform +pub fn iceberg::spec::Transform::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::Transform +pub fn iceberg::spec::Transform::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::Copy for iceberg::spec::Transform +impl core::marker::StructuralPartialEq for iceberg::spec::Transform +impl core::str::traits::FromStr for iceberg::spec::Transform +pub type iceberg::spec::Transform::Err = iceberg::Error +pub fn iceberg::spec::Transform::from_str(s: &str) -> iceberg::Result +impl serde_core::ser::Serialize for iceberg::spec::Transform +pub fn iceberg::spec::Transform::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::Transform +pub fn iceberg::spec::Transform::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::Type +pub iceberg::spec::Type::List(iceberg::spec::ListType) +pub iceberg::spec::Type::Map(iceberg::spec::MapType) +pub iceberg::spec::Type::Primitive(iceberg::spec::PrimitiveType) +pub iceberg::spec::Type::Struct(iceberg::spec::StructType) +pub iceberg::spec::Type::Variant(iceberg::spec::VariantType) +impl iceberg::spec::Type +pub fn iceberg::spec::Type::as_primitive_type(&self) -> core::option::Option<&iceberg::spec::PrimitiveType> +pub fn iceberg::spec::Type::decimal(precision: u32, scale: u32) -> iceberg::Result +pub fn iceberg::spec::Type::decimal_max_precision(num_bytes: u32) -> iceberg::Result +pub fn iceberg::spec::Type::decimal_required_bytes(precision: u32) -> iceberg::Result +pub fn iceberg::spec::Type::is_floating_type(&self) -> bool +pub fn iceberg::spec::Type::is_nested(&self) -> bool +pub fn iceberg::spec::Type::is_primitive(&self) -> bool +pub fn iceberg::spec::Type::is_struct(&self) -> bool +pub fn iceberg::spec::Type::is_variant(&self) -> bool +pub fn iceberg::spec::Type::to_struct_type(self) -> core::option::Option +impl core::clone::Clone for iceberg::spec::Type +pub fn iceberg::spec::Type::clone(&self) -> iceberg::spec::Type +impl core::cmp::Eq for iceberg::spec::Type +impl core::cmp::PartialEq for iceberg::spec::Type +pub fn iceberg::spec::Type::eq(&self, other: &iceberg::spec::Type) -> bool +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(value: iceberg::spec::ListType) -> Self +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(value: iceberg::spec::MapType) -> Self +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(value: iceberg::spec::PrimitiveType) -> Self +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(value: iceberg::spec::StructType) -> Self +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(_: iceberg::spec::VariantType) -> Self +impl core::fmt::Debug for iceberg::spec::Type +pub fn iceberg::spec::Type::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::Type +pub fn iceberg::spec::Type::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::Type +impl serde_core::ser::Serialize for iceberg::spec::Type +pub fn iceberg::spec::Type::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::Type +pub fn iceberg::spec::Type::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +#[repr(u8)] pub enum iceberg::spec::ViewFormatVersion +pub iceberg::spec::ViewFormatVersion::V1 = 1 +impl core::clone::Clone for iceberg::spec::ViewFormatVersion +pub fn iceberg::spec::ViewFormatVersion::clone(&self) -> iceberg::spec::ViewFormatVersion +impl core::cmp::Eq for iceberg::spec::ViewFormatVersion +impl core::cmp::Ord for iceberg::spec::ViewFormatVersion +pub fn iceberg::spec::ViewFormatVersion::cmp(&self, other: &Self) -> core::cmp::Ordering +impl core::cmp::PartialEq for iceberg::spec::ViewFormatVersion +pub fn iceberg::spec::ViewFormatVersion::eq(&self, other: &iceberg::spec::ViewFormatVersion) -> bool +impl core::cmp::PartialOrd for iceberg::spec::ViewFormatVersion +pub fn iceberg::spec::ViewFormatVersion::partial_cmp(&self, other: &Self) -> core::option::Option +impl core::fmt::Debug for iceberg::spec::ViewFormatVersion +pub fn iceberg::spec::ViewFormatVersion::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::ViewFormatVersion +pub fn iceberg::spec::ViewFormatVersion::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::spec::ViewFormatVersion +impl core::marker::StructuralPartialEq for iceberg::spec::ViewFormatVersion +impl serde_core::ser::Serialize for iceberg::spec::ViewFormatVersion +pub fn iceberg::spec::ViewFormatVersion::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::ViewFormatVersion +pub fn iceberg::spec::ViewFormatVersion::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub enum iceberg::spec::ViewRepresentation +pub iceberg::spec::ViewRepresentation::Sql(iceberg::spec::SqlViewRepresentation) +impl core::clone::Clone for iceberg::spec::ViewRepresentation +pub fn iceberg::spec::ViewRepresentation::clone(&self) -> iceberg::spec::ViewRepresentation +impl core::cmp::Eq for iceberg::spec::ViewRepresentation +impl core::cmp::PartialEq for iceberg::spec::ViewRepresentation +pub fn iceberg::spec::ViewRepresentation::eq(&self, other: &iceberg::spec::ViewRepresentation) -> bool +impl core::convert::From for iceberg::spec::ViewRepresentation +pub fn iceberg::spec::ViewRepresentation::from(sql: iceberg::spec::SqlViewRepresentation) -> Self +impl core::fmt::Debug for iceberg::spec::ViewRepresentation +pub fn iceberg::spec::ViewRepresentation::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::ViewRepresentation +impl serde_core::ser::Serialize for iceberg::spec::ViewRepresentation +pub fn iceberg::spec::ViewRepresentation::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::ViewRepresentation +pub fn iceberg::spec::ViewRepresentation::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::BlobMetadata +pub iceberg::spec::BlobMetadata::fields: alloc::vec::Vec +pub iceberg::spec::BlobMetadata::properties: std::collections::hash::map::HashMap +pub iceberg::spec::BlobMetadata::sequence_number: i64 +pub iceberg::spec::BlobMetadata::snapshot_id: i64 +pub iceberg::spec::BlobMetadata::type: alloc::string::String +impl core::clone::Clone for iceberg::spec::BlobMetadata +pub fn iceberg::spec::BlobMetadata::clone(&self) -> iceberg::spec::BlobMetadata +impl core::cmp::Eq for iceberg::spec::BlobMetadata +impl core::cmp::PartialEq for iceberg::spec::BlobMetadata +pub fn iceberg::spec::BlobMetadata::eq(&self, other: &iceberg::spec::BlobMetadata) -> bool +impl core::fmt::Debug for iceberg::spec::BlobMetadata +pub fn iceberg::spec::BlobMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::BlobMetadata +impl serde_core::ser::Serialize for iceberg::spec::BlobMetadata +pub fn iceberg::spec::BlobMetadata::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::BlobMetadata +pub fn iceberg::spec::BlobMetadata::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::DataFile +impl iceberg::spec::DataFile +pub fn iceberg::spec::DataFile::column_sizes(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::DataFile::content_offset(&self) -> core::option::Option +pub fn iceberg::spec::DataFile::content_size_in_bytes(&self) -> core::option::Option +pub fn iceberg::spec::DataFile::content_type(&self) -> iceberg::spec::DataContentType +pub fn iceberg::spec::DataFile::equality_ids(&self) -> core::option::Option> +pub fn iceberg::spec::DataFile::file_format(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::spec::DataFile::file_path(&self) -> &str +pub fn iceberg::spec::DataFile::file_size_in_bytes(&self) -> u64 +pub fn iceberg::spec::DataFile::first_row_id(&self) -> core::option::Option +pub fn iceberg::spec::DataFile::key_metadata(&self) -> core::option::Option<&[u8]> +pub fn iceberg::spec::DataFile::lower_bounds(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::DataFile::nan_value_counts(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::DataFile::null_value_counts(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::DataFile::partition(&self) -> &iceberg::spec::Struct +pub fn iceberg::spec::DataFile::record_count(&self) -> u64 +pub fn iceberg::spec::DataFile::referenced_data_file(&self) -> core::option::Option +pub fn iceberg::spec::DataFile::sort_order_id(&self) -> core::option::Option +pub fn iceberg::spec::DataFile::split_offsets(&self) -> core::option::Option<&[i64]> +pub fn iceberg::spec::DataFile::upper_bounds(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::DataFile::value_counts(&self) -> &std::collections::hash::map::HashMap +impl core::clone::Clone for iceberg::spec::DataFile +pub fn iceberg::spec::DataFile::clone(&self) -> iceberg::spec::DataFile +impl core::cmp::Eq for iceberg::spec::DataFile +impl core::cmp::PartialEq for iceberg::spec::DataFile +pub fn iceberg::spec::DataFile::eq(&self, other: &iceberg::spec::DataFile) -> bool +impl core::fmt::Debug for iceberg::spec::DataFile +pub fn iceberg::spec::DataFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::DataFile +pub struct iceberg::spec::DataFileBuilder +impl iceberg::spec::DataFileBuilder +pub fn iceberg::spec::DataFileBuilder::build(&self) -> core::result::Result +pub fn iceberg::spec::DataFileBuilder::column_sizes(&mut self, value: std::collections::hash::map::HashMap) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::content(&mut self, value: iceberg::spec::DataContentType) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::content_offset(&mut self, value: core::option::Option) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::content_size_in_bytes(&mut self, value: core::option::Option) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::equality_ids(&mut self, value: core::option::Option>) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::file_format(&mut self, value: iceberg::spec::DataFileFormat) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::file_path(&mut self, value: alloc::string::String) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::file_size_in_bytes(&mut self, value: u64) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::first_row_id(&mut self, value: core::option::Option) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::key_metadata(&mut self, value: core::option::Option>) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::lower_bounds(&mut self, value: std::collections::hash::map::HashMap) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::nan_value_counts(&mut self, value: std::collections::hash::map::HashMap) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::null_value_counts(&mut self, value: std::collections::hash::map::HashMap) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::partition(&mut self, value: iceberg::spec::Struct) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::partition_spec_id(&mut self, value: i32) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::record_count(&mut self, value: u64) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::referenced_data_file(&mut self, value: core::option::Option) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::sort_order_id(&mut self, value: i32) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::split_offsets(&mut self, value: core::option::Option>) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::upper_bounds(&mut self, value: std::collections::hash::map::HashMap) -> &mut Self +pub fn iceberg::spec::DataFileBuilder::value_counts(&mut self, value: std::collections::hash::map::HashMap) -> &mut Self +impl core::clone::Clone for iceberg::spec::DataFileBuilder +pub fn iceberg::spec::DataFileBuilder::clone(&self) -> iceberg::spec::DataFileBuilder +impl core::default::Default for iceberg::spec::DataFileBuilder +pub fn iceberg::spec::DataFileBuilder::default() -> Self +pub struct iceberg::spec::Datum +impl iceberg::spec::Datum +pub fn iceberg::spec::Datum::binary>(input: I) -> Self +pub fn iceberg::spec::Datum::bool>(t: T) -> Self +pub fn iceberg::spec::Datum::bool_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Datum::data_type(&self) -> &iceberg::spec::PrimitiveType +pub fn iceberg::spec::Datum::date(days: i32) -> Self +pub fn iceberg::spec::Datum::date_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Datum::date_from_ymd(year: i32, month: u32, day: u32) -> iceberg::Result +pub fn iceberg::spec::Datum::decimal(value: iceberg::spec::Decimal) -> iceberg::Result +pub fn iceberg::spec::Datum::decimal_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Datum::decimal_with_precision(value: iceberg::spec::Decimal, precision: u32) -> iceberg::Result +pub fn iceberg::spec::Datum::double>(t: T) -> Self +pub fn iceberg::spec::Datum::fixed>(input: I) -> Self +pub fn iceberg::spec::Datum::float>(t: T) -> Self +pub fn iceberg::spec::Datum::int>(t: T) -> Self +pub fn iceberg::spec::Datum::is_nan(&self) -> bool +pub fn iceberg::spec::Datum::literal(&self) -> &iceberg::spec::PrimitiveLiteral +pub fn iceberg::spec::Datum::long>(t: T) -> Self +pub fn iceberg::spec::Datum::string(s: S) -> Self +pub fn iceberg::spec::Datum::time_from_hms_micro(hour: u32, min: u32, sec: u32, micro: u32) -> iceberg::Result +pub fn iceberg::spec::Datum::time_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Datum::time_micros(value: i64) -> iceberg::Result +pub fn iceberg::spec::Datum::timestamp_from_datetime(dt: chrono::naive::datetime::NaiveDateTime) -> Self +pub fn iceberg::spec::Datum::timestamp_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Datum::timestamp_micros(value: i64) -> Self +pub fn iceberg::spec::Datum::timestamp_nanos(value: i64) -> Self +pub fn iceberg::spec::Datum::timestamptz_from_datetime(dt: chrono::datetime::DateTime) -> Self +pub fn iceberg::spec::Datum::timestamptz_from_str>(s: S) -> iceberg::Result +pub fn iceberg::spec::Datum::timestamptz_micros(value: i64) -> Self +pub fn iceberg::spec::Datum::timestamptz_nanos(value: i64) -> Self +pub fn iceberg::spec::Datum::to(self, target_type: &iceberg::spec::Type) -> iceberg::Result +pub fn iceberg::spec::Datum::to_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::Datum::to_human_string(&self) -> alloc::string::String +pub fn iceberg::spec::Datum::try_from_bytes(bytes: &[u8], data_type: iceberg::spec::PrimitiveType) -> iceberg::Result +pub fn iceberg::spec::Datum::uuid(uuid: uuid::Uuid) -> Self +pub fn iceberg::spec::Datum::uuid_from_str>(s: S) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::Datum +pub fn iceberg::spec::Datum::clone(&self) -> iceberg::spec::Datum +impl core::cmp::Eq for iceberg::spec::Datum +impl core::cmp::PartialEq for iceberg::spec::Datum +pub fn iceberg::spec::Datum::eq(&self, other: &iceberg::spec::Datum) -> bool +impl core::cmp::PartialOrd for iceberg::spec::Datum +pub fn iceberg::spec::Datum::partial_cmp(&self, other: &Self) -> core::option::Option +impl core::convert::From for iceberg::spec::Literal +pub fn iceberg::spec::Literal::from(value: iceberg::spec::Datum) -> Self +impl core::convert::From for iceberg::spec::PrimitiveLiteral +pub fn iceberg::spec::PrimitiveLiteral::from(value: iceberg::spec::Datum) -> Self +impl core::fmt::Debug for iceberg::spec::Datum +pub fn iceberg::spec::Datum::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::Datum +pub fn iceberg::spec::Datum::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::Datum +pub fn iceberg::spec::Datum::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::Datum +impl serde_core::ser::Serialize for iceberg::spec::Datum +pub fn iceberg::spec::Datum::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::Datum +pub fn iceberg::spec::Datum::deserialize>(deserializer: D) -> core::result::Result::Error> +pub struct iceberg::spec::EncryptedKey +impl iceberg::spec::EncryptedKey +pub fn iceberg::spec::EncryptedKey::encrypted_by_id(&self) -> core::option::Option<&str> +pub fn iceberg::spec::EncryptedKey::encrypted_key_metadata(&self) -> &[u8] +pub fn iceberg::spec::EncryptedKey::key_id(&self) -> &str +pub fn iceberg::spec::EncryptedKey::properties(&self) -> &std::collections::hash::map::HashMap +impl core::clone::Clone for iceberg::spec::EncryptedKey +pub fn iceberg::spec::EncryptedKey::clone(&self) -> iceberg::spec::EncryptedKey +impl core::cmp::Eq for iceberg::spec::EncryptedKey +impl core::cmp::PartialEq for iceberg::spec::EncryptedKey +pub fn iceberg::spec::EncryptedKey::eq(&self, other: &iceberg::spec::EncryptedKey) -> bool +impl core::fmt::Debug for iceberg::spec::EncryptedKey +pub fn iceberg::spec::EncryptedKey::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::EncryptedKey +impl iceberg::spec::EncryptedKey +pub fn iceberg::spec::EncryptedKey::builder() -> EncryptedKeyBuilder<((), (), (), ())> +impl serde_core::ser::Serialize for iceberg::spec::EncryptedKey +pub fn iceberg::spec::EncryptedKey::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::EncryptedKey +pub fn iceberg::spec::EncryptedKey::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::FieldSummary +pub iceberg::spec::FieldSummary::contains_nan: core::option::Option +pub iceberg::spec::FieldSummary::contains_null: bool +pub iceberg::spec::FieldSummary::lower_bound: core::option::Option +pub iceberg::spec::FieldSummary::upper_bound: core::option::Option +impl core::clone::Clone for iceberg::spec::FieldSummary +pub fn iceberg::spec::FieldSummary::clone(&self) -> iceberg::spec::FieldSummary +impl core::cmp::Eq for iceberg::spec::FieldSummary +impl core::cmp::PartialEq for iceberg::spec::FieldSummary +pub fn iceberg::spec::FieldSummary::eq(&self, other: &iceberg::spec::FieldSummary) -> bool +impl core::default::Default for iceberg::spec::FieldSummary +pub fn iceberg::spec::FieldSummary::default() -> iceberg::spec::FieldSummary +impl core::fmt::Debug for iceberg::spec::FieldSummary +pub fn iceberg::spec::FieldSummary::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::FieldSummary +pub fn iceberg::spec::FieldSummary::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::FieldSummary +impl serde_core::ser::Serialize for iceberg::spec::FieldSummary +pub fn iceberg::spec::FieldSummary::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::FieldSummary +pub fn iceberg::spec::FieldSummary::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::GeographyType +impl iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::algorithm(&self) -> iceberg::spec::geospatial::EdgeInterpolationAlgorithm +pub fn iceberg::spec::geospatial::GeographyType::crs(&self) -> &str +pub fn iceberg::spec::geospatial::GeographyType::new(crs: core::option::Option, algorithm: iceberg::spec::geospatial::EdgeInterpolationAlgorithm) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::clone(&self) -> iceberg::spec::geospatial::GeographyType +impl core::cmp::Eq for iceberg::spec::geospatial::GeographyType +impl core::cmp::PartialEq for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::eq(&self, other: &iceberg::spec::geospatial::GeographyType) -> bool +impl core::default::Default for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::default() -> Self +impl core::fmt::Debug for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeographyType +impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeographyType +pub fn iceberg::spec::geospatial::GeographyType::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::GeometryType +impl iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::crs(&self) -> &str +pub fn iceberg::spec::geospatial::GeometryType::new(crs: core::option::Option) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::clone(&self) -> iceberg::spec::geospatial::GeometryType +impl core::cmp::Eq for iceberg::spec::geospatial::GeometryType +impl core::cmp::PartialEq for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::eq(&self, other: &iceberg::spec::geospatial::GeometryType) -> bool +impl core::default::Default for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::default() -> iceberg::spec::geospatial::GeometryType +impl core::fmt::Debug for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::geospatial::GeometryType +impl serde_core::ser::Serialize for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::geospatial::GeometryType +pub fn iceberg::spec::geospatial::GeometryType::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::ListType +pub iceberg::spec::ListType::element_field: iceberg::spec::NestedFieldRef +impl iceberg::spec::ListType +pub fn iceberg::spec::ListType::new(element_field: iceberg::spec::NestedFieldRef) -> Self +impl core::clone::Clone for iceberg::spec::ListType +pub fn iceberg::spec::ListType::clone(&self) -> iceberg::spec::ListType +impl core::cmp::Eq for iceberg::spec::ListType +impl core::cmp::PartialEq for iceberg::spec::ListType +pub fn iceberg::spec::ListType::eq(&self, other: &iceberg::spec::ListType) -> bool +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(value: iceberg::spec::ListType) -> Self +impl core::fmt::Debug for iceberg::spec::ListType +pub fn iceberg::spec::ListType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::ListType +pub struct iceberg::spec::Manifest +impl iceberg::spec::Manifest +pub fn iceberg::spec::Manifest::entries(&self) -> &[iceberg::spec::ManifestEntryRef] +pub fn iceberg::spec::Manifest::into_parts(self) -> (alloc::vec::Vec, iceberg::spec::ManifestMetadata) +pub fn iceberg::spec::Manifest::metadata(&self) -> &iceberg::spec::ManifestMetadata +pub fn iceberg::spec::Manifest::new(metadata: iceberg::spec::ManifestMetadata, entries: alloc::vec::Vec) -> Self +pub fn iceberg::spec::Manifest::parse_avro(bs: &[u8]) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::Manifest +pub fn iceberg::spec::Manifest::clone(&self) -> iceberg::spec::Manifest +impl core::cmp::Eq for iceberg::spec::Manifest +impl core::cmp::PartialEq for iceberg::spec::Manifest +pub fn iceberg::spec::Manifest::eq(&self, other: &iceberg::spec::Manifest) -> bool +impl core::fmt::Debug for iceberg::spec::Manifest +pub fn iceberg::spec::Manifest::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::Manifest +pub struct iceberg::spec::ManifestEntry +pub iceberg::spec::ManifestEntry::data_file: iceberg::spec::DataFile +pub iceberg::spec::ManifestEntry::file_sequence_number: core::option::Option +pub iceberg::spec::ManifestEntry::sequence_number: core::option::Option +pub iceberg::spec::ManifestEntry::snapshot_id: core::option::Option +pub iceberg::spec::ManifestEntry::status: iceberg::spec::ManifestStatus +impl iceberg::spec::ManifestEntry +pub fn iceberg::spec::ManifestEntry::content_type(&self) -> iceberg::spec::DataContentType +pub fn iceberg::spec::ManifestEntry::data_file(&self) -> &iceberg::spec::DataFile +pub fn iceberg::spec::ManifestEntry::file_format(&self) -> iceberg::spec::DataFileFormat +pub fn iceberg::spec::ManifestEntry::file_path(&self) -> &str +pub fn iceberg::spec::ManifestEntry::file_size_in_bytes(&self) -> u64 +pub fn iceberg::spec::ManifestEntry::is_alive(&self) -> bool +pub fn iceberg::spec::ManifestEntry::record_count(&self) -> u64 +pub fn iceberg::spec::ManifestEntry::sequence_number(&self) -> core::option::Option +pub fn iceberg::spec::ManifestEntry::snapshot_id(&self) -> core::option::Option +pub fn iceberg::spec::ManifestEntry::status(&self) -> iceberg::spec::ManifestStatus +impl core::clone::Clone for iceberg::spec::ManifestEntry +pub fn iceberg::spec::ManifestEntry::clone(&self) -> iceberg::spec::ManifestEntry +impl core::cmp::Eq for iceberg::spec::ManifestEntry +impl core::cmp::PartialEq for iceberg::spec::ManifestEntry +pub fn iceberg::spec::ManifestEntry::eq(&self, other: &iceberg::spec::ManifestEntry) -> bool +impl core::fmt::Debug for iceberg::spec::ManifestEntry +pub fn iceberg::spec::ManifestEntry::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::ManifestEntry +impl iceberg::spec::ManifestEntry +pub fn iceberg::spec::ManifestEntry::builder() -> ManifestEntryBuilder<((), (), (), (), ())> +pub struct iceberg::spec::ManifestFile +pub iceberg::spec::ManifestFile::added_files_count: core::option::Option +pub iceberg::spec::ManifestFile::added_rows_count: core::option::Option +pub iceberg::spec::ManifestFile::added_snapshot_id: i64 +pub iceberg::spec::ManifestFile::content: iceberg::spec::ManifestContentType +pub iceberg::spec::ManifestFile::deleted_files_count: core::option::Option +pub iceberg::spec::ManifestFile::deleted_rows_count: core::option::Option +pub iceberg::spec::ManifestFile::existing_files_count: core::option::Option +pub iceberg::spec::ManifestFile::existing_rows_count: core::option::Option +pub iceberg::spec::ManifestFile::first_row_id: core::option::Option +pub iceberg::spec::ManifestFile::key_metadata: core::option::Option> +pub iceberg::spec::ManifestFile::manifest_length: i64 +pub iceberg::spec::ManifestFile::manifest_path: alloc::string::String +pub iceberg::spec::ManifestFile::min_sequence_number: i64 +pub iceberg::spec::ManifestFile::partition_spec_id: i32 +pub iceberg::spec::ManifestFile::partitions: core::option::Option> +pub iceberg::spec::ManifestFile::sequence_number: i64 +impl iceberg::spec::ManifestFile +pub fn iceberg::spec::ManifestFile::has_added_files(&self) -> bool +pub fn iceberg::spec::ManifestFile::has_deleted_files(&self) -> bool +pub fn iceberg::spec::ManifestFile::has_existing_files(&self) -> bool +impl core::clone::Clone for iceberg::spec::ManifestFile +pub fn iceberg::spec::ManifestFile::clone(&self) -> iceberg::spec::ManifestFile +impl core::cmp::Eq for iceberg::spec::ManifestFile +impl core::cmp::PartialEq for iceberg::spec::ManifestFile +pub fn iceberg::spec::ManifestFile::eq(&self, other: &iceberg::spec::ManifestFile) -> bool +impl core::fmt::Debug for iceberg::spec::ManifestFile +pub fn iceberg::spec::ManifestFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::ManifestFile +pub fn iceberg::spec::ManifestFile::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::spec::ManifestFile +pub struct iceberg::spec::ManifestList +impl iceberg::spec::ManifestList +pub fn iceberg::spec::ManifestList::consume_entries(self) -> impl core::iter::traits::collect::IntoIterator +pub fn iceberg::spec::ManifestList::entries(&self) -> &[iceberg::spec::ManifestFile] +pub fn iceberg::spec::ManifestList::parse_with_version(bs: &[u8], version: iceberg::spec::FormatVersion) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::ManifestList +pub fn iceberg::spec::ManifestList::clone(&self) -> iceberg::spec::ManifestList +impl core::cmp::PartialEq for iceberg::spec::ManifestList +pub fn iceberg::spec::ManifestList::eq(&self, other: &iceberg::spec::ManifestList) -> bool +impl core::fmt::Debug for iceberg::spec::ManifestList +pub fn iceberg::spec::ManifestList::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::ManifestList +pub struct iceberg::spec::ManifestListReader +impl iceberg::spec::ManifestListReader +pub async fn iceberg::spec::ManifestListReader::load(&self) -> iceberg::Result +pub struct iceberg::spec::ManifestListWriter +impl iceberg::spec::ManifestListWriter +pub fn iceberg::spec::ManifestListWriter::add_manifests(&mut self, manifests: impl core::iter::traits::iterator::Iterator) -> iceberg::Result<()> +pub async fn iceberg::spec::ManifestListWriter::close(self) -> iceberg::Result +pub fn iceberg::spec::ManifestListWriter::next_row_id(&self) -> core::option::Option +pub fn iceberg::spec::ManifestListWriter::v1(writer: alloc::boxed::Box, snapshot_id: i64, parent_snapshot_id: core::option::Option) -> Self +pub fn iceberg::spec::ManifestListWriter::v2(writer: alloc::boxed::Box, snapshot_id: i64, parent_snapshot_id: core::option::Option, sequence_number: i64) -> Self +pub fn iceberg::spec::ManifestListWriter::v3(writer: alloc::boxed::Box, snapshot_id: i64, parent_snapshot_id: core::option::Option, sequence_number: i64, first_row_id: core::option::Option) -> Self +impl core::fmt::Debug for iceberg::spec::ManifestListWriter +pub fn iceberg::spec::ManifestListWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::ManifestMetadata +pub iceberg::spec::ManifestMetadata::content: iceberg::spec::ManifestContentType +pub iceberg::spec::ManifestMetadata::format_version: iceberg::spec::FormatVersion +pub iceberg::spec::ManifestMetadata::partition_spec: iceberg::spec::PartitionSpec +pub iceberg::spec::ManifestMetadata::schema: iceberg::spec::SchemaRef +pub iceberg::spec::ManifestMetadata::schema_id: iceberg::spec::SchemaId +impl iceberg::spec::ManifestMetadata +pub fn iceberg::spec::ManifestMetadata::content(&self) -> &iceberg::spec::ManifestContentType +pub fn iceberg::spec::ManifestMetadata::format_version(&self) -> &iceberg::spec::FormatVersion +pub fn iceberg::spec::ManifestMetadata::parse(meta: &std::collections::hash::map::HashMap>) -> iceberg::Result +pub fn iceberg::spec::ManifestMetadata::partition_spec(&self) -> &iceberg::spec::PartitionSpec +pub fn iceberg::spec::ManifestMetadata::schema(&self) -> &iceberg::spec::SchemaRef +pub fn iceberg::spec::ManifestMetadata::schema_id(&self) -> iceberg::spec::SchemaId +impl core::clone::Clone for iceberg::spec::ManifestMetadata +pub fn iceberg::spec::ManifestMetadata::clone(&self) -> iceberg::spec::ManifestMetadata +impl core::cmp::Eq for iceberg::spec::ManifestMetadata +impl core::cmp::PartialEq for iceberg::spec::ManifestMetadata +pub fn iceberg::spec::ManifestMetadata::eq(&self, other: &iceberg::spec::ManifestMetadata) -> bool +impl core::fmt::Debug for iceberg::spec::ManifestMetadata +pub fn iceberg::spec::ManifestMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::ManifestMetadata +impl iceberg::spec::ManifestMetadata +pub fn iceberg::spec::ManifestMetadata::builder() -> ManifestMetadataBuilder<((), (), (), (), ())> +pub struct iceberg::spec::ManifestReader +impl iceberg::spec::ManifestReader +pub async fn iceberg::spec::ManifestReader::read(self, manifest_file: &iceberg::spec::ManifestFile) -> iceberg::Result +pub struct iceberg::spec::ManifestWriter +impl iceberg::spec::ManifestWriter +pub fn iceberg::spec::ManifestWriter::add_delete_file(&mut self, data_file: iceberg::spec::DataFile, sequence_number: i64, file_sequence_number: core::option::Option) -> iceberg::Result<()> +pub fn iceberg::spec::ManifestWriter::add_existing_file(&mut self, data_file: iceberg::spec::DataFile, snapshot_id: i64, sequence_number: i64, file_sequence_number: core::option::Option) -> iceberg::Result<()> +pub fn iceberg::spec::ManifestWriter::add_file(&mut self, data_file: iceberg::spec::DataFile, sequence_number: i64) -> iceberg::Result<()> +pub async fn iceberg::spec::ManifestWriter::write_manifest_file(self) -> iceberg::Result +pub struct iceberg::spec::ManifestWriterBuilder +impl iceberg::spec::ManifestWriterBuilder +pub fn iceberg::spec::ManifestWriterBuilder::build_v1(self) -> iceberg::spec::ManifestWriter +pub fn iceberg::spec::ManifestWriterBuilder::build_v2_data(self) -> iceberg::spec::ManifestWriter +pub fn iceberg::spec::ManifestWriterBuilder::build_v2_deletes(self) -> iceberg::spec::ManifestWriter +pub fn iceberg::spec::ManifestWriterBuilder::build_v3_data(self) -> iceberg::spec::ManifestWriter +pub fn iceberg::spec::ManifestWriterBuilder::build_v3_deletes(self) -> iceberg::spec::ManifestWriter +pub fn iceberg::spec::ManifestWriterBuilder::new(output: iceberg::io::OutputFile, snapshot_id: core::option::Option, schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpec) -> Self +pub fn iceberg::spec::ManifestWriterBuilder::new_from_encrypted(encrypted_output: iceberg::encryption::EncryptedOutputFile, snapshot_id: core::option::Option, schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpec) -> iceberg::Result +pub struct iceberg::spec::Map +impl iceberg::spec::Map +pub fn iceberg::spec::Map::get(&self, key: &iceberg::spec::Literal) -> core::option::Option<&core::option::Option> +pub fn iceberg::spec::Map::has_same_content(&self, other: &iceberg::spec::Map) -> bool +pub fn iceberg::spec::Map::insert(&mut self, key: iceberg::spec::Literal, value: core::option::Option) -> core::option::Option> +pub fn iceberg::spec::Map::is_empty(&self) -> bool +pub fn iceberg::spec::Map::len(&self) -> usize +pub fn iceberg::spec::Map::new() -> Self +impl core::clone::Clone for iceberg::spec::Map +pub fn iceberg::spec::Map::clone(&self) -> iceberg::spec::Map +impl core::cmp::Eq for iceberg::spec::Map +impl core::cmp::PartialEq for iceberg::spec::Map +pub fn iceberg::spec::Map::eq(&self, other: &iceberg::spec::Map) -> bool +impl core::default::Default for iceberg::spec::Map +pub fn iceberg::spec::Map::default() -> Self +impl core::fmt::Debug for iceberg::spec::Map +pub fn iceberg::spec::Map::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::Map +pub fn iceberg::spec::Map::hash(&self, state: &mut H) +impl core::iter::traits::collect::FromIterator<(iceberg::spec::Literal, core::option::Option)> for iceberg::spec::Map +pub fn iceberg::spec::Map::from_iter)>>(iter: T) -> Self +impl core::iter::traits::collect::IntoIterator for iceberg::spec::Map +pub type iceberg::spec::Map::IntoIter = alloc::vec::into_iter::IntoIter<::Item> +pub type iceberg::spec::Map::Item = (iceberg::spec::Literal, core::option::Option) +pub fn iceberg::spec::Map::into_iter(self) -> Self::IntoIter +impl core::marker::StructuralPartialEq for iceberg::spec::Map +impl core::convert::From<[(iceberg::spec::Literal, core::option::Option); N]> for iceberg::spec::Map +pub fn iceberg::spec::Map::from(value: [(iceberg::spec::Literal, core::option::Option); N]) -> Self +pub struct iceberg::spec::MapType +pub iceberg::spec::MapType::key_field: iceberg::spec::NestedFieldRef +pub iceberg::spec::MapType::value_field: iceberg::spec::NestedFieldRef +impl iceberg::spec::MapType +pub fn iceberg::spec::MapType::new(key_field: iceberg::spec::NestedFieldRef, value_field: iceberg::spec::NestedFieldRef) -> Self +pub fn iceberg::spec::MapType::optional(key_id: i32, key_type: impl core::convert::Into, value_id: i32, value_type: impl core::convert::Into) -> Self +pub fn iceberg::spec::MapType::required(key_id: i32, key_type: impl core::convert::Into, value_id: i32, value_type: impl core::convert::Into) -> Self +impl core::clone::Clone for iceberg::spec::MapType +pub fn iceberg::spec::MapType::clone(&self) -> iceberg::spec::MapType +impl core::cmp::Eq for iceberg::spec::MapType +impl core::cmp::PartialEq for iceberg::spec::MapType +pub fn iceberg::spec::MapType::eq(&self, other: &iceberg::spec::MapType) -> bool +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(value: iceberg::spec::MapType) -> Self +impl core::fmt::Debug for iceberg::spec::MapType +pub fn iceberg::spec::MapType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::MapType +pub struct iceberg::spec::MappedField +impl iceberg::spec::MappedField +pub fn iceberg::spec::MappedField::field_id(&self) -> core::option::Option +pub fn iceberg::spec::MappedField::fields(&self) -> &[alloc::sync::Arc] +pub fn iceberg::spec::MappedField::names(&self) -> &[alloc::string::String] +pub fn iceberg::spec::MappedField::new(field_id: core::option::Option, names: alloc::vec::Vec, fields: alloc::vec::Vec) -> Self +impl core::clone::Clone for iceberg::spec::MappedField +pub fn iceberg::spec::MappedField::clone(&self) -> iceberg::spec::MappedField +impl core::cmp::Eq for iceberg::spec::MappedField +impl core::cmp::PartialEq for iceberg::spec::MappedField +pub fn iceberg::spec::MappedField::eq(&self, other: &iceberg::spec::MappedField) -> bool +impl core::fmt::Debug for iceberg::spec::MappedField +pub fn iceberg::spec::MappedField::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::MappedField +impl serde_core::ser::Serialize for iceberg::spec::MappedField +pub fn iceberg::spec::MappedField::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::MappedField +pub fn iceberg::spec::MappedField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::MetadataLog +pub iceberg::spec::MetadataLog::metadata_file: alloc::string::String +pub iceberg::spec::MetadataLog::timestamp_ms: i64 +impl core::clone::Clone for iceberg::spec::MetadataLog +pub fn iceberg::spec::MetadataLog::clone(&self) -> iceberg::spec::MetadataLog +impl core::cmp::Eq for iceberg::spec::MetadataLog +impl core::cmp::PartialEq for iceberg::spec::MetadataLog +pub fn iceberg::spec::MetadataLog::eq(&self, other: &iceberg::spec::MetadataLog) -> bool +impl core::fmt::Debug for iceberg::spec::MetadataLog +pub fn iceberg::spec::MetadataLog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::MetadataLog +impl serde_core::ser::Serialize for iceberg::spec::MetadataLog +pub fn iceberg::spec::MetadataLog::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::MetadataLog +pub fn iceberg::spec::MetadataLog::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::NameMapping +impl iceberg::spec::NameMapping +pub fn iceberg::spec::NameMapping::fields(&self) -> &[iceberg::spec::MappedField] +pub fn iceberg::spec::NameMapping::new(fields: alloc::vec::Vec) -> Self +impl core::clone::Clone for iceberg::spec::NameMapping +pub fn iceberg::spec::NameMapping::clone(&self) -> iceberg::spec::NameMapping +impl core::cmp::Eq for iceberg::spec::NameMapping +impl core::cmp::PartialEq for iceberg::spec::NameMapping +pub fn iceberg::spec::NameMapping::eq(&self, other: &iceberg::spec::NameMapping) -> bool +impl core::fmt::Debug for iceberg::spec::NameMapping +pub fn iceberg::spec::NameMapping::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::NameMapping +impl core::str::traits::FromStr for iceberg::spec::NameMapping +pub type iceberg::spec::NameMapping::Err = iceberg::Error +pub fn iceberg::spec::NameMapping::from_str(value: &str) -> iceberg::Result +impl serde_core::ser::Serialize for iceberg::spec::NameMapping +pub fn iceberg::spec::NameMapping::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NameMapping +pub fn iceberg::spec::NameMapping::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::NestedField +pub iceberg::spec::NestedField::doc: core::option::Option +pub iceberg::spec::NestedField::field_type: alloc::boxed::Box +pub iceberg::spec::NestedField::id: i32 +pub iceberg::spec::NestedField::initial_default: core::option::Option +pub iceberg::spec::NestedField::name: alloc::string::String +pub iceberg::spec::NestedField::required: bool +pub iceberg::spec::NestedField::write_default: core::option::Option +impl iceberg::spec::NestedField +pub fn iceberg::spec::NestedField::list_element(id: i32, field_type: impl core::convert::Into, required: bool) -> Self +pub fn iceberg::spec::NestedField::map_key_element(id: i32, field_type: impl core::convert::Into) -> Self +pub fn iceberg::spec::NestedField::map_value_element(id: i32, field_type: impl core::convert::Into, required: bool) -> Self +pub fn iceberg::spec::NestedField::new(id: i32, name: impl alloc::string::ToString, field_type: impl core::convert::Into, required: bool) -> Self +pub fn iceberg::spec::NestedField::optional(id: i32, name: impl alloc::string::ToString, field_type: impl core::convert::Into) -> Self +pub fn iceberg::spec::NestedField::required(id: i32, name: impl alloc::string::ToString, field_type: impl core::convert::Into) -> Self +pub fn iceberg::spec::NestedField::with_doc(self, doc: impl alloc::string::ToString) -> Self +pub fn iceberg::spec::NestedField::with_initial_default(self, value: iceberg::spec::Literal) -> Self +pub fn iceberg::spec::NestedField::with_write_default(self, value: iceberg::spec::Literal) -> Self +impl core::clone::Clone for iceberg::spec::NestedField +pub fn iceberg::spec::NestedField::clone(&self) -> iceberg::spec::NestedField +impl core::cmp::Eq for iceberg::spec::NestedField +impl core::cmp::PartialEq for iceberg::spec::NestedField +pub fn iceberg::spec::NestedField::eq(&self, other: &iceberg::spec::NestedField) -> bool +impl core::fmt::Debug for iceberg::spec::NestedField +pub fn iceberg::spec::NestedField::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::NestedField +pub fn iceberg::spec::NestedField::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::NestedField +impl serde_core::ser::Serialize for iceberg::spec::NestedField +pub fn iceberg::spec::NestedField::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::NestedField +pub fn iceberg::spec::NestedField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::PartitionField +pub iceberg::spec::PartitionField::field_id: i32 +pub iceberg::spec::PartitionField::name: alloc::string::String +pub iceberg::spec::PartitionField::source_id: i32 +pub iceberg::spec::PartitionField::transform: iceberg::spec::Transform +impl iceberg::spec::PartitionField +pub fn iceberg::spec::PartitionField::into_unbound(self) -> iceberg::spec::UnboundPartitionField +impl core::clone::Clone for iceberg::spec::PartitionField +pub fn iceberg::spec::PartitionField::clone(&self) -> iceberg::spec::PartitionField +impl core::cmp::Eq for iceberg::spec::PartitionField +impl core::cmp::PartialEq for iceberg::spec::PartitionField +pub fn iceberg::spec::PartitionField::eq(&self, other: &iceberg::spec::PartitionField) -> bool +impl core::convert::From for iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::from(field: iceberg::spec::PartitionField) -> Self +impl core::fmt::Debug for iceberg::spec::PartitionField +pub fn iceberg::spec::PartitionField::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::PartitionField +impl iceberg::spec::PartitionField +pub fn iceberg::spec::PartitionField::builder() -> PartitionFieldBuilder<((), (), (), ())> +impl serde_core::ser::Serialize for iceberg::spec::PartitionField +pub fn iceberg::spec::PartitionField::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::PartitionField +pub fn iceberg::spec::PartitionField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::PartitionKey +impl iceberg::spec::PartitionKey +pub fn iceberg::spec::PartitionKey::copy_with_data(&self, data: iceberg::spec::Struct) -> Self +pub fn iceberg::spec::PartitionKey::data(&self) -> &iceberg::spec::Struct +pub fn iceberg::spec::PartitionKey::is_effectively_none(partition_key: core::option::Option<&iceberg::spec::PartitionKey>) -> bool +pub fn iceberg::spec::PartitionKey::new(spec: iceberg::spec::PartitionSpec, schema: iceberg::spec::SchemaRef, data: iceberg::spec::Struct) -> Self +pub fn iceberg::spec::PartitionKey::schema(&self) -> &iceberg::spec::SchemaRef +pub fn iceberg::spec::PartitionKey::spec(&self) -> &iceberg::spec::PartitionSpec +pub fn iceberg::spec::PartitionKey::to_path(&self) -> alloc::string::String +impl core::clone::Clone for iceberg::spec::PartitionKey +pub fn iceberg::spec::PartitionKey::clone(&self) -> iceberg::spec::PartitionKey +impl core::fmt::Debug for iceberg::spec::PartitionKey +pub fn iceberg::spec::PartitionKey::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::PartitionSpec +impl iceberg::spec::PartitionSpec +pub fn iceberg::spec::PartitionSpec::builder(schema: impl core::convert::Into) -> iceberg::spec::PartitionSpecBuilder +pub fn iceberg::spec::PartitionSpec::fields(&self) -> &[iceberg::spec::PartitionField] +pub fn iceberg::spec::PartitionSpec::has_sequential_ids(&self) -> bool +pub fn iceberg::spec::PartitionSpec::highest_field_id(&self) -> core::option::Option +pub fn iceberg::spec::PartitionSpec::into_unbound(self) -> iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::PartitionSpec::is_compatible_with(&self, other: &iceberg::spec::PartitionSpec) -> bool +pub fn iceberg::spec::PartitionSpec::is_unpartitioned(&self) -> bool +pub fn iceberg::spec::PartitionSpec::partition_to_path(&self, data: &iceberg::spec::Struct, schema: iceberg::spec::SchemaRef) -> alloc::string::String +pub fn iceberg::spec::PartitionSpec::partition_type(&self, schema: &iceberg::spec::Schema) -> iceberg::Result +pub fn iceberg::spec::PartitionSpec::spec_id(&self) -> i32 +pub fn iceberg::spec::PartitionSpec::unpartition_spec() -> Self +pub fn iceberg::spec::PartitionSpec::with_spec_id(self, spec_id: i32) -> Self +impl core::clone::Clone for iceberg::spec::PartitionSpec +pub fn iceberg::spec::PartitionSpec::clone(&self) -> iceberg::spec::PartitionSpec +impl core::cmp::Eq for iceberg::spec::PartitionSpec +impl core::cmp::PartialEq for iceberg::spec::PartitionSpec +pub fn iceberg::spec::PartitionSpec::eq(&self, other: &iceberg::spec::PartitionSpec) -> bool +impl core::convert::From for iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpec::from(spec: iceberg::spec::PartitionSpec) -> Self +impl core::fmt::Debug for iceberg::spec::PartitionSpec +pub fn iceberg::spec::PartitionSpec::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::PartitionSpec +impl serde_core::ser::Serialize for iceberg::spec::PartitionSpec +pub fn iceberg::spec::PartitionSpec::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::PartitionSpec +pub fn iceberg::spec::PartitionSpec::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::PartitionSpecBuilder +impl iceberg::spec::PartitionSpecBuilder +pub fn iceberg::spec::PartitionSpecBuilder::add_partition_field(self, source_name: impl core::convert::AsRef, target_name: impl core::convert::Into, transform: iceberg::spec::Transform) -> iceberg::Result +pub fn iceberg::spec::PartitionSpecBuilder::add_unbound_field(self, field: iceberg::spec::UnboundPartitionField) -> iceberg::Result +pub fn iceberg::spec::PartitionSpecBuilder::add_unbound_fields(self, fields: impl core::iter::traits::collect::IntoIterator) -> iceberg::Result +pub fn iceberg::spec::PartitionSpecBuilder::build(self) -> iceberg::Result +pub fn iceberg::spec::PartitionSpecBuilder::new(schema: impl core::convert::Into) -> Self +pub fn iceberg::spec::PartitionSpecBuilder::new_from_unbound(unbound: iceberg::spec::UnboundPartitionSpec, schema: impl core::convert::Into) -> iceberg::Result +pub fn iceberg::spec::PartitionSpecBuilder::with_last_assigned_field_id(self, last_assigned_field_id: i32) -> Self +pub fn iceberg::spec::PartitionSpecBuilder::with_spec_id(self, spec_id: i32) -> Self +impl core::fmt::Debug for iceberg::spec::PartitionSpecBuilder +pub fn iceberg::spec::PartitionSpecBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::PartitionStatisticsFile +pub iceberg::spec::PartitionStatisticsFile::file_size_in_bytes: i64 +pub iceberg::spec::PartitionStatisticsFile::snapshot_id: i64 +pub iceberg::spec::PartitionStatisticsFile::statistics_path: alloc::string::String +impl core::clone::Clone for iceberg::spec::PartitionStatisticsFile +pub fn iceberg::spec::PartitionStatisticsFile::clone(&self) -> iceberg::spec::PartitionStatisticsFile +impl core::cmp::Eq for iceberg::spec::PartitionStatisticsFile +impl core::cmp::PartialEq for iceberg::spec::PartitionStatisticsFile +pub fn iceberg::spec::PartitionStatisticsFile::eq(&self, other: &iceberg::spec::PartitionStatisticsFile) -> bool +impl core::fmt::Debug for iceberg::spec::PartitionStatisticsFile +pub fn iceberg::spec::PartitionStatisticsFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::PartitionStatisticsFile +impl serde_core::ser::Serialize for iceberg::spec::PartitionStatisticsFile +pub fn iceberg::spec::PartitionStatisticsFile::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::PartitionStatisticsFile +pub fn iceberg::spec::PartitionStatisticsFile::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::Schema +impl iceberg::spec::Schema +pub fn iceberg::spec::Schema::accessor_by_field_id(&self, field_id: i32) -> core::option::Option> +pub fn iceberg::spec::Schema::as_struct(&self) -> &iceberg::spec::StructType +pub fn iceberg::spec::Schema::builder() -> iceberg::spec::SchemaBuilder +pub fn iceberg::spec::Schema::calc_min_compatible_format(&self) -> iceberg::spec::FormatVersion +pub fn iceberg::spec::Schema::check_format_compatibility(&self, format_version: iceberg::spec::FormatVersion) -> iceberg::Result<()> +pub fn iceberg::spec::Schema::field_by_alias(&self, alias: &str) -> core::option::Option<&iceberg::spec::NestedFieldRef> +pub fn iceberg::spec::Schema::field_by_id(&self, field_id: i32) -> core::option::Option<&iceberg::spec::NestedFieldRef> +pub fn iceberg::spec::Schema::field_by_name(&self, field_name: &str) -> core::option::Option<&iceberg::spec::NestedFieldRef> +pub fn iceberg::spec::Schema::field_by_name_case_insensitive(&self, field_name: &str) -> core::option::Option<&iceberg::spec::NestedFieldRef> +pub fn iceberg::spec::Schema::field_by_name_case_insensitive_checked(&self, field_name: &str) -> iceberg::Result> +pub fn iceberg::spec::Schema::field_id_by_name(&self, name: &str) -> core::option::Option +pub fn iceberg::spec::Schema::field_id_to_fields(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::Schema::field_id_to_name_map(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::Schema::highest_field_id(&self) -> i32 +pub fn iceberg::spec::Schema::identifier_field_ids(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator + '_ +pub fn iceberg::spec::Schema::into_builder(self) -> iceberg::spec::SchemaBuilder +pub fn iceberg::spec::Schema::name_by_field_id(&self, field_id: i32) -> core::option::Option<&str> +pub fn iceberg::spec::Schema::schema_id(&self) -> iceberg::spec::SchemaId +impl core::clone::Clone for iceberg::spec::Schema +pub fn iceberg::spec::Schema::clone(&self) -> iceberg::spec::Schema +impl core::cmp::Eq for iceberg::spec::Schema +impl core::cmp::PartialEq for iceberg::spec::Schema +pub fn iceberg::spec::Schema::eq(&self, other: &Self) -> bool +impl core::convert::TryFrom<&arrow_schema::schema::Schema> for iceberg::spec::Schema +pub type iceberg::spec::Schema::Error = iceberg::Error +pub fn iceberg::spec::Schema::try_from(schema: &arrow_schema::schema::Schema) -> iceberg::Result +impl core::convert::TryFrom<&iceberg::spec::Schema> for arrow_schema::schema::Schema +pub type arrow_schema::schema::Schema::Error = iceberg::Error +pub fn arrow_schema::schema::Schema::try_from(schema: &iceberg::spec::Schema) -> iceberg::Result +impl core::fmt::Debug for iceberg::spec::Schema +pub fn iceberg::spec::Schema::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::Schema +pub fn iceberg::spec::Schema::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg::spec::Schema +pub fn iceberg::spec::Schema::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::Schema +pub fn iceberg::spec::Schema::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::SchemaBuilder +impl iceberg::spec::SchemaBuilder +pub fn iceberg::spec::SchemaBuilder::build(self) -> iceberg::Result +pub fn iceberg::spec::SchemaBuilder::with_alias(self, alias_to_id: bimap::hash::BiHashMap) -> Self +pub fn iceberg::spec::SchemaBuilder::with_fields(self, fields: impl core::iter::traits::collect::IntoIterator) -> Self +pub fn iceberg::spec::SchemaBuilder::with_identifier_field_ids(self, ids: impl core::iter::traits::collect::IntoIterator) -> Self +pub fn iceberg::spec::SchemaBuilder::with_schema_id(self, schema_id: i32) -> Self +impl core::fmt::Debug for iceberg::spec::SchemaBuilder +pub fn iceberg::spec::SchemaBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::Snapshot +impl iceberg::spec::Snapshot +pub fn iceberg::spec::Snapshot::added_rows_count(&self) -> core::option::Option +pub fn iceberg::spec::Snapshot::encryption_key_id(&self) -> core::option::Option<&str> +pub fn iceberg::spec::Snapshot::first_row_id(&self) -> core::option::Option +pub fn iceberg::spec::Snapshot::manifest_list(&self) -> &str +pub fn iceberg::spec::Snapshot::parent_snapshot_id(&self) -> core::option::Option +pub fn iceberg::spec::Snapshot::row_range(&self) -> core::option::Option<(u64, u64)> +pub fn iceberg::spec::Snapshot::schema(&self, table_metadata: &iceberg::spec::TableMetadata) -> iceberg::Result +pub fn iceberg::spec::Snapshot::schema_id(&self) -> core::option::Option +pub fn iceberg::spec::Snapshot::sequence_number(&self) -> i64 +pub fn iceberg::spec::Snapshot::snapshot_id(&self) -> i64 +pub fn iceberg::spec::Snapshot::summary(&self) -> &iceberg::spec::Summary +pub fn iceberg::spec::Snapshot::timestamp(&self) -> iceberg::Result> +pub fn iceberg::spec::Snapshot::timestamp_ms(&self) -> i64 +impl core::clone::Clone for iceberg::spec::Snapshot +pub fn iceberg::spec::Snapshot::clone(&self) -> iceberg::spec::Snapshot +impl core::cmp::Eq for iceberg::spec::Snapshot +impl core::cmp::PartialEq for iceberg::spec::Snapshot +pub fn iceberg::spec::Snapshot::eq(&self, other: &iceberg::spec::Snapshot) -> bool +impl core::fmt::Debug for iceberg::spec::Snapshot +pub fn iceberg::spec::Snapshot::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::Snapshot +impl iceberg::spec::Snapshot +pub fn iceberg::spec::Snapshot::builder() -> SnapshotBuilder<((), (), (), (), (), (), (), (), ())> +pub struct iceberg::spec::SnapshotLog +pub iceberg::spec::SnapshotLog::snapshot_id: i64 +pub iceberg::spec::SnapshotLog::timestamp_ms: i64 +impl iceberg::spec::SnapshotLog +pub fn iceberg::spec::SnapshotLog::timestamp(self) -> iceberg::Result> +pub fn iceberg::spec::SnapshotLog::timestamp_ms(&self) -> i64 +impl core::clone::Clone for iceberg::spec::SnapshotLog +pub fn iceberg::spec::SnapshotLog::clone(&self) -> iceberg::spec::SnapshotLog +impl core::cmp::Eq for iceberg::spec::SnapshotLog +impl core::cmp::PartialEq for iceberg::spec::SnapshotLog +pub fn iceberg::spec::SnapshotLog::eq(&self, other: &iceberg::spec::SnapshotLog) -> bool +impl core::fmt::Debug for iceberg::spec::SnapshotLog +pub fn iceberg::spec::SnapshotLog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::SnapshotLog +impl serde_core::ser::Serialize for iceberg::spec::SnapshotLog +pub fn iceberg::spec::SnapshotLog::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::SnapshotLog +pub fn iceberg::spec::SnapshotLog::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::SnapshotReference +pub iceberg::spec::SnapshotReference::retention: iceberg::spec::SnapshotRetention +pub iceberg::spec::SnapshotReference::snapshot_id: i64 +impl iceberg::spec::SnapshotReference +pub fn iceberg::spec::SnapshotReference::is_branch(&self) -> bool +impl iceberg::spec::SnapshotReference +pub fn iceberg::spec::SnapshotReference::new(snapshot_id: i64, retention: iceberg::spec::SnapshotRetention) -> Self +impl core::clone::Clone for iceberg::spec::SnapshotReference +pub fn iceberg::spec::SnapshotReference::clone(&self) -> iceberg::spec::SnapshotReference +impl core::cmp::Eq for iceberg::spec::SnapshotReference +impl core::cmp::PartialEq for iceberg::spec::SnapshotReference +pub fn iceberg::spec::SnapshotReference::eq(&self, other: &iceberg::spec::SnapshotReference) -> bool +impl core::fmt::Debug for iceberg::spec::SnapshotReference +pub fn iceberg::spec::SnapshotReference::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::SnapshotReference +impl serde_core::ser::Serialize for iceberg::spec::SnapshotReference +pub fn iceberg::spec::SnapshotReference::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::SnapshotReference +pub fn iceberg::spec::SnapshotReference::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::SnapshotRowRange +pub iceberg::spec::SnapshotRowRange::added_rows: u64 +pub iceberg::spec::SnapshotRowRange::first_row_id: u64 +impl core::clone::Clone for iceberg::spec::SnapshotRowRange +pub fn iceberg::spec::SnapshotRowRange::clone(&self) -> iceberg::spec::SnapshotRowRange +impl core::cmp::Eq for iceberg::spec::SnapshotRowRange +impl core::cmp::PartialEq for iceberg::spec::SnapshotRowRange +pub fn iceberg::spec::SnapshotRowRange::eq(&self, other: &iceberg::spec::SnapshotRowRange) -> bool +impl core::fmt::Debug for iceberg::spec::SnapshotRowRange +pub fn iceberg::spec::SnapshotRowRange::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::SnapshotRowRange +pub struct iceberg::spec::SnapshotSummaryCollector +impl iceberg::spec::SnapshotSummaryCollector +pub fn iceberg::spec::SnapshotSummaryCollector::add_file(&mut self, data_file: &iceberg::spec::DataFile, schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpecRef) +pub fn iceberg::spec::SnapshotSummaryCollector::add_manifest(&mut self, manifest: &iceberg::spec::ManifestFile) +pub fn iceberg::spec::SnapshotSummaryCollector::build(&self) -> std::collections::hash::map::HashMap +pub fn iceberg::spec::SnapshotSummaryCollector::merge(&mut self, summary: iceberg::spec::SnapshotSummaryCollector) +pub fn iceberg::spec::SnapshotSummaryCollector::remove_file(&mut self, data_file: &iceberg::spec::DataFile, schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpecRef) +pub fn iceberg::spec::SnapshotSummaryCollector::set(&mut self, key: &str, value: &str) +pub fn iceberg::spec::SnapshotSummaryCollector::set_partition_summary_limit(&mut self, limit: u64) +pub fn iceberg::spec::SnapshotSummaryCollector::update_partition_metrics(&mut self, schema: iceberg::spec::SchemaRef, partition_spec: iceberg::spec::PartitionSpecRef, data_file: &iceberg::spec::DataFile, is_add_file: bool) +impl core::default::Default for iceberg::spec::SnapshotSummaryCollector +pub fn iceberg::spec::SnapshotSummaryCollector::default() -> iceberg::spec::SnapshotSummaryCollector +pub struct iceberg::spec::SortField +pub iceberg::spec::SortField::direction: iceberg::spec::SortDirection +pub iceberg::spec::SortField::null_order: iceberg::spec::NullOrder +pub iceberg::spec::SortField::source_id: i32 +pub iceberg::spec::SortField::transform: iceberg::spec::Transform +impl core::clone::Clone for iceberg::spec::SortField +pub fn iceberg::spec::SortField::clone(&self) -> iceberg::spec::SortField +impl core::cmp::Eq for iceberg::spec::SortField +impl core::cmp::PartialEq for iceberg::spec::SortField +pub fn iceberg::spec::SortField::eq(&self, other: &iceberg::spec::SortField) -> bool +impl core::fmt::Debug for iceberg::spec::SortField +pub fn iceberg::spec::SortField::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::SortField +pub fn iceberg::spec::SortField::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::SortField +impl iceberg::spec::SortField +pub fn iceberg::spec::SortField::builder() -> SortFieldBuilder<((), (), (), ())> +impl serde_core::ser::Serialize for iceberg::spec::SortField +pub fn iceberg::spec::SortField::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::SortField +pub fn iceberg::spec::SortField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::SortOrder +pub iceberg::spec::SortOrder::fields: alloc::vec::Vec +pub iceberg::spec::SortOrder::order_id: i64 +impl iceberg::spec::SortOrder +pub fn iceberg::spec::SortOrder::builder() -> iceberg::spec::SortOrderBuilder +pub fn iceberg::spec::SortOrder::is_unsorted(&self) -> bool +pub fn iceberg::spec::SortOrder::unsorted_order() -> iceberg::spec::SortOrder +pub fn iceberg::spec::SortOrder::with_order_id(self, order_id: i64) -> iceberg::spec::SortOrder +impl core::clone::Clone for iceberg::spec::SortOrder +pub fn iceberg::spec::SortOrder::clone(&self) -> iceberg::spec::SortOrder +impl core::cmp::Eq for iceberg::spec::SortOrder +impl core::cmp::PartialEq for iceberg::spec::SortOrder +pub fn iceberg::spec::SortOrder::eq(&self, other: &iceberg::spec::SortOrder) -> bool +impl core::default::Default for iceberg::spec::SortOrder +pub fn iceberg::spec::SortOrder::default() -> iceberg::spec::SortOrder +impl core::fmt::Debug for iceberg::spec::SortOrder +pub fn iceberg::spec::SortOrder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::SortOrder +impl serde_core::ser::Serialize for iceberg::spec::SortOrder +pub fn iceberg::spec::SortOrder::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::SortOrder +pub fn iceberg::spec::SortOrder::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::SortOrderBuilder +impl iceberg::spec::SortOrderBuilder +pub fn iceberg::spec::SortOrderBuilder::build(&self, schema: &iceberg::spec::Schema) -> iceberg::Result +pub fn iceberg::spec::SortOrderBuilder::build_unbound(&self) -> iceberg::Result +impl iceberg::spec::SortOrderBuilder +pub fn iceberg::spec::SortOrderBuilder::with_fields(&mut self, value: alloc::vec::Vec) -> &mut Self +pub fn iceberg::spec::SortOrderBuilder::with_order_id(&mut self, value: i64) -> &mut Self +pub fn iceberg::spec::SortOrderBuilder::with_sort_field(&mut self, item: VALUE) -> &mut Self where alloc::vec::Vec: core::default::Default + core::iter::traits::collect::Extend +impl core::clone::Clone for iceberg::spec::SortOrderBuilder +pub fn iceberg::spec::SortOrderBuilder::clone(&self) -> iceberg::spec::SortOrderBuilder +impl core::default::Default for iceberg::spec::SortOrderBuilder +pub fn iceberg::spec::SortOrderBuilder::default() -> Self +pub struct iceberg::spec::SqlViewRepresentation +pub iceberg::spec::SqlViewRepresentation::dialect: alloc::string::String +pub iceberg::spec::SqlViewRepresentation::sql: alloc::string::String +impl core::clone::Clone for iceberg::spec::SqlViewRepresentation +pub fn iceberg::spec::SqlViewRepresentation::clone(&self) -> iceberg::spec::SqlViewRepresentation +impl core::cmp::Eq for iceberg::spec::SqlViewRepresentation +impl core::cmp::PartialEq for iceberg::spec::SqlViewRepresentation +pub fn iceberg::spec::SqlViewRepresentation::eq(&self, other: &iceberg::spec::SqlViewRepresentation) -> bool +impl core::convert::From for iceberg::spec::ViewRepresentation +pub fn iceberg::spec::ViewRepresentation::from(sql: iceberg::spec::SqlViewRepresentation) -> Self +impl core::fmt::Debug for iceberg::spec::SqlViewRepresentation +pub fn iceberg::spec::SqlViewRepresentation::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::SqlViewRepresentation +impl serde_core::ser::Serialize for iceberg::spec::SqlViewRepresentation +pub fn iceberg::spec::SqlViewRepresentation::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::SqlViewRepresentation +pub fn iceberg::spec::SqlViewRepresentation::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::StatisticsFile +pub iceberg::spec::StatisticsFile::blob_metadata: alloc::vec::Vec +pub iceberg::spec::StatisticsFile::file_footer_size_in_bytes: i64 +pub iceberg::spec::StatisticsFile::file_size_in_bytes: i64 +pub iceberg::spec::StatisticsFile::key_metadata: core::option::Option +pub iceberg::spec::StatisticsFile::snapshot_id: i64 +pub iceberg::spec::StatisticsFile::statistics_path: alloc::string::String +impl core::clone::Clone for iceberg::spec::StatisticsFile +pub fn iceberg::spec::StatisticsFile::clone(&self) -> iceberg::spec::StatisticsFile +impl core::cmp::Eq for iceberg::spec::StatisticsFile +impl core::cmp::PartialEq for iceberg::spec::StatisticsFile +pub fn iceberg::spec::StatisticsFile::eq(&self, other: &iceberg::spec::StatisticsFile) -> bool +impl core::fmt::Debug for iceberg::spec::StatisticsFile +pub fn iceberg::spec::StatisticsFile::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::StatisticsFile +impl serde_core::ser::Serialize for iceberg::spec::StatisticsFile +pub fn iceberg::spec::StatisticsFile::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::StatisticsFile +pub fn iceberg::spec::StatisticsFile::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::Struct +impl iceberg::spec::Struct +pub fn iceberg::spec::Struct::empty() -> Self +pub fn iceberg::spec::Struct::fields(&self) -> &[core::option::Option] +pub fn iceberg::spec::Struct::is_null_at_index(&self, index: usize) -> bool +pub fn iceberg::spec::Struct::iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator> +impl core::clone::Clone for iceberg::spec::Struct +pub fn iceberg::spec::Struct::clone(&self) -> iceberg::spec::Struct +impl core::cmp::Eq for iceberg::spec::Struct +impl core::cmp::PartialEq for iceberg::spec::Struct +pub fn iceberg::spec::Struct::eq(&self, other: &iceberg::spec::Struct) -> bool +impl core::fmt::Debug for iceberg::spec::Struct +pub fn iceberg::spec::Struct::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::Struct +pub fn iceberg::spec::Struct::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::iter::traits::collect::FromIterator> for iceberg::spec::Struct +pub fn iceberg::spec::Struct::from_iter>>(iter: I) -> Self +impl core::iter::traits::collect::IntoIterator for iceberg::spec::Struct +pub type iceberg::spec::Struct::IntoIter = alloc::vec::into_iter::IntoIter> +pub type iceberg::spec::Struct::Item = core::option::Option +pub fn iceberg::spec::Struct::into_iter(self) -> Self::IntoIter +impl core::marker::StructuralPartialEq for iceberg::spec::Struct +impl core::ops::index::Index for iceberg::spec::Struct +pub type iceberg::spec::Struct::Output = core::option::Option +pub fn iceberg::spec::Struct::index(&self, idx: usize) -> &Self::Output +pub struct iceberg::spec::StructType +impl iceberg::spec::StructType +pub fn iceberg::spec::StructType::field_by_id(&self, id: i32) -> core::option::Option<&iceberg::spec::NestedFieldRef> +pub fn iceberg::spec::StructType::field_by_name(&self, name: &str) -> core::option::Option<&iceberg::spec::NestedFieldRef> +pub fn iceberg::spec::StructType::fields(&self) -> &[iceberg::spec::NestedFieldRef] +pub fn iceberg::spec::StructType::new(fields: alloc::vec::Vec) -> Self +impl core::clone::Clone for iceberg::spec::StructType +pub fn iceberg::spec::StructType::clone(&self) -> iceberg::spec::StructType +impl core::cmp::Eq for iceberg::spec::StructType +impl core::cmp::PartialEq for iceberg::spec::StructType +pub fn iceberg::spec::StructType::eq(&self, other: &Self) -> bool +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(value: iceberg::spec::StructType) -> Self +impl core::default::Default for iceberg::spec::StructType +pub fn iceberg::spec::StructType::default() -> iceberg::spec::StructType +impl core::fmt::Debug for iceberg::spec::StructType +pub fn iceberg::spec::StructType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::StructType +pub fn iceberg::spec::StructType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::ops::index::Index for iceberg::spec::StructType +pub type iceberg::spec::StructType::Output = iceberg::spec::NestedField +pub fn iceberg::spec::StructType::index(&self, index: usize) -> &Self::Output +impl serde_core::ser::Serialize for iceberg::spec::StructType +pub fn iceberg::spec::StructType::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::StructType +pub fn iceberg::spec::StructType::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::Summary +pub iceberg::spec::Summary::additional_properties: std::collections::hash::map::HashMap +pub iceberg::spec::Summary::operation: iceberg::spec::Operation +impl core::clone::Clone for iceberg::spec::Summary +pub fn iceberg::spec::Summary::clone(&self) -> iceberg::spec::Summary +impl core::cmp::Eq for iceberg::spec::Summary +impl core::cmp::PartialEq for iceberg::spec::Summary +pub fn iceberg::spec::Summary::eq(&self, other: &iceberg::spec::Summary) -> bool +impl core::fmt::Debug for iceberg::spec::Summary +pub fn iceberg::spec::Summary::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::Summary +impl serde_core::ser::Serialize for iceberg::spec::Summary +pub fn iceberg::spec::Summary::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::Summary +pub fn iceberg::spec::Summary::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableMetadata +impl iceberg::spec::TableMetadata +pub fn iceberg::spec::TableMetadata::current_schema(&self) -> &iceberg::spec::SchemaRef +pub fn iceberg::spec::TableMetadata::current_schema_id(&self) -> iceberg::spec::SchemaId +pub fn iceberg::spec::TableMetadata::current_snapshot(&self) -> core::option::Option<&iceberg::spec::SnapshotRef> +pub fn iceberg::spec::TableMetadata::current_snapshot_id(&self) -> core::option::Option +pub fn iceberg::spec::TableMetadata::default_partition_spec(&self) -> &iceberg::spec::PartitionSpecRef +pub fn iceberg::spec::TableMetadata::default_partition_spec_id(&self) -> i32 +pub fn iceberg::spec::TableMetadata::default_partition_type(&self) -> &iceberg::spec::StructType +pub fn iceberg::spec::TableMetadata::default_sort_order(&self) -> &iceberg::spec::SortOrderRef +pub fn iceberg::spec::TableMetadata::default_sort_order_id(&self) -> i64 +pub fn iceberg::spec::TableMetadata::encryption_key(&self, key_id: &str) -> core::option::Option<&iceberg::spec::EncryptedKey> +pub fn iceberg::spec::TableMetadata::encryption_keys_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +pub fn iceberg::spec::TableMetadata::format_version(&self) -> iceberg::spec::FormatVersion +pub fn iceberg::spec::TableMetadata::history(&self) -> &[iceberg::spec::SnapshotLog] +pub fn iceberg::spec::TableMetadata::into_builder(self, current_file_location: core::option::Option) -> iceberg::spec::TableMetadataBuilder +pub fn iceberg::spec::TableMetadata::last_column_id(&self) -> i32 +pub fn iceberg::spec::TableMetadata::last_partition_id(&self) -> i32 +pub fn iceberg::spec::TableMetadata::last_sequence_number(&self) -> i64 +pub fn iceberg::spec::TableMetadata::last_updated_ms(&self) -> i64 +pub fn iceberg::spec::TableMetadata::last_updated_timestamp(&self) -> iceberg::Result> +pub fn iceberg::spec::TableMetadata::location(&self) -> &str +pub fn iceberg::spec::TableMetadata::metadata_compression_codec(&self) -> iceberg::Result +pub fn iceberg::spec::TableMetadata::metadata_location(&self) -> iceberg::Result +pub fn iceberg::spec::TableMetadata::metadata_log(&self) -> &[iceberg::spec::MetadataLog] +pub fn iceberg::spec::TableMetadata::next_row_id(&self) -> u64 +pub fn iceberg::spec::TableMetadata::next_sequence_number(&self) -> i64 +pub fn iceberg::spec::TableMetadata::partition_spec_by_id(&self, spec_id: i32) -> core::option::Option<&iceberg::spec::PartitionSpecRef> +pub fn iceberg::spec::TableMetadata::partition_specs_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +pub fn iceberg::spec::TableMetadata::partition_statistics_for_snapshot(&self, snapshot_id: i64) -> core::option::Option<&iceberg::spec::PartitionStatisticsFile> +pub fn iceberg::spec::TableMetadata::partition_statistics_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +pub fn iceberg::spec::TableMetadata::properties(&self) -> &std::collections::hash::map::HashMap +pub async fn iceberg::spec::TableMetadata::read_from(file_io: &iceberg::io::FileIO, metadata_location: impl core::convert::AsRef) -> iceberg::Result +pub fn iceberg::spec::TableMetadata::schema_by_id(&self, schema_id: iceberg::spec::SchemaId) -> core::option::Option<&iceberg::spec::SchemaRef> +pub fn iceberg::spec::TableMetadata::schemas_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +pub fn iceberg::spec::TableMetadata::snapshot_by_id(&self, snapshot_id: i64) -> core::option::Option<&iceberg::spec::SnapshotRef> +pub fn iceberg::spec::TableMetadata::snapshot_for_ref(&self, ref_name: &str) -> core::option::Option<&iceberg::spec::SnapshotRef> +pub fn iceberg::spec::TableMetadata::snapshots(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +pub fn iceberg::spec::TableMetadata::sort_order_by_id(&self, sort_order_id: i64) -> core::option::Option<&iceberg::spec::SortOrderRef> +pub fn iceberg::spec::TableMetadata::sort_orders_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +pub fn iceberg::spec::TableMetadata::statistics_for_snapshot(&self, snapshot_id: i64) -> core::option::Option<&iceberg::spec::StatisticsFile> +pub fn iceberg::spec::TableMetadata::statistics_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +pub fn iceberg::spec::TableMetadata::table_properties(&self) -> iceberg::spec::TableProperties<'_> +pub fn iceberg::spec::TableMetadata::unified_partition_type(&self, schema: &iceberg::spec::Schema) -> iceberg::Result +pub fn iceberg::spec::TableMetadata::uuid(&self) -> uuid::Uuid +pub async fn iceberg::spec::TableMetadata::write_to(&self, file_io: &iceberg::io::FileIO, metadata_location: &iceberg::MetadataLocation) -> iceberg::Result<()> +impl core::clone::Clone for iceberg::spec::TableMetadata +pub fn iceberg::spec::TableMetadata::clone(&self) -> iceberg::spec::TableMetadata +impl core::cmp::Eq for iceberg::spec::TableMetadata +impl core::cmp::PartialEq for iceberg::spec::TableMetadata +pub fn iceberg::spec::TableMetadata::eq(&self, other: &iceberg::spec::TableMetadata) -> bool +impl core::convert::From for iceberg::spec::TableMetadata +pub fn iceberg::spec::TableMetadata::from(result: iceberg::spec::TableMetadataBuildResult) -> Self +impl core::fmt::Debug for iceberg::spec::TableMetadata +pub fn iceberg::spec::TableMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::TableMetadata +impl serde_core::ser::Serialize for iceberg::spec::TableMetadata +pub fn iceberg::spec::TableMetadata::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::TableMetadata +pub fn iceberg::spec::TableMetadata::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::TableMetadataBuildResult +pub iceberg::spec::TableMetadataBuildResult::changes: alloc::vec::Vec +pub iceberg::spec::TableMetadataBuildResult::expired_metadata_logs: alloc::vec::Vec +pub iceberg::spec::TableMetadataBuildResult::metadata: iceberg::spec::TableMetadata +impl core::clone::Clone for iceberg::spec::TableMetadataBuildResult +pub fn iceberg::spec::TableMetadataBuildResult::clone(&self) -> iceberg::spec::TableMetadataBuildResult +impl core::cmp::PartialEq for iceberg::spec::TableMetadataBuildResult +pub fn iceberg::spec::TableMetadataBuildResult::eq(&self, other: &iceberg::spec::TableMetadataBuildResult) -> bool +impl core::convert::From for iceberg::spec::TableMetadata +pub fn iceberg::spec::TableMetadata::from(result: iceberg::spec::TableMetadataBuildResult) -> Self +impl core::fmt::Debug for iceberg::spec::TableMetadataBuildResult +pub fn iceberg::spec::TableMetadataBuildResult::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::TableMetadataBuildResult +pub struct iceberg::spec::TableMetadataBuilder +impl iceberg::spec::TableMetadataBuilder +pub const iceberg::spec::TableMetadataBuilder::LAST_ADDED: i32 +pub fn iceberg::spec::TableMetadataBuilder::add_current_schema(self, schema: iceberg::spec::Schema) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::add_default_partition_spec(self, unbound_spec: iceberg::spec::UnboundPartitionSpec) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::add_encryption_key(self, key: iceberg::spec::EncryptedKey) -> Self +pub fn iceberg::spec::TableMetadataBuilder::add_partition_spec(self, unbound_spec: iceberg::spec::UnboundPartitionSpec) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::add_schema(self, schema: iceberg::spec::Schema) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::add_snapshot(self, snapshot: iceberg::spec::Snapshot) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::add_sort_order(self, sort_order: iceberg::spec::SortOrder) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::assign_uuid(self, uuid: uuid::Uuid) -> Self +pub fn iceberg::spec::TableMetadataBuilder::build(self) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::from_table_creation(table_creation: iceberg::TableCreation) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::new(schema: iceberg::spec::Schema, spec: impl core::convert::Into, sort_order: iceberg::spec::SortOrder, location: alloc::string::String, format_version: iceberg::spec::FormatVersion, properties: std::collections::hash::map::HashMap) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::new_from_metadata(previous: iceberg::spec::TableMetadata, current_file_location: core::option::Option) -> Self +pub fn iceberg::spec::TableMetadataBuilder::remove_encryption_key(self, key_id: &str) -> Self +pub fn iceberg::spec::TableMetadataBuilder::remove_partition_specs(self, spec_ids: &[i32]) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::remove_partition_statistics(self, snapshot_id: i64) -> Self +pub fn iceberg::spec::TableMetadataBuilder::remove_properties(self, properties: &[alloc::string::String]) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::remove_ref(self, ref_name: &str) -> Self +pub fn iceberg::spec::TableMetadataBuilder::remove_schemas(self, schema_id_to_remove: &[i32]) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::remove_snapshots(self, snapshot_ids: &[i64]) -> Self +pub fn iceberg::spec::TableMetadataBuilder::remove_statistics(self, snapshot_id: i64) -> Self +pub fn iceberg::spec::TableMetadataBuilder::set_branch_snapshot(self, snapshot: iceberg::spec::Snapshot, branch: &str) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::set_current_schema(self, schema_id: i32) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::set_default_partition_spec(self, spec_id: i32) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::set_default_sort_order(self, sort_order_id: i64) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::set_location(self, location: alloc::string::String) -> Self +pub fn iceberg::spec::TableMetadataBuilder::set_partition_statistics(self, partition_statistics_file: iceberg::spec::PartitionStatisticsFile) -> Self +pub fn iceberg::spec::TableMetadataBuilder::set_properties(self, properties: std::collections::hash::map::HashMap) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::set_ref(self, ref_name: &str, reference: iceberg::spec::SnapshotReference) -> iceberg::Result +pub fn iceberg::spec::TableMetadataBuilder::set_statistics(self, statistics: iceberg::spec::StatisticsFile) -> Self +pub fn iceberg::spec::TableMetadataBuilder::upgrade_format_version(self, format_version: iceberg::spec::FormatVersion) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::TableMetadataBuilder +pub fn iceberg::spec::TableMetadataBuilder::clone(&self) -> iceberg::spec::TableMetadataBuilder +impl core::fmt::Debug for iceberg::spec::TableMetadataBuilder +pub fn iceberg::spec::TableMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::TableProperties<'properties> +impl iceberg::spec::TableProperties<'_> +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_NUM_RETRIES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SCHEMA: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SNAPSHOT_ID: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_FILE_FORMAT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_NAME_MAPPING: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_PARTITION_SPEC: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DEFAULT_SORT_ORDER: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_ENCRYPTION_KEY_ID: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_FORMAT_VERSION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_GC_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_GC_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_REF_AGE_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_SNAPSHOT_AGE_MS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_COMPRESSION_CODEC: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_NORM_LEVEL: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_COMPRESSION_CODEC: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_COMPRESSION_LEVEL: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_DICT_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_ROW_LIMIT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::PROPERTY_SNAPSHOT_COUNT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_UUID: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_DATA_LOCATION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_METADATA_PATH: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &'static str +pub const iceberg::spec::TableProperties<'_>::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize +pub const iceberg::spec::TableProperties<'_>::RESERVED_PROPERTIES: [&'static str; 9] +pub fn iceberg::spec::TableProperties<'_>::data_encryption_key_size(&self) -> iceberg::Result +impl<'properties> iceberg::spec::TableProperties<'properties> +pub fn iceberg::spec::TableProperties<'properties>::cdc_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::cdc_max_chunk_size(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::cdc_min_chunk_size(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::cdc_norm_level(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_max_retry_wait_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_min_retry_wait_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_num_retries(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::commit_total_retry_timeout_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::default_name_mapping(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::encryption_data_key_length(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::encryption_key_id(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::gc_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::max_ref_age_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::max_snapshot_age_ms(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::metadata_compression_codec(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::metadata_delete_after_commit_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::min_snapshots_to_keep(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::new(properties: &'properties std::collections::hash::map::HashMap) -> Self +pub fn iceberg::spec::TableProperties<'properties>::parquet_compression_codec(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_dict_size_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_page_row_limit(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_page_size_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::parquet_row_group_size_bytes(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_data_location(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_datafusion_fanout_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_folder_storage_location(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_format_default(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_metadata_path(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_enabled(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_location(&self) -> iceberg::Result> +pub fn iceberg::spec::TableProperties<'properties>::write_object_storage_partitioned_paths(&self) -> iceberg::Result +pub fn iceberg::spec::TableProperties<'properties>::write_target_file_size_bytes(&self) -> iceberg::Result +impl<'properties> core::fmt::Debug for iceberg::spec::TableProperties<'properties> +pub fn iceberg::spec::TableProperties<'properties>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::UnboundPartitionField +impl iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::field_id(&self) -> core::option::Option +pub fn iceberg::spec::UnboundPartitionField::name(&self) -> &str +pub fn iceberg::spec::UnboundPartitionField::source_id(&self) -> iceberg::Result +pub fn iceberg::spec::UnboundPartitionField::source_ids(&self) -> &[i32] +pub fn iceberg::spec::UnboundPartitionField::transform(&self) -> iceberg::spec::Transform +pub fn iceberg::spec::UnboundPartitionField::with_field_id(self, field_id: i32) -> Self +impl core::clone::Clone for iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::clone(&self) -> iceberg::spec::UnboundPartitionField +impl core::cmp::Eq for iceberg::spec::UnboundPartitionField +impl core::cmp::PartialEq for iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::eq(&self, other: &iceberg::spec::UnboundPartitionField) -> bool +impl core::convert::From for iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::from(field: iceberg::spec::PartitionField) -> Self +impl core::convert::From for iceberg::Result +pub fn iceberg::Result::from(field: iceberg::spec::UnboundPartitionField) -> Self +impl core::fmt::Debug for iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::UnboundPartitionField +impl iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::builder() -> UnboundPartitionFieldBuilder<((), (), (), ())> +impl serde_core::ser::Serialize for iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::UnboundPartitionField +pub fn iceberg::spec::UnboundPartitionField::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::UnboundPartitionSpec +impl iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpec::bind(self, schema: impl core::convert::Into) -> iceberg::Result +pub fn iceberg::spec::UnboundPartitionSpec::builder() -> iceberg::spec::UnboundPartitionSpecBuilder +pub fn iceberg::spec::UnboundPartitionSpec::fields(&self) -> &[iceberg::spec::UnboundPartitionField] +pub fn iceberg::spec::UnboundPartitionSpec::spec_id(&self) -> core::option::Option +pub fn iceberg::spec::UnboundPartitionSpec::with_spec_id(self, spec_id: i32) -> Self +impl core::clone::Clone for iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpec::clone(&self) -> iceberg::spec::UnboundPartitionSpec +impl core::cmp::Eq for iceberg::spec::UnboundPartitionSpec +impl core::cmp::PartialEq for iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpec::eq(&self, other: &iceberg::spec::UnboundPartitionSpec) -> bool +impl core::convert::From for iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpec::from(spec: iceberg::spec::PartitionSpec) -> Self +impl core::default::Default for iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpec::default() -> iceberg::spec::UnboundPartitionSpec +impl core::fmt::Debug for iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpec::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::UnboundPartitionSpec +impl serde_core::ser::Serialize for iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpec::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpec::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::UnboundPartitionSpecBuilder +impl iceberg::spec::UnboundPartitionSpecBuilder +pub fn iceberg::spec::UnboundPartitionSpecBuilder::add_partition_field(self, field: iceberg::spec::UnboundPartitionField) -> iceberg::Result +pub fn iceberg::spec::UnboundPartitionSpecBuilder::add_partition_fields(self, fields: impl core::iter::traits::collect::IntoIterator) -> iceberg::Result +pub fn iceberg::spec::UnboundPartitionSpecBuilder::build(self) -> iceberg::spec::UnboundPartitionSpec +pub fn iceberg::spec::UnboundPartitionSpecBuilder::new() -> Self +pub fn iceberg::spec::UnboundPartitionSpecBuilder::with_spec_id(self, spec_id: i32) -> Self +impl core::default::Default for iceberg::spec::UnboundPartitionSpecBuilder +pub fn iceberg::spec::UnboundPartitionSpecBuilder::default() -> iceberg::spec::UnboundPartitionSpecBuilder +impl core::fmt::Debug for iceberg::spec::UnboundPartitionSpecBuilder +pub fn iceberg::spec::UnboundPartitionSpecBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::VariantType +impl core::clone::Clone for iceberg::spec::VariantType +pub fn iceberg::spec::VariantType::clone(&self) -> iceberg::spec::VariantType +impl core::cmp::Eq for iceberg::spec::VariantType +impl core::cmp::PartialEq for iceberg::spec::VariantType +pub fn iceberg::spec::VariantType::eq(&self, other: &iceberg::spec::VariantType) -> bool +impl core::convert::From for iceberg::spec::Type +pub fn iceberg::spec::Type::from(_: iceberg::spec::VariantType) -> Self +impl core::fmt::Debug for iceberg::spec::VariantType +pub fn iceberg::spec::VariantType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::spec::VariantType +pub fn iceberg::spec::VariantType::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::spec::VariantType +pub fn iceberg::spec::VariantType::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::Copy for iceberg::spec::VariantType +impl core::marker::StructuralPartialEq for iceberg::spec::VariantType +impl serde_core::ser::Serialize for iceberg::spec::VariantType +pub fn iceberg::spec::VariantType::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::VariantType +pub fn iceberg::spec::VariantType::deserialize(deserializer: D) -> core::result::Result::Error> where D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::ViewMetadata +impl iceberg::spec::ViewMetadata +pub fn iceberg::spec::ViewMetadata::current_schema(&self) -> &iceberg::spec::SchemaRef +pub fn iceberg::spec::ViewMetadata::current_version(&self) -> &iceberg::spec::ViewVersionRef +pub fn iceberg::spec::ViewMetadata::current_version_id(&self) -> iceberg::spec::ViewVersionId +pub fn iceberg::spec::ViewMetadata::format_version(&self) -> iceberg::spec::ViewFormatVersion +pub fn iceberg::spec::ViewMetadata::history(&self) -> &[iceberg::spec::ViewVersionLog] +pub fn iceberg::spec::ViewMetadata::into_builder(self) -> iceberg::spec::ViewMetadataBuilder +pub fn iceberg::spec::ViewMetadata::location(&self) -> &str +pub fn iceberg::spec::ViewMetadata::properties(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::ViewMetadata::schema_by_id(&self, schema_id: iceberg::spec::SchemaId) -> core::option::Option<&iceberg::spec::SchemaRef> +pub fn iceberg::spec::ViewMetadata::schemas_iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +pub fn iceberg::spec::ViewMetadata::uuid(&self) -> uuid::Uuid +pub fn iceberg::spec::ViewMetadata::version_by_id(&self, version_id: iceberg::spec::ViewVersionId) -> core::option::Option<&iceberg::spec::ViewVersionRef> +pub fn iceberg::spec::ViewMetadata::versions(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +impl core::clone::Clone for iceberg::spec::ViewMetadata +pub fn iceberg::spec::ViewMetadata::clone(&self) -> iceberg::spec::ViewMetadata +impl core::cmp::Eq for iceberg::spec::ViewMetadata +impl core::cmp::PartialEq for iceberg::spec::ViewMetadata +pub fn iceberg::spec::ViewMetadata::eq(&self, other: &iceberg::spec::ViewMetadata) -> bool +impl core::fmt::Debug for iceberg::spec::ViewMetadata +pub fn iceberg::spec::ViewMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::ViewMetadata +impl serde_core::ser::Serialize for iceberg::spec::ViewMetadata +pub fn iceberg::spec::ViewMetadata::serialize(&self, serializer: S) -> core::result::Result<::Ok, ::Error> where S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::ViewMetadata +pub fn iceberg::spec::ViewMetadata::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::ViewMetadataBuilder +impl iceberg::spec::ViewMetadataBuilder +pub fn iceberg::spec::ViewMetadataBuilder::add_schema(self, schema: iceberg::spec::Schema) -> Self +pub fn iceberg::spec::ViewMetadataBuilder::add_version(self, view_version: iceberg::spec::ViewVersion) -> iceberg::Result +pub fn iceberg::spec::ViewMetadataBuilder::assign_uuid(self, uuid: uuid::Uuid) -> Self +pub fn iceberg::spec::ViewMetadataBuilder::build(self) -> iceberg::Result +pub fn iceberg::spec::ViewMetadataBuilder::from_view_creation(view_creation: iceberg::ViewCreation) -> iceberg::Result +pub fn iceberg::spec::ViewMetadataBuilder::new(location: alloc::string::String, schema: iceberg::spec::Schema, view_version: iceberg::spec::ViewVersion, format_version: iceberg::spec::ViewFormatVersion, properties: std::collections::hash::map::HashMap) -> iceberg::Result +pub fn iceberg::spec::ViewMetadataBuilder::new_from_metadata(previous: iceberg::spec::ViewMetadata) -> Self +pub fn iceberg::spec::ViewMetadataBuilder::remove_properties(self, removals: &[alloc::string::String]) -> Self +pub fn iceberg::spec::ViewMetadataBuilder::set_current_version(self, view_version: iceberg::spec::ViewVersion, schema: iceberg::spec::Schema) -> iceberg::Result +pub fn iceberg::spec::ViewMetadataBuilder::set_current_version_id(self, version_id: i32) -> iceberg::Result +pub fn iceberg::spec::ViewMetadataBuilder::set_location(self, location: alloc::string::String) -> Self +pub fn iceberg::spec::ViewMetadataBuilder::set_properties(self, updates: std::collections::hash::map::HashMap) -> iceberg::Result +pub fn iceberg::spec::ViewMetadataBuilder::upgrade_format_version(self, format_version: iceberg::spec::ViewFormatVersion) -> iceberg::Result +impl core::clone::Clone for iceberg::spec::ViewMetadataBuilder +pub fn iceberg::spec::ViewMetadataBuilder::clone(&self) -> iceberg::spec::ViewMetadataBuilder +impl core::fmt::Debug for iceberg::spec::ViewMetadataBuilder +pub fn iceberg::spec::ViewMetadataBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::spec::ViewRepresentations(_) +impl iceberg::spec::ViewRepresentations +pub fn iceberg::spec::ViewRepresentations::is_empty(&self) -> bool +pub fn iceberg::spec::ViewRepresentations::iter(&self) -> impl core::iter::traits::exact_size::ExactSizeIterator +pub fn iceberg::spec::ViewRepresentations::len(&self) -> usize +impl core::clone::Clone for iceberg::spec::ViewRepresentations +pub fn iceberg::spec::ViewRepresentations::clone(&self) -> iceberg::spec::ViewRepresentations +impl core::cmp::Eq for iceberg::spec::ViewRepresentations +impl core::cmp::PartialEq for iceberg::spec::ViewRepresentations +pub fn iceberg::spec::ViewRepresentations::eq(&self, other: &iceberg::spec::ViewRepresentations) -> bool +impl core::fmt::Debug for iceberg::spec::ViewRepresentations +pub fn iceberg::spec::ViewRepresentations::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::iter::traits::collect::IntoIterator for iceberg::spec::ViewRepresentations +pub type iceberg::spec::ViewRepresentations::IntoIter = alloc::vec::into_iter::IntoIter<::Item> +pub type iceberg::spec::ViewRepresentations::Item = iceberg::spec::ViewRepresentation +pub fn iceberg::spec::ViewRepresentations::into_iter(self) -> Self::IntoIter +impl core::marker::StructuralPartialEq for iceberg::spec::ViewRepresentations +impl serde_core::ser::Serialize for iceberg::spec::ViewRepresentations +pub fn iceberg::spec::ViewRepresentations::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::ViewRepresentations +pub fn iceberg::spec::ViewRepresentations::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::ViewVersion +impl iceberg::spec::ViewVersion +pub fn iceberg::spec::ViewVersion::default_catalog(&self) -> core::option::Option<&alloc::string::String> +pub fn iceberg::spec::ViewVersion::default_namespace(&self) -> &iceberg::NamespaceIdent +pub fn iceberg::spec::ViewVersion::representations(&self) -> &iceberg::spec::ViewRepresentations +pub fn iceberg::spec::ViewVersion::schema(&self, view_metadata: &iceberg::spec::ViewMetadata) -> iceberg::Result +pub fn iceberg::spec::ViewVersion::schema_id(&self) -> iceberg::spec::SchemaId +pub fn iceberg::spec::ViewVersion::summary(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::spec::ViewVersion::timestamp(&self) -> iceberg::Result> +pub fn iceberg::spec::ViewVersion::timestamp_ms(&self) -> i64 +pub fn iceberg::spec::ViewVersion::version_id(&self) -> iceberg::spec::ViewVersionId +pub fn iceberg::spec::ViewVersion::with_schema_id(self, schema_id: iceberg::spec::SchemaId) -> Self +pub fn iceberg::spec::ViewVersion::with_version_id(self, version_id: i32) -> Self +impl core::clone::Clone for iceberg::spec::ViewVersion +pub fn iceberg::spec::ViewVersion::clone(&self) -> iceberg::spec::ViewVersion +impl core::cmp::Eq for iceberg::spec::ViewVersion +impl core::cmp::PartialEq for iceberg::spec::ViewVersion +pub fn iceberg::spec::ViewVersion::eq(&self, other: &iceberg::spec::ViewVersion) -> bool +impl core::fmt::Debug for iceberg::spec::ViewVersion +pub fn iceberg::spec::ViewVersion::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::ViewVersion +impl iceberg::spec::ViewVersion +pub fn iceberg::spec::ViewVersion::builder() -> ViewVersionBuilder<((), (), (), (), (), (), ())> +impl serde_core::ser::Serialize for iceberg::spec::ViewVersion +pub fn iceberg::spec::ViewVersion::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::ViewVersion +pub fn iceberg::spec::ViewVersion::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::spec::ViewVersionLog +impl iceberg::spec::ViewVersionLog +pub fn iceberg::spec::ViewVersionLog::new(version_id: iceberg::spec::ViewVersionId, timestamp: i64) -> Self +pub fn iceberg::spec::ViewVersionLog::timestamp(&self) -> iceberg::Result> +pub fn iceberg::spec::ViewVersionLog::timestamp_ms(&self) -> i64 +pub fn iceberg::spec::ViewVersionLog::version_id(&self) -> iceberg::spec::ViewVersionId +impl core::clone::Clone for iceberg::spec::ViewVersionLog +pub fn iceberg::spec::ViewVersionLog::clone(&self) -> iceberg::spec::ViewVersionLog +impl core::cmp::Eq for iceberg::spec::ViewVersionLog +impl core::cmp::PartialEq for iceberg::spec::ViewVersionLog +pub fn iceberg::spec::ViewVersionLog::eq(&self, other: &iceberg::spec::ViewVersionLog) -> bool +impl core::fmt::Debug for iceberg::spec::ViewVersionLog +pub fn iceberg::spec::ViewVersionLog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::spec::ViewVersionLog +impl serde_core::ser::Serialize for iceberg::spec::ViewVersionLog +pub fn iceberg::spec::ViewVersionLog::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::spec::ViewVersionLog +pub fn iceberg::spec::ViewVersionLog::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub const iceberg::spec::DEFAULT_SCHEMA_ID: iceberg::spec::SchemaId +pub const iceberg::spec::INITIAL_ROW_ID: u64 +pub const iceberg::spec::LIST_FIELD_NAME: &str +pub const iceberg::spec::MAIN_BRANCH: &str +pub const iceberg::spec::MAP_KEY_FIELD_NAME: &str +pub const iceberg::spec::MAP_VALUE_FIELD_NAME: &str +pub const iceberg::spec::MIN_FORMAT_VERSION_ROW_LINEAGE: iceberg::spec::FormatVersion +pub const iceberg::spec::SCHEMA_NAME_DELIMITER: &str +pub const iceberg::spec::UNASSIGNED_SEQUENCE_NUMBER: i64 +pub const iceberg::spec::VIEW_PROPERTY_REPLACE_DROP_DIALECT_ALLOWED: &str +pub const iceberg::spec::VIEW_PROPERTY_REPLACE_DROP_DIALECT_ALLOWED_DEFAULT: bool +pub const iceberg::spec::VIEW_PROPERTY_VERSION_HISTORY_SIZE: &str +pub const iceberg::spec::VIEW_PROPERTY_VERSION_HISTORY_SIZE_DEFAULT: usize +pub trait iceberg::spec::PartnerAccessor

+pub fn iceberg::spec::PartnerAccessor::field_partner<'a>(&self, struct_partner: &'a P, field: &iceberg::spec::NestedField) -> iceberg::Result<&'a P> +pub fn iceberg::spec::PartnerAccessor::list_element_partner<'a>(&self, list_partner: &'a P) -> iceberg::Result<&'a P> +pub fn iceberg::spec::PartnerAccessor::map_key_partner<'a>(&self, map_partner: &'a P) -> iceberg::Result<&'a P> +pub fn iceberg::spec::PartnerAccessor::map_value_partner<'a>(&self, map_partner: &'a P) -> iceberg::Result<&'a P> +pub fn iceberg::spec::PartnerAccessor::struct_partner<'a>(&self, schema_partner: &'a P) -> iceberg::Result<&'a P> +impl iceberg::spec::PartnerAccessor> for iceberg::arrow::ArrowArrayAccessor +pub fn iceberg::arrow::ArrowArrayAccessor::field_partner<'a>(&self, struct_partner: &'a arrow_array::array::ArrayRef, field: &iceberg::spec::NestedField) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub fn iceberg::arrow::ArrowArrayAccessor::list_element_partner<'a>(&self, list_partner: &'a arrow_array::array::ArrayRef) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub fn iceberg::arrow::ArrowArrayAccessor::map_key_partner<'a>(&self, map_partner: &'a arrow_array::array::ArrayRef) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub fn iceberg::arrow::ArrowArrayAccessor::map_value_partner<'a>(&self, map_partner: &'a arrow_array::array::ArrayRef) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub fn iceberg::arrow::ArrowArrayAccessor::struct_partner<'a>(&self, schema_partner: &'a arrow_array::array::ArrayRef) -> iceberg::Result<&'a arrow_array::array::ArrayRef> +pub trait iceberg::spec::SchemaVisitor +pub type iceberg::spec::SchemaVisitor::T +pub fn iceberg::spec::SchemaVisitor::after_list_element(&mut self, _field: &iceberg::spec::NestedFieldRef) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaVisitor::after_map_key(&mut self, _field: &iceberg::spec::NestedFieldRef) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaVisitor::after_map_value(&mut self, _field: &iceberg::spec::NestedFieldRef) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaVisitor::after_struct_field(&mut self, _field: &iceberg::spec::NestedFieldRef) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaVisitor::before_list_element(&mut self, _field: &iceberg::spec::NestedFieldRef) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaVisitor::before_map_key(&mut self, _field: &iceberg::spec::NestedFieldRef) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaVisitor::before_map_value(&mut self, _field: &iceberg::spec::NestedFieldRef) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaVisitor::before_struct_field(&mut self, _field: &iceberg::spec::NestedFieldRef) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaVisitor::field(&mut self, field: &iceberg::spec::NestedFieldRef, value: Self::T) -> iceberg::Result +pub fn iceberg::spec::SchemaVisitor::list(&mut self, list: &iceberg::spec::ListType, value: Self::T) -> iceberg::Result +pub fn iceberg::spec::SchemaVisitor::map(&mut self, map: &iceberg::spec::MapType, key_value: Self::T, value: Self::T) -> iceberg::Result +pub fn iceberg::spec::SchemaVisitor::primitive(&mut self, p: &iceberg::spec::PrimitiveType) -> iceberg::Result +pub fn iceberg::spec::SchemaVisitor::schema(&mut self, schema: &iceberg::spec::Schema, value: Self::T) -> iceberg::Result +pub fn iceberg::spec::SchemaVisitor::struct(&mut self, struct: &iceberg::spec::StructType, results: alloc::vec::Vec) -> iceberg::Result +pub fn iceberg::spec::SchemaVisitor::variant(&mut self, v: &iceberg::spec::VariantType) -> iceberg::Result +pub trait iceberg::spec::SchemaWithPartnerVisitor

+pub type iceberg::spec::SchemaWithPartnerVisitor::T +pub fn iceberg::spec::SchemaWithPartnerVisitor::after_list_element(&mut self, _field: &iceberg::spec::NestedFieldRef, _partner: &P) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaWithPartnerVisitor::after_map_key(&mut self, _field: &iceberg::spec::NestedFieldRef, _partner: &P) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaWithPartnerVisitor::after_map_value(&mut self, _field: &iceberg::spec::NestedFieldRef, _partner: &P) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaWithPartnerVisitor::after_struct_field(&mut self, _field: &iceberg::spec::NestedFieldRef, _partner: &P) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaWithPartnerVisitor::before_list_element(&mut self, _field: &iceberg::spec::NestedFieldRef, _partner: &P) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaWithPartnerVisitor::before_map_key(&mut self, _field: &iceberg::spec::NestedFieldRef, _partner: &P) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaWithPartnerVisitor::before_map_value(&mut self, _field: &iceberg::spec::NestedFieldRef, _partner: &P) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaWithPartnerVisitor::before_struct_field(&mut self, _field: &iceberg::spec::NestedFieldRef, _partner: &P) -> iceberg::Result<()> +pub fn iceberg::spec::SchemaWithPartnerVisitor::field(&mut self, field: &iceberg::spec::NestedFieldRef, partner: &P, value: Self::T) -> iceberg::Result +pub fn iceberg::spec::SchemaWithPartnerVisitor::list(&mut self, list: &iceberg::spec::ListType, partner: &P, value: Self::T) -> iceberg::Result +pub fn iceberg::spec::SchemaWithPartnerVisitor::map(&mut self, map: &iceberg::spec::MapType, partner: &P, key_value: Self::T, value: Self::T) -> iceberg::Result +pub fn iceberg::spec::SchemaWithPartnerVisitor::primitive(&mut self, p: &iceberg::spec::PrimitiveType, partner: &P) -> iceberg::Result +pub fn iceberg::spec::SchemaWithPartnerVisitor::schema(&mut self, schema: &iceberg::spec::Schema, partner: &P, value: Self::T) -> iceberg::Result +pub fn iceberg::spec::SchemaWithPartnerVisitor::struct(&mut self, struct: &iceberg::spec::StructType, partner: &P, results: alloc::vec::Vec) -> iceberg::Result +pub fn iceberg::spec::SchemaWithPartnerVisitor::variant(&mut self, v: &iceberg::spec::VariantType, partner: &P) -> iceberg::Result +pub fn iceberg::spec::deserialize_data_file_from_json(json: &str, partition_spec_id: i32, partition_type: &iceberg::spec::StructType, schema: &iceberg::spec::Schema) -> iceberg::Result +pub fn iceberg::spec::prune_columns(schema: &iceberg::spec::Schema, selected: impl core::iter::traits::collect::IntoIterator, select_full_types: bool) -> iceberg::Result +pub fn iceberg::spec::read_data_files_from_avro(reader: &mut R, schema: &iceberg::spec::Schema, partition_spec_id: i32, partition_type: &iceberg::spec::StructType, version: iceberg::spec::FormatVersion) -> iceberg::Result> +pub fn iceberg::spec::serialize_data_file_to_json(data_file: iceberg::spec::DataFile, partition_type: &iceberg::spec::StructType, format_version: iceberg::spec::FormatVersion) -> iceberg::Result +pub fn iceberg::spec::visit_schema(schema: &iceberg::spec::Schema, visitor: &mut V) -> iceberg::Result<::T> +pub fn iceberg::spec::visit_schema_with_partner, A: iceberg::spec::PartnerAccessor

>(schema: &iceberg::spec::Schema, partner: &P, visitor: &mut V, accessor: &A) -> iceberg::Result<::T> +pub fn iceberg::spec::visit_struct(s: &iceberg::spec::StructType, visitor: &mut V) -> iceberg::Result<::T> +pub fn iceberg::spec::visit_struct_with_partner, A: iceberg::spec::PartnerAccessor

>(s: &iceberg::spec::StructType, partner: &P, visitor: &mut V, accessor: &A) -> iceberg::Result<::T> +pub fn iceberg::spec::write_data_files_to_avro(writer: &mut W, data_files: impl core::iter::traits::collect::IntoIterator, partition_type: &iceberg::spec::StructType, version: iceberg::spec::FormatVersion) -> iceberg::Result +pub type iceberg::spec::Decimal = fastnum::decimal::D128 +pub type iceberg::spec::ManifestEntryRef = alloc::sync::Arc +pub type iceberg::spec::NestedFieldRef = alloc::sync::Arc +pub type iceberg::spec::PartitionSpecRef = alloc::sync::Arc +pub type iceberg::spec::SchemaId = i32 +pub type iceberg::spec::SchemaRef = alloc::sync::Arc +pub type iceberg::spec::SnapshotRef = alloc::sync::Arc +pub type iceberg::spec::SortOrderRef = alloc::sync::Arc +pub type iceberg::spec::TableMetadataRef = alloc::sync::Arc +pub type iceberg::spec::UnboundPartitionSpecRef = alloc::sync::Arc +pub type iceberg::spec::ViewMetadataRef = alloc::sync::Arc +pub type iceberg::spec::ViewVersionId = i32 +pub type iceberg::spec::ViewVersionRef = alloc::sync::Arc +pub mod iceberg::table +pub struct iceberg::table::StaticTable(_) +impl iceberg::table::StaticTable +pub async fn iceberg::table::StaticTable::from_metadata(metadata: iceberg::spec::TableMetadata, table_ident: iceberg::TableIdent, file_io: iceberg::io::FileIO) -> iceberg::Result +pub async fn iceberg::table::StaticTable::from_metadata_file(metadata_location: &str, table_ident: iceberg::TableIdent, file_io: iceberg::io::FileIO) -> iceberg::Result +pub fn iceberg::table::StaticTable::into_table(self) -> iceberg::table::Table +pub fn iceberg::table::StaticTable::metadata(&self) -> iceberg::spec::TableMetadataRef +pub fn iceberg::table::StaticTable::reader_builder(&self) -> iceberg::arrow::ArrowReaderBuilder +pub fn iceberg::table::StaticTable::scan(&self) -> iceberg::scan::TableScanBuilder<'_> +impl core::clone::Clone for iceberg::table::StaticTable +pub fn iceberg::table::StaticTable::clone(&self) -> iceberg::table::StaticTable +impl core::fmt::Debug for iceberg::table::StaticTable +pub fn iceberg::table::StaticTable::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::table::Table +impl iceberg::table::Table +pub fn iceberg::table::Table::builder() -> iceberg::table::TableBuilder +pub fn iceberg::table::Table::current_schema_ref(&self) -> iceberg::spec::SchemaRef +pub fn iceberg::table::Table::encryption_manager(&self) -> core::option::Option<&alloc::sync::Arc> +pub fn iceberg::table::Table::file_io(&self) -> &iceberg::io::FileIO +pub fn iceberg::table::Table::identifier(&self) -> &iceberg::TableIdent +pub fn iceberg::table::Table::inspect(&self) -> iceberg::inspect::MetadataTable<'_> +pub fn iceberg::table::Table::manifest_list_reader(&self, snapshot: &iceberg::spec::SnapshotRef) -> iceberg::spec::ManifestListReader +pub fn iceberg::table::Table::manifest_reader(&self) -> iceberg::spec::ManifestReader +pub fn iceberg::table::Table::metadata(&self) -> &iceberg::spec::TableMetadata +pub fn iceberg::table::Table::metadata_location(&self) -> core::option::Option<&str> +pub fn iceberg::table::Table::metadata_location_result(&self) -> iceberg::Result<&str> +pub fn iceberg::table::Table::metadata_ref(&self) -> iceberg::spec::TableMetadataRef +pub fn iceberg::table::Table::reader_builder(&self) -> iceberg::arrow::ArrowReaderBuilder +pub fn iceberg::table::Table::readonly(&self) -> bool +pub fn iceberg::table::Table::scan(&self) -> iceberg::scan::TableScanBuilder<'_> +pub fn iceberg::table::Table::with_runtime(self, runtime: iceberg::Runtime) -> Self +impl core::clone::Clone for iceberg::table::Table +pub fn iceberg::table::Table::clone(&self) -> iceberg::table::Table +impl core::fmt::Debug for iceberg::table::Table +pub fn iceberg::table::Table::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::table::TableBuilder +impl iceberg::table::TableBuilder +pub fn iceberg::table::TableBuilder::build(self) -> iceberg::Result +pub fn iceberg::table::TableBuilder::cache_size_bytes(self, cache_size_bytes: u64) -> Self +pub fn iceberg::table::TableBuilder::disable_cache(self) -> Self +pub fn iceberg::table::TableBuilder::file_io(self, file_io: iceberg::io::FileIO) -> Self +pub fn iceberg::table::TableBuilder::identifier(self, identifier: iceberg::TableIdent) -> Self +pub fn iceberg::table::TableBuilder::kms_client(self, kms_client: alloc::sync::Arc) -> Self +pub fn iceberg::table::TableBuilder::metadata>(self, metadata: T) -> Self +pub fn iceberg::table::TableBuilder::metadata_location>(self, metadata_location: T) -> Self +pub fn iceberg::table::TableBuilder::readonly(self, readonly: bool) -> Self +pub fn iceberg::table::TableBuilder::runtime(self, runtime: iceberg::Runtime) -> Self +pub mod iceberg::transaction +pub struct iceberg::transaction::ActionCommit +impl iceberg::transaction::ActionCommit +pub fn iceberg::transaction::ActionCommit::new(updates: alloc::vec::Vec, requirements: alloc::vec::Vec) -> Self +pub fn iceberg::transaction::ActionCommit::take_requirements(&mut self) -> alloc::vec::Vec +pub fn iceberg::transaction::ActionCommit::take_updates(&mut self) -> alloc::vec::Vec +pub struct iceberg::transaction::AddColumn +impl iceberg::transaction::AddColumn +pub fn iceberg::transaction::AddColumn::optional(name: impl alloc::string::ToString, field_type: iceberg::spec::Type) -> Self +pub fn iceberg::transaction::AddColumn::required(name: impl alloc::string::ToString, field_type: iceberg::spec::Type, initial_default: iceberg::spec::Literal) -> Self +impl core::clone::Clone for iceberg::transaction::AddColumn +pub fn iceberg::transaction::AddColumn::clone(&self) -> iceberg::transaction::AddColumn +impl iceberg::transaction::AddColumn +pub fn iceberg::transaction::AddColumn::builder() -> AddColumnBuilder<((), (), (), (), (), (), ())> +pub struct iceberg::transaction::ExpireSnapshotsAction +impl iceberg::transaction::ExpireSnapshotsAction +pub fn iceberg::transaction::ExpireSnapshotsAction::expire_older_than_ms(self, older_than_ms: i64) -> Self +pub fn iceberg::transaction::ExpireSnapshotsAction::expire_snapshot_ids(self, snapshot_ids: impl core::iter::traits::collect::IntoIterator) -> Self +pub fn iceberg::transaction::ExpireSnapshotsAction::retain_last(self, retain_last: usize) -> Self +impl core::clone::Clone for iceberg::transaction::ExpireSnapshotsAction +pub fn iceberg::transaction::ExpireSnapshotsAction::clone(&self) -> iceberg::transaction::ExpireSnapshotsAction +pub struct iceberg::transaction::FastAppendAction +impl iceberg::transaction::FastAppendAction +pub fn iceberg::transaction::FastAppendAction::add_data_files(self, data_files: impl core::iter::traits::collect::IntoIterator) -> Self +pub fn iceberg::transaction::FastAppendAction::set_commit_uuid(self, commit_uuid: uuid::Uuid) -> Self +pub fn iceberg::transaction::FastAppendAction::set_snapshot_properties(self, snapshot_properties: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::transaction::FastAppendAction::with_check_duplicate(self, v: bool) -> Self +impl core::clone::Clone for iceberg::transaction::FastAppendAction +pub fn iceberg::transaction::FastAppendAction::clone(&self) -> iceberg::transaction::FastAppendAction +pub struct iceberg::transaction::ReplaceSortOrderAction +impl iceberg::transaction::ReplaceSortOrderAction +pub fn iceberg::transaction::ReplaceSortOrderAction::asc(self, name: &str, null_order: iceberg::spec::NullOrder) -> Self +pub fn iceberg::transaction::ReplaceSortOrderAction::asc_with_transform(self, name: &str, transform: iceberg::spec::Transform, null_order: iceberg::spec::NullOrder) -> Self +pub fn iceberg::transaction::ReplaceSortOrderAction::desc(self, name: &str, null_order: iceberg::spec::NullOrder) -> Self +pub fn iceberg::transaction::ReplaceSortOrderAction::desc_with_transform(self, name: &str, transform: iceberg::spec::Transform, null_order: iceberg::spec::NullOrder) -> Self +impl core::clone::Clone for iceberg::transaction::ReplaceSortOrderAction +pub fn iceberg::transaction::ReplaceSortOrderAction::clone(&self) -> iceberg::transaction::ReplaceSortOrderAction +pub struct iceberg::transaction::Transaction +impl iceberg::transaction::Transaction +pub async fn iceberg::transaction::Transaction::commit(self, catalog: &dyn iceberg::Catalog) -> iceberg::Result +pub fn iceberg::transaction::Transaction::expire_snapshots(&self) -> iceberg::transaction::ExpireSnapshotsAction +pub fn iceberg::transaction::Transaction::fast_append(&self) -> iceberg::transaction::FastAppendAction +pub fn iceberg::transaction::Transaction::new(table: &iceberg::table::Table) -> Self +pub fn iceberg::transaction::Transaction::replace_sort_order(&self) -> iceberg::transaction::ReplaceSortOrderAction +pub fn iceberg::transaction::Transaction::update_location(&self) -> iceberg::transaction::UpdateLocationAction +pub fn iceberg::transaction::Transaction::update_schema(&self) -> iceberg::transaction::UpdateSchemaAction +pub fn iceberg::transaction::Transaction::update_statistics(&self) -> iceberg::transaction::UpdateStatisticsAction +pub fn iceberg::transaction::Transaction::update_table_properties(&self) -> iceberg::transaction::UpdatePropertiesAction +pub fn iceberg::transaction::Transaction::upgrade_table_version(&self) -> iceberg::transaction::UpgradeFormatVersionAction +pub struct iceberg::transaction::UpdateLocationAction +impl iceberg::transaction::UpdateLocationAction +pub fn iceberg::transaction::UpdateLocationAction::set_location(self, location: alloc::string::String) -> Self +impl core::clone::Clone for iceberg::transaction::UpdateLocationAction +pub fn iceberg::transaction::UpdateLocationAction::clone(&self) -> iceberg::transaction::UpdateLocationAction +pub struct iceberg::transaction::UpdatePropertiesAction +impl iceberg::transaction::UpdatePropertiesAction +pub fn iceberg::transaction::UpdatePropertiesAction::remove(self, key: alloc::string::String) -> Self +pub fn iceberg::transaction::UpdatePropertiesAction::set(self, key: alloc::string::String, value: alloc::string::String) -> Self +impl core::clone::Clone for iceberg::transaction::UpdatePropertiesAction +pub fn iceberg::transaction::UpdatePropertiesAction::clone(&self) -> iceberg::transaction::UpdatePropertiesAction +pub struct iceberg::transaction::UpdateSchemaAction +impl iceberg::transaction::UpdateSchemaAction +pub fn iceberg::transaction::UpdateSchemaAction::add_column(self, add_column: iceberg::transaction::AddColumn) -> Self +pub fn iceberg::transaction::UpdateSchemaAction::delete_column(self, name: impl alloc::string::ToString) -> Self +impl core::clone::Clone for iceberg::transaction::UpdateSchemaAction +pub fn iceberg::transaction::UpdateSchemaAction::clone(&self) -> iceberg::transaction::UpdateSchemaAction +pub struct iceberg::transaction::UpdateStatisticsAction +impl iceberg::transaction::UpdateStatisticsAction +pub fn iceberg::transaction::UpdateStatisticsAction::remove_statistics(self, snapshot_id: i64) -> Self +pub fn iceberg::transaction::UpdateStatisticsAction::set_statistics(self, statistics_file: iceberg::spec::StatisticsFile) -> Self +impl core::clone::Clone for iceberg::transaction::UpdateStatisticsAction +pub fn iceberg::transaction::UpdateStatisticsAction::clone(&self) -> iceberg::transaction::UpdateStatisticsAction +pub struct iceberg::transaction::UpgradeFormatVersionAction +impl iceberg::transaction::UpgradeFormatVersionAction +pub fn iceberg::transaction::UpgradeFormatVersionAction::set_format_version(self, format_version: iceberg::spec::FormatVersion) -> Self +impl core::clone::Clone for iceberg::transaction::UpgradeFormatVersionAction +pub fn iceberg::transaction::UpgradeFormatVersionAction::clone(&self) -> iceberg::transaction::UpgradeFormatVersionAction +pub trait iceberg::transaction::ApplyTransactionAction +pub fn iceberg::transaction::ApplyTransactionAction::apply(self, tx: iceberg::transaction::Transaction) -> iceberg::Result +impl iceberg::transaction::ApplyTransactionAction for T +pub fn T::apply(self, tx: iceberg::transaction::Transaction) -> core::result::Result +pub mod iceberg::transform +pub trait iceberg::transform::TransformFunction: core::marker::Send + core::marker::Sync + core::fmt::Debug +pub fn iceberg::transform::TransformFunction::transform(&self, input: arrow_array::array::ArrayRef) -> iceberg::Result +pub fn iceberg::transform::TransformFunction::transform_literal(&self, input: &iceberg::spec::Datum) -> iceberg::Result> +pub fn iceberg::transform::TransformFunction::transform_literal_result(&self, input: &iceberg::spec::Datum) -> iceberg::Result +pub fn iceberg::transform::create_transform_function(transform: &iceberg::spec::Transform) -> iceberg::Result +pub type iceberg::transform::BoxedTransformFunction = alloc::boxed::Box +pub mod iceberg::util +pub mod iceberg::util::location +pub mod iceberg::util::snapshot +pub fn iceberg::util::snapshot::ancestors_between(table_metadata: &iceberg::spec::TableMetadataRef, latest_snapshot_id: i64, oldest_snapshot_id: core::option::Option) -> impl core::iter::traits::iterator::Iterator + core::marker::Send +pub fn iceberg::util::snapshot::ancestors_of(table_metadata: &iceberg::spec::TableMetadataRef, snapshot_id: i64) -> impl core::iter::traits::iterator::Iterator + core::marker::Send +pub fn iceberg::util::snapshot::snapshot_id_as_of_time(table_metadata: &iceberg::spec::TableMetadataRef, timestamp_ms: i64) -> iceberg::Result +pub mod iceberg::writer +pub mod iceberg::writer::base_writer +pub mod iceberg::writer::base_writer::data_file_writer +pub struct iceberg::writer::base_writer::data_file_writer::DataFileWriter +impl iceberg::writer::CurrentFileStatus for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_written_size(&self) -> usize +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::data_file_writer::DataFileWriter +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder +impl iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder) -> Self +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::R = iceberg::writer::base_writer::data_file_writer::DataFileWriter +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub mod iceberg::writer::base_writer::equality_delete_writer +pub struct iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder +impl iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder, config: iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig) -> Self +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::R = iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig +impl iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig::new(equality_ids: alloc::vec::Vec, original_schema: iceberg::spec::SchemaRef) -> iceberg::Result +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig::projected_arrow_schema_ref(&self) -> &arrow_schema::schema::SchemaRef +impl core::fmt::Debug for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteWriterConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub mod iceberg::writer::base_writer::position_delete_writer +pub struct iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder +impl iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::new(inner: iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder) -> Self +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::R = iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl core::fmt::Debug for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn iceberg::writer::base_writer::position_delete_writer::position_delete_schema() -> iceberg::spec::SchemaRef +pub mod iceberg::writer::file_writer +pub mod iceberg::writer::file_writer::location_generator +pub struct iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator +impl iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator::new(prefix: alloc::string::String, suffix: core::option::Option, format: iceberg::spec::DataFileFormat) -> Self +impl core::clone::Clone for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator::clone(&self) -> iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator +impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::writer::file_writer::location_generator::FileNameGenerator for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator::generate_file_name(&self) -> alloc::string::String +pub struct iceberg::writer::file_writer::location_generator::DefaultLocationGenerator +impl iceberg::writer::file_writer::location_generator::DefaultLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::new(table_metadata: &iceberg::spec::TableMetadata) -> iceberg::Result +pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::with_data_location(data_location: alloc::string::String) -> Self +impl core::clone::Clone for iceberg::writer::file_writer::location_generator::DefaultLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::clone(&self) -> iceberg::writer::file_writer::location_generator::DefaultLocationGenerator +impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::DefaultLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::DefaultLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +pub struct iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator +impl iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::new(table_metadata: &iceberg::spec::TableMetadata) -> iceberg::Result +impl core::clone::Clone for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::clone(&self) -> iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator +impl core::fmt::Debug for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +pub trait iceberg::writer::file_writer::location_generator::FileNameGenerator: core::clone::Clone + core::marker::Send + core::marker::Sync + 'static +pub fn iceberg::writer::file_writer::location_generator::FileNameGenerator::generate_file_name(&self) -> alloc::string::String +impl iceberg::writer::file_writer::location_generator::FileNameGenerator for iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultFileNameGenerator::generate_file_name(&self) -> alloc::string::String +pub trait iceberg::writer::file_writer::location_generator::LocationGenerator: core::clone::Clone + core::marker::Send + core::marker::Sync + 'static +pub fn iceberg::writer::file_writer::location_generator::LocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::DefaultLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::DefaultLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +impl iceberg::writer::file_writer::location_generator::LocationGenerator for iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator +pub fn iceberg::writer::file_writer::location_generator::ObjectStorageLocationGenerator::generate_location(&self, partition_key: core::option::Option<&iceberg::spec::PartitionKey>, file_name: &str) -> alloc::string::String +pub mod iceberg::writer::file_writer::rolling_writer +pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriter +impl iceberg::writer::file_writer::rolling_writer::RollingFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub async fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::close(self) -> iceberg::Result> +pub async fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::write(&mut self, partition_key: &core::option::Option, input: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<()> +impl core::fmt::Debug for iceberg::writer::file_writer::rolling_writer::RollingFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::rolling_writer::RollingFileWriter +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_written_size(&self) -> usize +pub struct iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder +impl iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::build(&self) -> iceberg::writer::file_writer::rolling_writer::RollingFileWriter +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::new(inner_builder: B, target_file_size: usize, file_io: iceberg::io::FileIO, location_generator: L, file_name_generator: F) -> Self +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::new_with_default_file_size(inner_builder: B, file_io: iceberg::io::FileIO, location_generator: L, file_name_generator: F) -> Self +impl core::clone::Clone for iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::clone(&self) -> iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder +impl core::fmt::Debug for iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::writer::file_writer::ParquetWriter +impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::ParquetWriter +pub fn iceberg::writer::file_writer::ParquetWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::file_writer::ParquetWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::file_writer::ParquetWriter::current_written_size(&self) -> usize +impl iceberg::writer::file_writer::FileWriter for iceberg::writer::file_writer::ParquetWriter +pub async fn iceberg::writer::file_writer::ParquetWriter::close(self) -> iceberg::Result> +pub async fn iceberg::writer::file_writer::ParquetWriter::write(&mut self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<()> +pub struct iceberg::writer::file_writer::ParquetWriterBuilder +impl iceberg::writer::file_writer::ParquetWriterBuilder +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::from_table_properties(table_props: &iceberg::spec::TableProperties<'_>, schema: iceberg::spec::SchemaRef) -> iceberg::Result +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef) -> Self +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::new_with_match_mode(props: parquet::file::properties::WriterProperties, schema: iceberg::spec::SchemaRef, match_mode: iceberg::arrow::FieldMatchMode) -> Self +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::with_encryption_manager(self, encryption_manager: alloc::sync::Arc) -> Self +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::with_match_mode(self, match_mode: iceberg::arrow::FieldMatchMode) -> Self +impl core::clone::Clone for iceberg::writer::file_writer::ParquetWriterBuilder +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::clone(&self) -> iceberg::writer::file_writer::ParquetWriterBuilder +impl core::fmt::Debug for iceberg::writer::file_writer::ParquetWriterBuilder +pub fn iceberg::writer::file_writer::ParquetWriterBuilder::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::writer::file_writer::FileWriterBuilder for iceberg::writer::file_writer::ParquetWriterBuilder +pub type iceberg::writer::file_writer::ParquetWriterBuilder::R = iceberg::writer::file_writer::ParquetWriter +pub async fn iceberg::writer::file_writer::ParquetWriterBuilder::build(&self, output_file: iceberg::io::OutputFile) -> iceberg::Result +pub trait iceberg::writer::file_writer::FileWriter: core::marker::Send + iceberg::writer::CurrentFileStatus + 'static +pub fn iceberg::writer::file_writer::FileWriter::close(self) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg::writer::file_writer::FileWriter::write(&mut self, batch: &arrow_array::record_batch::RecordBatch) -> impl core::future::future::Future> + core::marker::Send +impl iceberg::writer::file_writer::FileWriter for iceberg::writer::file_writer::ParquetWriter +pub async fn iceberg::writer::file_writer::ParquetWriter::close(self) -> iceberg::Result> +pub async fn iceberg::writer::file_writer::ParquetWriter::write(&mut self, batch: &arrow_array::record_batch::RecordBatch) -> iceberg::Result<()> +pub trait iceberg::writer::file_writer::FileWriterBuilder: core::clone::Clone + core::marker::Send + core::marker::Sync + 'static +pub type iceberg::writer::file_writer::FileWriterBuilder::R: iceberg::writer::file_writer::FileWriter +pub fn iceberg::writer::file_writer::FileWriterBuilder::build(&self, output_file: iceberg::io::OutputFile) -> impl core::future::future::Future> + core::marker::Send +impl iceberg::writer::file_writer::FileWriterBuilder for iceberg::writer::file_writer::ParquetWriterBuilder +pub type iceberg::writer::file_writer::ParquetWriterBuilder::R = iceberg::writer::file_writer::ParquetWriter +pub async fn iceberg::writer::file_writer::ParquetWriterBuilder::build(&self, output_file: iceberg::io::OutputFile) -> iceberg::Result +pub mod iceberg::writer::partitioning +pub mod iceberg::writer::partitioning::clustered_writer +pub struct iceberg::writer::partitioning::clustered_writer::ClusteredWriter where B: iceberg::writer::IcebergWriterBuilder, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item>, ::Item: core::clone::Clone +impl iceberg::writer::partitioning::clustered_writer::ClusteredWriter where B: iceberg::writer::IcebergWriterBuilder, I: core::marker::Send + 'static, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item>, ::Item: core::marker::Send + core::clone::Clone +pub fn iceberg::writer::partitioning::clustered_writer::ClusteredWriter::new(inner_builder: B) -> Self +impl iceberg::writer::partitioning::PartitioningWriter for iceberg::writer::partitioning::clustered_writer::ClusteredWriter where B: iceberg::writer::IcebergWriterBuilder, I: core::marker::Send + 'static, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item> + core::marker::Send + 'static, ::Item: core::marker::Send + core::clone::Clone +pub fn iceberg::writer::partitioning::clustered_writer::ClusteredWriter::close<'async_trait>(self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait +pub fn iceberg::writer::partitioning::clustered_writer::ClusteredWriter::write<'life0, 'async_trait>(&'life0 mut self, partition_key: iceberg::spec::PartitionKey, input: I) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub mod iceberg::writer::partitioning::fanout_writer +pub struct iceberg::writer::partitioning::fanout_writer::FanoutWriter where B: iceberg::writer::IcebergWriterBuilder, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item>, ::Item: core::clone::Clone +impl iceberg::writer::partitioning::fanout_writer::FanoutWriter where B: iceberg::writer::IcebergWriterBuilder, I: core::marker::Send + 'static, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item>, ::Item: core::marker::Send + core::clone::Clone +pub fn iceberg::writer::partitioning::fanout_writer::FanoutWriter::new(inner_builder: B) -> Self +impl iceberg::writer::partitioning::PartitioningWriter for iceberg::writer::partitioning::fanout_writer::FanoutWriter where B: iceberg::writer::IcebergWriterBuilder, I: core::marker::Send + 'static, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item> + core::marker::Send + 'static, ::Item: core::marker::Send + core::clone::Clone +pub fn iceberg::writer::partitioning::fanout_writer::FanoutWriter::close<'async_trait>(self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait +pub fn iceberg::writer::partitioning::fanout_writer::FanoutWriter::write<'life0, 'async_trait>(&'life0 mut self, partition_key: iceberg::spec::PartitionKey, input: I) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub mod iceberg::writer::partitioning::unpartitioned_writer +pub struct iceberg::writer::partitioning::unpartitioned_writer::UnpartitionedWriter where B: iceberg::writer::IcebergWriterBuilder, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item>, ::Item: core::clone::Clone +impl iceberg::writer::partitioning::unpartitioned_writer::UnpartitionedWriter where B: iceberg::writer::IcebergWriterBuilder, I: core::marker::Send + 'static, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item>, ::Item: core::marker::Send + core::clone::Clone +pub async fn iceberg::writer::partitioning::unpartitioned_writer::UnpartitionedWriter::close(self) -> iceberg::Result +pub fn iceberg::writer::partitioning::unpartitioned_writer::UnpartitionedWriter::new(inner_builder: B) -> Self +pub async fn iceberg::writer::partitioning::unpartitioned_writer::UnpartitionedWriter::write(&mut self, input: I) -> iceberg::Result<()> +pub trait iceberg::writer::partitioning::PartitioningWriter: core::marker::Send + 'static +pub fn iceberg::writer::partitioning::PartitioningWriter::close<'async_trait>(self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait +pub fn iceberg::writer::partitioning::PartitioningWriter::write<'life0, 'async_trait>(&'life0 mut self, partition_key: iceberg::spec::PartitionKey, input: I) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::partitioning::PartitioningWriter for iceberg::writer::partitioning::clustered_writer::ClusteredWriter where B: iceberg::writer::IcebergWriterBuilder, I: core::marker::Send + 'static, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item> + core::marker::Send + 'static, ::Item: core::marker::Send + core::clone::Clone +pub fn iceberg::writer::partitioning::clustered_writer::ClusteredWriter::close<'async_trait>(self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait +pub fn iceberg::writer::partitioning::clustered_writer::ClusteredWriter::write<'life0, 'async_trait>(&'life0 mut self, partition_key: iceberg::spec::PartitionKey, input: I) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::partitioning::PartitioningWriter for iceberg::writer::partitioning::fanout_writer::FanoutWriter where B: iceberg::writer::IcebergWriterBuilder, I: core::marker::Send + 'static, O: core::iter::traits::collect::IntoIterator + core::iter::traits::collect::FromIterator<::Item> + core::marker::Send + 'static, ::Item: core::marker::Send + core::clone::Clone +pub fn iceberg::writer::partitioning::fanout_writer::FanoutWriter::close<'async_trait>(self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait +pub fn iceberg::writer::partitioning::fanout_writer::FanoutWriter::write<'life0, 'async_trait>(&'life0 mut self, partition_key: iceberg::spec::PartitionKey, input: I) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub trait iceberg::writer::CurrentFileStatus +pub fn iceberg::writer::CurrentFileStatus::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::CurrentFileStatus::current_row_num(&self) -> usize +pub fn iceberg::writer::CurrentFileStatus::current_written_size(&self) -> usize +impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::ParquetWriter +pub fn iceberg::writer::file_writer::ParquetWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::file_writer::ParquetWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::file_writer::ParquetWriter::current_written_size(&self) -> usize +impl iceberg::writer::CurrentFileStatus for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::current_written_size(&self) -> usize +impl iceberg::writer::CurrentFileStatus for iceberg::writer::file_writer::rolling_writer::RollingFileWriter +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_file_path(&self) -> alloc::string::String +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_row_num(&self) -> usize +pub fn iceberg::writer::file_writer::rolling_writer::RollingFileWriter::current_written_size(&self) -> usize +pub trait iceberg::writer::IcebergWriter: core::marker::Send + 'static +pub fn iceberg::writer::IcebergWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::IcebergWriter::write<'life0, 'async_trait>(&'life0 mut self, input: I) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::data_file_writer::DataFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriter for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::close<'life0, 'async_trait>(&'life0 mut self) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter::write<'life0, 'async_trait>(&'life0 mut self, batch: arrow_array::record_batch::RecordBatch) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub trait iceberg::writer::IcebergWriterBuilder: core::marker::Send + core::marker::Sync + 'static +pub type iceberg::writer::IcebergWriterBuilder::R: iceberg::writer::IcebergWriter +pub fn iceberg::writer::IcebergWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::R = iceberg::writer::base_writer::data_file_writer::DataFileWriter +pub fn iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::R = iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriter +pub fn iceberg::writer::base_writer::equality_delete_writer::EqualityDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::writer::IcebergWriterBuilder for iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder where B: iceberg::writer::file_writer::FileWriterBuilder, L: iceberg::writer::file_writer::location_generator::LocationGenerator, F: iceberg::writer::file_writer::location_generator::FileNameGenerator +pub type iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::R = iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriter +pub fn iceberg::writer::base_writer::position_delete_writer::PositionDeleteFileWriterBuilder::build<'life0, 'async_trait>(&'life0 self, partition_key: core::option::Option) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub macro iceberg::ensure_data_valid! +#[non_exhaustive] pub enum iceberg::ErrorKind +pub iceberg::ErrorKind::CatalogCommitConflicts +pub iceberg::ErrorKind::DataInvalid +pub iceberg::ErrorKind::FeatureUnsupported +pub iceberg::ErrorKind::NamespaceAlreadyExists +pub iceberg::ErrorKind::NamespaceNotFound +pub iceberg::ErrorKind::PreconditionFailed +pub iceberg::ErrorKind::TableAlreadyExists +pub iceberg::ErrorKind::TableNotFound +pub iceberg::ErrorKind::Unexpected +impl iceberg::ErrorKind +pub fn iceberg::ErrorKind::into_static(self) -> &'static str +impl core::clone::Clone for iceberg::ErrorKind +pub fn iceberg::ErrorKind::clone(&self) -> iceberg::ErrorKind +impl core::cmp::Eq for iceberg::ErrorKind +impl core::cmp::PartialEq for iceberg::ErrorKind +pub fn iceberg::ErrorKind::eq(&self, other: &iceberg::ErrorKind) -> bool +impl core::convert::From for &'static str +pub fn &'static str::from(v: iceberg::ErrorKind) -> &'static str +impl core::fmt::Debug for iceberg::ErrorKind +pub fn iceberg::ErrorKind::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::ErrorKind +pub fn iceberg::ErrorKind::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Copy for iceberg::ErrorKind +impl core::marker::StructuralPartialEq for iceberg::ErrorKind +pub enum iceberg::TableRequirement +pub iceberg::TableRequirement::CurrentSchemaIdMatch +pub iceberg::TableRequirement::CurrentSchemaIdMatch::current_schema_id: iceberg::spec::SchemaId +pub iceberg::TableRequirement::DefaultSortOrderIdMatch +pub iceberg::TableRequirement::DefaultSortOrderIdMatch::default_sort_order_id: i64 +pub iceberg::TableRequirement::DefaultSpecIdMatch +pub iceberg::TableRequirement::DefaultSpecIdMatch::default_spec_id: i32 +pub iceberg::TableRequirement::LastAssignedFieldIdMatch +pub iceberg::TableRequirement::LastAssignedFieldIdMatch::last_assigned_field_id: i32 +pub iceberg::TableRequirement::LastAssignedPartitionIdMatch +pub iceberg::TableRequirement::LastAssignedPartitionIdMatch::last_assigned_partition_id: i32 +pub iceberg::TableRequirement::NotExist +pub iceberg::TableRequirement::RefSnapshotIdMatch +pub iceberg::TableRequirement::RefSnapshotIdMatch::ref: alloc::string::String +pub iceberg::TableRequirement::RefSnapshotIdMatch::snapshot_id: core::option::Option +pub iceberg::TableRequirement::UuidMatch +pub iceberg::TableRequirement::UuidMatch::uuid: uuid::Uuid +impl iceberg::TableRequirement +pub fn iceberg::TableRequirement::check(&self, metadata: core::option::Option<&iceberg::spec::TableMetadata>) -> iceberg::Result<()> +impl core::clone::Clone for iceberg::TableRequirement +pub fn iceberg::TableRequirement::clone(&self) -> iceberg::TableRequirement +impl core::cmp::PartialEq for iceberg::TableRequirement +pub fn iceberg::TableRequirement::eq(&self, other: &iceberg::TableRequirement) -> bool +impl core::fmt::Debug for iceberg::TableRequirement +pub fn iceberg::TableRequirement::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::TableRequirement +impl serde_core::ser::Serialize for iceberg::TableRequirement +pub fn iceberg::TableRequirement::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::TableRequirement +pub fn iceberg::TableRequirement::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::TableUpdate +pub iceberg::TableUpdate::AddEncryptionKey +pub iceberg::TableUpdate::AddEncryptionKey::encryption_key: iceberg::spec::EncryptedKey +pub iceberg::TableUpdate::AddSchema +pub iceberg::TableUpdate::AddSchema::schema: iceberg::spec::Schema +pub iceberg::TableUpdate::AddSnapshot +pub iceberg::TableUpdate::AddSnapshot::snapshot: iceberg::spec::Snapshot +pub iceberg::TableUpdate::AddSortOrder +pub iceberg::TableUpdate::AddSortOrder::sort_order: iceberg::spec::SortOrder +pub iceberg::TableUpdate::AddSpec +pub iceberg::TableUpdate::AddSpec::spec: iceberg::spec::UnboundPartitionSpec +pub iceberg::TableUpdate::AssignUuid +pub iceberg::TableUpdate::AssignUuid::uuid: uuid::Uuid +pub iceberg::TableUpdate::RemoveEncryptionKey +pub iceberg::TableUpdate::RemoveEncryptionKey::key_id: alloc::string::String +pub iceberg::TableUpdate::RemovePartitionSpecs +pub iceberg::TableUpdate::RemovePartitionSpecs::spec_ids: alloc::vec::Vec +pub iceberg::TableUpdate::RemovePartitionStatistics +pub iceberg::TableUpdate::RemovePartitionStatistics::snapshot_id: i64 +pub iceberg::TableUpdate::RemoveProperties +pub iceberg::TableUpdate::RemoveProperties::removals: alloc::vec::Vec +pub iceberg::TableUpdate::RemoveSchemas +pub iceberg::TableUpdate::RemoveSchemas::schema_ids: alloc::vec::Vec +pub iceberg::TableUpdate::RemoveSnapshotRef +pub iceberg::TableUpdate::RemoveSnapshotRef::ref_name: alloc::string::String +pub iceberg::TableUpdate::RemoveSnapshots +pub iceberg::TableUpdate::RemoveSnapshots::snapshot_ids: alloc::vec::Vec +pub iceberg::TableUpdate::RemoveStatistics +pub iceberg::TableUpdate::RemoveStatistics::snapshot_id: i64 +pub iceberg::TableUpdate::SetCurrentSchema +pub iceberg::TableUpdate::SetCurrentSchema::schema_id: i32 +pub iceberg::TableUpdate::SetDefaultSortOrder +pub iceberg::TableUpdate::SetDefaultSortOrder::sort_order_id: i64 +pub iceberg::TableUpdate::SetDefaultSpec +pub iceberg::TableUpdate::SetDefaultSpec::spec_id: i32 +pub iceberg::TableUpdate::SetLocation +pub iceberg::TableUpdate::SetLocation::location: alloc::string::String +pub iceberg::TableUpdate::SetPartitionStatistics +pub iceberg::TableUpdate::SetPartitionStatistics::partition_statistics: iceberg::spec::PartitionStatisticsFile +pub iceberg::TableUpdate::SetProperties +pub iceberg::TableUpdate::SetProperties::updates: std::collections::hash::map::HashMap +pub iceberg::TableUpdate::SetSnapshotRef +pub iceberg::TableUpdate::SetSnapshotRef::ref_name: alloc::string::String +pub iceberg::TableUpdate::SetSnapshotRef::reference: iceberg::spec::SnapshotReference +pub iceberg::TableUpdate::SetStatistics +pub iceberg::TableUpdate::SetStatistics::statistics: iceberg::spec::StatisticsFile +pub iceberg::TableUpdate::UpgradeFormatVersion +pub iceberg::TableUpdate::UpgradeFormatVersion::format_version: iceberg::spec::FormatVersion +impl iceberg::TableUpdate +pub fn iceberg::TableUpdate::apply(self, builder: iceberg::spec::TableMetadataBuilder) -> iceberg::Result +impl core::clone::Clone for iceberg::TableUpdate +pub fn iceberg::TableUpdate::clone(&self) -> iceberg::TableUpdate +impl core::cmp::PartialEq for iceberg::TableUpdate +pub fn iceberg::TableUpdate::eq(&self, other: &iceberg::TableUpdate) -> bool +impl core::fmt::Debug for iceberg::TableUpdate +pub fn iceberg::TableUpdate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::TableUpdate +impl serde_core::ser::Serialize for iceberg::TableUpdate +pub fn iceberg::TableUpdate::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::TableUpdate +pub fn iceberg::TableUpdate::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg::ViewUpdate +pub iceberg::ViewUpdate::AddSchema +pub iceberg::ViewUpdate::AddSchema::last_column_id: core::option::Option +pub iceberg::ViewUpdate::AddSchema::schema: iceberg::spec::Schema +pub iceberg::ViewUpdate::AddViewVersion +pub iceberg::ViewUpdate::AddViewVersion::view_version: iceberg::spec::ViewVersion +pub iceberg::ViewUpdate::AssignUuid +pub iceberg::ViewUpdate::AssignUuid::uuid: uuid::Uuid +pub iceberg::ViewUpdate::RemoveProperties +pub iceberg::ViewUpdate::RemoveProperties::removals: alloc::vec::Vec +pub iceberg::ViewUpdate::SetCurrentViewVersion +pub iceberg::ViewUpdate::SetCurrentViewVersion::view_version_id: i32 +pub iceberg::ViewUpdate::SetLocation +pub iceberg::ViewUpdate::SetLocation::location: alloc::string::String +pub iceberg::ViewUpdate::SetProperties +pub iceberg::ViewUpdate::SetProperties::updates: std::collections::hash::map::HashMap +pub iceberg::ViewUpdate::UpgradeFormatVersion +pub iceberg::ViewUpdate::UpgradeFormatVersion::format_version: iceberg::spec::ViewFormatVersion +impl core::clone::Clone for iceberg::ViewUpdate +pub fn iceberg::ViewUpdate::clone(&self) -> iceberg::ViewUpdate +impl core::cmp::PartialEq for iceberg::ViewUpdate +pub fn iceberg::ViewUpdate::eq(&self, other: &iceberg::ViewUpdate) -> bool +impl core::fmt::Debug for iceberg::ViewUpdate +pub fn iceberg::ViewUpdate::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::ViewUpdate +impl serde_core::ser::Serialize for iceberg::ViewUpdate +pub fn iceberg::ViewUpdate::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::ViewUpdate +pub fn iceberg::ViewUpdate::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::Error +impl iceberg::Error +pub fn iceberg::Error::backtrace(&self) -> &std::backtrace::Backtrace +pub fn iceberg::Error::kind(&self) -> iceberg::ErrorKind +pub fn iceberg::Error::message(&self) -> &str +pub fn iceberg::Error::new(kind: iceberg::ErrorKind, message: impl core::convert::Into) -> Self +pub fn iceberg::Error::retryable(&self) -> bool +pub fn iceberg::Error::with_context(self, key: &'static str, value: impl core::convert::Into) -> Self +pub fn iceberg::Error::with_retryable(self, retryable: bool) -> Self +pub fn iceberg::Error::with_source(self, src: impl core::convert::Into) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: apache_avro::error::Error) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: arrow_schema::error::ArrowError) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: chrono::format::ParseError) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: core::array::TryFromSliceError) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: core::num::error::ParseIntError) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: core::num::error::TryFromIntError) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: core::str::error::Utf8Error) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: futures_channel::mpsc::SendError) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: parquet::errors::ParquetError) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: reqwest::error::Error) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: serde_json::error::Error) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: std::io::error::Error) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: url::parser::ParseError) -> Self +impl core::convert::From for iceberg::Error +pub fn iceberg::Error::from(v: uuid::error::Error) -> Self +impl core::error::Error for iceberg::Error +pub fn iceberg::Error::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)> +impl core::fmt::Debug for iceberg::Error +pub fn iceberg::Error::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::Error +pub fn iceberg::Error::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::JoinHandle(_) +impl core::future::future::Future for iceberg::JoinHandle +pub type iceberg::JoinHandle::Output = core::result::Result +pub fn iceberg::JoinHandle::poll(self: core::pin::Pin<&mut Self>, cx: &mut core::task::wake::Context<'_>) -> core::task::poll::Poll +impl core::marker::Unpin for iceberg::JoinHandle +pub struct iceberg::MemoryCatalog +impl core::fmt::Debug for iceberg::memory::MemoryCatalog +pub fn iceberg::memory::MemoryCatalog::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::Catalog for iceberg::memory::MemoryCatalog +pub fn iceberg::memory::MemoryCatalog::create_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::create_table<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent, table_creation: iceberg::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::drop_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::drop_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::get_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::list_namespaces<'life0, 'life1, 'async_trait>(&'life0 self, maybe_parent: core::option::Option<&'life1 iceberg::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::list_tables<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::load_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::namespace_exists<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::purge_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::register_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::rename_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, src_table_ident: &'life1 iceberg::TableIdent, dst_table_ident: &'life2 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::memory::MemoryCatalog::table_exists<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::update_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub struct iceberg::MetadataLocation +impl iceberg::MetadataLocation +pub fn iceberg::MetadataLocation::compression_codec(&self) -> iceberg::compression::CompressionCodec +pub fn iceberg::MetadataLocation::try_new_with_metadata(metadata: &iceberg::spec::TableMetadata) -> iceberg::Result +pub fn iceberg::MetadataLocation::try_with_new_metadata(&self, new_metadata: &iceberg::spec::TableMetadata) -> iceberg::Result +pub fn iceberg::MetadataLocation::with_next_version(&self) -> Self +impl core::clone::Clone for iceberg::MetadataLocation +pub fn iceberg::MetadataLocation::clone(&self) -> iceberg::MetadataLocation +impl core::cmp::PartialEq for iceberg::MetadataLocation +pub fn iceberg::MetadataLocation::eq(&self, other: &iceberg::MetadataLocation) -> bool +impl core::fmt::Debug for iceberg::MetadataLocation +pub fn iceberg::MetadataLocation::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::MetadataLocation +pub fn iceberg::MetadataLocation::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::MetadataLocation +impl core::str::traits::FromStr for iceberg::MetadataLocation +pub type iceberg::MetadataLocation::Err = iceberg::Error +pub fn iceberg::MetadataLocation::from_str(s: &str) -> iceberg::Result +pub struct iceberg::Namespace +impl iceberg::Namespace +pub fn iceberg::Namespace::name(&self) -> &iceberg::NamespaceIdent +pub fn iceberg::Namespace::new(name: iceberg::NamespaceIdent) -> Self +pub fn iceberg::Namespace::properties(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::Namespace::with_properties(name: iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> Self +impl core::clone::Clone for iceberg::Namespace +pub fn iceberg::Namespace::clone(&self) -> iceberg::Namespace +impl core::cmp::Eq for iceberg::Namespace +impl core::cmp::PartialEq for iceberg::Namespace +pub fn iceberg::Namespace::eq(&self, other: &iceberg::Namespace) -> bool +impl core::fmt::Debug for iceberg::Namespace +pub fn iceberg::Namespace::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::Namespace +pub struct iceberg::NamespaceIdent(_) +impl iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::from_strs(iter: impl core::iter::traits::collect::IntoIterator) -> iceberg::Result +pub fn iceberg::NamespaceIdent::from_vec(names: alloc::vec::Vec) -> iceberg::Result +pub fn iceberg::NamespaceIdent::inner(self) -> alloc::vec::Vec +pub fn iceberg::NamespaceIdent::new(name: alloc::string::String) -> Self +pub fn iceberg::NamespaceIdent::parent(&self) -> core::option::Option +pub fn iceberg::NamespaceIdent::to_url_string(&self) -> alloc::string::String +impl core::clone::Clone for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::clone(&self) -> iceberg::NamespaceIdent +impl core::cmp::Eq for iceberg::NamespaceIdent +impl core::cmp::Ord for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::cmp(&self, other: &iceberg::NamespaceIdent) -> core::cmp::Ordering +impl core::cmp::PartialEq for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::eq(&self, other: &iceberg::NamespaceIdent) -> bool +impl core::cmp::PartialOrd for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::partial_cmp(&self, other: &iceberg::NamespaceIdent) -> core::option::Option +impl core::convert::AsRef> for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::as_ref(&self) -> &alloc::vec::Vec +impl core::fmt::Debug for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::NamespaceIdent +impl core::ops::deref::Deref for iceberg::NamespaceIdent +pub type iceberg::NamespaceIdent::Target = [alloc::string::String] +pub fn iceberg::NamespaceIdent::deref(&self) -> &Self::Target +impl serde_core::ser::Serialize for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::NamespaceIdent +pub fn iceberg::NamespaceIdent::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::Runtime +impl iceberg::Runtime +pub fn iceberg::Runtime::cpu(&self) -> &iceberg::RuntimeHandle +pub fn iceberg::Runtime::current() -> Self +pub fn iceberg::Runtime::io(&self) -> &iceberg::RuntimeHandle +pub fn iceberg::Runtime::new(runtime: &tokio::runtime::runtime::Runtime) -> Self +pub fn iceberg::Runtime::new_with_split(io_runtime: &tokio::runtime::runtime::Runtime, cpu_runtime: &tokio::runtime::runtime::Runtime) -> Self +pub fn iceberg::Runtime::try_current() -> iceberg::Result +impl core::clone::Clone for iceberg::Runtime +pub fn iceberg::Runtime::clone(&self) -> iceberg::Runtime +impl core::fmt::Debug for iceberg::Runtime +pub fn iceberg::Runtime::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::RuntimeHandle +impl iceberg::RuntimeHandle +pub fn iceberg::RuntimeHandle::spawn(&self, future: F) -> iceberg::JoinHandle<::Output> where F: core::future::future::Future + core::marker::Send + 'static, ::Output: core::marker::Send + 'static +pub fn iceberg::RuntimeHandle::spawn_blocking(&self, f: F) -> iceberg::JoinHandle where F: core::ops::function::FnOnce() -> T + core::marker::Send + 'static, T: core::marker::Send + 'static +impl core::clone::Clone for iceberg::RuntimeHandle +pub fn iceberg::RuntimeHandle::clone(&self) -> iceberg::RuntimeHandle +impl core::fmt::Debug for iceberg::RuntimeHandle +pub fn iceberg::RuntimeHandle::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg::SessionContext +impl iceberg::SessionContext +pub fn iceberg::SessionContext::credentials(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::SessionContext::empty() -> Self +pub fn iceberg::SessionContext::identity(&self) -> core::option::Option<&str> +pub fn iceberg::SessionContext::properties(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::SessionContext::session_id(&self) -> &str +impl core::clone::Clone for iceberg::SessionContext +pub fn iceberg::SessionContext::clone(&self) -> iceberg::SessionContext +impl core::fmt::Debug for iceberg::SessionContext +pub fn iceberg::SessionContext::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::SessionContext +pub fn iceberg::SessionContext::builder() -> SessionContextBuilder<((), (), (), ())> +pub struct iceberg::TableCommit +impl iceberg::TableCommit +pub fn iceberg::TableCommit::apply(self, table: iceberg::table::Table) -> iceberg::Result +pub fn iceberg::TableCommit::identifier(&self) -> &iceberg::TableIdent +pub fn iceberg::TableCommit::take_requirements(&mut self) -> alloc::vec::Vec +pub fn iceberg::TableCommit::take_updates(&mut self) -> alloc::vec::Vec +impl core::fmt::Debug for iceberg::TableCommit +pub fn iceberg::TableCommit::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::TableCommit +pub fn iceberg::TableCommit::builder() -> TableCommitBuilder<((), (), ())> +pub struct iceberg::TableCreation +pub iceberg::TableCreation::format_version: iceberg::spec::FormatVersion +pub iceberg::TableCreation::location: core::option::Option +pub iceberg::TableCreation::name: alloc::string::String +pub iceberg::TableCreation::partition_spec: core::option::Option +pub iceberg::TableCreation::properties: std::collections::hash::map::HashMap +pub iceberg::TableCreation::schema: iceberg::spec::Schema +pub iceberg::TableCreation::sort_order: core::option::Option +impl core::fmt::Debug for iceberg::TableCreation +pub fn iceberg::TableCreation::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::TableCreation +pub fn iceberg::TableCreation::builder() -> TableCreationBuilder<((), (), (), (), (), (), ())> +pub struct iceberg::TableIdent +pub iceberg::TableIdent::name: alloc::string::String +pub iceberg::TableIdent::namespace: iceberg::NamespaceIdent +impl iceberg::TableIdent +pub fn iceberg::TableIdent::from_strs(iter: impl core::iter::traits::collect::IntoIterator) -> iceberg::Result +pub fn iceberg::TableIdent::name(&self) -> &str +pub fn iceberg::TableIdent::namespace(&self) -> &iceberg::NamespaceIdent +pub fn iceberg::TableIdent::new(namespace: iceberg::NamespaceIdent, name: alloc::string::String) -> Self +impl core::clone::Clone for iceberg::TableIdent +pub fn iceberg::TableIdent::clone(&self) -> iceberg::TableIdent +impl core::cmp::Eq for iceberg::TableIdent +impl core::cmp::Ord for iceberg::TableIdent +pub fn iceberg::TableIdent::cmp(&self, other: &iceberg::TableIdent) -> core::cmp::Ordering +impl core::cmp::PartialEq for iceberg::TableIdent +pub fn iceberg::TableIdent::eq(&self, other: &iceberg::TableIdent) -> bool +impl core::cmp::PartialOrd for iceberg::TableIdent +pub fn iceberg::TableIdent::partial_cmp(&self, other: &iceberg::TableIdent) -> core::option::Option +impl core::fmt::Debug for iceberg::TableIdent +pub fn iceberg::TableIdent::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg::TableIdent +pub fn iceberg::TableIdent::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::hash::Hash for iceberg::TableIdent +pub fn iceberg::TableIdent::hash<__H: core::hash::Hasher>(&self, state: &mut __H) +impl core::marker::StructuralPartialEq for iceberg::TableIdent +impl serde_core::ser::Serialize for iceberg::TableIdent +pub fn iceberg::TableIdent::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::TableIdent +pub fn iceberg::TableIdent::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::ViewCreation +pub iceberg::ViewCreation::default_catalog: core::option::Option +pub iceberg::ViewCreation::default_namespace: iceberg::NamespaceIdent +pub iceberg::ViewCreation::location: alloc::string::String +pub iceberg::ViewCreation::name: alloc::string::String +pub iceberg::ViewCreation::properties: std::collections::hash::map::HashMap +pub iceberg::ViewCreation::representations: iceberg::spec::ViewRepresentations +pub iceberg::ViewCreation::schema: iceberg::spec::Schema +pub iceberg::ViewCreation::summary: std::collections::hash::map::HashMap +impl core::fmt::Debug for iceberg::ViewCreation +pub fn iceberg::ViewCreation::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::ViewCreation +pub fn iceberg::ViewCreation::builder() -> ViewCreationBuilder<((), (), (), (), (), (), (), ())> +pub trait iceberg::Catalog: core::fmt::Debug + core::marker::Sync + core::marker::Send +pub fn iceberg::Catalog::create_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::create_table<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::NamespaceIdent, creation: iceberg::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::drop_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::drop_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::get_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::list_namespaces<'life0, 'life1, 'async_trait>(&'life0 self, parent: core::option::Option<&'life1 iceberg::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::list_tables<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::load_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::namespace_exists<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::purge_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::register_table<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::rename_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, src: &'life1 iceberg::TableIdent, dest: &'life2 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::Catalog::table_exists<'life0, 'life1, 'async_trait>(&'life0 self, table: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::update_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace: &'life1 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::Catalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +impl iceberg::Catalog for iceberg::memory::MemoryCatalog +pub fn iceberg::memory::MemoryCatalog::create_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::create_table<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent, table_creation: iceberg::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::drop_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::drop_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::get_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::list_namespaces<'life0, 'life1, 'async_trait>(&'life0 self, maybe_parent: core::option::Option<&'life1 iceberg::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::list_tables<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::load_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::namespace_exists<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::purge_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::register_table<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::rename_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, src_table_ident: &'life1 iceberg::TableIdent, dst_table_ident: &'life2 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::memory::MemoryCatalog::table_exists<'life0, 'life1, 'async_trait>(&'life0 self, table_ident: &'life1 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::update_namespace<'life0, 'life1, 'async_trait>(&'life0 self, namespace_ident: &'life1 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg::memory::MemoryCatalog::update_table<'life0, 'async_trait>(&'life0 self, commit: iceberg::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub trait iceberg::CatalogBuilder: core::default::Default + core::fmt::Debug + core::marker::Send + core::marker::Sync +pub type iceberg::CatalogBuilder::C: iceberg::Catalog +pub fn iceberg::CatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg::CatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg::CatalogBuilder::with_runtime(self, runtime: iceberg::Runtime) -> Self +pub fn iceberg::CatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +impl iceberg::CatalogBuilder for iceberg::memory::MemoryCatalogBuilder +pub type iceberg::memory::MemoryCatalogBuilder::C = iceberg::memory::MemoryCatalog +pub fn iceberg::memory::MemoryCatalogBuilder::load(self, name: impl core::convert::Into, props: std::collections::hash::map::HashMap) -> impl core::future::future::Future> + core::marker::Send +pub fn iceberg::memory::MemoryCatalogBuilder::with_kms_client_factory(self, kms_client_factory: alloc::sync::Arc) -> Self +pub fn iceberg::memory::MemoryCatalogBuilder::with_runtime(self, runtime: iceberg::Runtime) -> Self +pub fn iceberg::memory::MemoryCatalogBuilder::with_storage_factory(self, storage_factory: alloc::sync::Arc) -> Self +pub trait iceberg::SessionCatalog: core::fmt::Debug + core::marker::Send + core::marker::Sync +pub fn iceberg::SessionCatalog::create_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, namespace: &'life2 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::create_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, namespace: &'life2 iceberg::NamespaceIdent, creation: iceberg::TableCreation) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::drop_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, namespace: &'life2 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::drop_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, table: &'life2 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::get_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, namespace: &'life2 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::list_namespaces<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, parent: core::option::Option<&'life2 iceberg::NamespaceIdent>) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::list_tables<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, namespace: &'life2 iceberg::NamespaceIdent) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::load_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, table: &'life2 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::namespace_exists<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, namespace: &'life2 iceberg::NamespaceIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::purge_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, table: &'life2 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::register_table<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, table: &'life2 iceberg::TableIdent, metadata_location: alloc::string::String) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::rename_table<'life0, 'life1, 'life2, 'life3, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, src: &'life2 iceberg::TableIdent, dest: &'life3 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait, 'life3: 'async_trait +pub fn iceberg::SessionCatalog::table_exists<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, table: &'life2 iceberg::TableIdent) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::update_namespace<'life0, 'life1, 'life2, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, namespace: &'life2 iceberg::NamespaceIdent, properties: std::collections::hash::map::HashMap) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait +pub fn iceberg::SessionCatalog::update_table<'life0, 'life1, 'async_trait>(&'life0 self, context: &'life1 iceberg::SessionContext, commit: iceberg::TableCommit) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub async fn iceberg::delete_removed_metadata_files(io: &iceberg::io::FileIO, base: &iceberg::spec::TableMetadata, new: &iceberg::spec::TableMetadata) +pub async fn iceberg::drop_table_data(table_info: &iceberg::table::Table) -> iceberg::Result<()> +pub type iceberg::Result = core::result::Result diff --git a/crates/iceberg/src/arrow/caching_delete_file_loader.rs b/crates/iceberg/src/arrow/caching_delete_file_loader.rs index 9cf605680e..ef6f9f1817 100644 --- a/crates/iceberg/src/arrow/caching_delete_file_loader.rs +++ b/crates/iceberg/src/arrow/caching_delete_file_loader.rs @@ -16,25 +16,29 @@ // under the License. use std::collections::{HashMap, HashSet}; -use std::ops::Not; use std::sync::Arc; use arrow_array::{Array, ArrayRef, Int64Array, StringArray, StructArray}; +use bytes::Bytes; use futures::{StreamExt, TryStreamExt}; use tokio::sync::oneshot::{Receiver, channel}; -use super::delete_filter::DeleteFilter; +use super::delete_filter::{DeleteFilter, PosDelLoadAction, PosDelLoadGuard}; use crate::arrow::delete_file_loader::BasicDeleteFileLoader; +use crate::arrow::scan_metrics::ScanMetrics; use crate::arrow::{arrow_primitive_to_literal, arrow_schema_to_schema}; use crate::delete_vector::DeleteVector; +use crate::encryption::{EncryptedInputFile, StandardKeyMetadata}; +use crate::error::invalid_data; use crate::expr::Predicate::AlwaysTrue; use crate::expr::{Predicate, Reference}; use crate::io::FileIO; +use crate::runtime::Runtime; use crate::scan::{ArrowRecordBatchStream, FileScanTaskDeleteFile}; use crate::spec::{ - DataContentType, Datum, ListType, MapType, NestedField, NestedFieldRef, PartnerAccessor, - PrimitiveType, Schema, SchemaRef, SchemaWithPartnerVisitor, StructType, Type, - visit_schema_with_partner, + DataContentType, DataFileFormat, Datum, ListType, MapType, NestedField, NestedFieldRef, + PartnerAccessor, PrimitiveType, Schema, SchemaRef, SchemaWithPartnerVisitor, StructType, Type, + VariantType, visit_schema_with_partner, }; use crate::{Error, ErrorKind, Result}; @@ -42,36 +46,76 @@ use crate::{Error, ErrorKind, Result}; pub(crate) struct CachingDeleteFileLoader { basic_delete_file_loader: BasicDeleteFileLoader, concurrency_limit_data_files: usize, + /// Shared filter state to allow caching loaded deletes across multiple + /// calls to `load_deletes` (e.g., across multiple file scan tasks). + delete_filter: DeleteFilter, + runtime: Runtime, } // Intermediate context during processing of a delete file task. enum DeleteFileContext { - // TODO: Delete Vector loader from Puffin files ExistingEqDel, - PosDels(ArrowRecordBatchStream), + ExistingPosDel, + PosDels { + guard: PosDelLoadGuard, + stream: ArrowRecordBatchStream, + }, FreshEqDel { batch_stream: ArrowRecordBatchStream, equality_ids: HashSet, sender: tokio::sync::oneshot::Sender, }, + // A V3 deletion vector: the raw deletion-vector-v1 blob bytes, the data file whose rows it + // deletes, and the manifest's expected cardinality. The blob is decoded and validated in the + // parse phase. + DelVec { + data_file_path: String, + blob: Bytes, + record_count: u64, + dv_path: String, + }, } // Final result of the processing of a delete file task before // results are fully merged into the DeleteFileManager's state enum ParsedDeleteFileContext { - DelVecs(HashMap), + DelVecs { + guard: PosDelLoadGuard, + results: HashMap, + }, + // A single deletion vector decoded from a Puffin blob, keyed by the data file it applies to. + DelVec { + data_file_path: String, + delete_vector: DeleteVector, + }, EqDel, + ExistingPosDel, } #[allow(unused_variables)] impl CachingDeleteFileLoader { - pub(crate) fn new(file_io: FileIO, concurrency_limit_data_files: usize) -> Self { + pub(crate) fn new( + file_io: FileIO, + concurrency_limit_data_files: usize, + runtime: Runtime, + ) -> Self { + let scan_metrics = ScanMetrics::new(); CachingDeleteFileLoader { - basic_delete_file_loader: BasicDeleteFileLoader::new(file_io), + basic_delete_file_loader: BasicDeleteFileLoader::new(file_io, scan_metrics), concurrency_limit_data_files, + delete_filter: DeleteFilter::new(runtime.clone()), + runtime, } } + pub(crate) fn with_scan_metrics(mut self, scan_metrics: ScanMetrics) -> Self { + self.basic_delete_file_loader = BasicDeleteFileLoader::new( + self.basic_delete_file_loader.file_io().clone(), + scan_metrics, + ); + self + } + /// Initiates loading of all deletes for all the specified tasks /// /// Returned future completes once all positional deletes and delete vectors @@ -89,12 +133,13 @@ impl CachingDeleteFileLoader { /// tasks from starting to load the same equality delete file. We spawn a task to load /// the EQ delete's record batch stream, convert it to a predicate, update the delete filter, /// and notify any task that was waiting for it. - /// * When this gets updated to add support for delete vectors, the load phase will return - /// a PuffinReader for them. + /// * For a V3 deletion vector, the load phase reads the blob's byte range directly from its + /// Puffin file (decrypting first if the entry carries key metadata), and the parse phase + /// decodes it into a single `DeleteVector`. /// * The parse phase parses each record batch stream according to its associated data type. /// The result of this is a map of data file paths to delete vectors for the positional - /// delete tasks (and in future for the delete vector tasks). For equality delete - /// file tasks, this results in an unbound Predicate. + /// delete tasks, or a single (data file path, delete vector) pair for a deletion vector + /// task. For equality delete file tasks, this results in an unbound Predicate. /// * The unbound Predicates resulting from equality deletes are sent to their associated oneshot /// channel to store them in the right place in the delete file managers state. /// * The results of all of these futures are awaited on in parallel with the specified @@ -115,10 +160,10 @@ impl CachingDeleteFileLoader { /// | /// | /// +-----------------------------+--------------------------+ - /// Pos Del Del Vec (Not yet Implemented) EQ Del + /// Pos Del Del Vec EQ Del /// | | | /// [parse pos del stream] [parse del vec puffin] [parse eq del] - /// HashMap HashMap (Predicate, Sender) + /// HashMap DeleteVector (Predicate, Sender) /// | | | /// | | [persist to state] /// | | () @@ -142,7 +187,6 @@ impl CachingDeleteFileLoader { schema: SchemaRef, ) -> Receiver> { let (tx, rx) = channel(); - let del_filter = DeleteFilter::default(); let stream_items = delete_file_entries .iter() @@ -150,22 +194,22 @@ impl CachingDeleteFileLoader { ( t.clone(), self.basic_delete_file_loader.clone(), - del_filter.clone(), + self.delete_filter.clone(), schema.clone(), ) }) .collect::>(); let task_stream = futures::stream::iter(stream_items); - let del_filter = del_filter.clone(); + let del_filter = self.delete_filter.clone(); let concurrency_limit_data_files = self.concurrency_limit_data_files; let basic_delete_file_loader = self.basic_delete_file_loader.clone(); - crate::runtime::spawn(async move { + self.runtime.io().spawn(async move { let result = async move { let mut del_filter = del_filter; let basic_delete_file_loader = basic_delete_file_loader.clone(); - let results: Vec = task_stream + let mut results_stream = task_stream .map(move |(task, file_io, del_filter, schema)| { let basic_delete_file_loader = basic_delete_file_loader.clone(); async move { @@ -181,15 +225,25 @@ impl CachingDeleteFileLoader { .map(move |ctx| { Ok(async { Self::parse_file_content_for_task(ctx.await?).await }) }) - .try_buffer_unordered(concurrency_limit_data_files) - .try_collect::>() - .await?; - - for item in results { - if let ParsedDeleteFileContext::DelVecs(hash_map) = item { - for (data_file_path, delete_vector) in hash_map.into_iter() { + .try_buffer_unordered(concurrency_limit_data_files); + + while let Some(item) = results_stream.next().await { + match item? { + ParsedDeleteFileContext::DelVecs { guard, results } => { + for (data_file_path, delete_vector) in results.into_iter() { + del_filter.upsert_delete_vector(data_file_path, delete_vector); + } + // Mark the positional delete file as fully loaded so waiters can proceed + guard.finish(); + } + ParsedDeleteFileContext::DelVec { + data_file_path, + delete_vector, + } => { del_filter.upsert_delete_vector(data_file_path, delete_vector); } + ParsedDeleteFileContext::EqDel + | ParsedDeleteFileContext::ExistingPosDel => {} } } @@ -209,31 +263,76 @@ impl CachingDeleteFileLoader { del_filter: DeleteFilter, schema: SchemaRef, ) -> Result { - match task.file_type { - DataContentType::PositionDeletes => Ok(DeleteFileContext::PosDels( - basic_delete_file_loader - .parquet_to_batch_stream(&task.file_path) - .await?, - )), + match task.file_type() { + DataContentType::PositionDeletes => { + // A V3 deletion vector arrives as a PositionDeletes entry whose deletes live in + // a Puffin blob, not in a positional-delete parquet file. + if task.file_format() == DataFileFormat::Puffin { + return Self::load_deletion_vector(task, basic_delete_file_loader).await; + } + + loop { + match del_filter.try_start_pos_del_load(task.file_path()) { + PosDelLoadAction::AlreadyLoaded => { + return Ok(DeleteFileContext::ExistingPosDel); + } + PosDelLoadAction::WaitFor(wait) => { + // Positional deletes are accessed synchronously by ArrowReader. + // We must wait here to ensure the data is ready before returning, + // otherwise ArrowReader might get an empty/partial result. + // A failed load returns its error here. A load dropped without + // one (cancelled) ends the wait too, so check again. + wait.wait().await?; + } + PosDelLoadAction::Load(guard) => { + let stream = match basic_delete_file_loader + .parquet_to_batch_stream( + task.file_path(), + task.file_size_in_bytes(), + task.key_metadata(), + ) + .await + { + Ok(stream) => stream, + Err(err) => { + guard.fail(&err); + return Err(err); + } + }; + return Ok(DeleteFileContext::PosDels { guard, stream }); + } + } + } + } DataContentType::EqualityDeletes => { - let Some(notify) = del_filter.try_start_eq_del_load(&task.file_path) else { + let Some(notify) = del_filter.try_start_eq_del_load(task.file_path()) else { return Ok(DeleteFileContext::ExistingEqDel); }; let (sender, receiver) = channel(); - del_filter.insert_equality_delete(&task.file_path, receiver); + del_filter.insert_equality_delete(task.file_path(), notify, receiver); + + // Per the Iceberg spec, evolve schema for equality deletes but only for the + // equality_ids columns, not all table columns. + let equality_ids_vec = task.equality_ids().unwrap().to_vec(); + let evolved_stream = BasicDeleteFileLoader::evolve_schema( + basic_delete_file_loader + .parquet_to_batch_stream( + task.file_path(), + task.file_size_in_bytes(), + task.key_metadata(), + ) + .await?, + schema, + &equality_ids_vec, + ) + .await?; Ok(DeleteFileContext::FreshEqDel { - batch_stream: BasicDeleteFileLoader::evolve_schema( - basic_delete_file_loader - .parquet_to_batch_stream(&task.file_path) - .await?, - schema, - ) - .await?, + batch_stream: evolved_stream, sender, - equality_ids: HashSet::from_iter(task.equality_ids.clone().unwrap()), + equality_ids: HashSet::from_iter(equality_ids_vec), }) } @@ -244,15 +343,135 @@ impl CachingDeleteFileLoader { } } + /// Extracts a deletion-vector task's metadata as the values needed by the read: + /// `(start, len, referenced data file path, expected cardinality)`. + /// + /// Both builder construction and deserialization validate the task's required fields. + /// The fallible extraction here converts the optional fields into values for the + /// byte-range read, preserving `DataInvalid` errors instead of unwrapping them. + fn validate_deletion_vector_task( + task: &FileScanTaskDeleteFile, + ) -> Result<(u64, u64, String, u64)> { + let content_offset = task.content_offset().ok_or_else(|| { + invalid_data!( + "deletion vector {} is missing content_offset", + task.file_path() + ) + })?; + let content_size = task.content_size_in_bytes().ok_or_else(|| { + invalid_data!( + "deletion vector {} is missing content_size_in_bytes", + task.file_path() + ) + })?; + let data_file_path = task + .referenced_data_file() + .map(ToOwned::to_owned) + .ok_or_else(|| { + invalid_data!( + "deletion vector {} is missing referenced_data_file", + task.file_path() + ) + })?; + let record_count = task.record_count().ok_or_else(|| { + invalid_data!( + "deletion vector {} is missing record_count", + task.file_path() + ) + })?; + + // content_offset / content_size_in_bytes are u64 on the task, and the + // manifest -> task conversion already rejected a negative, so there is + // nothing left to convert or re-check here. + Ok((content_offset, content_size, data_file_path, record_count)) + } + + /// Validates a decoded deletion vector's cardinality against the manifest entry's + /// `record_count`, mirroring Iceberg-Java's `BitmapPositionDeleteIndex.deserializeBitmap`. + fn validate_deletion_vector_cardinality( + delete_vector: &DeleteVector, + expected: u64, + dv_path: &str, + ) -> Result<()> { + let actual = delete_vector.len(); + if actual != expected { + return Err(invalid_data!( + "deletion vector {dv_path} decoded to {actual} positions, expected {expected} from record_count" + )); + } + Ok(()) + } + + /// Reads a V3 deletion vector blob directly from its Puffin file. + /// + /// The spec requires a delete manifest entry's `content_offset` / `content_size_in_bytes` to + /// match the blob's offset and length in the Puffin footer, so the blob is read by range + /// without parsing the footer. It is decoded into a [`DeleteVector`] in the parse phase. + /// + /// Decrypts the range read when `task.key_metadata` is set, the same way `ManifestReader` + /// decrypts a manifest file (`spec/manifest/reader.rs`): the coordinate space of + /// `content_offset` / `content_size_in_bytes` is the plaintext file, which is what + /// `EncryptedInputFile` reads over. + async fn load_deletion_vector( + task: &FileScanTaskDeleteFile, + basic_delete_file_loader: BasicDeleteFileLoader, + ) -> Result { + let (start, len, data_file_path, record_count) = Self::validate_deletion_vector_task(task)?; + + let input_file = basic_delete_file_loader + .file_io() + .new_input(task.file_path())?; + let blob = match task.key_metadata() { + Some(key_metadata) => { + let key_metadata = StandardKeyMetadata::decode(key_metadata)?; + EncryptedInputFile::new(input_file, key_metadata) + .reader() + .await? + .read(start..start + len) + .await? + } + None => input_file.reader().await?.read(start..start + len).await?, + }; + + Ok(DeleteFileContext::DelVec { + data_file_path, + blob, + record_count, + dv_path: task.file_path().to_string(), + }) + } + async fn parse_file_content_for_task( ctx: DeleteFileContext, ) -> Result { match ctx { DeleteFileContext::ExistingEqDel => Ok(ParsedDeleteFileContext::EqDel), - DeleteFileContext::PosDels(batch_stream) => { - let del_vecs = - Self::parse_positional_deletes_record_batch_stream(batch_stream).await?; - Ok(ParsedDeleteFileContext::DelVecs(del_vecs)) + DeleteFileContext::ExistingPosDel => Ok(ParsedDeleteFileContext::ExistingPosDel), + DeleteFileContext::PosDels { guard, stream } => { + match Self::parse_positional_deletes_record_batch_stream(stream).await { + Ok(del_vecs) => Ok(ParsedDeleteFileContext::DelVecs { + guard, + results: del_vecs, + }), + Err(err) => { + guard.fail(&err); + Err(err) + } + } + } + DeleteFileContext::DelVec { + data_file_path, + blob, + record_count, + dv_path, + } => { + let delete_vector = DeleteVector::deserialize(&blob)?; + Self::validate_deletion_vector_cardinality(&delete_vector, record_count, &dv_path)?; + + Ok(ParsedDeleteFileContext::DelVec { + data_file_path, + delete_vector, + }) } DeleteFileContext::FreshEqDel { sender, @@ -283,48 +502,104 @@ impl CachingDeleteFileLoader { mut stream: ArrowRecordBatchStream, ) -> Result> { let mut result: HashMap = HashMap::default(); + let mut run_positions: Vec = Vec::new(); while let Some(batch) = stream.next().await { + // run_positions is reused across batches, so the end-of-batch flush + // below must drain it before the next batch opens a new run. + debug_assert!(run_positions.is_empty()); + let batch = batch?; - let schema = batch.schema(); let columns = batch.columns(); let Some(file_paths) = columns[0].as_any().downcast_ref::() else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Could not downcast file paths array to StringArray", + return Err(invalid_data!( + "Could not downcast file paths array to StringArray" )); }; let Some(positions) = columns[1].as_any().downcast_ref::() else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Could not downcast positions array to Int64Array", + return Err(invalid_data!( + "Could not downcast positions array to Int64Array" )); }; + // Within a batch, positional deletes are sorted by (file_path, pos), + // so the rows for one data file form a contiguous run. Buffer each + // run and merge it with a single map lookup, allocating and hashing + // the key once per run instead of once per row. Grouping is per + // batch, not across the whole stream: a run never spans batch + // boundaries, so a path that also appears in another batch merges + // into its existing delete vector (order does not affect the result). + let mut run_path: Option<&str> = None; + for (file_path, pos) in file_paths.iter().zip(positions.iter()) { let (Some(file_path), Some(pos)) = (file_path, pos) else { - return Err(Error::new( - ErrorKind::DataInvalid, - "null values in delete file", - )); + return Err(invalid_data!("null values in delete file")); }; + if pos < 0 { + return Err(invalid_data!( + "negative position in delete file {file_path}: {pos}" + )); + } + + if run_path != Some(file_path) { + if let Some(prev_path) = run_path { + Self::merge_delete_positions(&mut result, prev_path, &run_positions); + run_positions.clear(); + } + + run_path = Some(file_path); + } - result - .entry(file_path.to_string()) - .or_default() - .insert(pos as u64); + run_positions.push(pos as u64); + } + + if let Some(prev_path) = run_path { + Self::merge_delete_positions(&mut result, prev_path, &run_positions); + run_positions.clear(); } } Ok(result) } + /// Marks every position in `positions` as deleted for `file_path`, merging + /// into any delete vector already recorded for that file. + fn merge_delete_positions( + result: &mut HashMap, + file_path: &str, + positions: &[u64], + ) { + // Callers only flush a run after pushing at least one position onto it. + debug_assert!(!positions.is_empty()); + + let delete_vector = result.entry(file_path.to_string()).or_default(); + // In spec-compliant files rows are sorted by (file_path, pos), so a run is + // usually ascending with no value already recorded, which `insert_positions` + // bulk-appends in one pass. Its precondition is stricter than the spec, + // though: it rejects duplicate positions (sorted means non-decreasing, so + // ties are compliant) as well as out-of-order rows and runs that overlap + // positions from an earlier batch. Fall back to per-position inserts in + // those cases; `insert` is idempotent, so re-inserting any prefix the failed + // append already added is harmless. + if let Err(err) = delete_vector.insert_positions(positions) { + tracing::debug!( + file_path, + run_len = positions.len(), + error = %err, + "positional delete run fell back to per-position insert" + ); + for &pos in positions { + delete_vector.insert(pos); + } + } + } + async fn parse_equality_deletes_record_batch_stream( mut stream: ArrowRecordBatchStream, equality_ids: HashSet, ) -> Result { - let mut result_predicate = AlwaysTrue; + let mut row_predicates = Vec::new(); let mut batch_schema_iceberg: Option = None; let accessor = EqDelRecordBatchPartnerAccessor; @@ -354,24 +629,50 @@ impl CachingDeleteFileLoader { continue; } - // Process the collected columns in lockstep + // Iceberg spec (Equality Delete Files): a null data value never equals a non-null + // delete value, so a row with a null equality column must be kept. Build the keep + // predicate as `col IS NULL OR col != v` (`col IS NOT NULL` for a null delete value); + // a bare `col != v` drops nulls. #[allow(clippy::len_zero)] while datum_columns_with_names[0].0.len() > 0 { - let mut row_predicate = AlwaysTrue; + let mut row_keep_predicate = Predicate::AlwaysFalse; for &mut (ref mut column, ref field_name) in &mut datum_columns_with_names { if let Some(item) = column.next() { - let cell_predicate = if let Some(datum) = item? { - Reference::new(field_name.clone()).equal_to(datum.clone()) + let reference = Reference::new(field_name.clone()); + let cell_keep_predicate = if let Some(datum) = item? { + reference + .clone() + .is_null() + .or(reference.not_equal_to(datum.clone())) } else { - Reference::new(field_name.clone()).is_null() + reference.is_not_null() }; - row_predicate = row_predicate.and(cell_predicate) + row_keep_predicate = row_keep_predicate.or(cell_keep_predicate); } } - result_predicate = result_predicate.and(row_predicate.not()); + row_predicates.push(row_keep_predicate); + } + } + + // All row predicates are combined to a single predicate by creating a balanced binary tree. + // Using a simple fold would result in a deeply nested predicate that can cause a stack overflow. + while row_predicates.len() > 1 { + let mut next_level = Vec::with_capacity(row_predicates.len().div_ceil(2)); + let mut iter = row_predicates.into_iter(); + while let Some(p1) = iter.next() { + if let Some(p2) = iter.next() { + next_level.push(p1.and(p2)); + } else { + next_level.push(p1); + } } + row_predicates = next_level; + } + + match row_predicates.pop() { + Some(p) => Ok(p), + None => Ok(AlwaysTrue), } - Ok(result_predicate.rewrite_not()) } } @@ -474,6 +775,10 @@ impl SchemaWithPartnerVisitor for EqDelColumnProcessor<'_> { fn primitive(&mut self, _primitive: &PrimitiveType, _partner: &ArrayRef) -> Result<()> { Ok(()) } + + fn variant(&mut self, _v: &VariantType, _partner: &ArrayRef) -> Result<()> { + Ok(()) + } } struct EqDelRecordBatchPartnerAccessor; @@ -536,7 +841,10 @@ mod tests { use std::fs::File; use std::sync::Arc; - use arrow_array::{ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray, StructArray}; + use arrow_array::cast::AsArray; + use arrow_array::{ + ArrayRef, BinaryArray, Int32Array, Int64Array, RecordBatch, StringArray, StructArray, + }; use arrow_schema::{DataType, Field, Fields}; use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; use parquet::basic::Compression; @@ -545,22 +853,50 @@ mod tests { use super::*; use crate::arrow::delete_filter::tests::setup; + use crate::scan::FileScanTaskDeleteFile; + use crate::spec::{DataContentType, DataFileFormat, Schema}; + use crate::test_utils::encode_dv_blob; + + #[test] + fn test_deserialized_deletion_vector_rejects_missing_fields() { + // A deletion vector that omits its required coordinates fails validation + // during deserialization (the try_from mirror routes through `build()`), + // before the loader ever sees the task. + let err = serde_json::from_value::(serde_json::json!({ + "file_path": "dv.puffin", + "file_size_in_bytes": 100, + "file_type": "PositionDeletes", + "file_format": "Puffin", + "partition_spec_id": 0 + })) + .unwrap_err(); + + assert!( + err.to_string().contains("missing referenced_data_file"), + "expected a missing-field validation error, got `{err}`" + ); + } #[tokio::test] async fn test_delete_file_loader_parse_equality_deletes() { let tmp_dir = TempDir::new().unwrap(); let table_location = tmp_dir.path().as_os_str().to_str().unwrap(); - let file_io = FileIO::from_path(table_location).unwrap().build().unwrap(); + let file_io = FileIO::new_with_fs(); let eq_delete_file_path = setup_write_equality_delete_file_1(table_location); - let basic_delete_file_loader = BasicDeleteFileLoader::new(file_io.clone()); + let basic_delete_file_loader = + BasicDeleteFileLoader::new(file_io.clone(), ScanMetrics::new()); let record_batch_stream = basic_delete_file_loader - .parquet_to_batch_stream(&eq_delete_file_path) + .parquet_to_batch_stream( + &eq_delete_file_path, + std::fs::metadata(&eq_delete_file_path).unwrap().len(), + None, + ) .await .expect("could not get batch stream"); - let eq_ids = HashSet::from_iter(vec![2, 3, 4, 6]); + let eq_ids = HashSet::from_iter(vec![2, 3, 4, 6, 8]); let parsed_eq_delete = CachingDeleteFileLoader::parse_equality_deletes_record_batch_stream( record_batch_stream, @@ -568,16 +904,131 @@ mod tests { ) .await .expect("error parsing batch stream"); - println!("{}", parsed_eq_delete); - let expected = "((((y != 1) OR (z != 100)) OR (a != \"HELP\")) OR (sa != 4)) AND ((((y != 2) OR (z IS NOT NULL)) OR (a IS NOT NULL)) OR (sa != 5))".to_string(); + let expected = "((((((y IS NULL) OR (y != 1)) OR ((z IS NULL) OR (z != 100))) OR ((a IS NULL) OR (a != \"HELP\"))) OR ((sa IS NULL) OR (sa != 4))) OR ((b IS NULL) OR (b != 62696E6172795F64617461))) AND ((((((y IS NULL) OR (y != 2)) OR (z IS NOT NULL)) OR (a IS NOT NULL)) OR ((sa IS NULL) OR (sa != 5))) OR (b IS NOT NULL))".to_string(); assert_eq!(parsed_eq_delete.to_string(), expected); } + // An equality delete keyed on a nullable column must not delete rows whose value in that + // column is null: per the Iceberg spec (Equality Delete Files), a null matches only a null + // delete value. Mirrors Iceberg-Java's + // TestSparkReaderDeletes.testEqualityDeleteWithSchemaEvolution. + #[tokio::test] + async fn test_equality_delete_predicate_preserves_null_rows() { + let schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "status", + DataType::Utf8, + true, + "3", + )])); + let batch = + RecordBatch::try_new(schema, vec![ + Arc::new(StringArray::from(vec![Some("INACTIVE")])) as ArrayRef, + ]) + .unwrap(); + let stream: ArrowRecordBatchStream = futures::stream::iter(vec![Ok(batch)]).boxed(); + + let predicate = CachingDeleteFileLoader::parse_equality_deletes_record_batch_stream( + stream, + HashSet::from_iter(vec![3]), + ) + .await + .expect("error parsing equality delete stream"); + + assert_eq!( + predicate.to_string(), + "(status IS NULL) OR (status != \"INACTIVE\")" + ); + } + + // A delete row with a null value in the column matches only rows whose value is null (Iceberg + // spec, Equality Delete Files), so the keep predicate is `col IS NOT NULL`. + #[tokio::test] + async fn test_equality_delete_predicate_matches_null_delete_value() { + let schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "status", + DataType::Utf8, + true, + "3", + )])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(StringArray::from(vec![ + None as Option<&str>, + ])) as ArrayRef]) + .unwrap(); + let stream: ArrowRecordBatchStream = futures::stream::iter(vec![Ok(batch)]).boxed(); + + let predicate = CachingDeleteFileLoader::parse_equality_deletes_record_batch_stream( + stream, + HashSet::from_iter(vec![3]), + ) + .await + .expect("error parsing equality delete stream"); + + assert_eq!(predicate.to_string(), "status IS NOT NULL"); + } + + // A delete row with several equality columns keeps a data row that differs in any one of them, + // so the per-column keep predicates are OR-ed. + #[tokio::test] + async fn test_equality_delete_predicate_multiple_columns() { + let schema = Arc::new(arrow_schema::Schema::new(vec![ + simple_field("id", DataType::Int64, true, "1"), + simple_field("status", DataType::Utf8, true, "3"), + ])); + let batch = RecordBatch::try_new(schema, vec![ + Arc::new(Int64Array::from(vec![1])) as ArrayRef, + Arc::new(StringArray::from(vec![Some("X")])) as ArrayRef, + ]) + .unwrap(); + let stream: ArrowRecordBatchStream = futures::stream::iter(vec![Ok(batch)]).boxed(); + + let predicate = CachingDeleteFileLoader::parse_equality_deletes_record_batch_stream( + stream, + HashSet::from_iter(vec![1, 3]), + ) + .await + .expect("error parsing equality delete stream"); + + assert_eq!( + predicate.to_string(), + "((id IS NULL) OR (id != 1)) OR ((status IS NULL) OR (status != \"X\"))" + ); + } + + // A data row is kept only if it matches none of the delete rows, so the per-row keep + // predicates are AND-ed. + #[tokio::test] + async fn test_equality_delete_predicate_multiple_delete_rows() { + let schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "status", + DataType::Utf8, + true, + "3", + )])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(StringArray::from(vec![ + Some("A"), + Some("B"), + ])) as ArrayRef]) + .unwrap(); + let stream: ArrowRecordBatchStream = futures::stream::iter(vec![Ok(batch)]).boxed(); + + let predicate = CachingDeleteFileLoader::parse_equality_deletes_record_batch_stream( + stream, + HashSet::from_iter(vec![3]), + ) + .await + .expect("error parsing equality delete stream"); + + assert_eq!( + predicate.to_string(), + "((status IS NULL) OR (status != \"A\")) AND ((status IS NULL) OR (status != \"B\"))" + ); + } + /// Create a simple field with metadata. fn simple_field(name: &str, ty: DataType, nullable: bool, value: &str) -> Field { - arrow_schema::Field::new(name, ty, nullable).with_metadata(HashMap::from([( + Field::new(name, ty, nullable).with_metadata(HashMap::from([( PARQUET_FIELD_ID_META_KEY.to_string(), value.to_string(), )])) @@ -604,6 +1055,9 @@ mod tests { ), ])); + let col_b_vals = vec![Some(&b"binary_data"[..]), None]; + let col_b = Arc::new(BinaryArray::from(col_b_vals)) as ArrayRef; + let equality_delete_schema = { let struct_field = DataType::Struct(Fields::from(vec![ simple_field("sa", DataType::Int32, false, "6"), @@ -611,22 +1065,26 @@ mod tests { ])); let fields = vec![ - Field::new("y", arrow_schema::DataType::Int64, true).with_metadata(HashMap::from( - [(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())], - )), - Field::new("z", arrow_schema::DataType::Int64, true).with_metadata(HashMap::from( - [(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())], - )), - Field::new("a", arrow_schema::DataType::Utf8, true).with_metadata(HashMap::from([ - (PARQUET_FIELD_ID_META_KEY.to_string(), "4".to_string()), - ])), + Field::new("y", DataType::Int64, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + Field::new("z", DataType::Int64, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "3".to_string(), + )])), + Field::new("a", DataType::Utf8, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "4".to_string(), + )])), simple_field("s", struct_field, false, "5"), + simple_field("b", DataType::Binary, true, "8"), ]; Arc::new(arrow_schema::Schema::new(fields)) }; let equality_deletes_to_write = RecordBatch::try_new(equality_delete_schema.clone(), vec![ - col_y, col_z, col_a, col_s, + col_y, col_z, col_a, col_s, col_b, ]) .unwrap(); @@ -659,17 +1117,18 @@ mod tests { async fn test_caching_delete_file_loader_load_deletes() { let tmp_dir = TempDir::new().unwrap(); let table_location = tmp_dir.path(); - let file_io = FileIO::from_path(table_location.as_os_str().to_str().unwrap()) - .unwrap() - .build() - .unwrap(); + let file_io = FileIO::new_with_fs(); - let delete_file_loader = CachingDeleteFileLoader::new(file_io.clone(), 10); + let delete_file_loader = + CachingDeleteFileLoader::new(file_io.clone(), 10, Runtime::current()); let file_scan_tasks = setup(table_location); let delete_filter = delete_file_loader - .load_deletes(&file_scan_tasks[0].deletes, file_scan_tasks[0].schema_ref()) + .load_deletes( + file_scan_tasks[0].deletes(), + file_scan_tasks[0].schema_ref(), + ) .await .unwrap() .unwrap(); @@ -686,4 +1145,1038 @@ mod tests { let result = delete_filter.get_delete_vector(&file_scan_tasks[1]); assert!(result.is_none()); // no pos dels for file 3 } + + #[tokio::test] + async fn test_parse_positional_deletes_rejects_negative_positions() { + let schema = crate::arrow::delete_filter::tests::create_pos_del_schema(); + let file_path_col = Arc::new(StringArray::from_iter_values(vec!["data.parquet"])); + let pos_col = Arc::new(Int64Array::from_iter_values(vec![-1i64])); + let batch = RecordBatch::try_new(schema, vec![file_path_col, pos_col]).unwrap(); + let stream = futures::stream::iter(vec![Ok(batch)]).boxed(); + + let err = CachingDeleteFileLoader::parse_positional_deletes_record_batch_stream(stream) + .await + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("negative position")); + } + + fn sorted_positions(dv: &DeleteVector) -> Vec { + let mut positions: Vec = dv.iter().collect(); + positions.sort_unstable(); + positions + } + + /// Spec-compliant input: rows sorted by (file_path, pos). Exercises the + /// common shape: multi-position runs, several files in one batch, and a path + /// "b" whose positions span two batches, so its two runs merge into one + /// delete vector. + #[tokio::test] + async fn test_parse_positional_deletes_merges_sorted_runs() { + let schema = crate::arrow::delete_filter::tests::create_pos_del_schema(); + + let batch1 = RecordBatch::try_new(schema.clone(), vec![ + Arc::new(StringArray::from_iter_values(vec!["a", "a", "a", "b"])), + Arc::new(Int64Array::from_iter_values(vec![1i64, 3, 5, 2])), + ]) + .unwrap(); + let batch2 = RecordBatch::try_new(schema, vec![ + Arc::new(StringArray::from_iter_values(vec!["b", "c"])), + Arc::new(Int64Array::from_iter_values(vec![4i64, 0])), + ]) + .unwrap(); + let stream = futures::stream::iter(vec![Ok(batch1), Ok(batch2)]).boxed(); + + let result = CachingDeleteFileLoader::parse_positional_deletes_record_batch_stream(stream) + .await + .unwrap(); + + assert_eq!(result.len(), 3); + assert_eq!(sorted_positions(&result["a"]), vec![1, 3, 5]); + assert_eq!(sorted_positions(&result["b"]), vec![2, 4]); + assert_eq!(sorted_positions(&result["c"]), vec![0]); + } + + /// Deliberately unsorted input. The spec requires position delete rows to be + /// sorted by (file_path, pos), but the reader must not depend on it: run + /// buffering only groups *contiguous* rows, so a path split into + /// non-contiguous runs (here "a" before and after "b") must still merge into + /// a single delete vector rather than silently dropping positions. + #[tokio::test] + async fn test_parse_positional_deletes_merges_spec_noncompliant_unsorted_runs() { + let schema = crate::arrow::delete_filter::tests::create_pos_del_schema(); + + let batch = RecordBatch::try_new(schema, vec![ + Arc::new(StringArray::from_iter_values(vec!["a", "b", "a"])), + Arc::new(Int64Array::from_iter_values(vec![3i64, 2, 1])), + ]) + .unwrap(); + let stream = futures::stream::iter(vec![Ok(batch)]).boxed(); + + let result = CachingDeleteFileLoader::parse_positional_deletes_record_batch_stream(stream) + .await + .unwrap(); + + assert_eq!(result.len(), 2); + assert_eq!(sorted_positions(&result["a"]), vec![1, 3]); + assert_eq!(sorted_positions(&result["b"]), vec![2]); + } + + /// Cross-batch overlap: batch2 carries lower positions for "a" than batch1 + /// already recorded, so the run fails the append precondition (every value + /// must exceed all recorded) and drops to the per-position fallback. All + /// positions must still merge. + #[tokio::test] + async fn test_parse_positional_deletes_merges_overlapping_cross_batch_runs() { + let schema = crate::arrow::delete_filter::tests::create_pos_del_schema(); + + let batch1 = RecordBatch::try_new(schema.clone(), vec![ + Arc::new(StringArray::from_iter_values(vec!["a", "a"])), + Arc::new(Int64Array::from_iter_values(vec![5i64, 10])), + ]) + .unwrap(); + let batch2 = RecordBatch::try_new(schema, vec![ + Arc::new(StringArray::from_iter_values(vec!["a", "a"])), + Arc::new(Int64Array::from_iter_values(vec![3i64, 7])), + ]) + .unwrap(); + let stream = futures::stream::iter(vec![Ok(batch1), Ok(batch2)]).boxed(); + + let result = CachingDeleteFileLoader::parse_positional_deletes_record_batch_stream(stream) + .await + .unwrap(); + + assert_eq!(result.len(), 1); + assert_eq!(sorted_positions(&result["a"]), vec![3, 5, 7, 10]); + } + + /// Spec-compliant duplicate positions: sorted by (file_path, pos) is only + /// non-decreasing, so a repeated position is valid input. It fails the + /// strictly-ascending append precondition and exercises the fallback, whose + /// idempotent inserts collapse the duplicate. + #[tokio::test] + async fn test_parse_positional_deletes_merges_duplicate_positions() { + let schema = crate::arrow::delete_filter::tests::create_pos_del_schema(); + + let batch = RecordBatch::try_new(schema, vec![ + Arc::new(StringArray::from_iter_values(vec!["a", "a", "a"])), + Arc::new(Int64Array::from_iter_values(vec![3i64, 3, 7])), + ]) + .unwrap(); + let stream = futures::stream::iter(vec![Ok(batch)]).boxed(); + + let result = CachingDeleteFileLoader::parse_positional_deletes_record_batch_stream(stream) + .await + .unwrap(); + + assert_eq!(result.len(), 1); + assert_eq!(sorted_positions(&result["a"]), vec![3, 7]); + } + + /// Verifies that evolve_schema on partial-schema equality deletes works correctly + /// when only equality_ids columns are evolved, not all table columns. + /// + /// Per the [Iceberg spec](https://iceberg.apache.org/spec/#equality-delete-files), + /// equality delete files can contain only a subset of columns. + #[tokio::test] + async fn test_partial_schema_equality_deletes_evolve_succeeds() { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().as_os_str().to_str().unwrap(); + + // Create table schema with REQUIRED fields + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + // Write equality delete file with PARTIAL schema (only 'data' column) + let delete_file_path = { + let data_vals = vec!["a", "d", "g"]; + let data_col = Arc::new(StringArray::from(data_vals)) as ArrayRef; + + let delete_schema = Arc::new(arrow_schema::Schema::new(vec![simple_field( + "data", + DataType::Utf8, + false, + "2", // field ID + )])); + + let delete_batch = RecordBatch::try_new(delete_schema.clone(), vec![data_col]).unwrap(); + + let path = format!("{}/partial-eq-deletes.parquet", &table_location); + let file = File::create(&path).unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let mut writer = + ArrowWriter::try_new(file, delete_batch.schema(), Some(props)).unwrap(); + writer.write(&delete_batch).expect("Writing batch"); + writer.close().unwrap(); + path + }; + + let file_io = FileIO::new_with_fs(); + let basic_delete_file_loader = + BasicDeleteFileLoader::new(file_io.clone(), ScanMetrics::new()); + + let batch_stream = basic_delete_file_loader + .parquet_to_batch_stream( + &delete_file_path, + std::fs::metadata(&delete_file_path).unwrap().len(), + None, + ) + .await + .unwrap(); + + // Only evolve the equality_ids columns (field 2), not all table columns + let equality_ids = vec![2]; + let evolved_stream = + BasicDeleteFileLoader::evolve_schema(batch_stream, table_schema, &equality_ids) + .await + .unwrap(); + + let result = evolved_stream.try_collect::>().await; + + assert!( + result.is_ok(), + "Expected success when evolving only equality_ids columns, got error: {:?}", + result.err() + ); + + let batches = result.unwrap(); + assert_eq!(batches.len(), 1); + + let batch = &batches[0]; + assert_eq!(batch.num_rows(), 3); + assert_eq!(batch.num_columns(), 1); // Only 'data' column + + // Verify the actual values are preserved after schema evolution + let data_col = batch.column(0).as_string::(); + assert_eq!(data_col.value(0), "a"); + assert_eq!(data_col.value(1), "d"); + assert_eq!(data_col.value(2), "g"); + } + + /// Test loading a FileScanTask with BOTH positional and equality deletes. + /// Verifies the fix for the inverted condition that caused "Missing predicate for equality delete file" errors. + #[tokio::test] + async fn test_load_deletes_with_mixed_types() { + use crate::scan::FileScanTask; + use crate::spec::{DataFileFormat, Schema}; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path(); + let file_io = FileIO::new_with_fs(); + + // Create the data file schema + let data_file_schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(2, "y", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(3, "z", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + + // Write positional delete file + let positional_delete_schema = crate::arrow::delete_filter::tests::create_pos_del_schema(); + let file_path_values = + vec![format!("{}/data-1.parquet", table_location.to_str().unwrap()); 4]; + let file_path_col = Arc::new(StringArray::from_iter_values(&file_path_values)); + let pos_col = Arc::new(Int64Array::from_iter_values(vec![0i64, 1, 2, 3])); + + let positional_deletes_to_write = + RecordBatch::try_new(positional_delete_schema.clone(), vec![ + file_path_col, + pos_col, + ]) + .unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let pos_del_path = format!("{}/pos-del-mixed.parquet", table_location.to_str().unwrap()); + let file = File::create(&pos_del_path).unwrap(); + let mut writer = ArrowWriter::try_new( + file, + positional_deletes_to_write.schema(), + Some(props.clone()), + ) + .unwrap(); + writer.write(&positional_deletes_to_write).unwrap(); + writer.close().unwrap(); + + // Write equality delete file + let eq_delete_path = setup_write_equality_delete_file_1(table_location.to_str().unwrap()); + + // Create FileScanTask with BOTH positional and equality deletes + let pos_del = FileScanTaskDeleteFile::builder() + .with_file_path(pos_del_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&pos_del_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(); + + let eq_del = FileScanTaskDeleteFile::builder() + .with_file_path(eq_delete_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&eq_delete_path).unwrap().len()) + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_equality_ids(Some(vec![2, 3])) // Only use field IDs that exist in both schemas + .build() + .unwrap(); + + let file_scan_task = FileScanTask::builder() + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(0) + .with_data_file_path(format!( + "{}/data-1.parquet", + table_location.to_str().unwrap() + )) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(data_file_schema.clone()) + .with_project_field_ids(vec![2, 3]) + .with_deletes(vec![pos_del, eq_del]) + .with_case_sensitive(false) + .build() + .unwrap(); + + // Load the deletes - should handle both types without error + let delete_file_loader = + CachingDeleteFileLoader::new(file_io.clone(), 10, Runtime::current()); + let delete_filter = delete_file_loader + .load_deletes(file_scan_task.deletes(), file_scan_task.schema_ref()) + .await + .unwrap() + .unwrap(); + + // Verify both delete types can be processed together + let result = delete_filter + .build_equality_delete_predicate(&file_scan_task) + .await; + assert!( + result.is_ok(), + "Failed to build equality delete predicate: {:?}", + result.err() + ); + } + + #[tokio::test] + async fn test_large_equality_delete_batch_stack_overflow() { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().as_os_str().to_str().unwrap(); + let file_io = FileIO::new_with_fs(); + + // Create a large batch of equality deletes + let num_rows = 20_000; + let col_y_vals: Vec = (0..num_rows).collect(); + let col_y = Arc::new(Int64Array::from(col_y_vals)) as ArrayRef; + + let schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new("y", DataType::Int64, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + + let record_batch = RecordBatch::try_new(schema.clone(), vec![col_y]).unwrap(); + + // Write to file + let path = format!("{}/large-eq-deletes.parquet", &table_location); + let file = File::create(&path).unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let mut writer = ArrowWriter::try_new(file, schema, Some(props)).unwrap(); + writer.write(&record_batch).unwrap(); + writer.close().unwrap(); + + let basic_delete_file_loader = + BasicDeleteFileLoader::new(file_io.clone(), ScanMetrics::new()); + let record_batch_stream = basic_delete_file_loader + .parquet_to_batch_stream(&path, std::fs::metadata(&path).unwrap().len(), None) + .await + .expect("could not get batch stream"); + + let eq_ids = HashSet::from_iter(vec![2]); + + let result = CachingDeleteFileLoader::parse_equality_deletes_record_batch_stream( + record_batch_stream, + eq_ids, + ) + .await; + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_caching_delete_file_loader_caches_results() { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path(); + let file_io = FileIO::new_with_fs(); + + let delete_file_loader = + CachingDeleteFileLoader::new(file_io.clone(), 10, Runtime::current()); + + let file_scan_tasks = setup(table_location); + + // Load deletes for the first time + let delete_filter_1 = delete_file_loader + .load_deletes( + file_scan_tasks[0].deletes(), + file_scan_tasks[0].schema_ref(), + ) + .await + .unwrap() + .unwrap(); + + // Load deletes for the second time (same task/files) + let delete_filter_2 = delete_file_loader + .load_deletes( + file_scan_tasks[0].deletes(), + file_scan_tasks[0].schema_ref(), + ) + .await + .unwrap() + .unwrap(); + + let dv1 = delete_filter_1 + .get_delete_vector(&file_scan_tasks[0]) + .unwrap(); + let dv2 = delete_filter_2 + .get_delete_vector(&file_scan_tasks[0]) + .unwrap(); + + // Verify that the delete vectors point to the same memory location, + // confirming that the second load reused the result from the first. + assert!(Arc::ptr_eq(&dv1, &dv2)); + } + + fn dv_task( + dv_path: String, + file_size: u64, + data_file_path: String, + content_offset: u64, + content_size: u64, + record_count: u64, + key_metadata: Option>, + ) -> FileScanTaskDeleteFile { + FileScanTaskDeleteFile::builder() + .with_file_path(dv_path) + .with_file_size_in_bytes(file_size) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some(data_file_path)) + .with_content_offset(Some(content_offset)) + .with_content_size_in_bytes(Some(content_size)) + .with_record_count(Some(record_count)) + .with_key_metadata(key_metadata) + .build() + .expect("deletion vector task should be valid") + } + + #[tokio::test] + async fn test_load_deletes_applies_deletion_vector() { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let blob = encode_dv_blob([0u64, 1, 5]); + + // Embed the blob in a Puffin-like file behind leading bytes so content_offset is + // non-zero, then let the loader read it back by range. + let content_offset = 12u64; + let content_size = blob.len() as u64; + let mut file_bytes = vec![0u8; content_offset as usize]; + file_bytes.extend_from_slice(&blob); + file_bytes.extend_from_slice(&[0u8; 8]); + let dv_path = format!("{table_location}/deletes.puffin"); + std::fs::write(&dv_path, &file_bytes).unwrap(); + + let data_file_path = format!("{table_location}/data-1.parquet"); + let dv = dv_task( + dv_path.clone(), + std::fs::metadata(&dv_path).unwrap().len(), + data_file_path.clone(), + content_offset, + content_size, + 3, + None, + ); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + + let loader = CachingDeleteFileLoader::new(file_io, 10, Runtime::current()); + let delete_filter = loader.load_deletes(&[dv], schema).await.unwrap().unwrap(); + + let delete_vector = delete_filter + .get_delete_vector_for_path(&data_file_path) + .expect("a delete vector should be indexed for the referenced data file"); + let mut positions: Vec = delete_vector.lock().unwrap().iter().collect(); + positions.sort_unstable(); + assert_eq!(positions, vec![0, 1, 5]); + } + + #[tokio::test] + async fn test_load_deletes_decrypts_deletion_vector() { + use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata}; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + + let blob = encode_dv_blob([2u64, 4]); + let plaintext_size = blob.len() as u64; + let dv_path = format!("{table_location}/deletes.puffin"); + let output = EncryptedOutputFile::new(file_io.new_output(&dv_path).unwrap(), key_metadata); + let file_metadata = output.write(Bytes::from(blob)).await.unwrap(); + let encoded_key_metadata = output + .key_metadata_with_saved_file_metadata(&file_metadata) + .encode() + .unwrap(); + + // content_offset / content_size_in_bytes are in the plaintext coordinate space, distinct + // from the ciphertext's on-disk size (header, nonce, and tag overhead). + let file_size = std::fs::metadata(&dv_path).unwrap().len(); + let data_file_path = format!("{table_location}/data-1.parquet"); + let dv = dv_task( + dv_path.clone(), + file_size, + data_file_path.clone(), + 0, + plaintext_size, + 2, + Some(encoded_key_metadata), + ); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + + let loader = CachingDeleteFileLoader::new(file_io, 10, Runtime::current()); + let delete_filter = loader.load_deletes(&[dv], schema).await.unwrap().unwrap(); + + let delete_vector = delete_filter + .get_delete_vector_for_path(&data_file_path) + .expect("a delete vector should be indexed for the referenced data file"); + let mut positions: Vec = delete_vector.lock().unwrap().iter().collect(); + positions.sort_unstable(); + assert_eq!(positions, vec![2, 4]); + } + + #[tokio::test] + async fn test_load_deletes_rejects_deletion_vector_cardinality_mismatch() { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let blob = encode_dv_blob([0u64, 1, 5]); + let dv_path = format!("{table_location}/deletes.puffin"); + std::fs::write(&dv_path, &blob).unwrap(); + + let data_file_path = format!("{table_location}/data-1.parquet"); + // record_count says 2 positions, but the blob decodes to 3. + let dv = dv_task( + dv_path.clone(), + std::fs::metadata(&dv_path).unwrap().len(), + data_file_path, + 0, + blob.len() as u64, + 2, + None, + ); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + + let loader = CachingDeleteFileLoader::new(file_io, 10, Runtime::current()); + let err = loader + .load_deletes(&[dv], schema) + .await + .unwrap() + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("expected 2 from record_count")); + } + + // A well-formed deletion-vector task, for tests that then corrupt one field after builder + // validation. + fn valid_dv_task() -> FileScanTaskDeleteFile { + dv_task( + "deletes.puffin".to_string(), + 100, + "data.parquet".to_string(), + 4, + 40, + 2, + None, + ) + } + + // The coordinate fields are unsigned, so a negative byte offset is rejected during + // deserialization: a hand-written or corrupted scan plan fails before any loader + // code runs. + #[test] + fn test_deserializing_negative_content_offset_is_rejected() { + let mut task = serde_json::to_value(valid_dv_task()).unwrap(); + task["content_offset"] = serde_json::json!(-1); + + let err = serde_json::from_value::(task).unwrap_err(); + assert!( + err.to_string().contains("invalid value"), + "expected an out-of-range integer error, got `{err}`" + ); + } + + #[test] + fn test_deserializing_negative_content_size_is_rejected() { + let mut task = serde_json::to_value(valid_dv_task()).unwrap(); + task["content_size_in_bytes"] = serde_json::json!(-1); + + let err = serde_json::from_value::(task).unwrap_err(); + assert!( + err.to_string().contains("invalid value"), + "expected an out-of-range integer error, got `{err}`" + ); + } + + #[test] + fn test_validate_deletion_vector_task_accepts_valid_coordinates() { + let (start, len, data_file_path, record_count) = + CachingDeleteFileLoader::validate_deletion_vector_task(&valid_dv_task()).unwrap(); + assert_eq!(start, 4); + assert_eq!(len, 40); + assert_eq!(data_file_path, "data.parquet"); + assert_eq!(record_count, 2); + } + + #[test] + fn test_validate_deletion_vector_cardinality_accepts_matching_count() { + let mut dv = DeleteVector::default(); + dv.insert(1); + dv.insert(2); + + CachingDeleteFileLoader::validate_deletion_vector_cardinality(&dv, 2, "deletes.puffin") + .unwrap(); + } + + #[test] + fn test_validate_deletion_vector_cardinality_rejects_mismatched_count() { + let mut dv = DeleteVector::default(); + dv.insert(1); + + let err = + CachingDeleteFileLoader::validate_deletion_vector_cardinality(&dv, 2, "deletes.puffin") + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("expected 2 from record_count")); + } + + #[tokio::test] + async fn test_pos_del_load_can_be_retried_after_failure() { + let tmp_dir = TempDir::new().unwrap(); + let file_scan_tasks = setup(tmp_dir.path()); + let task = &file_scan_tasks[0]; + let pos_del = task + .deletes() + .iter() + .find(|delete| delete.file_type() == DataContentType::PositionDeletes) + .unwrap() + .clone(); + let path = pos_del.file_path().to_string(); + let hidden = format!("{path}.hidden"); + let load = |loader: &CachingDeleteFileLoader| { + loader.load_deletes(std::slice::from_ref(&pos_del), task.schema_ref()) + }; + + // The first load fails because the file is not there yet. + std::fs::rename(&path, &hidden).unwrap(); + let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current()); + assert!(load(&loader).await.unwrap().is_err()); + + // Once it is, the same loader loads it instead of waiting on the failed load. + std::fs::rename(&hidden, &path).unwrap(); + let retried = tokio::time::timeout(std::time::Duration::from_secs(5), load(&loader)) + .await + .expect("retry waited on the failed load") + .unwrap() + .unwrap(); + let fresh = load(&CachingDeleteFileLoader::new( + FileIO::new_with_fs(), + 10, + Runtime::current(), + )) + .await + .unwrap() + .unwrap(); + + let positions = |filter: &DeleteFilter| -> Vec { + filter + .get_delete_vector(task) + .unwrap() + .lock() + .unwrap() + .iter() + .collect() + }; + assert!(!positions(&fresh).is_empty()); + assert_eq!(positions(&retried), positions(&fresh)); + } + + /// The load of a positional delete file taken by the test, and tasks polled until + /// they wait on it. + struct PendingPosDelLoad { + guard: PosDelLoadGuard, + waiters: Vec>>, + path: String, + } + + /// Takes the load of the first test task's positional delete file, then polls + /// `load_file_for_task` for the same file from `waiters` tasks until each waits on it. + async fn wait_on_pos_del_load(tmp_dir: &TempDir, waiters: usize) -> PendingPosDelLoad { + let task = setup(tmp_dir.path()).remove(0); + let pos_del = task + .deletes() + .iter() + .find(|delete| delete.file_type() == DataContentType::PositionDeletes) + .unwrap() + .clone(); + let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current()); + + let PosDelLoadAction::Load(guard) = loader + .delete_filter + .try_start_pos_del_load(pos_del.file_path()) + else { + panic!("expected Load for a new file"); + }; + let mut pending = Vec::with_capacity(waiters); + for _ in 0..waiters { + let (pos_del, schema) = (pos_del.clone(), task.schema_ref()); + let (basic, filter) = ( + loader.basic_delete_file_loader.clone(), + loader.delete_filter.clone(), + ); + let mut waiter: futures::future::BoxFuture<'static, _> = Box::pin(async move { + CachingDeleteFileLoader::load_file_for_task(&pos_del, basic, filter, schema).await + }); + assert!(futures::poll!(&mut waiter).is_pending()); + pending.push(waiter); + } + PendingPosDelLoad { + guard, + waiters: pending, + path: pos_del.file_path().to_string(), + } + } + + #[tokio::test] + async fn test_pos_del_waiter_loads_file_after_cancelled_load() { + let tmp_dir = TempDir::new().unwrap(); + let PendingPosDelLoad { + guard, mut waiters, .. + } = wait_on_pos_del_load(&tmp_dir, 1).await; + + // Dropped without an error, as when a sibling's error stops `load_deletes`. + drop(guard); + + let ctx = tokio::time::timeout(std::time::Duration::from_secs(5), waiters.remove(0)) + .await + .expect("waiter did not wake after the load was cancelled") + .unwrap(); + assert!(matches!(ctx, DeleteFileContext::PosDels { .. })); + } + + #[tokio::test] + async fn test_pos_del_waiters_return_the_error_of_a_failed_load() { + let tmp_dir = TempDir::new().unwrap(); + let PendingPosDelLoad { guard, waiters, .. } = wait_on_pos_del_load(&tmp_dir, 2).await; + + // The file is still there, so a waiter that opened it again would succeed. + guard.fail(&Error::new(ErrorKind::Unexpected, "load failed")); + + for waiter in waiters { + let err = tokio::time::timeout(std::time::Duration::from_secs(5), waiter) + .await + .expect("waiter did not wake after the load failed") + .err() + .unwrap(); + assert_eq!(err.kind(), ErrorKind::Unexpected); + assert!(err.message().contains("load failed"), "{err}"); + } + } + + #[tokio::test] + async fn test_pos_del_waiters_retry_after_cancelled_load() { + let tmp_dir = TempDir::new().unwrap(); + let PendingPosDelLoad { + guard, + waiters, + path, + } = wait_on_pos_del_load(&tmp_dir, 2).await; + + // Each waiter loads the file itself after a cancelled load, and gets its own error. + std::fs::remove_file(&path).unwrap(); + drop(guard); + + for waiter in waiters { + let err = tokio::time::timeout(std::time::Duration::from_secs(5), waiter) + .await + .expect("waiter did not wake after the load was cancelled") + .err() + .unwrap(); + assert!(err.to_string().contains("Failed to open file"), "{err}"); + } + } + + /// Local filesystem storage that yields before every read, so the loads of several + /// delete files in one `load_deletes` call start together, and holds reads of + /// `gated.parquet` until the gate is opened, so one of them is still in flight when + /// another fails. `LocalFsStorage` never yields, so on the current-thread test + /// runtime each load would otherwise finish before the next one starts. + mod gated_storage { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + use bytes::Bytes; + use futures::stream::BoxStream; + + use crate::Result; + use crate::io::{ + FileMetadata, FileRead, FileWrite, InputFile, LocalFsStorage, OutputFile, Storage, + StorageConfig, StorageFactory, + }; + + #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] + pub(super) struct GatedStorage { + #[serde(skip)] + inner: LocalFsStorage, + #[serde(skip)] + open: Arc, + } + + impl GatedStorage { + async fn wait_for_gate(&self, path: &str) { + tokio::task::yield_now().await; + while path.ends_with("gated.parquet") && !self.open.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + } + } + + #[async_trait::async_trait] + #[typetag::serde(name = "test_gated_storage")] + impl Storage for GatedStorage { + async fn metadata(&self, path: &str) -> Result { + self.inner.metadata(path).await + } + + fn new_input(&self, path: &str) -> Result { + Ok(InputFile::new(Arc::new(self.clone()), path.to_string())) + } + + fn new_output(&self, path: &str) -> Result { + Ok(OutputFile::new(Arc::new(self.clone()), path.to_string())) + } + + async fn exists(&self, path: &str) -> Result { + self.inner.exists(path).await + } + + async fn read(&self, path: &str) -> Result { + self.wait_for_gate(path).await; + self.inner.read(path).await + } + + async fn reader(&self, path: &str) -> Result> { + self.wait_for_gate(path).await; + self.inner.reader(path).await + } + + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { + self.inner.write(path, bs).await + } + + async fn writer(&self, path: &str) -> Result> { + self.inner.writer(path).await + } + + async fn delete(&self, path: &str) -> Result<()> { + self.inner.delete(path).await + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + self.inner.delete_prefix(path).await + } + + async fn delete_stream(&self, paths: BoxStream<'static, String>) -> Result<()> { + self.inner.delete_stream(paths).await + } + } + + #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] + pub(super) struct GatedStorageFactory { + #[serde(skip)] + pub(super) open: Arc, + } + + #[typetag::serde(name = "test_gated_storage_factory")] + impl StorageFactory for GatedStorageFactory { + fn build(&self, _config: &StorageConfig) -> Result> { + Ok(Arc::new(GatedStorage { + inner: LocalFsStorage, + open: Arc::clone(&self.open), + })) + } + } + } + + /// Writes a position delete file at `path` for `data.parquet` with `positions`. + fn write_pos_del_file(path: &str, positions: Vec) -> FileScanTaskDeleteFile { + let batch = RecordBatch::try_new( + crate::arrow::delete_filter::tests::create_pos_del_schema(), + vec![ + Arc::new(StringArray::from_iter_values(vec![ + "data.parquet"; + positions.len() + ])), + Arc::new(Int64Array::from_iter_values(positions)), + ], + ) + .unwrap(); + let mut writer = + ArrowWriter::try_new(File::create(path).unwrap(), batch.schema(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + FileScanTaskDeleteFile::builder() + .with_file_path(path.to_string()) + .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap() + } + + /// A failing positional delete file ends `load_deletes` early while the load of the + /// other file in the same call is still in flight, dropping its guard. That file + /// still loads later. + #[tokio::test] + async fn test_pos_del_load_dropped_by_a_failing_sibling_can_be_loaded_later() { + let tmp_dir = TempDir::new().unwrap(); + let schema = Arc::new(Schema::builder().build().unwrap()); + let invalid = write_pos_del_file( + tmp_dir.path().join("invalid.parquet").to_str().unwrap(), + vec![-1], + ); + // Its read waits for the gate, so its load is in flight when `invalid` fails. + let valid = write_pos_del_file( + tmp_dir.path().join("gated.parquet").to_str().unwrap(), + vec![1, 3], + ); + let factory = Arc::new(gated_storage::GatedStorageFactory::default()); + let gate = Arc::clone(&factory.open); + let file_io = crate::io::FileIOBuilder::new(factory).build(); + let loader = CachingDeleteFileLoader::new(file_io, 10, Runtime::current()); + + let err = loader + .load_deletes(&[invalid, valid.clone()], Arc::clone(&schema)) + .await + .unwrap() + .err() + .unwrap(); + assert!(err.message().contains("negative position"), "{err}"); + + gate.store(true, std::sync::atomic::Ordering::SeqCst); + let filter = tokio::time::timeout( + std::time::Duration::from_secs(5), + loader.load_deletes(&[valid], schema), + ) + .await + .expect("load waited on the dropped load") + .unwrap() + .unwrap(); + let positions: Vec = filter + .get_delete_vector_for_path("data.parquet") + .unwrap() + .lock() + .unwrap() + .iter() + .collect(); + assert_eq!(positions, vec![1, 3]); + } + + #[tokio::test] + async fn test_pos_del_load_can_be_retried_after_parse_failure() { + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().join("pos-del.parquet"); + let path = path.to_str().unwrap(); + let schema = Arc::new(Schema::builder().build().unwrap()); + let loader = CachingDeleteFileLoader::new(FileIO::new_with_fs(), 10, Runtime::current()); + + // The file opens but fails to parse, so the load is abandoned after the open. + let invalid = write_pos_del_file(path, vec![-1]); + let err = loader + .load_deletes(&[invalid], Arc::clone(&schema)) + .await + .unwrap() + .err() + .unwrap(); + assert!(err.message().contains("negative position"), "{err}"); + + // Once the file is valid, the same loader loads it instead of waiting. + let valid = write_pos_del_file(path, vec![1, 3]); + let filter = tokio::time::timeout( + std::time::Duration::from_secs(5), + loader.load_deletes(&[valid], schema), + ) + .await + .expect("retry waited on the failed load") + .unwrap() + .unwrap(); + let positions: Vec = filter + .get_delete_vector_for_path("data.parquet") + .unwrap() + .lock() + .unwrap() + .iter() + .collect(); + assert_eq!(positions, vec![1, 3]); + } } diff --git a/crates/iceberg/src/arrow/delete_file_loader.rs b/crates/iceberg/src/arrow/delete_file_loader.rs index 592ef2eb4a..ec61710a42 100644 --- a/crates/iceberg/src/arrow/delete_file_loader.rs +++ b/crates/iceberg/src/arrow/delete_file_loader.rs @@ -18,9 +18,12 @@ use std::sync::Arc; use futures::{StreamExt, TryStreamExt}; +use parquet::arrow::ParquetRecordBatchStreamBuilder; use crate::arrow::ArrowReader; -use crate::arrow::record_batch_transformer::RecordBatchTransformer; +use crate::arrow::reader::ParquetReadOptions; +use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; +use crate::arrow::scan_metrics::ScanMetrics; use crate::io::FileIO; use crate::scan::{ArrowRecordBatchStream, FileScanTaskDeleteFile}; use crate::spec::{Schema, SchemaRef}; @@ -43,48 +46,64 @@ pub trait DeleteFileLoader { #[derive(Clone, Debug)] pub(crate) struct BasicDeleteFileLoader { file_io: FileIO, + scan_metrics: ScanMetrics, } #[allow(unused_variables)] impl BasicDeleteFileLoader { - pub fn new(file_io: FileIO) -> Self { - BasicDeleteFileLoader { file_io } + pub fn new(file_io: FileIO, scan_metrics: ScanMetrics) -> Self { + BasicDeleteFileLoader { + file_io, + scan_metrics, + } } + + pub(crate) fn file_io(&self) -> &FileIO { + &self.file_io + } + /// Loads a RecordBatchStream for a given datafile. pub(crate) async fn parquet_to_batch_stream( &self, data_file_path: &str, + file_size_in_bytes: u64, + key_metadata: Option<&[u8]>, ) -> Result { /* Essentially a super-cut-down ArrowReader. We can't use ArrowReader directly as that introduces a circular dependency. */ - let record_batch_stream = ArrowReader::create_parquet_record_batch_stream_builder( + let parquet_read_options = ParquetReadOptions::builder().build(); + + let (parquet_file_reader, arrow_metadata) = ArrowReader::open_parquet_file( data_file_path, - self.file_io.clone(), - false, + &self.file_io, + file_size_in_bytes, + parquet_read_options, + self.scan_metrics.bytes_read_counter(), + key_metadata, ) - .await? - .build()? - .map_err(|e| Error::new(ErrorKind::Unexpected, format!("{}", e))); + .await?; + + let record_batch_stream = + ParquetRecordBatchStreamBuilder::new_with_metadata(parquet_file_reader, arrow_metadata) + .build()? + .map_err(|e| Error::new(ErrorKind::Unexpected, format!("{e}"))); Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream) } - /// Evolves the schema of the RecordBatches from an equality delete file + /// Evolves the schema of the RecordBatches from an equality delete file. + /// + /// Per the [Iceberg spec](https://iceberg.apache.org/spec/#equality-delete-files), + /// only evolves the specified `equality_ids` columns, not all table columns. pub(crate) async fn evolve_schema( record_batch_stream: ArrowRecordBatchStream, target_schema: Arc, + equality_ids: &[i32], ) -> Result { - let eq_ids = target_schema - .as_ref() - .field_id_to_name_map() - .keys() - .cloned() - .collect::>(); - let mut record_batch_transformer = - RecordBatchTransformer::build(target_schema.clone(), &eq_ids); + RecordBatchTransformerBuilder::new(target_schema.clone(), equality_ids).build(); let record_batch_stream = record_batch_stream.map(move |record_batch| { record_batch.and_then(|record_batch| { @@ -103,9 +122,22 @@ impl DeleteFileLoader for BasicDeleteFileLoader { task: &FileScanTaskDeleteFile, schema: SchemaRef, ) -> Result { - let raw_batch_stream = self.parquet_to_batch_stream(&task.file_path).await?; + let raw_batch_stream = self + .parquet_to_batch_stream( + task.file_path(), + task.file_size_in_bytes(), + task.key_metadata(), + ) + .await?; - Self::evolve_schema(raw_batch_stream, schema).await + // For equality deletes, only evolve the equality_ids columns. + // For positional deletes (equality_ids is None), use all field IDs. + let field_ids = match task.equality_ids() { + Some(ids) => ids.to_vec(), + None => schema.field_id_to_name_map().keys().cloned().collect(), + }; + + Self::evolve_schema(raw_batch_stream, schema, &field_ids).await } } @@ -115,23 +147,22 @@ mod tests { use super::*; use crate::arrow::delete_filter::tests::setup; + use crate::arrow::test_utils::write_encrypted_parquet; #[tokio::test] async fn test_basic_delete_file_loader_read_delete_file() { let tmp_dir = TempDir::new().unwrap(); let table_location = tmp_dir.path(); - let file_io = FileIO::from_path(table_location.as_os_str().to_str().unwrap()) - .unwrap() - .build() - .unwrap(); + let file_io = FileIO::new_with_fs(); - let delete_file_loader = BasicDeleteFileLoader::new(file_io.clone()); + let scan_metrics = ScanMetrics::new(); + let delete_file_loader = BasicDeleteFileLoader::new(file_io.clone(), scan_metrics); let file_scan_tasks = setup(table_location); let result = delete_file_loader .read_delete_file( - &file_scan_tasks[0].deletes[0], + &file_scan_tasks[0].deletes()[0], file_scan_tasks[0].schema_ref(), ) .await @@ -141,4 +172,160 @@ mod tests { assert_eq!(result.len(), 1); } + + #[tokio::test] + async fn test_read_encrypted_positional_delete_file() { + use std::sync::Arc; + + use arrow_array::{Int64Array, RecordBatch, StringArray}; + + use crate::arrow::delete_filter::tests::create_pos_del_schema; + use crate::encryption::StandardKeyMetadata; + use crate::scan::FileScanTaskDeleteFile; + use crate::spec::{DataContentType, DataFileFormat}; + + let encryption_key = b"0123456789abcdef"; + let aad_prefix = b"aad_prefix"; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap(); + let file_io = FileIO::new_with_fs(); + + let positional_delete_schema = create_pos_del_schema(); + let file_path_col = Arc::new(StringArray::from_iter_values(vec!["data.parquet"; 4])); + let pos_col = Arc::new(Int64Array::from(vec![0i64, 1, 5, 10])); + let batch = RecordBatch::try_new(positional_delete_schema.clone(), vec![ + file_path_col, + pos_col, + ]) + .unwrap(); + + let del_path = format!("{table_location}/encrypted-pos-del.parquet"); + write_encrypted_parquet(&del_path, &batch, encryption_key, Some(aad_prefix)); + + let key_metadata = StandardKeyMetadata::try_new(encryption_key) + .unwrap() + .with_aad_prefix(aad_prefix) + .encode() + .unwrap(); + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + crate::spec::NestedField::required( + 2147483546, + "file_path", + crate::spec::Type::Primitive(crate::spec::PrimitiveType::String), + ) + .into(), + crate::spec::NestedField::required( + 2147483545, + "pos", + crate::spec::Type::Primitive(crate::spec::PrimitiveType::Long), + ) + .into(), + ]) + .build() + .unwrap(), + ); + + let task = FileScanTaskDeleteFile::builder() + .with_file_path(del_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_key_metadata(Some(Box::from(key_metadata.as_ref()))) + .build() + .unwrap(); + + let scan_metrics = ScanMetrics::new(); + let delete_file_loader = BasicDeleteFileLoader::new(file_io, scan_metrics); + + let result = delete_file_loader + .read_delete_file(&task, schema) + .await + .unwrap(); + + let batches: Vec<_> = result.try_collect().await.unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 4); + } + + #[tokio::test] + async fn test_read_encrypted_equality_delete_file() { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{Int64Array, RecordBatch}; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + + use crate::encryption::StandardKeyMetadata; + use crate::scan::FileScanTaskDeleteFile; + use crate::spec::{DataContentType, DataFileFormat}; + + let encryption_key = b"0123456789abcdef"; + let aad_prefix = b"my-table-uuid!!"; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap(); + let file_io = FileIO::new_with_fs(); + + let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![ + arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]), + ), + ])); + + let id_col = Arc::new(Int64Array::from(vec![100i64, 200, 300])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_col]).unwrap(); + + let del_path = format!("{table_location}/encrypted-eq-del.parquet"); + write_encrypted_parquet(&del_path, &batch, encryption_key, Some(aad_prefix)); + + let key_metadata = StandardKeyMetadata::try_new(encryption_key) + .unwrap() + .with_aad_prefix(aad_prefix) + .encode() + .unwrap(); + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + crate::spec::NestedField::required( + 1, + "id", + crate::spec::Type::Primitive(crate::spec::PrimitiveType::Long), + ) + .into(), + ]) + .build() + .unwrap(), + ); + + let task = FileScanTaskDeleteFile::builder() + .with_file_path(del_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len()) + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_equality_ids(Some(vec![1])) + .with_key_metadata(Some(Box::from(key_metadata.as_ref()))) + .build() + .unwrap(); + + let scan_metrics = ScanMetrics::new(); + let delete_file_loader = BasicDeleteFileLoader::new(file_io, scan_metrics); + + let result = delete_file_loader + .read_delete_file(&task, schema) + .await + .unwrap(); + + let batches: Vec<_> = result.try_collect().await.unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 3); + } } diff --git a/crates/iceberg/src/arrow/delete_filter.rs b/crates/iceberg/src/arrow/delete_filter.rs index b853baa993..1b49e12e27 100644 --- a/crates/iceberg/src/arrow/delete_filter.rs +++ b/crates/iceberg/src/arrow/delete_filter.rs @@ -16,14 +16,16 @@ // under the License. use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError, RwLock}; use tokio::sync::Notify; +use tokio::sync::futures::OwnedNotified; use tokio::sync::oneshot::Receiver; use crate::delete_vector::DeleteVector; use crate::expr::Predicate::AlwaysTrue; use crate::expr::{Bind, BoundPredicate, Predicate}; +use crate::runtime::Runtime; use crate::scan::{FileScanTask, FileScanTaskDeleteFile}; use crate::spec::DataContentType; use crate::{Error, ErrorKind, Result}; @@ -34,18 +36,119 @@ enum EqDelState { Loaded(Predicate), } +/// State tracking for positional delete files. +/// Unlike equality deletes, positional deletes must be fully loaded before +/// the ArrowReader proceeds because retrieval is synchronous and non-blocking. +#[derive(Debug)] +enum PosDelState { + /// The file is currently being loaded by a task. + /// Other tasks wait on the attempt for it to finish or fail. + Loading(Arc), + /// The file has been fully loaded and merged into the delete vector map. + Loaded, +} + #[derive(Debug, Default)] struct DeleteFileFilterState { delete_vectors: HashMap>>, equality_deletes: HashMap, + positional_deletes: HashMap, } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug)] pub(crate) struct DeleteFilter { state: Arc>, + runtime: Runtime, +} + +/// One attempt to load a positional delete file, shared with the tasks waiting on it. +#[derive(Debug, Default)] +pub(crate) struct PosDelAttempt { + notify: Arc, + /// Set when the load fails, so the tasks waiting on it get the error instead of + /// each opening the file again. + failure: OnceLock<(ErrorKind, String)>, +} + +/// A wait on another task's positional delete load, from [`PosDelLoadAction::WaitFor`]. +pub(crate) struct PosDelWait { + notified: OwnedNotified, + attempt: Arc, +} + +impl PosDelWait { + /// Waits for the load to end. Returns its error if it failed. `Ok` means it finished + /// or was dropped without an error, so the caller must call + /// [`DeleteFilter::try_start_pos_del_load`] again rather than assume the file is loaded. + pub(crate) async fn wait(self) -> Result<()> { + self.notified.await; + match self.attempt.failure.get() { + Some((kind, message)) => Err(Error::new(*kind, message.clone())), + None => Ok(()), + } + } +} + +/// Action to take when trying to start loading a positional delete file +pub(crate) enum PosDelLoadAction { + /// The file is not loaded, the caller should load it and call + /// [`PosDelLoadGuard::finish`] once its delete vectors are merged. + Load(PosDelLoadGuard), + /// The file is already loaded, nothing to do. + AlreadyLoaded, + /// The file is currently being loaded by another task. + /// The caller *must* await [`PosDelWait::wait`] to ensure data availability before + /// returning, as subsequent access (get_delete_vector) is synchronous. The + /// wait is created under the state lock so it cannot miss the loader's + /// `notify_waiters()` (which stores no permit). + WaitFor(PosDelWait), +} + +/// Ownership of an in-progress positional delete load, returned by +/// [`DeleteFilter::try_start_pos_del_load`]. +/// +/// [`finish`](Self::finish) marks the file loaded. [`fail`](Self::fail) hands the error to +/// the tasks waiting on this attempt. Dropping the guard without finishing removes the +/// `Loading` marker and wakes the waiters, so a later attempt can load the file instead of +/// waiting forever. +pub(crate) struct PosDelLoadGuard { + filter: DeleteFilter, + file_path: String, + attempt: Arc, + finished: bool, +} + +impl PosDelLoadGuard { + /// Marks the file as loaded and wakes any waiters. + pub(crate) fn finish(mut self) { + self.finished = true; + self.filter.finish_pos_del_load(&self.file_path); + } + + /// Ends the load with `error`, which the tasks waiting on it return. Later calls to + /// [`DeleteFilter::try_start_pos_del_load`] start a new attempt. + pub(crate) fn fail(self, error: &Error) { + let _ = self.attempt.failure.set((error.kind(), error.to_string())); + } +} + +impl Drop for PosDelLoadGuard { + fn drop(&mut self) { + if !self.finished { + self.filter.abandon_pos_del_load(&self.file_path); + } + } } impl DeleteFilter { + /// Create a new DeleteFilter with the given runtime. + pub(crate) fn new(runtime: Runtime) -> Self { + Self { + state: Arc::new(RwLock::new(DeleteFileFilterState::default())), + runtime, + } + } + /// Retrieve a delete vector for the data file associated with a given file scan task pub(crate) fn get_delete_vector( &self, @@ -57,21 +160,23 @@ impl DeleteFilter { /// Retrieve a delete vector for a data file pub(crate) fn get_delete_vector_for_path( &self, - delete_file_path: &str, + data_file_path: &str, ) -> Option>> { self.state .read() .ok() - .and_then(|st| st.delete_vectors.get(delete_file_path).cloned()) + .and_then(|st| st.delete_vectors.get(data_file_path).cloned()) } pub(crate) fn try_start_eq_del_load(&self, file_path: &str) -> Option> { let mut state = self.state.write().unwrap(); - if !state.equality_deletes.contains_key(file_path) { + // Skip if already loaded/loading - another task owns it + if state.equality_deletes.contains_key(file_path) { return None; } + // Mark as loading to prevent duplicate work let notifier = Arc::new(Notify::new()); state .equality_deletes @@ -80,22 +185,95 @@ impl DeleteFilter { Some(notifier) } + /// Attempts to mark a positional delete file as "loading". + /// + /// Returns an action dictating whether the caller should load the file, + /// wait for another task to load it, or do nothing. + pub(crate) fn try_start_pos_del_load(&self, file_path: &str) -> PosDelLoadAction { + let mut state = self.state.write().unwrap(); + + if let Some(state) = state.positional_deletes.get(file_path) { + match state { + PosDelState::Loaded => return PosDelLoadAction::AlreadyLoaded, + PosDelState::Loading(attempt) => { + return PosDelLoadAction::WaitFor(PosDelWait { + notified: attempt.notify.clone().notified_owned(), + attempt: attempt.clone(), + }); + } + } + } + + let attempt = Arc::new(PosDelAttempt::default()); + state + .positional_deletes + .insert(file_path.to_string(), PosDelState::Loading(attempt.clone())); + + PosDelLoadAction::Load(PosDelLoadGuard { + filter: self.clone(), + file_path: file_path.to_string(), + attempt, + finished: false, + }) + } + + /// Marks a positional delete file as successfully loaded and notifies any waiting tasks. + fn finish_pos_del_load(&self, file_path: &str) { + let notify = { + let mut state = self.state.write().unwrap(); + if let Some(PosDelState::Loading(attempt)) = state + .positional_deletes + .insert(file_path.to_string(), PosDelState::Loaded) + { + Some(attempt) + } else { + None + } + }; + + if let Some(attempt) = notify { + attempt.notify.notify_waiters(); + } + } + + /// Removes the `Loading` marker of a positional delete file whose load failed or was + /// dropped, and wakes the tasks waiting on it. + fn abandon_pos_del_load(&self, file_path: &str) { + // This runs from `Drop`, possibly while unwinding from a panic that poisoned the + // lock; a second panic here would abort. + let removed = self + .state + .write() + .unwrap_or_else(PoisonError::into_inner) + .positional_deletes + .remove(file_path); + debug_assert!(matches!(removed, Some(PosDelState::Loading(_)))); + if let Some(PosDelState::Loading(attempt)) = removed { + attempt.notify.notify_waiters(); + } + } + /// Retrieve the equality delete predicate for a given eq delete file path pub(crate) async fn get_equality_delete_predicate_for_delete_file_path( &self, file_path: &str, ) -> Option { - let notifier = { + // Build the `Notified` while holding the read lock. `notified_owned()` records tokio's + // `notify_waiters_calls` counter at construction and completes on first poll if that + // counter has since advanced. Reading the counter under the lock guarantees it is taken + // before `insert_equality_delete` can advance it via `notify_waiters()`, so the + // notification is never missed even though we `.await` after releasing the lock. + let notified = { match self.state.read().unwrap().equality_deletes.get(file_path) { None => return None, - Some(EqDelState::Loading(notifier)) => notifier.clone(), + Some(EqDelState::Loading(notifier)) => notifier.clone().notified_owned(), Some(EqDelState::Loaded(predicate)) => { return Some(predicate.clone()); } } }; - notifier.notified().await; + notified.await; match self.state.read().unwrap().equality_deletes.get(file_path) { Some(EqDelState::Loaded(predicate)) => Some(predicate.clone()), @@ -114,20 +292,20 @@ impl DeleteFilter { // * Bind the predicate to the task's schema to get a `BoundPredicate` let mut combined_predicate = AlwaysTrue; - for delete in &file_scan_task.deletes { + for delete in file_scan_task.deletes() { if !is_equality_delete(delete) { continue; } let Some(predicate) = self - .get_equality_delete_predicate_for_delete_file_path(&delete.file_path) + .get_equality_delete_predicate_for_delete_file_path(delete.file_path()) .await else { return Err(Error::new( ErrorKind::Unexpected, format!( "Missing predicate for equality delete file '{}'", - delete.file_path + delete.file_path() ), )); }; @@ -139,8 +317,8 @@ impl DeleteFilter { return Ok(None); } - // TODO: handle case-insensitive case - let bound_predicate = combined_predicate.bind(file_scan_task.schema.clone(), false)?; + let bound_predicate = combined_predicate + .bind(file_scan_task.schema_ref(), file_scan_task.case_sensitive())?; Ok(Some(bound_predicate)) } @@ -161,23 +339,21 @@ impl DeleteFilter { *entry.lock().unwrap() |= delete_vector; } + /// Stores the equality delete predicate that `eq_del` delivers, and wakes the tasks + /// waiting on `notify`, the notifier from [`Self::try_start_eq_del_load`]. pub(crate) fn insert_equality_delete( &self, delete_file_path: &str, + notify: Arc, eq_del: Receiver, ) { - let notify = Arc::new(Notify::new()); - { - let mut state = self.state.write().unwrap(); - state.equality_deletes.insert( - delete_file_path.to_string(), - EqDelState::Loading(notify.clone()), - ); - } - + debug_assert!(matches!( + self.state.read().unwrap().equality_deletes.get(delete_file_path), + Some(EqDelState::Loading(stored)) if Arc::ptr_eq(stored, ¬ify) + )); let state = self.state.clone(); let delete_file_path = delete_file_path.to_string(); - crate::runtime::spawn(async move { + self.runtime.cpu().spawn(async move { let eq_del = eq_del.await.unwrap(); { let mut state = state.write().unwrap(); @@ -191,7 +367,7 @@ impl DeleteFilter { } pub(crate) fn is_equality_delete(f: &FileScanTaskDeleteFile) -> bool { - matches!(f.file_type, DataContentType::EqualityDeletes) + matches!(f.file_type(), DataContentType::EqualityDeletes) } #[cfg(test)] @@ -209,29 +385,150 @@ pub(crate) mod tests { use super::*; use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader; + use crate::expr::Reference; use crate::io::FileIO; - use crate::spec::{DataFileFormat, Schema}; + use crate::spec::{DataFileFormat, Datum, NestedField, PrimitiveType, Schema, Type}; type ArrowSchemaRef = Arc; const FIELD_ID_POSITIONAL_DELETE_FILE_PATH: u64 = 2147483546; const FIELD_ID_POSITIONAL_DELETE_POS: u64 = 2147483545; + // Regression test for the positional-delete lost-wakeup hang. + // + // Drives the real API through the losing interleaving: the loader fires + // `notify_waiters()` (via `finish_pos_del_load`) *before* the waiter awaits the notifier + // handed back by `WaitFor`. `notify_waiters()` stores no permit, so this only completes if + // the waiter's `Notified` was created before the signal. Because `WaitFor` now carries an + // `OwnedNotified` created under the lock in `try_start_pos_del_load`, it is; on the old + // `WaitFor(Arc)` contract the waiter created its `Notified` too late and hung. + #[tokio::test] + async fn test_wait_for_completes_when_load_finishes_before_await() { + let filter = DeleteFilter::new(Runtime::current()); + let path = "s3://bucket/pos-delete.parquet"; + + let PosDelLoadAction::Load(guard) = filter.try_start_pos_del_load(path) else { + panic!("expected Load for a new file"); + }; + + let PosDelLoadAction::WaitFor(wait) = filter.try_start_pos_del_load(path) else { + panic!("expected WaitFor for an in-progress load"); + }; + + // Loader completes and signals before the waiter awaits. + guard.finish(); + + let waited = tokio::time::timeout(std::time::Duration::from_secs(5), wait.wait()).await; + assert!( + matches!(waited, Ok(Ok(()))), + "WaitFor future must resolve after finish_pos_del_load" + ); + } + + #[tokio::test] + async fn test_cancelled_pos_del_load_wakes_waiters_and_can_be_retried() { + let filter = DeleteFilter::new(Runtime::current()); + let path = "s3://bucket/pos-delete.parquet"; + + let PosDelLoadAction::Load(guard) = filter.try_start_pos_del_load(path) else { + panic!("expected Load for a new file"); + }; + let PosDelLoadAction::WaitFor(wait) = filter.try_start_pos_del_load(path) else { + panic!("expected WaitFor for an in-progress load"); + }; + + // The load is cancelled: its guard is dropped without `finish` or `fail`. + drop(guard); + + let waited = tokio::time::timeout(std::time::Duration::from_secs(5), wait.wait()).await; + assert!( + matches!(waited, Ok(Ok(()))), + "WaitFor future must resolve when the load is dropped" + ); + + // The file is not loaded, so the next caller is told to load it. + assert!(matches!( + filter.try_start_pos_del_load(path), + PosDelLoadAction::Load(_) + )); + } + + #[tokio::test] + async fn test_failed_pos_del_load_returns_its_error_to_waiters() { + let filter = DeleteFilter::new(Runtime::current()); + let path = "s3://bucket/pos-delete.parquet"; + + let PosDelLoadAction::Load(guard) = filter.try_start_pos_del_load(path) else { + panic!("expected Load for a new file"); + }; + let waits: Vec<_> = (0..2) + .map(|_| match filter.try_start_pos_del_load(path) { + PosDelLoadAction::WaitFor(wait) => wait, + _ => panic!("expected WaitFor for an in-progress load"), + }) + .collect(); + + guard.fail( + &Error::new(ErrorKind::DataInvalid, "bad file") + .with_source(std::io::Error::other("403 Forbidden")), + ); + + for wait in waits { + let err = tokio::time::timeout(std::time::Duration::from_secs(5), wait.wait()) + .await + .expect("WaitFor future must resolve when the load fails") + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("bad file"), "{err}"); + assert!(err.message().contains("403 Forbidden"), "{err}"); + } + + // Only the waiters of that attempt get the error; a later call loads again. + assert!(matches!( + filter.try_start_pos_del_load(path), + PosDelLoadAction::Load(_) + )); + } + + // A task that finds the load started before the loader calls `insert_equality_delete` + // must be woken when the predicate arrives. + #[tokio::test] + async fn test_eq_del_waiter_registered_before_insert_is_woken() { + let filter = DeleteFilter::new(Runtime::current()); + let path = "s3://bucket/eq-delete.parquet"; + let notify = filter.try_start_eq_del_load(path).unwrap(); + + let mut waiter = Box::pin(filter.get_equality_delete_predicate_for_delete_file_path(path)); + // Registers the waiter on the notifier that `try_start_eq_del_load` stored. + assert!(futures::poll!(&mut waiter).is_pending()); + + let (tx, rx) = tokio::sync::oneshot::channel(); + filter.insert_equality_delete(path, notify, rx); + let predicate = Reference::new("id").equal_to(Datum::long(1)); + tx.send(predicate.clone()).unwrap(); + + let loaded = tokio::time::timeout(std::time::Duration::from_secs(5), waiter) + .await + .expect("waiter did not wake after the predicate arrived"); + assert_eq!(loaded, Some(predicate)); + } + #[tokio::test] async fn test_delete_file_filter_load_deletes() { let tmp_dir = TempDir::new().unwrap(); let table_location = tmp_dir.path(); - let file_io = FileIO::from_path(table_location.as_os_str().to_str().unwrap()) - .unwrap() - .build() - .unwrap(); + let file_io = FileIO::new_with_fs(); - let delete_file_loader = CachingDeleteFileLoader::new(file_io.clone(), 10); + let delete_file_loader = + CachingDeleteFileLoader::new(file_io.clone(), 10, Runtime::current()); let file_scan_tasks = setup(table_location); let delete_filter = delete_file_loader - .load_deletes(&file_scan_tasks[0].deletes, file_scan_tasks[0].schema_ref()) + .load_deletes( + file_scan_tasks[0].deletes(), + file_scan_tasks[0].schema_ref(), + ) .await .unwrap() .unwrap(); @@ -242,7 +539,10 @@ pub(crate) mod tests { assert_eq!(result.lock().unwrap().len(), 12); // pos dels from pos del file 1 and 2 let delete_filter = delete_file_loader - .load_deletes(&file_scan_tasks[1].deletes, file_scan_tasks[1].schema_ref()) + .load_deletes( + file_scan_tasks[1].deletes(), + file_scan_tasks[1].schema_ref(), + ) .await .unwrap() .unwrap(); @@ -307,50 +607,88 @@ pub(crate) mod tests { writer.close().unwrap(); } - let pos_del_1 = FileScanTaskDeleteFile { - file_path: format!("{}/pos-del-1.parquet", table_location.to_str().unwrap()), - file_type: DataContentType::PositionDeletes, - partition_spec_id: 0, - equality_ids: None, - }; + let pos_del_1 = FileScanTaskDeleteFile::builder() + .with_file_path(format!( + "{}/pos-del-1.parquet", + table_location.to_str().unwrap() + )) + .with_file_size_in_bytes( + std::fs::metadata(format!( + "{}/pos-del-1.parquet", + table_location.to_str().unwrap() + )) + .unwrap() + .len(), + ) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(); - let pos_del_2 = FileScanTaskDeleteFile { - file_path: format!("{}/pos-del-2.parquet", table_location.to_str().unwrap()), - file_type: DataContentType::PositionDeletes, - partition_spec_id: 0, - equality_ids: None, - }; + let pos_del_2 = FileScanTaskDeleteFile::builder() + .with_file_path(format!( + "{}/pos-del-2.parquet", + table_location.to_str().unwrap() + )) + .with_file_size_in_bytes( + std::fs::metadata(format!( + "{}/pos-del-2.parquet", + table_location.to_str().unwrap() + )) + .unwrap() + .len(), + ) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(); - let pos_del_3 = FileScanTaskDeleteFile { - file_path: format!("{}/pos-del-3.parquet", table_location.to_str().unwrap()), - file_type: DataContentType::PositionDeletes, - partition_spec_id: 0, - equality_ids: None, - }; + let pos_del_3 = FileScanTaskDeleteFile::builder() + .with_file_path(format!( + "{}/pos-del-3.parquet", + table_location.to_str().unwrap() + )) + .with_file_size_in_bytes( + std::fs::metadata(format!( + "{}/pos-del-3.parquet", + table_location.to_str().unwrap() + )) + .unwrap() + .len(), + ) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(); let file_scan_tasks = vec![ - FileScanTask { - start: 0, - length: 0, - record_count: None, - data_file_path: format!("{}/1.parquet", table_location.to_str().unwrap()), - data_file_format: DataFileFormat::Parquet, - schema: data_file_schema.clone(), - project_field_ids: vec![], - predicate: None, - deletes: vec![pos_del_1, pos_del_2.clone()], - }, - FileScanTask { - start: 0, - length: 0, - record_count: None, - data_file_path: format!("{}/2.parquet", table_location.to_str().unwrap()), - data_file_format: DataFileFormat::Parquet, - schema: data_file_schema.clone(), - project_field_ids: vec![], - predicate: None, - deletes: vec![pos_del_3], - }, + FileScanTask::builder() + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{}/1.parquet", table_location.to_str().unwrap())) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(data_file_schema.clone()) + .with_project_field_ids(vec![]) + .with_deletes(vec![pos_del_1, pos_del_2.clone()]) + .with_case_sensitive(false) + .build() + .unwrap(), + FileScanTask::builder() + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{}/2.parquet", table_location.to_str().unwrap())) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(data_file_schema.clone()) + .with_project_field_ids(vec![]) + .with_deletes(vec![pos_del_3]) + .with_case_sensitive(false) + .build() + .unwrap(), ]; file_scan_tasks @@ -372,4 +710,59 @@ pub(crate) mod tests { ]; Arc::new(arrow_schema::Schema::new(fields)) } + + #[tokio::test] + async fn test_build_equality_delete_predicate_case_sensitive() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "Id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + + // ---------- fake FileScanTask ---------- + let task = FileScanTask::builder() + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(0) + .with_data_file_path("data.parquet".to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![]) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_path("eq-del.parquet".to_string()) + .with_file_size_in_bytes(1) // never read; this test fails before opening the file + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(), + ]) + .with_case_sensitive(true) + .build() + .unwrap(); + + let filter = DeleteFilter::new(Runtime::current()); + + // ---------- insert equality delete predicate ---------- + let pred = Reference::new("id").equal_to(Datum::long(10)); + + let notify = filter.try_start_eq_del_load("eq-del.parquet").unwrap(); + let (tx, rx) = tokio::sync::oneshot::channel(); + filter.insert_equality_delete("eq-del.parquet", notify, rx); + + tx.send(pred).unwrap(); + + // ---------- should FAIL ---------- + let result = filter.build_equality_delete_predicate(&task).await; + + assert!( + result.is_err(), + "case_sensitive=true should fail when column case mismatches" + ); + } } diff --git a/crates/iceberg/src/arrow/int96.rs b/crates/iceberg/src/arrow/int96.rs new file mode 100644 index 0000000000..6add023098 --- /dev/null +++ b/crates/iceberg/src/arrow/int96.rs @@ -0,0 +1,577 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! INT96 timestamp coercion for Parquet files. + +use std::sync::Arc; + +use arrow_schema::{ + DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, TimeUnit, +}; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + +use crate::arrow::schema::{ArrowSchemaVisitor, DEFAULT_MAP_FIELD_NAME, visit_schema}; +use crate::error::Result; +use crate::spec::{PrimitiveType, Schema, Type}; +use crate::{Error, ErrorKind}; + +/// Coerce Arrow schema types for INT96 columns to match the Iceberg table schema. +/// +/// arrow-rs defaults INT96 to `Timestamp(Nanosecond)`, which overflows i64 for dates outside +/// ~1677-2262. We use arrow-rs's schema hint mechanism to read INT96 at the resolution +/// specified by the Iceberg schema (`timestamp` → microsecond, `timestamp_ns` → nanosecond). +/// +/// Iceberg Java handles this differently: it bypasses parquet-mr with a custom column reader +/// (`GenericParquetReaders.TimestampInt96Reader`). We achieve the same result via schema hints. +/// +/// References: +/// - Iceberg spec primitive types: +/// - arrow-rs schema hint support: +pub(crate) fn coerce_int96_timestamps( + arrow_schema: &ArrowSchemaRef, + iceberg_schema: &Schema, +) -> Option> { + let mut visitor = Int96CoercionVisitor::new(iceberg_schema); + let coerced = visit_schema(arrow_schema, &mut visitor).ok()?; + if visitor.changed { + Some(Arc::new(coerced)) + } else { + None + } +} + +/// Visitor that coerces `Timestamp(Nanosecond)` Arrow fields to the resolution +/// indicated by the Iceberg schema. +struct Int96CoercionVisitor<'a> { + iceberg_schema: &'a Schema, + field_stack: Vec, + changed: bool, +} + +impl<'a> Int96CoercionVisitor<'a> { + fn new(iceberg_schema: &'a Schema) -> Self { + Self { + iceberg_schema, + field_stack: Vec::new(), + changed: false, + } + } + + /// Determine the target TimeUnit for a Timestamp(Nanosecond) field based on the + /// Iceberg schema. Falls back to microsecond when field IDs are unavailable, + /// matching Iceberg Java behavior. + fn target_unit(&self, field: &FieldRef) -> Option { + if !matches!( + field.data_type(), + DataType::Timestamp(TimeUnit::Nanosecond, _) + ) { + return None; + } + + let target = field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .and_then(|id_str| id_str.parse::().ok()) + .and_then(|field_id| self.iceberg_schema.field_by_id(field_id)) + .and_then(|f| match &*f.field_type { + Type::Primitive(PrimitiveType::Timestamp | PrimitiveType::Timestamptz) => { + Some(TimeUnit::Microsecond) + } + Type::Primitive(PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs) => { + Some(TimeUnit::Nanosecond) + } + _ => None, + }) + // Iceberg Java reads INT96 as microseconds by default + .unwrap_or(TimeUnit::Microsecond); + + if target == TimeUnit::Nanosecond { + None + } else { + Some(target) + } + } +} + +impl ArrowSchemaVisitor for Int96CoercionVisitor<'_> { + type T = Field; + type U = ArrowSchema; + + fn before_field(&mut self, field: &FieldRef) -> Result<()> { + self.field_stack.push(field.clone()); + Ok(()) + } + + fn after_field(&mut self, _field: &FieldRef) -> Result<()> { + self.field_stack.pop(); + Ok(()) + } + + fn before_list_element(&mut self, field: &FieldRef) -> Result<()> { + self.field_stack.push(field.clone()); + Ok(()) + } + + fn after_list_element(&mut self, _field: &FieldRef) -> Result<()> { + self.field_stack.pop(); + Ok(()) + } + + fn before_map_key(&mut self, field: &FieldRef) -> Result<()> { + self.field_stack.push(field.clone()); + Ok(()) + } + + fn after_map_key(&mut self, _field: &FieldRef) -> Result<()> { + self.field_stack.pop(); + Ok(()) + } + + fn before_map_value(&mut self, field: &FieldRef) -> Result<()> { + self.field_stack.push(field.clone()); + Ok(()) + } + + fn after_map_value(&mut self, _field: &FieldRef) -> Result<()> { + self.field_stack.pop(); + Ok(()) + } + + fn schema(&mut self, schema: &ArrowSchema, values: Vec) -> Result { + Ok(ArrowSchema::new_with_metadata( + values, + schema.metadata().clone(), + )) + } + + fn r#struct(&mut self, _fields: &Fields, results: Vec) -> Result { + let field_info = self + .field_stack + .last() + .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Field stack underflow in struct"))?; + Ok(Field::new( + field_info.name(), + DataType::Struct(Fields::from(results)), + field_info.is_nullable(), + ) + .with_metadata(field_info.metadata().clone())) + } + + fn list(&mut self, list: &DataType, value: Field) -> Result { + let field_info = self + .field_stack + .last() + .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Field stack underflow in list"))?; + let list_type = match list { + DataType::List(_) => DataType::List(Arc::new(value)), + DataType::LargeList(_) => DataType::LargeList(Arc::new(value)), + DataType::FixedSizeList(_, size) => DataType::FixedSizeList(Arc::new(value), *size), + _ => { + return Err(Error::new( + ErrorKind::Unexpected, + format!("Expected list type, got {list}"), + )); + } + }; + Ok( + Field::new(field_info.name(), list_type, field_info.is_nullable()) + .with_metadata(field_info.metadata().clone()), + ) + } + + fn map(&mut self, map: &DataType, key_value: Field, value: Field) -> Result { + let field_info = self + .field_stack + .last() + .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Field stack underflow in map"))?; + let sorted = match map { + DataType::Map(_, sorted) => *sorted, + _ => { + return Err(Error::new( + ErrorKind::Unexpected, + format!("Expected map type, got {map}"), + )); + } + }; + let struct_field = Field::new( + DEFAULT_MAP_FIELD_NAME, + DataType::Struct(Fields::from(vec![key_value, value])), + false, + ); + Ok(Field::new( + field_info.name(), + DataType::Map(Arc::new(struct_field), sorted), + field_info.is_nullable(), + ) + .with_metadata(field_info.metadata().clone())) + } + + fn primitive(&mut self, p: &DataType) -> Result { + let field_info = self.field_stack.last().ok_or_else(|| { + Error::new(ErrorKind::Unexpected, "Field stack underflow in primitive") + })?; + + if let Some(target_unit) = self.target_unit(field_info) { + let tz = match field_info.data_type() { + DataType::Timestamp(_, tz) => tz.clone(), + _ => None, + }; + self.changed = true; + Ok(Field::new( + field_info.name(), + DataType::Timestamp(target_unit, tz), + field_info.is_nullable(), + ) + .with_metadata(field_info.metadata().clone())) + } else { + Ok( + Field::new(field_info.name(), p.clone(), field_info.is_nullable()) + .with_metadata(field_info.metadata().clone()), + ) + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + + use super::coerce_int96_timestamps; + use crate::spec::{ListType, MapType, NestedField, PrimitiveType, Schema, StructType, Type}; + + fn iceberg_schema_with_timestamp() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp)).into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap() + } + + fn field_id_meta(id: i32) -> HashMap { + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string())]) + } + + #[test] + fn test_coerce_timestamp_ns_to_us() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), true) + .with_metadata(field_id_meta(1)), + Field::new("id", DataType::Int32, false).with_metadata(field_id_meta(2)), + ])); + let iceberg = iceberg_schema_with_timestamp(); + + let coerced = coerce_int96_timestamps(&arrow_schema, &iceberg).unwrap(); + assert_eq!( + coerced.field(0).data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, None) + ); + // Non-timestamp field unchanged + assert_eq!(coerced.field(1).data_type(), &DataType::Int32); + } + + #[test] + fn test_coerce_timestamptz_ns_to_us() { + let iceberg = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamptz)).into(), + ]) + .build() + .unwrap(); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + "ts", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + true, + ) + .with_metadata(field_id_meta(1)), + ])); + + let coerced = coerce_int96_timestamps(&arrow_schema, &iceberg).unwrap(); + assert_eq!( + coerced.field(0).data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())) + ); + } + + #[test] + fn test_no_coercion_when_iceberg_is_timestamp_ns() { + let iceberg = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::TimestampNs)).into(), + ]) + .build() + .unwrap(); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), true) + .with_metadata(field_id_meta(1)), + ])); + + assert!(coerce_int96_timestamps(&arrow_schema, &iceberg).is_none()); + } + + #[test] + fn test_no_coercion_when_iceberg_is_timestamptz_ns() { + let iceberg = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::TimestamptzNs)) + .into(), + ]) + .build() + .unwrap(); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + "ts", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + true, + ) + .with_metadata(field_id_meta(1)), + ])); + + assert!(coerce_int96_timestamps(&arrow_schema, &iceberg).is_none()); + } + + #[test] + fn test_no_coercion_when_already_microsecond() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("ts", DataType::Timestamp(TimeUnit::Microsecond, None), true) + .with_metadata(field_id_meta(1)), + Field::new("id", DataType::Int32, false).with_metadata(field_id_meta(2)), + ])); + let iceberg = iceberg_schema_with_timestamp(); + + assert!(coerce_int96_timestamps(&arrow_schema, &iceberg).is_none()); + } + + // Without field IDs, the visitor can't look up the Iceberg type and falls back + // to microsecond to match Iceberg Java behavior. + #[test] + fn test_defaults_to_us_without_field_ids() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Nanosecond, None), + true, + )])); + let iceberg = iceberg_schema_with_timestamp(); + + let coerced = coerce_int96_timestamps(&arrow_schema, &iceberg).unwrap(); + assert_eq!( + coerced.field(0).data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, None) + ); + } + + // Field ID exists but points to a non-timestamp Iceberg type. The field_by_id + // lookup succeeds but the match arm returns None, so unwrap_or falls back to + // microsecond. + #[test] + fn test_defaults_to_us_when_iceberg_type_is_not_timestamp() { + let iceberg = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), true) + .with_metadata(field_id_meta(1)), + ])); + + let coerced = coerce_int96_timestamps(&arrow_schema, &iceberg).unwrap(); + assert_eq!( + coerced.field(0).data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, None) + ); + } + + #[test] + fn test_coerce_preserves_field_metadata() { + let mut meta = field_id_meta(1); + meta.insert("custom_key".to_string(), "custom_value".to_string()); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), true) + .with_metadata(meta.clone()), + ])); + let iceberg = iceberg_schema_with_timestamp(); + + let coerced = coerce_int96_timestamps(&arrow_schema, &iceberg).unwrap(); + assert_eq!(coerced.field(0).metadata(), &meta); + } + + #[test] + fn test_coerce_timestamp_in_struct() { + let iceberg = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "data", + Type::Struct(StructType::new(vec![ + NestedField::optional(2, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + "data", + DataType::Struct( + vec![ + Field::new("ts", DataType::Timestamp(TimeUnit::Nanosecond, None), true) + .with_metadata(field_id_meta(2)), + ] + .into(), + ), + false, + ) + .with_metadata(field_id_meta(1)), + ])); + + let coerced = coerce_int96_timestamps(&arrow_schema, &iceberg).unwrap(); + let inner = match coerced.field(0).data_type() { + DataType::Struct(fields) => fields, + other => panic!("Expected Struct, got {other}"), + }; + assert_eq!( + inner[0].data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, None) + ); + } + + #[test] + fn test_coerce_timestamp_in_list() { + let iceberg = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional( + 1, + "timestamps", + Type::List(ListType { + element_field: NestedField::optional( + 2, + "element", + Type::Primitive(PrimitiveType::Timestamp), + ) + .into(), + }), + ) + .into(), + ]) + .build() + .unwrap(); + + let element_field = Field::new( + "element", + DataType::Timestamp(TimeUnit::Nanosecond, None), + true, + ) + .with_metadata(field_id_meta(2)); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("timestamps", DataType::List(Arc::new(element_field)), true) + .with_metadata(field_id_meta(1)), + ])); + + let coerced = coerce_int96_timestamps(&arrow_schema, &iceberg).unwrap(); + let element_dt = match coerced.field(0).data_type() { + DataType::List(f) => f.data_type(), + other => panic!("Expected List, got {other}"), + }; + assert_eq!( + element_dt, + &DataType::Timestamp(TimeUnit::Microsecond, None) + ); + } + + #[test] + fn test_coerce_timestamp_in_map_value() { + let iceberg = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional( + 1, + "ts_map", + Type::Map(MapType { + key_field: NestedField::required( + 2, + "key", + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::optional( + 3, + "value", + Type::Primitive(PrimitiveType::Timestamp), + ) + .into(), + }), + ) + .into(), + ]) + .build() + .unwrap(); + + let key_field = Field::new("key", DataType::Utf8, false).with_metadata(field_id_meta(2)); + let value_field = Field::new( + "value", + DataType::Timestamp(TimeUnit::Nanosecond, None), + true, + ) + .with_metadata(field_id_meta(3)); + let entries_field = Field::new( + "key_value", + DataType::Struct(vec![key_field, value_field].into()), + false, + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + "ts_map", + DataType::Map(Arc::new(entries_field), false), + true, + ) + .with_metadata(field_id_meta(1)), + ])); + + let coerced = coerce_int96_timestamps(&arrow_schema, &iceberg).unwrap(); + let value_dt = match coerced.field(0).data_type() { + DataType::Map(entries, _) => match entries.data_type() { + DataType::Struct(fields) => fields[1].data_type().clone(), + other => panic!("Expected Struct inside Map, got {other}"), + }, + other => panic!("Expected Map, got {other}"), + }; + assert_eq!(value_dt, DataType::Timestamp(TimeUnit::Microsecond, None)); + } +} diff --git a/crates/iceberg/src/arrow/manifest.rs b/crates/iceberg/src/arrow/manifest.rs new file mode 100644 index 0000000000..08bd9ed4cd --- /dev/null +++ b/crates/iceberg/src/arrow/manifest.rs @@ -0,0 +1,1897 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Reading manifests and manifest lists into Arrow. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_arith::boolean::{and, is_null}; +use arrow_array::builder::Int64Builder; +use arrow_array::cast::AsArray; +use arrow_array::types::{TimestampMicrosecondType, TimestampNanosecondType}; +use arrow_array::{ + Array, ArrayRef, BooleanArray, Int32Array, Int64Array, ListArray, MapArray, RecordBatch, + StructArray, new_null_array, +}; +use arrow_avro::reader::{Reader, ReaderBuilder}; +use arrow_avro::schema::AvroSchema; +use arrow_cast::{CastOptions, cast_with_options}; +use arrow_ord::cmp::eq; +use arrow_schema::{ArrowError, DataType, Field, Fields, TimeUnit}; +use arrow_select::zip::zip; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; +use serde_json::Value as JsonValue; + +use crate::arrow::schema_to_arrow_schema; +use crate::error::invalid_data; +use crate::spec::{ + DATA_FILE_CONTENT_FIELD_ID, DATA_FILE_FIELD_ID, DATA_FILE_PARTITION_FIELD_ID, + FILE_SEQUENCE_NUMBER_FIELD_ID, FormatVersion, MANIFEST_CONTENT_FIELD_ID, + MANIFEST_SEQUENCE_NUMBER_FIELD_ID, MIN_SEQUENCE_NUMBER_FIELD_ID, ManifestContentType, + ManifestFile, ManifestMetadata, ManifestStatus, SEQUENCE_NUMBER_FIELD_ID, Schema, + manifest_entry_fields, manifest_file_fields, +}; +use crate::{Error, Result}; + +/// Reads the entries of a manifest file into a [`RecordBatch`] with one row +/// per entry. +/// +/// `bytes` is the manifest file and `manifest_file` is its entry in the +/// manifest list. The schema is the spec's `manifest_entry` struct for the +/// latest format version, with field IDs in the Arrow field metadata, so +/// manifests of every format version read into the same schema apart from +/// `data_file.partition`, which follows the manifest's partition spec. +/// +/// Fields are matched to the manifest's Avro schema by field ID. A v1 +/// manifest reads with the spec's v2+ defaults, and null snapshot IDs, +/// sequence numbers, and first row IDs are inherited from `manifest_file`. +/// Lower and upper bounds keep the binary single-value serialization they were +/// written with. +/// +/// # Example +/// +/// ``` +/// use iceberg::arrow::read_manifest_entries; +/// use iceberg::spec::{ManifestContentType, ManifestFile}; +/// +/// let bytes = std::fs::read(concat!( +/// env!("CARGO_MANIFEST_DIR"), +/// "/testdata/manifests/pyiceberg-v2-data.avro" +/// ))?; +/// let manifest_file = ManifestFile { +/// manifest_path: "s3://bucket/metadata/manifest.avro".to_string(), +/// manifest_length: bytes.len() as i64, +/// partition_spec_id: 0, +/// content: ManifestContentType::Data, +/// sequence_number: 1, +/// min_sequence_number: 1, +/// added_snapshot_id: 1, +/// added_files_count: None, +/// existing_files_count: None, +/// deleted_files_count: None, +/// added_rows_count: None, +/// existing_rows_count: None, +/// deleted_rows_count: None, +/// partitions: None, +/// key_metadata: None, +/// first_row_id: None, +/// }; +/// +/// let batch = read_manifest_entries(&bytes, &manifest_file)?; +/// assert_eq!(batch.num_rows(), 2); +/// # Ok::<(), Box>(()) +/// ``` +pub fn read_manifest_entries(bytes: &[u8], manifest_file: &ManifestFile) -> Result { + // usize::MAX decodes the whole manifest into one batch. First row ID + // inheritance is a running sum over every entry in the file, and an empty + // manifest still returns its schema. arrow-avro sizes its buffers + // independently of the batch size, so this doesn't preallocate. + let builder = || ReaderBuilder::new().with_batch_size(usize::MAX); + let mut reader = builder().build(bytes).map_err(invalid_manifest)?; + let header = reader.avro_header(); + let metadata: HashMap> = header + .metadata() + .map(|(key, value)| (String::from_utf8_lossy(key).into_owned(), value.to_vec())) + .collect(); + let metadata = ManifestMetadata::parse(&metadata)?; + let mut avro_schema: JsonValue = serde_json::from_slice( + header + .get("avro.schema") + .ok_or_else(|| invalid_data!("manifest has no avro.schema"))?, + )?; + let partition_type = metadata + .partition_spec() + .partition_type(metadata.schema())?; + let schema = Arc::new(schema_to_arrow_schema( + &Schema::builder() + .with_fields(manifest_entry_fields(&partition_type)) + .build()?, + )?); + + // TODO(#3257): arrow-avro 59 can't decode a record with no fields, which + // is the partition of every unpartitioned manifest, so project it out and + // build it in `missing_field`. arrow-avro 60 fixes this + // (apache/arrow-rs#10771). + if remove_empty_partition(&mut avro_schema) { + reader = builder() + .with_reader_schema(AvroSchema::new(avro_schema.to_string())) + .build(bytes) + .map_err(invalid_manifest)?; + } + + let Some(batch) = only_batch(reader, invalid_manifest)? else { + return Ok(RecordBatch::new_empty(schema)); + }; + // v1 manifests have no content or sequence number columns, which v2+ + // readers read as 0. A v2+ manifest may lack file_sequence_number (Java + // wrote none before 1.1.0), which reads as null and is inherited. + let zero_defaults: &[i32] = match metadata.format_version() { + FormatVersion::V1 => &[ + DATA_FILE_CONTENT_FIELD_ID, + SEQUENCE_NUMBER_FIELD_ID, + FILE_SEQUENCE_NUMBER_FIELD_ID, + ], + _ => &[DATA_FILE_CONTENT_FIELD_ID], + }; + let entries = project_struct( + &StructArray::from(batch), + &avro_schema, + schema.fields(), + zero_defaults, + )?; + Ok(RecordBatch::from(inherit(entries, manifest_file)?)) +} + +/// Reads the entries of a manifest list into a [`RecordBatch`] with one row +/// per manifest. +/// +/// The schema is the spec's `manifest_file` struct for the latest format +/// version, with field IDs in the Arrow field metadata, so manifest lists of +/// every format version read into the same schema. Fields are matched to the +/// manifest list's Avro schema by field ID. The format version comes from the +/// `format-version` key that writers put in the file's metadata, and a +/// manifest list without one is v1. A v1 manifest list reads with the spec's +/// v2+ defaults. Partition summary bounds keep the binary single-value +/// serialization they were written with. +/// +/// # Example +/// +/// ``` +/// use iceberg::arrow::read_manifest_list; +/// +/// let bytes = std::fs::read(concat!( +/// env!("CARGO_MANIFEST_DIR"), +/// "/testdata/manifests_lists/manifest-list-v2-2.avro" +/// ))?; +/// let batch = read_manifest_list(&bytes)?; +/// assert_eq!(batch.num_rows(), 1); +/// # Ok::<(), Box>(()) +/// ``` +pub fn read_manifest_list(bytes: &[u8]) -> Result { + // usize::MAX decodes the whole manifest list into one batch, which + // `only_batch` checks. + let reader = ReaderBuilder::new() + .with_batch_size(usize::MAX) + .build(bytes) + .map_err(invalid_manifest_list)?; + let header = reader.avro_header(); + let avro_schema: JsonValue = serde_json::from_slice( + header + .get("avro.schema") + .ok_or_else(|| invalid_data!("manifest list has no avro.schema"))?, + )?; + // The spec doesn't define this key, but Java, PyIceberg, and iceberg-rust + // all write it, and rejecting an unknown version keeps a newer manifest + // list from reading as an older one. + let format_version = match header.get("format-version") { + Some(bytes) => serde_json::from_slice::(bytes).map_err(|err| { + invalid_data!("failed to parse format version in manifest list metadata") + .with_source(err) + })?, + None => FormatVersion::V1, + }; + let schema = Arc::new(schema_to_arrow_schema( + &Schema::builder() + .with_fields(manifest_file_fields()) + .build()?, + )?); + let Some(batch) = only_batch(reader, invalid_manifest_list)? else { + return Ok(RecordBatch::new_empty(schema)); + }; + // v1 manifest lists have no content or sequence number columns, which v2+ + // readers read as 0. + let zero_defaults: &[i32] = match format_version { + FormatVersion::V1 => &[ + MANIFEST_CONTENT_FIELD_ID, + MANIFEST_SEQUENCE_NUMBER_FIELD_ID, + MIN_SEQUENCE_NUMBER_FIELD_ID, + ], + _ => &[], + }; + let manifests = project_struct( + &StructArray::from(batch), + &avro_schema, + schema.fields(), + zero_defaults, + )?; + Ok(RecordBatch::from(manifests)) +} + +/// Returns the only batch that `reader` decodes, or `None` for a file without +/// records. +fn only_batch( + mut reader: Reader<&[u8]>, + invalid: fn(ArrowError) -> Error, +) -> Result> { + let Some(batch) = reader.next() else { + return Ok(None); + }; + let batch = batch.map_err(invalid)?; + // Taking only the first batch relies on arrow-avro filling it to the batch + // size, so fail instead of dropping records if that changes. + if reader.next().is_some() { + return Err(invalid_data!( + "arrow-avro split the file into more than one batch" + )); + } + Ok(Some(batch)) +} + +/// Removes `data_file.partition` from a manifest's Avro schema if it has no +/// fields, and returns whether it did. +fn remove_empty_partition(avro_schema: &mut JsonValue) -> bool { + let has_field_id = |field: &JsonValue, id: i32| field["field-id"].as_i64() == Some(id.into()); + let Some(data_file_fields) = avro_schema["fields"] + .as_array_mut() + .and_then(|fields| { + fields + .iter_mut() + .find(|field| has_field_id(field, DATA_FILE_FIELD_ID)) + }) + .and_then(|data_file| data_file["type"]["fields"].as_array_mut()) + else { + return false; + }; + let len = data_file_fields.len(); + data_file_fields.retain(|field| { + !(has_field_id(field, DATA_FILE_PARTITION_FIELD_ID) + && field["type"]["fields"] + .as_array() + .is_some_and(Vec::is_empty)) + }); + data_file_fields.len() != len +} + +fn invalid_manifest(err: ArrowError) -> Error { + invalid_data!("failed to read manifest").with_source(err) +} + +fn invalid_manifest_list(err: ArrowError) -> Error { + invalid_data!("failed to read manifest list").with_source(err) +} + +fn invalid_column(err: ArrowError) -> Error { + invalid_data!("failed to build Arrow column from Avro").with_source(err) +} + +/// Returns the field ID of a field in the schema built from +/// `manifest_entry_fields` or `manifest_file_fields`, which give every field +/// one. +fn field_id(field: &Field) -> i32 { + field.metadata()[PARQUET_FIELD_ID_META_KEY].parse().unwrap() +} + +/// Returns the type of a nullable Avro field, `["null", T]`, as `T`. +fn non_null_type(avro_type: &JsonValue) -> &JsonValue { + match avro_type.as_array().map(Vec::as_slice) { + Some([JsonValue::String(null), branch] | [branch, JsonValue::String(null)]) + if null == "null" => + { + branch + } + _ => avro_type, + } +} + +/// Builds each of `fields` from the column of `array` that the Avro record +/// `record` gives the same field ID. A field that `record` lacks reads as 0 if +/// its ID is in `zero_defaults`, and as null otherwise. +fn project_struct( + array: &StructArray, + record: &JsonValue, + fields: &Fields, + zero_defaults: &[i32], +) -> Result { + let avro_fields = record["fields"] + .as_array() + .ok_or_else(|| invalid_data!("Avro type {record} is not a record"))?; + let mut columns = Vec::with_capacity(fields.len()); + for field in fields { + let id = field_id(field); + columns.push( + match avro_fields + .iter() + .position(|avro_field| avro_field["field-id"].as_i64() == Some(id.into())) + { + Some(index) => project( + array.column(index), + non_null_type(&avro_fields[index]["type"]), + field, + zero_defaults, + )?, + None => missing_field(field, array.len(), zero_defaults)?, + }, + ); + } + StructArray::try_new_with_length(fields.clone(), columns, array.nulls().cloned(), array.len()) + .map_err(invalid_column) +} + +/// Builds the column for a field that the Avro file doesn't have. +fn missing_field(field: &Field, len: usize, zero_defaults: &[i32]) -> Result { + let id = field_id(field); + let zero = zero_defaults.contains(&id); + match field.data_type() { + DataType::Int32 if zero => Ok(Arc::new(Int32Array::from_value(0, len))), + DataType::Int64 if zero => Ok(Arc::new(Int64Array::from_value(0, len))), + _ if field.is_nullable() => Ok(new_null_array(field.data_type(), len)), + DataType::Struct(fields) if fields.is_empty() => { + Ok(Arc::new(StructArray::new_empty_fields(len, None))) + } + _ => Err(invalid_data!( + "Avro file has no required field {} with field ID {id}", + field.name() + )), + } +} + +/// Converts the decoded `array` of Avro type `avro_type` to `field`, the +/// spec's Arrow field for it. +fn project( + array: &ArrayRef, + avro_type: &JsonValue, + field: &Field, + zero_defaults: &[i32], +) -> Result { + let type_mismatch = || { + invalid_data!( + "field {} has Avro type {avro_type}, which doesn't match {}", + field.name(), + field.data_type() + ) + }; + Ok(match field.data_type() { + DataType::Struct(fields) => Arc::new(project_struct( + array.as_struct_opt().ok_or_else(type_mismatch)?, + avro_type, + fields, + zero_defaults, + )?), + DataType::List(element) => { + let list = array.as_list_opt::().ok_or_else(type_mismatch)?; + let values = project( + list.values(), + non_null_type(&avro_type["items"]), + element, + zero_defaults, + )?; + Arc::new( + ListArray::try_new( + element.clone(), + list.offsets().clone(), + values, + list.nulls().cloned(), + ) + .map_err(invalid_column)?, + ) + } + // Avro stores maps with int keys as arrays of key-value records. + DataType::Map(entries, _) => { + let list = array.as_list_opt::().ok_or_else(type_mismatch)?; + let DataType::Struct(key_value) = entries.data_type() else { + unreachable!() + }; + let entries_array = project_struct( + list.values().as_struct_opt().ok_or_else(type_mismatch)?, + &avro_type["items"], + key_value, + zero_defaults, + )?; + Arc::new( + MapArray::try_new( + entries.clone(), + list.offsets().clone(), + entries_array, + list.nulls().cloned(), + false, + ) + .map_err(invalid_column)?, + ) + } + // arrow-avro gives every timestamp a UTC time zone, so only the type + // changes. + DataType::Timestamp(TimeUnit::Microsecond, time_zone) => Arc::new( + array + .as_primitive_opt::() + .ok_or_else(type_mismatch)? + .clone() + .with_timezone_opt(time_zone.clone()), + ), + DataType::Timestamp(TimeUnit::Nanosecond, time_zone) => Arc::new( + array + .as_primitive_opt::() + .ok_or_else(type_mismatch)? + .clone() + .with_timezone_opt(time_zone.clone()), + ), + data_type if array.data_type() == data_type => array.clone(), + // Without `safe: false`, a value that doesn't fit the spec's type would + // become null instead of an error. + data_type => cast_with_options(array, data_type, &CastOptions { + safe: false, + ..Default::default() + }) + .map_err(invalid_column)?, + }) +} + +/// Fills in the values that the spec says entries inherit from the manifest +/// list. +fn inherit(entries: StructArray, manifest_file: &ManifestFile) -> Result { + let (fields, mut columns, nulls) = entries.into_parts(); + let index = |name: &str| fields.find(name).unwrap().0; + let (status, snapshot_id, sequence_number, file_sequence_number, data_file) = ( + index("status"), + index("snapshot_id"), + index("sequence_number"), + index("file_sequence_number"), + index("data_file"), + ); + + columns[snapshot_id] = + fill_nulls(&columns[snapshot_id], None, manifest_file.added_snapshot_id)?; + if columns[sequence_number].null_count() > 0 || columns[file_sequence_number].null_count() > 0 { + let added = eq( + &columns[status], + &Int32Array::new_scalar(ManifestStatus::Added as i32), + )?; + for column in [sequence_number, file_sequence_number] { + columns[column] = fill_nulls( + &columns[column], + Some(&added), + manifest_file.sequence_number, + )?; + } + } + if manifest_file.content == ManifestContentType::Data { + let (data_file_fields, mut data_file_columns, data_file_nulls) = + columns[data_file].as_struct().clone().into_parts(); + let index = |name: &str| data_file_fields.find(name).unwrap().0; + let first_row_id = index("first_row_id"); + data_file_columns[first_row_id] = inherit_first_row_ids( + data_file_columns[first_row_id].as_primitive(), + columns[status].as_primitive(), + data_file_columns[index("record_count")].as_primitive(), + manifest_file, + )?; + columns[data_file] = Arc::new(StructArray::new( + data_file_fields, + data_file_columns, + data_file_nulls, + )); + } + Ok(StructArray::new(fields, columns, nulls)) +} + +/// Replaces the nulls of `array` with `value`, only in the rows that `rows` +/// selects when given. +fn fill_nulls(array: &ArrayRef, rows: Option<&BooleanArray>, value: i64) -> Result { + if array.null_count() == 0 { + return Ok(array.clone()); + } + let mask = match rows { + Some(rows) => and(&is_null(array)?, rows)?, + None => is_null(array)?, + }; + Ok(zip(&mask, &Int64Array::new_scalar(value), array)?) +} + +/// Assigns first row IDs to the data files that lack one, following first row +/// ID inheritance. Deleted entries take no row IDs, because the spec's +/// suggested manifest `first_row_id` increment, `added_rows_count` plus +/// `existing_rows_count`, leaves no room for them. A manifest without a first +/// row ID belongs to a snapshot without row lineage, so all its row IDs are +/// null. +fn inherit_first_row_ids( + first_row_ids: &Int64Array, + status: &Int32Array, + record_counts: &Int64Array, + manifest_file: &ManifestFile, +) -> Result { + let Some(manifest_first_row_id) = manifest_file.first_row_id else { + return Ok(new_null_array(&DataType::Int64, first_row_ids.len())); + }; + if first_row_ids.null_count() == 0 { + return Ok(Arc::new(first_row_ids.clone())); + } + let mut next_row_id = i64::try_from(manifest_first_row_id)?; + let mut builder = Int64Builder::with_capacity(first_row_ids.len()); + for ((first_row_id, &status), &record_count) in first_row_ids + .iter() + .zip(status.values()) + .zip(record_counts.values()) + { + match first_row_id { + None if status != ManifestStatus::Deleted as i32 => { + if record_count < 0 { + return Err(invalid_data!( + "negative record_count {record_count} assigning first row IDs in {}", + manifest_file.manifest_path + )); + } + builder.append_value(next_row_id); + next_row_id = next_row_id.checked_add(record_count).ok_or_else(|| { + invalid_data!( + "row ID overflow assigning first row IDs in {}", + manifest_file.manifest_path + ) + })?; + } + first_row_id => builder.append_option(first_row_id), + } + } + Ok(Arc::new(builder.finish())) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use apache_avro::types::Value as AvroValue; + use apache_avro::{Codec, DeflateSettings, ZstandardSettings}; + use arrow_array::cast::AsArray; + use arrow_array::types::{Int32Type, Int64Type}; + use arrow_array::{ + Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Decimal128Array, + FixedSizeBinaryArray, Float32Array, Float64Array, Int32Array, Int64Array, StringArray, + Time64MicrosecondArray, TimestampMicrosecondArray, TimestampNanosecondArray, + }; + use arrow_buffer::{NullBuffer, OffsetBuffer}; + + use super::*; + use crate::ErrorKind; + use crate::arrow::arrow_schema_to_schema; + use crate::io::FileIO; + use crate::spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Datum, FieldSummary, ListType, + Literal, Manifest, ManifestEntry, ManifestList, ManifestListWriter, ManifestStatus, + ManifestWriter, ManifestWriterBuilder, MapType, NestedField, PartitionSpec, + PrimitiveLiteral, PrimitiveType, Schema, Struct, StructType, Transform, Type, + UNASSIGNED_SEQUENCE_NUMBER, + }; + + fn table_schema() -> Schema { + let columns = [ + (1, "b", PrimitiveType::Boolean), + (2, "i", PrimitiveType::Int), + (3, "l", PrimitiveType::Long), + (4, "f", PrimitiveType::Float), + (5, "d", PrimitiveType::Double), + (6, "dec", PrimitiveType::Decimal { + precision: 38, + scale: 10, + }), + (7, "date", PrimitiveType::Date), + (8, "time", PrimitiveType::Time), + (9, "ts", PrimitiveType::Timestamp), + (10, "tstz", PrimitiveType::Timestamptz), + (11, "ts_ns", PrimitiveType::TimestampNs), + (12, "tstz_ns", PrimitiveType::TimestamptzNs), + (13, "s", PrimitiveType::String), + (14, "fx", PrimitiveType::Fixed(3)), + (15, "bin", PrimitiveType::Binary), + ]; + Schema::builder() + .with_fields( + columns.into_iter().map(|(id, name, ty)| { + NestedField::optional(id, name, Type::Primitive(ty)).into() + }), + ) + .build() + .unwrap() + } + + fn partition_spec(fields: &[(&str, Transform)]) -> PartitionSpec { + fields + .iter() + .fold( + PartitionSpec::builder(table_schema()).with_spec_id(0), + |builder, (source, transform)| { + builder + .add_partition_field(*source, format!("{source}_{transform}"), *transform) + .unwrap() + }, + ) + .build() + .unwrap() + } + + fn data_file(content: DataContentType, path: &str) -> DataFileBuilder { + let mut builder = DataFileBuilder::default(); + builder + .content(content) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .record_count(1) + .file_size_in_bytes(10); + builder + } + + /// Writes a manifest with `ManifestWriter` and returns its bytes and its + /// manifest list entry. + async fn write_manifest( + spec: PartitionSpec, + snapshot_id: Option, + build: impl FnOnce(ManifestWriterBuilder) -> ManifestWriter, + add: impl FnOnce(&mut ManifestWriter), + ) -> (Vec, ManifestFile) { + let io = FileIO::new_with_memory(); + let output = io.new_output("memory:///manifest.avro").unwrap(); + let mut writer = build(ManifestWriterBuilder::new( + output, + snapshot_id, + Arc::new(table_schema()), + spec, + )); + add(&mut writer); + let manifest_file = writer.write_manifest_file().await.unwrap(); + let bytes = io + .new_input("memory:///manifest.avro") + .unwrap() + .read() + .await + .unwrap() + .to_vec(); + (bytes, manifest_file) + } + + fn column(batch: &RecordBatch, name: &str) -> ArrayRef { + batch.column_by_name(name).unwrap().clone() + } + + fn data_file_column(batch: &RecordBatch, name: &str) -> ArrayRef { + column(batch, "data_file") + .as_struct() + .column_by_name(name) + .unwrap() + .clone() + } + + /// Entries of the map in `row` of a `map` column, sorted by key. + fn long_map(column: &ArrayRef, row: usize) -> Vec<(i32, i64)> { + let entries = column.as_map().value(row); + let keys = entries.column(0).as_primitive::(); + let values = entries.column(1).as_primitive::(); + let mut pairs: Vec<_> = keys + .values() + .iter() + .copied() + .zip(values.values().iter().copied()) + .collect(); + pairs.sort_unstable(); + pairs + } + + /// Entries of the map in `row` of a `map` column, sorted by key. + fn binary_map(column: &ArrayRef, row: usize) -> Vec<(i32, Vec)> { + let entries = column.as_map().value(row); + let keys = entries.column(0).as_primitive::(); + let values = entries.column(1).as_binary::(); + let mut pairs: Vec<_> = keys + .values() + .iter() + .copied() + .zip(values.iter().map(|v| v.unwrap().to_vec())) + .collect(); + pairs.sort_unstable(); + pairs + } + + fn int_map_type(key_id: i32, value_id: i32, value: PrimitiveType) -> Type { + Type::Map(MapType::new( + NestedField::map_key_element(key_id, Type::Primitive(PrimitiveType::Int)).into(), + NestedField::map_value_element(value_id, Type::Primitive(value), true).into(), + )) + } + + fn list_type(element_id: i32, element: PrimitiveType) -> Type { + Type::List(ListType::new( + NestedField::list_element(element_id, Type::Primitive(element), true).into(), + )) + } + + /// Rewrites an Avro file with the user metadata `key` set to `value`, or + /// without it if `value` is `None`. + fn set_avro_metadata(bytes: &[u8], key: &str, value: Option<&str>) -> Vec { + let reader = apache_avro::Reader::new(bytes).unwrap(); + let schema = reader.writer_schema().clone(); + let mut metadata = reader.user_metadata().clone(); + match value { + Some(value) => metadata.insert(key.to_string(), value.as_bytes().to_vec()), + None => metadata.remove(key), + }; + let mut writer = apache_avro::Writer::new(&schema, Vec::new()).unwrap(); + for (key, value) in metadata { + writer.add_user_metadata(key, value).unwrap(); + } + for record in reader { + writer.append_value(record.unwrap()).unwrap(); + } + writer.into_inner().unwrap() + } + + /// Rewrites a manifest with `codec`, `edit_schema` applied to its Avro + /// schema, and `edit_record` applied to each record. + fn rewrite_manifest( + bytes: &[u8], + codec: Codec, + edit_schema: impl Fn(&mut serde_json::Value), + edit_record: impl Fn(&mut AvroValue), + ) -> Vec { + let reader = apache_avro::Reader::new(bytes).unwrap(); + let mut schema = serde_json::to_value(reader.writer_schema()).unwrap(); + edit_schema(&mut schema); + let schema = apache_avro::Schema::parse(&schema).unwrap(); + let metadata = reader.user_metadata().clone(); + let mut writer = apache_avro::Writer::with_codec(&schema, Vec::new(), codec).unwrap(); + for (key, value) in metadata { + writer.add_user_metadata(key, value).unwrap(); + } + for record in reader { + let mut record = record.unwrap(); + edit_record(&mut record); + writer.append_value(record).unwrap(); + } + writer.into_inner().unwrap() + } + + /// Calls `edit` on every record field object in an Avro schema. + fn for_each_avro_field(schema: &mut serde_json::Value, edit: &impl Fn(&mut serde_json::Value)) { + match schema { + serde_json::Value::Object(object) => { + if let Some(serde_json::Value::Array(fields)) = object.get_mut("fields") { + for field in fields { + edit(field); + for_each_avro_field(field.get_mut("type").unwrap(), edit); + } + } + if let Some(items) = object.get_mut("items") { + for_each_avro_field(items, edit); + } + } + serde_json::Value::Array(branches) => { + for branch in branches { + for_each_avro_field(branch, edit); + } + } + _ => {} + } + } + + /// Renames the record field `from` to `to` in every Avro record value. + fn rename_avro_field(value: &mut AvroValue, from: &str, to: &str) { + match value { + AvroValue::Record(fields) => { + for (name, field) in fields { + if name == from { + *name = to.to_string(); + } + rename_avro_field(field, from, to); + } + } + AvroValue::Union(_, value) => rename_avro_field(value, from, to), + AvroValue::Array(items) => { + for item in items { + rename_avro_field(item, from, to); + } + } + _ => {} + } + } + + /// Renames record fields in an Avro file's schema and records, keeping + /// their field IDs. + fn rename_fields(bytes: &[u8], renames: &[(&str, &str)]) -> Vec { + rewrite_manifest( + bytes, + Codec::Null, + |schema| { + for_each_avro_field(schema, &|field| { + for (from, to) in renames { + if field["name"] == *from { + field["name"] = (*to).into(); + } + } + }) + }, + |record| { + for (from, to) in renames { + rename_avro_field(record, from, to); + } + }, + ) + } + + #[tokio::test] + async fn test_schema_is_spec_manifest_entry() { + let spec = partition_spec(&[("i", Transform::Identity), ("s", Transform::Truncate(2))]); + let (bytes, manifest_file) = + write_manifest(spec, Some(1), |b| b.build_v2_data(), |_| {}).await; + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + assert_eq!(batch.num_rows(), 0); + + let long = || Type::Primitive(PrimitiveType::Long); + let int = || Type::Primitive(PrimitiveType::Int); + let string = || Type::Primitive(PrimitiveType::String); + let data_file = StructType::new(vec![ + NestedField::required(134, "content", int()).into(), + NestedField::required(100, "file_path", string()).into(), + NestedField::required(101, "file_format", string()).into(), + NestedField::required( + 102, + "partition", + Type::Struct(StructType::new(vec![ + NestedField::optional(1000, "i_identity", int()).into(), + NestedField::optional(1001, "s_truncate[2]", string()).into(), + ])), + ) + .into(), + NestedField::required(103, "record_count", long()).into(), + NestedField::required(104, "file_size_in_bytes", long()).into(), + NestedField::optional( + 108, + "column_sizes", + int_map_type(117, 118, PrimitiveType::Long), + ) + .into(), + NestedField::optional( + 109, + "value_counts", + int_map_type(119, 120, PrimitiveType::Long), + ) + .into(), + NestedField::optional( + 110, + "null_value_counts", + int_map_type(121, 122, PrimitiveType::Long), + ) + .into(), + NestedField::optional( + 137, + "nan_value_counts", + int_map_type(138, 139, PrimitiveType::Long), + ) + .into(), + NestedField::optional( + 125, + "lower_bounds", + int_map_type(126, 127, PrimitiveType::Binary), + ) + .into(), + NestedField::optional( + 128, + "upper_bounds", + int_map_type(129, 130, PrimitiveType::Binary), + ) + .into(), + NestedField::optional(131, "key_metadata", Type::Primitive(PrimitiveType::Binary)) + .into(), + NestedField::optional(132, "split_offsets", list_type(133, PrimitiveType::Long)).into(), + NestedField::optional(135, "equality_ids", list_type(136, PrimitiveType::Int)).into(), + NestedField::optional(140, "sort_order_id", int()).into(), + NestedField::optional(142, "first_row_id", long()).into(), + NestedField::optional(143, "referenced_data_file", string()).into(), + NestedField::optional(144, "content_offset", long()).into(), + NestedField::optional(145, "content_size_in_bytes", long()).into(), + ]); + let expected = Schema::builder() + .with_fields(vec![ + NestedField::required(0, "status", int()).into(), + NestedField::optional(1, "snapshot_id", long()).into(), + NestedField::optional(3, "sequence_number", long()).into(), + NestedField::optional(4, "file_sequence_number", long()).into(), + NestedField::required(2, "data_file", Type::Struct(data_file)).into(), + ]) + .build() + .unwrap(); + assert_eq!( + arrow_schema_to_schema(&batch.schema()).unwrap().as_struct(), + expected.as_struct() + ); + } + + #[tokio::test] + async fn test_data_file_columns() { + let added = DataFile { + column_sizes: HashMap::from([(1, 11)]), + value_counts: HashMap::from([(1, 10), (3, 9)]), + null_value_counts: HashMap::from([(1, 1)]), + nan_value_counts: HashMap::from([(4, 0)]), + lower_bounds: HashMap::from([(2, Datum::int(-1)), (13, Datum::string("abc"))]), + upper_bounds: HashMap::from([(2, Datum::int(7))]), + key_metadata: Some(vec![1, 2]), + split_offsets: Some(vec![4, 100]), + sort_order_id: Some(0), + ..data_file(DataContentType::Data, "s3://b/data/0.parquet") + .record_count(10) + .file_size_in_bytes(100) + .build() + .unwrap() + }; + let existing = data_file(DataContentType::Data, "s3://b/data/1.parquet") + .build() + .unwrap(); + let (bytes, manifest_file) = write_manifest( + partition_spec(&[]), + Some(1), + |b| b.build_v2_data(), + |writer| { + writer.add_file(added, 2).unwrap(); + writer.add_existing_file(existing, 5, 3, Some(3)).unwrap(); + }, + ) + .await; + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + assert_eq!(batch.num_rows(), 2); + + assert_eq!( + column(&batch, "status").as_ref(), + &Int32Array::from(vec![1, 0]) as &dyn Array + ); + assert_eq!( + column(&batch, "snapshot_id").as_ref(), + &Int64Array::from(vec![1, 5]) as &dyn Array + ); + assert_eq!( + column(&batch, "sequence_number").as_ref(), + &Int64Array::from(vec![2, 3]) as &dyn Array + ); + + let column = |name: &str| data_file_column(&batch, name); + assert_eq!( + column("content").as_ref(), + &Int32Array::from(vec![0, 0]) as &dyn Array + ); + assert_eq!( + column("file_path").as_ref(), + &StringArray::from(vec!["s3://b/data/0.parquet", "s3://b/data/1.parquet"]) + as &dyn Array + ); + assert_eq!( + column("file_format").as_ref(), + &StringArray::from(vec!["PARQUET", "PARQUET"]) as &dyn Array + ); + assert_eq!( + column("record_count").as_ref(), + &Int64Array::from(vec![10, 1]) as &dyn Array + ); + assert_eq!( + column("file_size_in_bytes").as_ref(), + &Int64Array::from(vec![100, 10]) as &dyn Array + ); + assert_eq!(long_map(&column("column_sizes"), 0), vec![(1, 11)]); + assert_eq!(long_map(&column("value_counts"), 0), vec![(1, 10), (3, 9)]); + assert_eq!(long_map(&column("null_value_counts"), 0), vec![(1, 1)]); + assert_eq!(long_map(&column("nan_value_counts"), 0), vec![(4, 0)]); + assert_eq!(long_map(&column("value_counts"), 1), vec![]); + assert_eq!(binary_map(&column("lower_bounds"), 0), vec![ + (2, (-1i32).to_le_bytes().to_vec()), + (13, b"abc".to_vec()), + ]); + assert_eq!(binary_map(&column("upper_bounds"), 0), vec![( + 2, + 7i32.to_le_bytes().to_vec() + )]); + assert_eq!( + column("key_metadata").as_ref(), + &BinaryArray::from(vec![Some(&[1u8, 2][..]), None]) as &dyn Array + ); + let split_offsets = column("split_offsets"); + assert_eq!( + split_offsets.as_list::().value(0).as_ref(), + &Int64Array::from(vec![4, 100]) as &dyn Array + ); + assert_eq!( + column("sort_order_id").as_ref(), + &Int32Array::from(vec![Some(0), None]) as &dyn Array + ); + } + + #[tokio::test] + async fn test_delete_file_columns() { + let position_deletes = DataFile { + file_format: DataFileFormat::Puffin, + referenced_data_file: Some("s3://b/data/0.parquet".to_string()), + content_offset: Some(4), + content_size_in_bytes: Some(40), + ..data_file(DataContentType::PositionDeletes, "s3://b/data/1.puffin") + .build() + .unwrap() + }; + let equality_deletes = DataFile { + equality_ids: Some(vec![1, 3]), + ..data_file(DataContentType::EqualityDeletes, "s3://b/data/2.parquet") + .build() + .unwrap() + }; + let (bytes, manifest_file) = write_manifest( + partition_spec(&[]), + Some(1), + |b| b.build_v3_deletes(), + |writer| { + writer.add_file(position_deletes, 2).unwrap(); + writer.add_file(equality_deletes, 2).unwrap(); + }, + ) + .await; + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + + let column = |name: &str| data_file_column(&batch, name); + assert_eq!( + column("content").as_ref(), + &Int32Array::from(vec![1, 2]) as &dyn Array + ); + assert_eq!( + column("file_format").as_ref(), + &StringArray::from(vec!["PUFFIN", "PARQUET"]) as &dyn Array + ); + assert_eq!( + column("referenced_data_file").as_ref(), + &StringArray::from(vec![Some("s3://b/data/0.parquet"), None]) as &dyn Array + ); + assert_eq!( + column("content_offset").as_ref(), + &Int64Array::from(vec![Some(4), None]) as &dyn Array + ); + assert_eq!( + column("content_size_in_bytes").as_ref(), + &Int64Array::from(vec![Some(40), None]) as &dyn Array + ); + let equality_ids = column("equality_ids"); + let equality_ids = equality_ids.as_list::(); + assert!(equality_ids.is_null(0)); + assert_eq!( + equality_ids.value(1).as_ref(), + &Int32Array::from(vec![1, 3]) as &dyn Array + ); + } + + #[tokio::test] + async fn test_partition_values() { + let identity_columns = [ + "b", "i", "l", "f", "d", "dec", "date", "time", "ts", "tstz", "ts_ns", "tstz_ns", "s", + "fx", "bin", + ]; + let mut fields: Vec<_> = identity_columns + .iter() + .map(|column| (*column, Transform::Identity)) + .collect(); + fields.extend([ + ("i", Transform::Bucket(4)), + ("ts", Transform::Day), + ("s", Transform::Truncate(2)), + ("l", Transform::Void), + ]); + let spec = partition_spec(&fields); + + let values = Struct::from_iter([ + Some(Literal::bool(true)), + Some(Literal::int(-7)), + Some(Literal::long(1i64 << 40)), + Some(Literal::float(1.5f32)), + Some(Literal::double(-2.25)), + Some(Literal::decimal(-12_345)), + Some(Literal::date(19_000)), + Some(Literal::time(3_600_000_000)), + Some(Literal::timestamp(1_700_000_000_000_000)), + Some(Literal::timestamptz(1_700_000_000_000_001)), + Some(Literal::timestamp_nano(1_700_000_000_000_000_002)), + Some(Literal::timestamptz_nano(1_700_000_000_000_000_003)), + Some(Literal::string("abc")), + Some(Literal::fixed([1u8, 2, 3])), + Some(Literal::binary([9u8])), + Some(Literal::int(3)), + Some(Literal::date(19_675)), + Some(Literal::string("ab")), + None, + ]); + let nulls = Struct::from_iter(fields.iter().map(|_| None)); + let (bytes, manifest_file) = write_manifest( + spec, + Some(1), + |b| b.build_v2_data(), + |writer| { + for (i, partition) in [values, nulls].into_iter().enumerate() { + let file = DataFile { + partition, + ..data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet")) + .build() + .unwrap() + }; + writer.add_file(file, 1).unwrap(); + } + }, + ) + .await; + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + + let partition = data_file_column(&batch, "partition"); + let partition = partition.as_struct(); + let expected: Vec = vec![ + Arc::new(BooleanArray::from(vec![Some(true), None])), + Arc::new(Int32Array::from(vec![Some(-7), None])), + Arc::new(Int64Array::from(vec![Some(1i64 << 40), None])), + Arc::new(Float32Array::from(vec![Some(1.5), None])), + Arc::new(Float64Array::from(vec![Some(-2.25), None])), + Arc::new( + Decimal128Array::from(vec![Some(-12_345), None]) + .with_precision_and_scale(38, 10) + .unwrap(), + ), + Arc::new(Date32Array::from(vec![Some(19_000), None])), + Arc::new(Time64MicrosecondArray::from(vec![ + Some(3_600_000_000), + None, + ])), + Arc::new(TimestampMicrosecondArray::from(vec![ + Some(1_700_000_000_000_000), + None, + ])), + Arc::new( + TimestampMicrosecondArray::from(vec![Some(1_700_000_000_000_001), None]) + .with_timezone("+00:00"), + ), + Arc::new(TimestampNanosecondArray::from(vec![ + Some(1_700_000_000_000_000_002), + None, + ])), + Arc::new( + TimestampNanosecondArray::from(vec![Some(1_700_000_000_000_000_003), None]) + .with_timezone("+00:00"), + ), + Arc::new(StringArray::from(vec![Some("abc"), None])), + Arc::new( + FixedSizeBinaryArray::try_from_sparse_iter_with_size( + [Some(vec![1u8, 2, 3]), None].into_iter(), + 3, + ) + .unwrap(), + ), + Arc::new(BinaryArray::from(vec![Some(&[9u8][..]), None])), + Arc::new(Int32Array::from(vec![Some(3), None])), + Arc::new(Date32Array::from(vec![Some(19_675), None])), + Arc::new(StringArray::from(vec![Some("ab"), None])), + Arc::new(Int64Array::from(vec![None, None])), + ]; + assert_eq!(partition.num_columns(), expected.len()); + for (index, expected) in expected.iter().enumerate() { + assert_eq!( + partition.column(index).as_ref(), + expected.as_ref(), + "partition field {}", + partition.fields()[index].name() + ); + } + } + + #[tokio::test] + async fn test_v1_manifest_reads_with_v2_defaults() { + let file = data_file(DataContentType::Data, "s3://b/data/0.parquet") + .build() + .unwrap(); + let (bytes, mut manifest_file) = write_manifest( + partition_spec(&[]), + Some(1), + |b| b.build_v1(), + |writer| writer.add_file(file, -1).unwrap(), + ) + .await; + // The v1 default must win over inheritance, so give the manifest list + // entry a sequence number that inheritance would use. + manifest_file.sequence_number = 9; + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + + assert_eq!( + data_file_column(&batch, "content").as_ref(), + &Int32Array::from(vec![0]) as &dyn Array + ); + for name in ["sequence_number", "file_sequence_number"] { + assert_eq!( + column(&batch, name).as_ref(), + &Int64Array::from(vec![0]) as &dyn Array, + "{name}" + ); + } + assert_eq!( + column(&batch, "snapshot_id").as_ref(), + &Int64Array::from(vec![1]) as &dyn Array + ); + } + + #[tokio::test] + async fn test_snapshot_id_and_sequence_number_inheritance() { + let file = |i: usize| { + data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet")) + .build() + .unwrap() + }; + let (bytes, mut manifest_file) = write_manifest( + partition_spec(&[]), + None, + |b| b.build_v2_data(), + |writer| { + writer.add_file(file(0), -1).unwrap(); + writer.add_delete_file(file(1), 4, Some(4)).unwrap(); + writer.add_existing_file(file(2), 3, 5, Some(5)).unwrap(); + }, + ) + .await; + // ManifestWriter rejects an existing entry without sequence numbers. + let bytes = rewrite_manifest( + &bytes, + Codec::Null, + |_| {}, + |record| { + let AvroValue::Record(fields) = record else { + panic!("manifest entry is not a record"); + }; + if fields.contains(&("status".to_string(), AvroValue::Int(0))) { + for (name, value) in fields { + if name.ends_with("sequence_number") { + *value = AvroValue::Union(0, Box::new(AvroValue::Null)); + } + } + } + }, + ); + manifest_file.added_snapshot_id = 42; + manifest_file.sequence_number = 7; + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + + assert_eq!( + column(&batch, "snapshot_id").as_ref(), + &Int64Array::from(vec![42, 42, 3]) as &dyn Array + ); + for name in ["sequence_number", "file_sequence_number"] { + assert_eq!( + column(&batch, name).as_ref(), + &Int64Array::from(vec![Some(7), Some(4), None]) as &dyn Array, + "{name}" + ); + } + } + + #[tokio::test] + async fn test_v2_manifest_without_file_sequence_number_inherits() { + let file = |i: usize| { + data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet")) + .build() + .unwrap() + }; + let (bytes, mut manifest_file) = write_manifest( + partition_spec(&[]), + Some(1), + |b| b.build_v2_data(), + |writer| { + writer.add_file(file(0), -1).unwrap(); + writer.add_existing_file(file(1), 3, 5, Some(5)).unwrap(); + }, + ) + .await; + // Java wrote v2 manifests without file_sequence_number before 1.1.0. + let bytes = rewrite_manifest( + &bytes, + Codec::Null, + |schema| { + schema["fields"] + .as_array_mut() + .unwrap() + .retain(|field| field["name"] != "file_sequence_number") + }, + |record| { + let AvroValue::Record(fields) = record else { + panic!("manifest entry is not a record"); + }; + fields.retain(|(name, _)| name != "file_sequence_number"); + }, + ); + manifest_file.sequence_number = 7; + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + + assert_eq!( + column(&batch, "file_sequence_number").as_ref(), + &Int64Array::from(vec![Some(7), None]) as &dyn Array + ); + } + + async fn first_row_id_manifest() -> (Vec, ManifestFile) { + let file = |i: usize, record_count: u64, first_row_id: Option| DataFile { + first_row_id, + ..data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet")) + .record_count(record_count) + .build() + .unwrap() + }; + let entry = |status, data_file| ManifestEntry { + status, + snapshot_id: Some(1), + sequence_number: Some(1), + file_sequence_number: Some(1), + data_file, + }; + write_manifest( + partition_spec(&[]), + Some(1), + |b| b.build_v3_data(), + |writer| { + writer.add_file(file(0, 10, None), 1).unwrap(); + writer + .add_existing_entry(entry(ManifestStatus::Existing, file(1, 3, Some(5)))) + .unwrap(); + writer + .add_delete_entry(entry(ManifestStatus::Deleted, file(2, 4, None))) + .unwrap(); + writer + .add_existing_entry(entry(ManifestStatus::Existing, file(3, 6, None))) + .unwrap(); + writer.add_file(file(4, 2, None), 1).unwrap(); + }, + ) + .await + } + + #[tokio::test] + async fn test_first_row_id_inheritance() { + let (bytes, mut manifest_file) = first_row_id_manifest().await; + manifest_file.first_row_id = Some(100); + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + // Deleted entries don't take row IDs, and entries with a first_row_id keep it. + assert_eq!( + data_file_column(&batch, "first_row_id").as_ref(), + &Int64Array::from(vec![Some(100), Some(5), None, Some(110), Some(116)]) as &dyn Array + ); + + manifest_file.first_row_id = None; + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + assert_eq!(data_file_column(&batch, "first_row_id").null_count(), 5); + + manifest_file.first_row_id = Some(i64::MAX as u64 - 5); + let err = read_manifest_entries(&bytes, &manifest_file).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + + // A negative record_count would assign overlapping row IDs. + let bytes = rewrite_manifest( + &bytes, + Codec::Null, + |_| {}, + |record| { + let AvroValue::Record(fields) = record else { + panic!("manifest entry is not a record"); + }; + let Some((_, AvroValue::Record(data_file))) = + fields.iter_mut().find(|(name, _)| name == "data_file") + else { + panic!("manifest entry has no data_file record"); + }; + for (name, value) in data_file { + if name == "record_count" && *value == AvroValue::Long(10) { + *value = AvroValue::Long(-10); + } + } + }, + ); + manifest_file.first_row_id = Some(100); + let err = read_manifest_entries(&bytes, &manifest_file).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_compressed_manifests() { + let (bytes, manifest_file) = first_row_id_manifest().await; + let expected = read_manifest_entries(&bytes, &manifest_file).unwrap(); + for codec in [ + Codec::Deflate(DeflateSettings::default()), + Codec::Snappy, + Codec::Zstandard(ZstandardSettings::default()), + ] { + let bytes = rewrite_manifest(&bytes, codec, |_| {}, |_| {}); + assert_eq!( + read_manifest_entries(&bytes, &manifest_file).unwrap(), + expected, + "{codec:?}" + ); + } + } + + #[tokio::test] + async fn test_fields_are_matched_by_field_id() { + let file = DataFile { + partition: Struct::from_iter([Some(Literal::int(7))]), + ..data_file(DataContentType::Data, "s3://b/data/0.parquet") + .build() + .unwrap() + }; + let (bytes, manifest_file) = write_manifest( + partition_spec(&[("i", Transform::Identity)]), + Some(1), + |b| b.build_v2_data(), + |writer| writer.add_file(file, 1).unwrap(), + ) + .await; + let bytes = rename_fields(&bytes, &[ + ("file_path", "path"), + ("i_identity", "renamed_partition"), + ]); + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + + assert_eq!( + data_file_column(&batch, "file_path").as_ref(), + &StringArray::from(vec!["s3://b/data/0.parquet"]) as &dyn Array + ); + let partition = data_file_column(&batch, "partition"); + assert_eq!( + partition.as_struct().column(0).as_ref(), + &Int32Array::from(vec![7]) as &dyn Array + ); + } + + #[tokio::test] + async fn test_missing_required_field_is_an_error() { + let file = data_file(DataContentType::Data, "s3://b/data/0.parquet") + .build() + .unwrap(); + let (bytes, manifest_file) = write_manifest( + partition_spec(&[]), + Some(1), + |b| b.build_v2_data(), + |writer| writer.add_file(file, 1).unwrap(), + ) + .await; + // A file_path under another field ID leaves the required file_path missing. + let bytes = rewrite_manifest( + &bytes, + Codec::Null, + |schema| { + for_each_avro_field(schema, &|field| { + if field["name"] == "file_path" { + field["field-id"] = 999.into(); + } + }) + }, + |_| {}, + ); + let err = read_manifest_entries(&bytes, &manifest_file).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_manifest_written_by_pyiceberg() { + let bytes = std::fs::read(format!( + "{}/testdata/manifests/pyiceberg-v2-data.avro", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let manifest = Manifest::parse_avro(&bytes).unwrap(); + let manifest_file = ManifestFile { + manifest_path: "s3://bucket/metadata/manifest.avro".to_string(), + manifest_length: bytes.len() as i64, + partition_spec_id: 0, + content: ManifestContentType::Data, + sequence_number: 1, + min_sequence_number: 1, + added_snapshot_id: 1, + added_files_count: None, + existing_files_count: None, + deleted_files_count: None, + added_rows_count: None, + existing_rows_count: None, + deleted_rows_count: None, + partitions: None, + key_metadata: None, + first_row_id: None, + }; + let batch = read_manifest_entries(&bytes, &manifest_file).unwrap(); + + assert_eq!(batch.num_rows(), manifest.entries().len()); + let paths = data_file_column(&batch, "file_path"); + let partition = data_file_column(&batch, "partition"); + let category = partition.as_struct().column(0).as_string::().clone(); + for (row, entry) in manifest.entries().iter().enumerate() { + let data_file = entry.data_file(); + assert_eq!(paths.as_string::().value(row), data_file.file_path()); + let Some(Literal::Primitive(PrimitiveLiteral::String(expected))) = + &data_file.partition()[0] + else { + panic!("partition value is not a string"); + }; + assert_eq!(category.value(row), expected); + let mut lower_bounds: Vec<_> = data_file + .lower_bounds() + .iter() + .map(|(id, bound)| (*id, bound.to_bytes().unwrap().to_vec())) + .collect(); + lower_bounds.sort_unstable(); + assert!(!lower_bounds.is_empty()); + assert_eq!( + binary_map(&data_file_column(&batch, "lower_bounds"), row), + lower_bounds + ); + } + } + + /// A manifest list entry that a manifest list writer for snapshot 1 + /// assigns sequence numbers to. + fn listed_manifest(path: &str, content: ManifestContentType) -> ManifestFile { + ManifestFile { + manifest_path: path.to_string(), + manifest_length: 100, + partition_spec_id: 0, + content, + sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + added_snapshot_id: 1, + added_files_count: Some(1), + existing_files_count: Some(2), + deleted_files_count: Some(3), + added_rows_count: Some(10), + existing_rows_count: Some(20), + deleted_rows_count: Some(30), + partitions: None, + key_metadata: None, + first_row_id: None, + } + } + + /// Writes a manifest list for snapshot 1 with sequence number 5 and, in + /// v3, first row ID 100. + async fn write_manifest_list(version: FormatVersion, manifests: Vec) -> Vec { + let io = FileIO::new_with_memory(); + let path = "memory:///manifest-list.avro"; + let output = io.new_output(path).unwrap().writer().await.unwrap(); + let mut writer = match version { + FormatVersion::V1 => ManifestListWriter::v1(output, 1, None), + FormatVersion::V2 => ManifestListWriter::v2(output, 1, None, 5), + FormatVersion::V3 => ManifestListWriter::v3(output, 1, None, 5, Some(100)), + }; + writer.add_manifests(manifests.into_iter()).unwrap(); + writer.close().await.unwrap(); + io.new_input(path).unwrap().read().await.unwrap().to_vec() + } + + #[tokio::test] + async fn test_schema_is_spec_manifest_file() { + let bytes = write_manifest_list(FormatVersion::V3, vec![]).await; + let batch = read_manifest_list(&bytes).unwrap(); + assert_eq!(batch.num_rows(), 0); + + let long = || Type::Primitive(PrimitiveType::Long); + let int = || Type::Primitive(PrimitiveType::Int); + let boolean = || Type::Primitive(PrimitiveType::Boolean); + let binary = || Type::Primitive(PrimitiveType::Binary); + let field_summary = StructType::new(vec![ + NestedField::required(509, "contains_null", boolean()).into(), + NestedField::optional(518, "contains_nan", boolean()).into(), + NestedField::optional(510, "lower_bound", binary()).into(), + NestedField::optional(511, "upper_bound", binary()).into(), + ]); + let expected = Schema::builder() + .with_fields(vec![ + NestedField::required(500, "manifest_path", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::required(501, "manifest_length", long()).into(), + NestedField::required(502, "partition_spec_id", int()).into(), + NestedField::required(517, "content", int()).into(), + NestedField::required(515, "sequence_number", long()).into(), + NestedField::required(516, "min_sequence_number", long()).into(), + NestedField::required(503, "added_snapshot_id", long()).into(), + NestedField::optional(504, "added_files_count", int()).into(), + NestedField::optional(505, "existing_files_count", int()).into(), + NestedField::optional(506, "deleted_files_count", int()).into(), + NestedField::optional(512, "added_rows_count", long()).into(), + NestedField::optional(513, "existing_rows_count", long()).into(), + NestedField::optional(514, "deleted_rows_count", long()).into(), + NestedField::optional( + 507, + "partitions", + Type::List(ListType::new( + NestedField::list_element(508, Type::Struct(field_summary), true).into(), + )), + ) + .into(), + NestedField::optional(519, "key_metadata", binary()).into(), + NestedField::optional(520, "first_row_id", long()).into(), + ]) + .build() + .unwrap(); + assert_eq!( + arrow_schema_to_schema(&batch.schema()).unwrap().as_struct(), + expected.as_struct() + ); + } + + #[tokio::test] + async fn test_manifest_list_columns() { + let data = ManifestFile { + partitions: Some(vec![FieldSummary { + contains_null: true, + contains_nan: Some(false), + lower_bound: Some(Datum::int(1).to_bytes().unwrap()), + upper_bound: Some(Datum::int(9).to_bytes().unwrap()), + }]), + key_metadata: Some(vec![1, 2]), + ..listed_manifest("s3://b/metadata/data.avro", ManifestContentType::Data) + }; + let existing_deletes = ManifestFile { + partition_spec_id: 1, + sequence_number: 3, + min_sequence_number: 2, + added_snapshot_id: 0, + added_files_count: Some(0), + ..listed_manifest("s3://b/metadata/deletes.avro", ManifestContentType::Deletes) + }; + let bytes = write_manifest_list(FormatVersion::V3, vec![data, existing_deletes]).await; + let batch = read_manifest_list(&bytes).unwrap(); + + let schema = batch.schema(); + let DataType::List(summary) = schema.field_with_name("partitions").unwrap().data_type() + else { + panic!("partitions is not a list"); + }; + let DataType::Struct(summary_fields) = summary.data_type() else { + panic!("partitions element is not a struct"); + }; + let summaries = StructArray::new( + summary_fields.clone(), + vec![ + Arc::new(BooleanArray::from(vec![true])), + Arc::new(BooleanArray::from(vec![Some(false)])), + Arc::new(BinaryArray::from(vec![&1i32.to_le_bytes()[..]])), + Arc::new(BinaryArray::from(vec![&9i32.to_le_bytes()[..]])), + ], + None, + ); + let columns: Vec = vec![ + Arc::new(StringArray::from(vec![ + "s3://b/metadata/data.avro", + "s3://b/metadata/deletes.avro", + ])), + Arc::new(Int64Array::from(vec![100, 100])), + Arc::new(Int32Array::from(vec![0, 1])), + Arc::new(Int32Array::from(vec![0, 1])), + Arc::new(Int64Array::from(vec![5, 3])), + Arc::new(Int64Array::from(vec![5, 2])), + Arc::new(Int64Array::from(vec![1, 0])), + Arc::new(Int32Array::from(vec![1, 0])), + Arc::new(Int32Array::from(vec![2, 2])), + Arc::new(Int32Array::from(vec![3, 3])), + Arc::new(Int64Array::from(vec![10, 10])), + Arc::new(Int64Array::from(vec![20, 20])), + Arc::new(Int64Array::from(vec![30, 30])), + Arc::new(ListArray::new( + summary.clone(), + OffsetBuffer::from_lengths([1, 0]), + Arc::new(summaries), + Some(NullBuffer::from(vec![true, false])), + )), + Arc::new(BinaryArray::from(vec![Some(&[1u8, 2][..]), None])), + // The writer assigns the data manifest the snapshot's first row ID. + Arc::new(Int64Array::from(vec![Some(100), None])), + ]; + assert_eq!(batch, RecordBatch::try_new(schema, columns).unwrap()); + } + + #[tokio::test] + async fn test_v1_manifest_list_reads_with_v2_defaults() { + let unknown_counts = ManifestFile { + added_files_count: None, + existing_files_count: None, + deleted_files_count: None, + added_rows_count: None, + existing_rows_count: None, + deleted_rows_count: None, + ..listed_manifest("s3://b/metadata/1.avro", ManifestContentType::Data) + }; + let bytes = write_manifest_list(FormatVersion::V1, vec![ + listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data), + unknown_counts, + ]) + .await; + // A manifest list without a format version reads as v1. + let without_format_version = set_avro_metadata(&bytes, "format-version", None); + for bytes in [bytes, without_format_version] { + let batch = read_manifest_list(&bytes).unwrap(); + + assert_eq!( + column(&batch, "content").as_ref(), + &Int32Array::from(vec![0, 0]) as &dyn Array + ); + for name in ["sequence_number", "min_sequence_number"] { + assert_eq!( + column(&batch, name).as_ref(), + &Int64Array::from(vec![0, 0]) as &dyn Array, + "{name}" + ); + } + assert_eq!( + column(&batch, "added_files_count").as_ref(), + &Int32Array::from(vec![Some(1), None]) as &dyn Array + ); + assert_eq!( + column(&batch, "deleted_rows_count").as_ref(), + &Int64Array::from(vec![Some(30), None]) as &dyn Array + ); + assert_eq!(column(&batch, "first_row_id").null_count(), 2); + } + } + + #[tokio::test] + async fn test_manifest_list_fields_are_matched_by_field_id() { + let manifest = ManifestFile { + partitions: Some(vec![FieldSummary { + contains_null: true, + contains_nan: None, + lower_bound: None, + upper_bound: None, + }]), + ..listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data) + }; + let bytes = write_manifest_list(FormatVersion::V2, vec![manifest]).await; + let bytes = rename_fields(&bytes, &[ + ("manifest_path", "path"), + ("added_files_count", "added_data_files_count"), + ("contains_null", "has_null"), + ]); + let batch = read_manifest_list(&bytes).unwrap(); + + assert_eq!( + column(&batch, "manifest_path").as_ref(), + &StringArray::from(vec!["s3://b/metadata/0.avro"]) as &dyn Array + ); + assert_eq!( + column(&batch, "added_files_count").as_ref(), + &Int32Array::from(vec![1]) as &dyn Array + ); + let partitions = column(&batch, "partitions"); + assert_eq!( + partitions + .as_list::() + .value(0) + .as_struct() + .column(0) + .as_ref(), + &BooleanArray::from(vec![true]) as &dyn Array + ); + } + + #[tokio::test] + async fn test_invalid_manifest_list_is_an_error() { + let err = read_manifest_list(&[]).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + + let bytes = write_manifest_list(FormatVersion::V2, vec![listed_manifest( + "s3://b/metadata/0.avro", + ManifestContentType::Data, + )]) + .await; + // A manifest_path under another field ID leaves the required + // manifest_path missing. + let moved_manifest_path = rewrite_manifest( + &bytes, + Codec::Null, + |schema| { + for_each_avro_field(schema, &|field| { + if field["name"] == "manifest_path" { + field["field-id"] = 999.into(); + } + }) + }, + |_| {}, + ); + // Only v1 manifest lists read a missing sequence number as 0. + let without_sequence_number = rewrite_manifest( + &bytes, + Codec::Null, + |schema| { + schema["fields"] + .as_array_mut() + .unwrap() + .retain(|field| field["name"] != "sequence_number") + }, + |record| { + let AvroValue::Record(fields) = record else { + panic!("manifest list entry is not a record"); + }; + fields.retain(|(name, _)| name != "sequence_number"); + }, + ); + let unknown_format_version = set_avro_metadata(&bytes, "format-version", Some("4")); + for bytes in [ + moved_manifest_path, + without_sequence_number, + unknown_format_version, + ] { + let err = read_manifest_list(&bytes).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + } + + #[test] + fn test_manifest_lists_written_by_java() { + // Written by Java 1.4.0, with the v1 file count names, and 1.5.0. + for name in ["manifest-list-v2-1.avro", "manifest-list-v2-2.avro"] { + let bytes = std::fs::read(format!( + "{}/testdata/manifests_lists/{name}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let manifest_list = + ManifestList::parse_with_version(&bytes, FormatVersion::V2).unwrap(); + let batch = read_manifest_list(&bytes).unwrap(); + + assert_eq!(batch.num_rows(), manifest_list.entries().len(), "{name}"); + let paths = column(&batch, "manifest_path"); + let sequence_numbers = column(&batch, "sequence_number"); + let added_files_counts = column(&batch, "added_files_count"); + let partitions = column(&batch, "partitions"); + for (row, manifest) in manifest_list.entries().iter().enumerate() { + assert_eq!( + paths.as_string::().value(row), + manifest.manifest_path, + "{name}" + ); + assert_eq!( + sequence_numbers.as_primitive::().value(row), + manifest.sequence_number, + "{name}" + ); + assert_eq!( + added_files_counts.as_primitive::().value(row), + manifest.added_files_count.unwrap() as i32, + "{name}" + ); + let summaries = partitions.as_list::().value(row); + let lower_bounds: Vec<_> = summaries + .as_struct() + .column_by_name("lower_bound") + .unwrap() + .as_binary::() + .iter() + .map(|bound| bound.map(<[u8]>::to_vec)) + .collect(); + let expected: Vec<_> = manifest + .partitions + .iter() + .flatten() + .map(|summary| summary.lower_bound.as_ref().map(|bound| bound.to_vec())) + .collect(); + assert_eq!(lower_bounds, expected, "{name}"); + } + } + } +} diff --git a/crates/iceberg/src/arrow/mod.rs b/crates/iceberg/src/arrow/mod.rs index d32cbeb8f9..d8343ef1ef 100644 --- a/crates/iceberg/src/arrow/mod.rs +++ b/crates/iceberg/src/arrow/mod.rs @@ -27,11 +27,26 @@ pub(crate) mod caching_delete_file_loader; pub mod delete_file_loader; pub(crate) mod delete_filter; +mod int96; +mod manifest; +pub use manifest::{read_manifest_entries, read_manifest_list}; mod reader; -pub(crate) mod record_batch_projector; +/// RecordBatch projection utilities +pub mod record_batch_projector; pub(crate) mod record_batch_transformer; +pub(crate) use record_batch_transformer::build_partition_constant; +mod scan_metrics; mod value; pub use reader::*; +pub use scan_metrics::{ScanMetrics, ScanResult}; pub use value::*; -pub(crate) mod record_batch_partition_splitter; +/// Partition value calculator for computing partition values +pub mod partition_value_calculator; +pub use partition_value_calculator::*; +/// Record batch partition splitter for partitioned tables +pub mod record_batch_partition_splitter; +pub use record_batch_partition_splitter::*; + +#[cfg(test)] +pub(crate) mod test_utils; diff --git a/crates/iceberg/src/arrow/nan_val_cnt_visitor.rs b/crates/iceberg/src/arrow/nan_val_cnt_visitor.rs index e514457887..d01f3e9e56 100644 --- a/crates/iceberg/src/arrow/nan_val_cnt_visitor.rs +++ b/crates/iceberg/src/arrow/nan_val_cnt_visitor.rs @@ -28,7 +28,7 @@ use crate::Result; use crate::arrow::{ArrowArrayAccessor, FieldMatchMode}; use crate::spec::{ ListType, MapType, NestedFieldRef, PrimitiveType, Schema, SchemaRef, SchemaWithPartnerVisitor, - StructType, visit_struct_with_partner, + StructType, VariantType, visit_struct_with_partner, }; macro_rules! cast_and_update_cnt_map { @@ -122,6 +122,10 @@ impl SchemaWithPartnerVisitor for NanValueCountVisitor { Ok(()) } + fn variant(&mut self, _v: &VariantType, _col: &ArrayRef) -> Result { + Ok(()) + } + fn after_struct_field(&mut self, field: &NestedFieldRef, partner: &ArrayRef) -> Result<()> { let field_id = field.id; count_float_nans!(partner, self, field_id); diff --git a/crates/iceberg/src/arrow/partition_value_calculator.rs b/crates/iceberg/src/arrow/partition_value_calculator.rs new file mode 100644 index 0000000000..142de42cdd --- /dev/null +++ b/crates/iceberg/src/arrow/partition_value_calculator.rs @@ -0,0 +1,308 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Partition value calculation for Iceberg tables. +//! +//! This module provides utilities for calculating partition values from record batches +//! based on a partition specification. + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch, StructArray}; +use arrow_schema::DataType; + +use super::record_batch_projector::RecordBatchProjector; +use super::type_to_arrow_type; +use crate::Result; +use crate::error::invalid_data; +use crate::spec::{PartitionSpec, Schema, StructType, Type}; +use crate::transform::{BoxedTransformFunction, create_transform_function}; + +/// Calculator for partition values in Iceberg tables. +/// +/// This struct handles the projection of source columns and application of +/// partition transforms to compute partition values for a given record batch. +#[derive(Debug)] +pub struct PartitionValueCalculator { + projector: RecordBatchProjector, + transform_functions: Vec, + partition_type: StructType, + partition_arrow_type: DataType, +} + +impl PartitionValueCalculator { + /// Create a new PartitionValueCalculator. + /// + /// # Arguments + /// + /// * `partition_spec` - The partition specification + /// * `table_schema` - The Iceberg table schema + /// + /// # Returns + /// + /// Returns a new `PartitionValueCalculator` instance or an error if initialization fails. + /// + /// # Errors + /// + /// Returns an error if: + /// - The partition spec is unpartitioned + /// - Transform function creation fails + /// - Projector initialization fails + pub fn try_new(partition_spec: &PartitionSpec, table_schema: &Schema) -> Result { + if partition_spec.is_unpartitioned() { + return Err(invalid_data!( + "Cannot create partition calculator for unpartitioned table" + )); + } + + // Create transform functions for each partition field + let transform_functions: Vec = partition_spec + .fields() + .iter() + .map(|pf| create_transform_function(&pf.transform)) + .collect::>>()?; + + // Extract source field IDs for projection + let source_field_ids: Vec = partition_spec + .fields() + .iter() + .map(|pf| pf.source_id) + .collect(); + + // Create projector for extracting source columns + let projector = RecordBatchProjector::from_iceberg_schema( + Arc::new(table_schema.clone()), + &source_field_ids, + )?; + + // Get partition type information + let partition_type = partition_spec.partition_type(table_schema)?; + let partition_arrow_type = type_to_arrow_type(&Type::Struct(partition_type.clone()))?; + + Ok(Self { + projector, + transform_functions, + partition_type, + partition_arrow_type, + }) + } + + /// Get the partition type as an Iceberg StructType. + pub fn partition_type(&self) -> &StructType { + &self.partition_type + } + + /// Get the partition type as an Arrow DataType. + pub fn partition_arrow_type(&self) -> &DataType { + &self.partition_arrow_type + } + + /// Calculate partition values for a record batch. + /// + /// This method: + /// 1. Projects the source columns from the batch + /// 2. Applies partition transforms to each source column + /// 3. Constructs a StructArray containing the partition values + /// + /// # Arguments + /// + /// * `batch` - The record batch to calculate partition values for + /// + /// # Returns + /// + /// Returns an ArrayRef containing a StructArray of partition values, or an error if calculation fails. + /// + /// # Errors + /// + /// Returns an error if: + /// - Column projection fails + /// - Transform application fails + /// - StructArray construction fails + pub fn calculate(&self, batch: &RecordBatch) -> Result { + // Project source columns from the batch + let source_columns = self.projector.project_column(batch.columns())?; + + // Get expected struct fields for the result + let expected_struct_fields = match &self.partition_arrow_type { + DataType::Struct(fields) => fields.clone(), + _ => { + return Err(invalid_data!("Expected partition type must be a struct")); + } + }; + + // Apply transforms to each source column + let mut partition_values = Vec::with_capacity(self.transform_functions.len()); + for (source_column, transform_fn) in source_columns.iter().zip(&self.transform_functions) { + let partition_value = transform_fn.transform(source_column.clone())?; + partition_values.push(partition_value); + } + + // Construct the StructArray + let struct_array = StructArray::try_new(expected_struct_fields, partition_values, None) + .map_err(|e| invalid_data!("Failed to create partition struct array: {e}"))?; + + Ok(Arc::new(struct_array)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{BinaryArray, Int32Array, LargeBinaryArray, RecordBatch, StringArray}; + use arrow_schema::{Field, Schema as ArrowSchema}; + + use super::*; + use crate::spec::{NestedField, PartitionSpecBuilder, PrimitiveType, Transform}; + + #[test] + fn test_partition_calculator_identity_transform() { + let table_schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + let partition_spec = PartitionSpecBuilder::new(Arc::new(table_schema.clone())) + .add_partition_field("id", "id_partition", Transform::Identity) + .unwrap() + .build() + .unwrap(); + + let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); + + // Verify partition type + assert_eq!(calculator.partition_type().fields().len(), 1); + assert_eq!(calculator.partition_type().fields()[0].name, "id_partition"); + + // Create test batch + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + ])); + + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(vec![10, 20, 30])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ]) + .unwrap(); + + // Calculate partition values + let result = calculator.calculate(&batch).unwrap(); + let struct_array = result.as_any().downcast_ref::().unwrap(); + + let id_partition = struct_array + .column_by_name("id_partition") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(id_partition.value(0), 10); + assert_eq!(id_partition.value(1), 20); + assert_eq!(id_partition.value(2), 30); + } + + #[test] + fn test_partition_calculator_binary_partition_accepts_large_binary_input() { + let table_schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "key", Type::Primitive(PrimitiveType::Binary)).into(), + ]) + .build() + .unwrap(); + + let partition_spec = PartitionSpecBuilder::new(Arc::new(table_schema.clone())) + .add_partition_field("key", "key_identity", Transform::Identity) + .unwrap() + .add_partition_field("key", "key_trunc", Transform::Truncate(2)) + .unwrap() + .build() + .unwrap(); + let calculator = PartitionValueCalculator::try_new(&partition_spec, &table_schema).unwrap(); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("key", DataType::LargeBinary, true), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(LargeBinaryArray::from_opt_vec(vec![ + Some(b"abc"), + None, + Some(b"z"), + ])), + ]) + .unwrap(); + + let result = calculator.calculate(&batch).unwrap(); + + let DataType::Struct(expected_fields) = calculator.partition_arrow_type() else { + unreachable!() + }; + let expected = StructArray::try_new( + expected_fields.clone(), + vec![ + Arc::new(BinaryArray::from_opt_vec(vec![ + Some(b"abc"), + None, + Some(b"z"), + ])), + Arc::new(BinaryArray::from_opt_vec(vec![ + Some(b"ab"), + None, + Some(b"z"), + ])), + ], + None, + ) + .unwrap(); + assert_eq!( + result.as_any().downcast_ref::().unwrap(), + &expected + ); + } + + #[test] + fn test_partition_calculator_unpartitioned_error() { + let table_schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let partition_spec = PartitionSpecBuilder::new(Arc::new(table_schema.clone())) + .build() + .unwrap(); + + let result = PartitionValueCalculator::try_new(&partition_spec, &table_schema); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("unpartitioned table") + ); + } +} diff --git a/crates/iceberg/src/arrow/reader.rs b/crates/iceberg/src/arrow/reader.rs deleted file mode 100644 index 05aa6a4c9f..0000000000 --- a/crates/iceberg/src/arrow/reader.rs +++ /dev/null @@ -1,1961 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Parquet file data reader - -use std::collections::{HashMap, HashSet}; -use std::ops::Range; -use std::str::FromStr; -use std::sync::Arc; - -use arrow_arith::boolean::{and, and_kleene, is_not_null, is_null, not, or, or_kleene}; -use arrow_array::{Array, ArrayRef, BooleanArray, Datum as ArrowDatum, RecordBatch, Scalar}; -use arrow_cast::cast::cast; -use arrow_ord::cmp::{eq, gt, gt_eq, lt, lt_eq, neq}; -use arrow_schema::{ - ArrowError, DataType, FieldRef, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, -}; -use arrow_string::like::starts_with; -use bytes::Bytes; -use fnv::FnvHashSet; -use futures::future::BoxFuture; -use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, try_join}; -use parquet::arrow::arrow_reader::{ - ArrowPredicateFn, ArrowReaderOptions, RowFilter, RowSelection, RowSelector, -}; -use parquet::arrow::async_reader::AsyncFileReader; -use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder, ProjectionMask}; -use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader, RowGroupMetaData}; -use parquet::schema::types::{SchemaDescriptor, Type as ParquetType}; - -use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader; -use crate::arrow::record_batch_transformer::RecordBatchTransformer; -use crate::arrow::{arrow_schema_to_schema, get_arrow_datum}; -use crate::delete_vector::DeleteVector; -use crate::error::Result; -use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; -use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; -use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; -use crate::expr::{BoundPredicate, BoundReference}; -use crate::io::{FileIO, FileMetadata, FileRead}; -use crate::scan::{ArrowRecordBatchStream, FileScanTask, FileScanTaskStream}; -use crate::spec::{Datum, NestedField, PrimitiveType, Schema, Type}; -use crate::utils::available_parallelism; -use crate::{Error, ErrorKind}; - -/// Builder to create ArrowReader -pub struct ArrowReaderBuilder { - batch_size: Option, - file_io: FileIO, - concurrency_limit_data_files: usize, - row_group_filtering_enabled: bool, - row_selection_enabled: bool, -} - -impl ArrowReaderBuilder { - /// Create a new ArrowReaderBuilder - pub(crate) fn new(file_io: FileIO) -> Self { - let num_cpus = available_parallelism().get(); - - ArrowReaderBuilder { - batch_size: None, - file_io, - concurrency_limit_data_files: num_cpus, - row_group_filtering_enabled: true, - row_selection_enabled: false, - } - } - - /// Sets the max number of in flight data files that are being fetched - pub fn with_data_file_concurrency_limit(mut self, val: usize) -> Self { - self.concurrency_limit_data_files = val; - self - } - - /// Sets the desired size of batches in the response - /// to something other than the default - pub fn with_batch_size(mut self, batch_size: usize) -> Self { - self.batch_size = Some(batch_size); - self - } - - /// Determines whether to enable row group filtering. - pub fn with_row_group_filtering_enabled(mut self, row_group_filtering_enabled: bool) -> Self { - self.row_group_filtering_enabled = row_group_filtering_enabled; - self - } - - /// Determines whether to enable row selection. - pub fn with_row_selection_enabled(mut self, row_selection_enabled: bool) -> Self { - self.row_selection_enabled = row_selection_enabled; - self - } - - /// Build the ArrowReader. - pub fn build(self) -> ArrowReader { - ArrowReader { - batch_size: self.batch_size, - file_io: self.file_io.clone(), - delete_file_loader: CachingDeleteFileLoader::new( - self.file_io.clone(), - self.concurrency_limit_data_files, - ), - concurrency_limit_data_files: self.concurrency_limit_data_files, - row_group_filtering_enabled: self.row_group_filtering_enabled, - row_selection_enabled: self.row_selection_enabled, - } - } -} - -/// Reads data from Parquet files -#[derive(Clone)] -pub struct ArrowReader { - batch_size: Option, - file_io: FileIO, - delete_file_loader: CachingDeleteFileLoader, - - /// the maximum number of data files that can be fetched at the same time - concurrency_limit_data_files: usize, - - row_group_filtering_enabled: bool, - row_selection_enabled: bool, -} - -impl ArrowReader { - /// Take a stream of FileScanTasks and reads all the files. - /// Returns a stream of Arrow RecordBatches containing the data from the files - pub fn read(self, tasks: FileScanTaskStream) -> Result { - let file_io = self.file_io.clone(); - let batch_size = self.batch_size; - let concurrency_limit_data_files = self.concurrency_limit_data_files; - let row_group_filtering_enabled = self.row_group_filtering_enabled; - let row_selection_enabled = self.row_selection_enabled; - - let stream = tasks - .map_ok(move |task| { - let file_io = file_io.clone(); - - Self::process_file_scan_task( - task, - batch_size, - file_io, - self.delete_file_loader.clone(), - row_group_filtering_enabled, - row_selection_enabled, - ) - }) - .map_err(|err| { - Error::new(ErrorKind::Unexpected, "file scan task generate failed").with_source(err) - }) - .try_buffer_unordered(concurrency_limit_data_files) - .try_flatten_unordered(concurrency_limit_data_files); - - Ok(Box::pin(stream) as ArrowRecordBatchStream) - } - - #[allow(clippy::too_many_arguments)] - async fn process_file_scan_task( - task: FileScanTask, - batch_size: Option, - file_io: FileIO, - delete_file_loader: CachingDeleteFileLoader, - row_group_filtering_enabled: bool, - row_selection_enabled: bool, - ) -> Result { - let should_load_page_index = - (row_selection_enabled && task.predicate.is_some()) || !task.deletes.is_empty(); - - let delete_filter_rx = delete_file_loader.load_deletes(&task.deletes, task.schema.clone()); - - let mut record_batch_stream_builder = Self::create_parquet_record_batch_stream_builder( - &task.data_file_path, - file_io.clone(), - should_load_page_index, - ) - .await?; - - // Create a projection mask for the batch stream to select which columns in the - // Parquet file that we want in the response - let projection_mask = Self::get_arrow_projection_mask( - &task.project_field_ids, - &task.schema, - record_batch_stream_builder.parquet_schema(), - record_batch_stream_builder.schema(), - )?; - record_batch_stream_builder = record_batch_stream_builder.with_projection(projection_mask); - - // RecordBatchTransformer performs any transformations required on the RecordBatches - // that come back from the file, such as type promotion, default column insertion - // and column re-ordering - let mut record_batch_transformer = - RecordBatchTransformer::build(task.schema_ref(), task.project_field_ids()); - - if let Some(batch_size) = batch_size { - record_batch_stream_builder = record_batch_stream_builder.with_batch_size(batch_size); - } - - let delete_filter = delete_filter_rx.await.unwrap()?; - let delete_predicate = delete_filter.build_equality_delete_predicate(&task).await?; - - // In addition to the optional predicate supplied in the `FileScanTask`, - // we also have an optional predicate resulting from equality delete files. - // If both are present, we logical-AND them together to form a single filter - // predicate that we can pass to the `RecordBatchStreamBuilder`. - let final_predicate = match (&task.predicate, delete_predicate) { - (None, None) => None, - (Some(predicate), None) => Some(predicate.clone()), - (None, Some(ref predicate)) => Some(predicate.clone()), - (Some(filter_predicate), Some(delete_predicate)) => { - Some(filter_predicate.clone().and(delete_predicate)) - } - }; - - // There are two possible sources both for potential lists of selected RowGroup indices, - // and for `RowSelection`s. - // Selected RowGroup index lists can come from two sources: - // * When there are equality delete files that are applicable; - // * When there is a scan predicate and row_group_filtering_enabled = true. - // `RowSelection`s can be created in either or both of the following cases: - // * When there are positional delete files that are applicable; - // * When there is a scan predicate and row_selection_enabled = true - // Note that, in the former case we only perform row group filtering when - // there is a scan predicate AND row_group_filtering_enabled = true, - // but we perform row selection filtering if there are applicable - // equality delete files OR (there is a scan predicate AND row_selection_enabled), - // since the only implemented method of applying positional deletes is - // by using a `RowSelection`. - let mut selected_row_group_indices = None; - let mut row_selection = None; - - if let Some(predicate) = final_predicate { - let (iceberg_field_ids, field_id_map) = Self::build_field_id_set_and_map( - record_batch_stream_builder.parquet_schema(), - &predicate, - )?; - - let row_filter = Self::get_row_filter( - &predicate, - record_batch_stream_builder.parquet_schema(), - &iceberg_field_ids, - &field_id_map, - )?; - record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter); - - if row_group_filtering_enabled { - let result = Self::get_selected_row_group_indices( - &predicate, - record_batch_stream_builder.metadata(), - &field_id_map, - &task.schema, - )?; - - selected_row_group_indices = Some(result); - } - - if row_selection_enabled { - row_selection = Some(Self::get_row_selection_for_filter_predicate( - &predicate, - record_batch_stream_builder.metadata(), - &selected_row_group_indices, - &field_id_map, - &task.schema, - )?); - } - } - - let positional_delete_indexes = delete_filter.get_delete_vector(&task); - - if let Some(positional_delete_indexes) = positional_delete_indexes { - let delete_row_selection = { - let positional_delete_indexes = positional_delete_indexes.lock().unwrap(); - - Self::build_deletes_row_selection( - record_batch_stream_builder.metadata().row_groups(), - &selected_row_group_indices, - &positional_delete_indexes, - ) - }?; - - // merge the row selection from the delete files with the row selection - // from the filter predicate, if there is one from the filter predicate - row_selection = match row_selection { - None => Some(delete_row_selection), - Some(filter_row_selection) => { - Some(filter_row_selection.intersection(&delete_row_selection)) - } - }; - } - - if let Some(row_selection) = row_selection { - record_batch_stream_builder = - record_batch_stream_builder.with_row_selection(row_selection); - } - - if let Some(selected_row_group_indices) = selected_row_group_indices { - record_batch_stream_builder = - record_batch_stream_builder.with_row_groups(selected_row_group_indices); - } - - // Build the batch stream and send all the RecordBatches that it generates - // to the requester. - let record_batch_stream = - record_batch_stream_builder - .build()? - .map(move |batch| match batch { - Ok(batch) => record_batch_transformer.process_record_batch(batch), - Err(err) => Err(err.into()), - }); - - Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream) - } - - pub(crate) async fn create_parquet_record_batch_stream_builder( - data_file_path: &str, - file_io: FileIO, - should_load_page_index: bool, - ) -> Result>> { - // Get the metadata for the Parquet file we need to read and build - // a reader for the data within - let parquet_file = file_io.new_input(data_file_path)?; - let (parquet_metadata, parquet_reader) = - try_join!(parquet_file.metadata(), parquet_file.reader())?; - let parquet_file_reader = ArrowFileReader::new(parquet_metadata, parquet_reader) - .with_preload_column_index(true) - .with_preload_offset_index(true) - .with_preload_page_index(should_load_page_index); - - // Create the record batch stream builder, which wraps the parquet file reader - let record_batch_stream_builder = ParquetRecordBatchStreamBuilder::new_with_options( - parquet_file_reader, - ArrowReaderOptions::new(), - ) - .await?; - Ok(record_batch_stream_builder) - } - - /// computes a `RowSelection` from positional delete indices. - /// - /// Using the Parquet page index, we build a `RowSelection` that rejects rows that are indicated - /// as having been deleted by a positional delete, taking into account any row groups that have - /// been skipped entirely by the filter predicate - fn build_deletes_row_selection( - row_group_metadata_list: &[RowGroupMetaData], - selected_row_groups: &Option>, - positional_deletes: &DeleteVector, - ) -> Result { - let mut results: Vec = Vec::new(); - let mut selected_row_groups_idx = 0; - let mut current_row_group_base_idx: u64 = 0; - let mut delete_vector_iter = positional_deletes.iter(); - let mut next_deleted_row_idx_opt = delete_vector_iter.next(); - - for (idx, row_group_metadata) in row_group_metadata_list.iter().enumerate() { - let row_group_num_rows = row_group_metadata.num_rows() as u64; - let next_row_group_base_idx = current_row_group_base_idx + row_group_num_rows; - - // if row group selection is enabled, - if let Some(selected_row_groups) = selected_row_groups { - // if we've consumed all the selected row groups, we're done - if selected_row_groups_idx == selected_row_groups.len() { - break; - } - - if idx == selected_row_groups[selected_row_groups_idx] { - // we're in a selected row group. Increment selected_row_groups_idx - // so that next time around the for loop we're looking for the next - // selected row group - selected_row_groups_idx += 1; - } else { - // remove any positional deletes from the skipped page so that - // `positional.deletes.min()` can be used - delete_vector_iter.advance_to(next_row_group_base_idx); - next_deleted_row_idx_opt = delete_vector_iter.next(); - - // still increment the current page base index but then skip to the next row group - // in the file - current_row_group_base_idx += row_group_num_rows; - continue; - } - } - - let mut next_deleted_row_idx = match next_deleted_row_idx_opt { - Some(next_deleted_row_idx) => { - // if the index of the next deleted row is beyond this row group, add a selection for - // the remainder of this row group and skip to the next row group - if next_deleted_row_idx >= next_row_group_base_idx { - results.push(RowSelector::select(row_group_num_rows as usize)); - continue; - } - - next_deleted_row_idx - } - - // If there are no more pos deletes, add a selector for the entirety of this row group. - _ => { - results.push(RowSelector::select(row_group_num_rows as usize)); - continue; - } - }; - - let mut current_idx = current_row_group_base_idx; - 'chunks: while next_deleted_row_idx < next_row_group_base_idx { - // `select` all rows that precede the next delete index - if current_idx < next_deleted_row_idx { - let run_length = next_deleted_row_idx - current_idx; - results.push(RowSelector::select(run_length as usize)); - current_idx += run_length; - } - - // `skip` all consecutive deleted rows in the current row group - let mut run_length = 0; - while next_deleted_row_idx == current_idx - && next_deleted_row_idx < next_row_group_base_idx - { - run_length += 1; - current_idx += 1; - - next_deleted_row_idx_opt = delete_vector_iter.next(); - next_deleted_row_idx = match next_deleted_row_idx_opt { - Some(next_deleted_row_idx) => next_deleted_row_idx, - _ => { - // We've processed the final positional delete. - // Conclude the skip and then break so that we select the remaining - // rows in the row group and move on to the next row group - results.push(RowSelector::skip(run_length)); - break 'chunks; - } - }; - } - if run_length > 0 { - results.push(RowSelector::skip(run_length)); - } - } - - if current_idx < next_row_group_base_idx { - results.push(RowSelector::select( - (next_row_group_base_idx - current_idx) as usize, - )); - } - - current_row_group_base_idx += row_group_num_rows; - } - - Ok(results.into()) - } - - fn build_field_id_set_and_map( - parquet_schema: &SchemaDescriptor, - predicate: &BoundPredicate, - ) -> Result<(HashSet, HashMap)> { - // Collects all Iceberg field IDs referenced in the filter predicate - let mut collector = CollectFieldIdVisitor { - field_ids: HashSet::default(), - }; - visit(&mut collector, predicate)?; - - let iceberg_field_ids = collector.field_ids(); - let field_id_map = build_field_id_map(parquet_schema)?; - - Ok((iceberg_field_ids, field_id_map)) - } - - /// Insert the leaf field id into the field_ids using for projection. - /// For nested type, it will recursively insert the leaf field id. - fn include_leaf_field_id(field: &NestedField, field_ids: &mut Vec) { - match field.field_type.as_ref() { - Type::Primitive(_) => { - field_ids.push(field.id); - } - Type::Struct(struct_type) => { - for nested_field in struct_type.fields() { - Self::include_leaf_field_id(nested_field, field_ids); - } - } - Type::List(list_type) => { - Self::include_leaf_field_id(&list_type.element_field, field_ids); - } - Type::Map(map_type) => { - Self::include_leaf_field_id(&map_type.key_field, field_ids); - Self::include_leaf_field_id(&map_type.value_field, field_ids); - } - } - } - - fn get_arrow_projection_mask( - field_ids: &[i32], - iceberg_schema_of_task: &Schema, - parquet_schema: &SchemaDescriptor, - arrow_schema: &ArrowSchemaRef, - ) -> Result { - fn type_promotion_is_valid( - file_type: Option<&PrimitiveType>, - projected_type: Option<&PrimitiveType>, - ) -> bool { - match (file_type, projected_type) { - (Some(lhs), Some(rhs)) if lhs == rhs => true, - (Some(PrimitiveType::Int), Some(PrimitiveType::Long)) => true, - (Some(PrimitiveType::Float), Some(PrimitiveType::Double)) => true, - ( - Some(PrimitiveType::Decimal { - precision: file_precision, - scale: file_scale, - }), - Some(PrimitiveType::Decimal { - precision: requested_precision, - scale: requested_scale, - }), - ) if requested_precision >= file_precision && file_scale == requested_scale => true, - // Uuid will be store as Fixed(16) in parquet file, so the read back type will be Fixed(16). - (Some(PrimitiveType::Fixed(16)), Some(PrimitiveType::Uuid)) => true, - _ => false, - } - } - - let mut leaf_field_ids = vec![]; - for field_id in field_ids { - let field = iceberg_schema_of_task.field_by_id(*field_id); - if let Some(field) = field { - Self::include_leaf_field_id(field, &mut leaf_field_ids); - } - } - - if leaf_field_ids.is_empty() { - Ok(ProjectionMask::all()) - } else { - // Build the map between field id and column index in Parquet schema. - let mut column_map = HashMap::new(); - - let fields = arrow_schema.fields(); - // Pre-project only the fields that have been selected, possibly avoiding converting - // some Arrow types that are not yet supported. - let mut projected_fields: HashMap = HashMap::new(); - let projected_arrow_schema = ArrowSchema::new_with_metadata( - fields.filter_leaves(|_, f| { - f.metadata() - .get(PARQUET_FIELD_ID_META_KEY) - .and_then(|field_id| i32::from_str(field_id).ok()) - .is_some_and(|field_id| { - projected_fields.insert((*f).clone(), field_id); - leaf_field_ids.contains(&field_id) - }) - }), - arrow_schema.metadata().clone(), - ); - let iceberg_schema = arrow_schema_to_schema(&projected_arrow_schema)?; - - fields.filter_leaves(|idx, field| { - let Some(field_id) = projected_fields.get(field).cloned() else { - return false; - }; - - let iceberg_field = iceberg_schema_of_task.field_by_id(field_id); - let parquet_iceberg_field = iceberg_schema.field_by_id(field_id); - - if iceberg_field.is_none() || parquet_iceberg_field.is_none() { - return false; - } - - if !type_promotion_is_valid( - parquet_iceberg_field - .unwrap() - .field_type - .as_primitive_type(), - iceberg_field.unwrap().field_type.as_primitive_type(), - ) { - return false; - } - - column_map.insert(field_id, idx); - true - }); - - if column_map.len() != leaf_field_ids.len() { - let missing_fields = leaf_field_ids - .iter() - .filter(|field_id| !column_map.contains_key(field_id)) - .collect::>(); - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Parquet schema {} and Iceberg schema {} do not match.", - iceberg_schema, iceberg_schema_of_task - ), - ) - .with_context("column_map", format! {"{:?}", column_map}) - .with_context("field_ids", format! {"{:?}", leaf_field_ids}) - .with_context("missing_fields", format! {"{:?}", missing_fields})); - } - - let mut indices = vec![]; - for field_id in leaf_field_ids { - if let Some(col_idx) = column_map.get(&field_id) { - indices.push(*col_idx); - } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Field {} is not found in Parquet schema.", field_id), - )); - } - } - Ok(ProjectionMask::leaves(parquet_schema, indices)) - } - } - - fn get_row_filter( - predicates: &BoundPredicate, - parquet_schema: &SchemaDescriptor, - iceberg_field_ids: &HashSet, - field_id_map: &HashMap, - ) -> Result { - // Collect Parquet column indices from field ids. - // If the field id is not found in Parquet schema, it will be ignored due to schema evolution. - let mut column_indices = iceberg_field_ids - .iter() - .filter_map(|field_id| field_id_map.get(field_id).cloned()) - .collect::>(); - column_indices.sort(); - - // The converter that converts `BoundPredicates` to `ArrowPredicates` - let mut converter = PredicateConverter { - parquet_schema, - column_map: field_id_map, - column_indices: &column_indices, - }; - - // After collecting required leaf column indices used in the predicate, - // creates the projection mask for the Arrow predicates. - let projection_mask = ProjectionMask::leaves(parquet_schema, column_indices.clone()); - let predicate_func = visit(&mut converter, predicates)?; - let arrow_predicate = ArrowPredicateFn::new(projection_mask, predicate_func); - Ok(RowFilter::new(vec![Box::new(arrow_predicate)])) - } - - fn get_selected_row_group_indices( - predicate: &BoundPredicate, - parquet_metadata: &Arc, - field_id_map: &HashMap, - snapshot_schema: &Schema, - ) -> Result> { - let row_groups_metadata = parquet_metadata.row_groups(); - let mut results = Vec::with_capacity(row_groups_metadata.len()); - - for (idx, row_group_metadata) in row_groups_metadata.iter().enumerate() { - if RowGroupMetricsEvaluator::eval( - predicate, - row_group_metadata, - field_id_map, - snapshot_schema, - )? { - results.push(idx); - } - } - - Ok(results) - } - - fn get_row_selection_for_filter_predicate( - predicate: &BoundPredicate, - parquet_metadata: &Arc, - selected_row_groups: &Option>, - field_id_map: &HashMap, - snapshot_schema: &Schema, - ) -> Result { - let Some(column_index) = parquet_metadata.column_index() else { - return Err(Error::new( - ErrorKind::Unexpected, - "Parquet file metadata does not contain a column index", - )); - }; - - let Some(offset_index) = parquet_metadata.offset_index() else { - return Err(Error::new( - ErrorKind::Unexpected, - "Parquet file metadata does not contain an offset index", - )); - }; - - let mut selected_row_groups_idx = 0; - - let page_index = column_index - .iter() - .enumerate() - .zip(offset_index) - .zip(parquet_metadata.row_groups()); - - let mut results = Vec::new(); - for (((idx, column_index), offset_index), row_group_metadata) in page_index { - if let Some(selected_row_groups) = selected_row_groups { - // skip row groups that aren't present in selected_row_groups - if idx == selected_row_groups[selected_row_groups_idx] { - selected_row_groups_idx += 1; - } else { - continue; - } - } - - let selections_for_page = PageIndexEvaluator::eval( - predicate, - column_index, - offset_index, - row_group_metadata, - field_id_map, - snapshot_schema, - )?; - - results.push(selections_for_page); - - if let Some(selected_row_groups) = selected_row_groups { - if selected_row_groups_idx == selected_row_groups.len() { - break; - } - } - } - - Ok(results.into_iter().flatten().collect::>().into()) - } -} - -/// Build the map of parquet field id to Parquet column index in the schema. -fn build_field_id_map(parquet_schema: &SchemaDescriptor) -> Result> { - let mut column_map = HashMap::new(); - for (idx, field) in parquet_schema.columns().iter().enumerate() { - let field_type = field.self_type(); - match field_type { - ParquetType::PrimitiveType { basic_info, .. } => { - if !basic_info.has_id() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Leave column idx: {}, name: {}, type {:?} in schema doesn't have field id", - idx, - basic_info.name(), - field_type - ), - )); - } - column_map.insert(basic_info.id(), idx); - } - ParquetType::GroupType { .. } => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Leave column in schema should be primitive type but got {:?}", - field_type - ), - )); - } - }; - } - - Ok(column_map) -} - -/// A visitor to collect field ids from bound predicates. -struct CollectFieldIdVisitor { - field_ids: HashSet, -} - -impl CollectFieldIdVisitor { - fn field_ids(self) -> HashSet { - self.field_ids - } -} - -impl BoundPredicateVisitor for CollectFieldIdVisitor { - type T = (); - - fn always_true(&mut self) -> Result<()> { - Ok(()) - } - - fn always_false(&mut self) -> Result<()> { - Ok(()) - } - - fn and(&mut self, _lhs: (), _rhs: ()) -> Result<()> { - Ok(()) - } - - fn or(&mut self, _lhs: (), _rhs: ()) -> Result<()> { - Ok(()) - } - - fn not(&mut self, _inner: ()) -> Result<()> { - Ok(()) - } - - fn is_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn not_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn is_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn not_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn less_than( - &mut self, - reference: &BoundReference, - _literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn less_than_or_eq( - &mut self, - reference: &BoundReference, - _literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn greater_than( - &mut self, - reference: &BoundReference, - _literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn greater_than_or_eq( - &mut self, - reference: &BoundReference, - _literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn eq( - &mut self, - reference: &BoundReference, - _literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn not_eq( - &mut self, - reference: &BoundReference, - _literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn starts_with( - &mut self, - reference: &BoundReference, - _literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn not_starts_with( - &mut self, - reference: &BoundReference, - _literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn r#in( - &mut self, - reference: &BoundReference, - _literals: &FnvHashSet, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } - - fn not_in( - &mut self, - reference: &BoundReference, - _literals: &FnvHashSet, - _predicate: &BoundPredicate, - ) -> Result<()> { - self.field_ids.insert(reference.field().id); - Ok(()) - } -} - -/// A visitor to convert Iceberg bound predicates to Arrow predicates. -struct PredicateConverter<'a> { - /// The Parquet schema descriptor. - pub parquet_schema: &'a SchemaDescriptor, - /// The map between field id and leaf column index in Parquet schema. - pub column_map: &'a HashMap, - /// The required column indices in Parquet schema for the predicates. - pub column_indices: &'a Vec, -} - -impl PredicateConverter<'_> { - /// When visiting a bound reference, we return index of the leaf column in the - /// required column indices which is used to project the column in the record batch. - /// Return None if the field id is not found in the column map, which is possible - /// due to schema evolution. - fn bound_reference(&mut self, reference: &BoundReference) -> Result> { - // The leaf column's index in Parquet schema. - if let Some(column_idx) = self.column_map.get(&reference.field().id) { - if self.parquet_schema.get_column_root(*column_idx).is_group() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Leave column `{}` in predicates isn't a root column in Parquet schema.", - reference.field().name - ), - )); - } - - // The leaf column's index in the required column indices. - let index = self - .column_indices - .iter() - .position(|&idx| idx == *column_idx) - .ok_or(Error::new( - ErrorKind::DataInvalid, - format!( - "Leave column `{}` in predicates cannot be found in the required column indices.", - reference.field().name - ), - ))?; - - Ok(Some(index)) - } else { - Ok(None) - } - } - - /// Build an Arrow predicate that always returns true. - fn build_always_true(&self) -> Result> { - Ok(Box::new(|batch| { - Ok(BooleanArray::from(vec![true; batch.num_rows()])) - })) - } - - /// Build an Arrow predicate that always returns false. - fn build_always_false(&self) -> Result> { - Ok(Box::new(|batch| { - Ok(BooleanArray::from(vec![false; batch.num_rows()])) - })) - } -} - -/// Gets the leaf column from the record batch for the required column index. Only -/// supports top-level columns for now. -fn project_column( - batch: &RecordBatch, - column_idx: usize, -) -> std::result::Result { - let column = batch.column(column_idx); - - match column.data_type() { - DataType::Struct(_) => Err(ArrowError::SchemaError( - "Does not support struct column yet.".to_string(), - )), - _ => Ok(column.clone()), - } -} - -type PredicateResult = - dyn FnMut(RecordBatch) -> std::result::Result + Send + 'static; - -impl BoundPredicateVisitor for PredicateConverter<'_> { - type T = Box; - - fn always_true(&mut self) -> Result> { - self.build_always_true() - } - - fn always_false(&mut self) -> Result> { - self.build_always_false() - } - - fn and( - &mut self, - mut lhs: Box, - mut rhs: Box, - ) -> Result> { - Ok(Box::new(move |batch| { - let left = lhs(batch.clone())?; - let right = rhs(batch)?; - and_kleene(&left, &right) - })) - } - - fn or( - &mut self, - mut lhs: Box, - mut rhs: Box, - ) -> Result> { - Ok(Box::new(move |batch| { - let left = lhs(batch.clone())?; - let right = rhs(batch)?; - or_kleene(&left, &right) - })) - } - - fn not(&mut self, mut inner: Box) -> Result> { - Ok(Box::new(move |batch| { - let pred_ret = inner(batch)?; - not(&pred_ret) - })) - } - - fn is_null( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - Ok(Box::new(move |batch| { - let column = project_column(&batch, idx)?; - is_null(&column) - })) - } else { - // A missing column, treating it as null. - self.build_always_true() - } - } - - fn not_null( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - Ok(Box::new(move |batch| { - let column = project_column(&batch, idx)?; - is_not_null(&column) - })) - } else { - // A missing column, treating it as null. - self.build_always_false() - } - } - - fn is_nan( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> Result> { - if self.bound_reference(reference)?.is_some() { - self.build_always_true() - } else { - // A missing column, treating it as null. - self.build_always_false() - } - } - - fn not_nan( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> Result> { - if self.bound_reference(reference)?.is_some() { - self.build_always_false() - } else { - // A missing column, treating it as null. - self.build_always_true() - } - } - - fn less_than( - &mut self, - reference: &BoundReference, - literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literal = get_arrow_datum(literal)?; - - Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; - let literal = try_cast_literal(&literal, left.data_type())?; - lt(&left, literal.as_ref()) - })) - } else { - // A missing column, treating it as null. - self.build_always_true() - } - } - - fn less_than_or_eq( - &mut self, - reference: &BoundReference, - literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literal = get_arrow_datum(literal)?; - - Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; - let literal = try_cast_literal(&literal, left.data_type())?; - lt_eq(&left, literal.as_ref()) - })) - } else { - // A missing column, treating it as null. - self.build_always_true() - } - } - - fn greater_than( - &mut self, - reference: &BoundReference, - literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literal = get_arrow_datum(literal)?; - - Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; - let literal = try_cast_literal(&literal, left.data_type())?; - gt(&left, literal.as_ref()) - })) - } else { - // A missing column, treating it as null. - self.build_always_false() - } - } - - fn greater_than_or_eq( - &mut self, - reference: &BoundReference, - literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literal = get_arrow_datum(literal)?; - - Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; - let literal = try_cast_literal(&literal, left.data_type())?; - gt_eq(&left, literal.as_ref()) - })) - } else { - // A missing column, treating it as null. - self.build_always_false() - } - } - - fn eq( - &mut self, - reference: &BoundReference, - literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literal = get_arrow_datum(literal)?; - - Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; - let literal = try_cast_literal(&literal, left.data_type())?; - eq(&left, literal.as_ref()) - })) - } else { - // A missing column, treating it as null. - self.build_always_false() - } - } - - fn not_eq( - &mut self, - reference: &BoundReference, - literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literal = get_arrow_datum(literal)?; - - Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; - let literal = try_cast_literal(&literal, left.data_type())?; - neq(&left, literal.as_ref()) - })) - } else { - // A missing column, treating it as null. - self.build_always_false() - } - } - - fn starts_with( - &mut self, - reference: &BoundReference, - literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literal = get_arrow_datum(literal)?; - - Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; - let literal = try_cast_literal(&literal, left.data_type())?; - starts_with(&left, literal.as_ref()) - })) - } else { - // A missing column, treating it as null. - self.build_always_false() - } - } - - fn not_starts_with( - &mut self, - reference: &BoundReference, - literal: &Datum, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literal = get_arrow_datum(literal)?; - - Ok(Box::new(move |batch| { - let left = project_column(&batch, idx)?; - let literal = try_cast_literal(&literal, left.data_type())?; - // update here if arrow ever adds a native not_starts_with - not(&starts_with(&left, literal.as_ref())?) - })) - } else { - // A missing column, treating it as null. - self.build_always_true() - } - } - - fn r#in( - &mut self, - reference: &BoundReference, - literals: &FnvHashSet, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literals: Vec<_> = literals - .iter() - .map(|lit| get_arrow_datum(lit).unwrap()) - .collect(); - - Ok(Box::new(move |batch| { - // update this if arrow ever adds a native is_in kernel - let left = project_column(&batch, idx)?; - - let mut acc = BooleanArray::from(vec![false; batch.num_rows()]); - for literal in &literals { - let literal = try_cast_literal(literal, left.data_type())?; - acc = or(&acc, &eq(&left, literal.as_ref())?)? - } - - Ok(acc) - })) - } else { - // A missing column, treating it as null. - self.build_always_false() - } - } - - fn not_in( - &mut self, - reference: &BoundReference, - literals: &FnvHashSet, - _predicate: &BoundPredicate, - ) -> Result> { - if let Some(idx) = self.bound_reference(reference)? { - let literals: Vec<_> = literals - .iter() - .map(|lit| get_arrow_datum(lit).unwrap()) - .collect(); - - Ok(Box::new(move |batch| { - // update this if arrow ever adds a native not_in kernel - let left = project_column(&batch, idx)?; - let mut acc = BooleanArray::from(vec![true; batch.num_rows()]); - for literal in &literals { - let literal = try_cast_literal(literal, left.data_type())?; - acc = and(&acc, &neq(&left, literal.as_ref())?)? - } - - Ok(acc) - })) - } else { - // A missing column, treating it as null. - self.build_always_true() - } - } -} - -/// ArrowFileReader is a wrapper around a FileRead that impls parquets AsyncFileReader. -pub struct ArrowFileReader { - meta: FileMetadata, - preload_column_index: bool, - preload_offset_index: bool, - preload_page_index: bool, - metadata_size_hint: Option, - r: R, -} - -impl ArrowFileReader { - /// Create a new ArrowFileReader - pub fn new(meta: FileMetadata, r: R) -> Self { - Self { - meta, - preload_column_index: false, - preload_offset_index: false, - preload_page_index: false, - metadata_size_hint: None, - r, - } - } - - /// Enable or disable preloading of the column index - pub fn with_preload_column_index(mut self, preload: bool) -> Self { - self.preload_column_index = preload; - self - } - - /// Enable or disable preloading of the offset index - pub fn with_preload_offset_index(mut self, preload: bool) -> Self { - self.preload_offset_index = preload; - self - } - - /// Enable or disable preloading of the page index - pub fn with_preload_page_index(mut self, preload: bool) -> Self { - self.preload_page_index = preload; - self - } - - /// Provide a hint as to the number of bytes to prefetch for parsing the Parquet metadata - /// - /// This hint can help reduce the number of fetch requests. For more details see the - /// [ParquetMetaDataReader documentation](https://docs.rs/parquet/latest/parquet/file/metadata/struct.ParquetMetaDataReader.html#method.with_prefetch_hint). - pub fn with_metadata_size_hint(mut self, hint: usize) -> Self { - self.metadata_size_hint = Some(hint); - self - } -} - -impl AsyncFileReader for ArrowFileReader { - fn get_bytes(&mut self, range: Range) -> BoxFuture<'_, parquet::errors::Result> { - Box::pin( - self.r - .read(range.start..range.end) - .map_err(|err| parquet::errors::ParquetError::External(Box::new(err))), - ) - } - - // TODO: currently we don't respect `ArrowReaderOptions` cause it don't expose any method to access the option field - // we will fix it after `v55.1.0` is released in https://github.com/apache/arrow-rs/issues/7393 - fn get_metadata( - &mut self, - _options: Option<&'_ ArrowReaderOptions>, - ) -> BoxFuture<'_, parquet::errors::Result>> { - async move { - let reader = ParquetMetaDataReader::new() - .with_prefetch_hint(self.metadata_size_hint) - .with_column_indexes(self.preload_column_index) - .with_page_indexes(self.preload_page_index) - .with_offset_indexes(self.preload_offset_index); - let size = self.meta.size; - let meta = reader.load_and_finish(self, size).await?; - - Ok(Arc::new(meta)) - } - .boxed() - } -} - -/// The Arrow type of an array that the Parquet reader reads may not match the exact Arrow type -/// that Iceberg uses for literals - but they are effectively the same logical type, -/// i.e. LargeUtf8 and Utf8 or Utf8View and Utf8 or Utf8View and LargeUtf8. -/// -/// The Arrow compute kernels that we use must match the type exactly, so first cast the literal -/// into the type of the batch we read from Parquet before sending it to the compute kernel. -fn try_cast_literal( - literal: &Arc, - column_type: &DataType, -) -> std::result::Result, ArrowError> { - let literal_array = literal.get().0; - - // No cast required - if literal_array.data_type() == column_type { - return Ok(Arc::clone(literal)); - } - - let literal_array = cast(literal_array, column_type)?; - Ok(Arc::new(Scalar::new(literal_array))) -} - -#[cfg(test)] -mod tests { - use std::collections::{HashMap, HashSet}; - use std::fs::File; - use std::sync::Arc; - - use arrow_array::cast::AsArray; - use arrow_array::{ArrayRef, LargeStringArray, RecordBatch, StringArray}; - use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; - use futures::TryStreamExt; - use parquet::arrow::arrow_reader::{RowSelection, RowSelector}; - use parquet::arrow::{ArrowWriter, ProjectionMask}; - use parquet::basic::Compression; - use parquet::file::metadata::{ColumnChunkMetaData, RowGroupMetaData}; - use parquet::file::properties::WriterProperties; - use parquet::schema::parser::parse_message_type; - use parquet::schema::types::{SchemaDescPtr, SchemaDescriptor}; - use roaring::RoaringTreemap; - use tempfile::TempDir; - - use crate::ErrorKind; - use crate::arrow::reader::{CollectFieldIdVisitor, PARQUET_FIELD_ID_META_KEY}; - use crate::arrow::{ArrowReader, ArrowReaderBuilder}; - use crate::delete_vector::DeleteVector; - use crate::expr::visitors::bound_predicate_visitor::visit; - use crate::expr::{Bind, Predicate, Reference}; - use crate::io::FileIO; - use crate::scan::{FileScanTask, FileScanTaskStream}; - use crate::spec::{DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type}; - - fn table_schema_simple() -> SchemaRef { - Arc::new( - Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![2]) - .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), - NestedField::optional(4, "qux", Type::Primitive(PrimitiveType::Float)).into(), - ]) - .build() - .unwrap(), - ) - } - - #[test] - fn test_collect_field_id() { - let schema = table_schema_simple(); - let expr = Reference::new("qux").is_null(); - let bound_expr = expr.bind(schema, true).unwrap(); - - let mut visitor = CollectFieldIdVisitor { - field_ids: HashSet::default(), - }; - visit(&mut visitor, &bound_expr).unwrap(); - - let mut expected = HashSet::default(); - expected.insert(4_i32); - - assert_eq!(visitor.field_ids, expected); - } - - #[test] - fn test_collect_field_id_with_and() { - let schema = table_schema_simple(); - let expr = Reference::new("qux") - .is_null() - .and(Reference::new("baz").is_null()); - let bound_expr = expr.bind(schema, true).unwrap(); - - let mut visitor = CollectFieldIdVisitor { - field_ids: HashSet::default(), - }; - visit(&mut visitor, &bound_expr).unwrap(); - - let mut expected = HashSet::default(); - expected.insert(4_i32); - expected.insert(3); - - assert_eq!(visitor.field_ids, expected); - } - - #[test] - fn test_collect_field_id_with_or() { - let schema = table_schema_simple(); - let expr = Reference::new("qux") - .is_null() - .or(Reference::new("baz").is_null()); - let bound_expr = expr.bind(schema, true).unwrap(); - - let mut visitor = CollectFieldIdVisitor { - field_ids: HashSet::default(), - }; - visit(&mut visitor, &bound_expr).unwrap(); - - let mut expected = HashSet::default(); - expected.insert(4_i32); - expected.insert(3); - - assert_eq!(visitor.field_ids, expected); - } - - #[test] - fn test_arrow_projection_mask() { - let schema = Arc::new( - Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![1]) - .with_fields(vec![ - NestedField::required(1, "c1", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(2, "c2", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional( - 3, - "c3", - Type::Primitive(PrimitiveType::Decimal { - precision: 38, - scale: 3, - }), - ) - .into(), - ]) - .build() - .unwrap(), - ); - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("c1", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - // Type not supported - Field::new("c2", DataType::Duration(TimeUnit::Microsecond), true).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())]), - ), - // Precision is beyond the supported range - Field::new("c3", DataType::Decimal128(39, 3), true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - ])); - - let message_type = " -message schema { - required binary c1 (STRING) = 1; - optional int32 c2 (INTEGER(8,true)) = 2; - optional fixed_len_byte_array(17) c3 (DECIMAL(39,3)) = 3; -} - "; - let parquet_type = parse_message_type(message_type).expect("should parse schema"); - let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); - - // Try projecting the fields c2 and c3 with the unsupported data types - let err = ArrowReader::get_arrow_projection_mask( - &[1, 2, 3], - &schema, - &parquet_schema, - &arrow_schema, - ) - .unwrap_err(); - - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert_eq!( - err.to_string(), - "DataInvalid => Unsupported Arrow data type: Duration(Microsecond)".to_string() - ); - - // Omitting field c2, we still get an error due to c3 being selected - let err = ArrowReader::get_arrow_projection_mask( - &[1, 3], - &schema, - &parquet_schema, - &arrow_schema, - ) - .unwrap_err(); - - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert_eq!( - err.to_string(), - "DataInvalid => Failed to create decimal type, source: DataInvalid => Decimals with precision larger than 38 are not supported: 39".to_string() - ); - - // Finally avoid selecting fields with unsupported data types - let mask = - ArrowReader::get_arrow_projection_mask(&[1], &schema, &parquet_schema, &arrow_schema) - .expect("Some ProjectionMask"); - assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0])); - } - - #[tokio::test] - async fn test_kleene_logic_or_behaviour() { - // a IS NULL OR a = 'foo' - let predicate = Reference::new("a") - .is_null() - .or(Reference::new("a").equal_to(Datum::string("foo"))); - - // Table data: [NULL, "foo", "bar"] - let data_for_col_a = vec![None, Some("foo".to_string()), Some("bar".to_string())]; - - // Expected: [NULL, "foo"]. - let expected = vec![None, Some("foo".to_string())]; - - let (file_io, schema, table_location, _temp_dir) = - setup_kleene_logic(data_for_col_a, DataType::Utf8); - let reader = ArrowReaderBuilder::new(file_io).build(); - - let result_data = test_perform_read(predicate, schema, table_location, reader).await; - - assert_eq!(result_data, expected); - } - - #[tokio::test] - async fn test_kleene_logic_and_behaviour() { - // a IS NOT NULL AND a != 'foo' - let predicate = Reference::new("a") - .is_not_null() - .and(Reference::new("a").not_equal_to(Datum::string("foo"))); - - // Table data: [NULL, "foo", "bar"] - let data_for_col_a = vec![None, Some("foo".to_string()), Some("bar".to_string())]; - - // Expected: ["bar"]. - let expected = vec![Some("bar".to_string())]; - - let (file_io, schema, table_location, _temp_dir) = - setup_kleene_logic(data_for_col_a, DataType::Utf8); - let reader = ArrowReaderBuilder::new(file_io).build(); - - let result_data = test_perform_read(predicate, schema, table_location, reader).await; - - assert_eq!(result_data, expected); - } - - #[tokio::test] - async fn test_predicate_cast_literal() { - let predicates = vec![ - // a == 'foo' - (Reference::new("a").equal_to(Datum::string("foo")), vec![ - Some("foo".to_string()), - ]), - // a != 'foo' - ( - Reference::new("a").not_equal_to(Datum::string("foo")), - vec![Some("bar".to_string())], - ), - // STARTS_WITH(a, 'foo') - (Reference::new("a").starts_with(Datum::string("f")), vec![ - Some("foo".to_string()), - ]), - // NOT STARTS_WITH(a, 'foo') - ( - Reference::new("a").not_starts_with(Datum::string("f")), - vec![Some("bar".to_string())], - ), - // a < 'foo' - (Reference::new("a").less_than(Datum::string("foo")), vec![ - Some("bar".to_string()), - ]), - // a <= 'foo' - ( - Reference::new("a").less_than_or_equal_to(Datum::string("foo")), - vec![Some("foo".to_string()), Some("bar".to_string())], - ), - // a > 'foo' - ( - Reference::new("a").greater_than(Datum::string("bar")), - vec![Some("foo".to_string())], - ), - // a >= 'foo' - ( - Reference::new("a").greater_than_or_equal_to(Datum::string("foo")), - vec![Some("foo".to_string())], - ), - // a IN ('foo', 'bar') - ( - Reference::new("a").is_in([Datum::string("foo"), Datum::string("baz")]), - vec![Some("foo".to_string())], - ), - // a NOT IN ('foo', 'bar') - ( - Reference::new("a").is_not_in([Datum::string("foo"), Datum::string("baz")]), - vec![Some("bar".to_string())], - ), - ]; - - // Table data: ["foo", "bar"] - let data_for_col_a = vec![Some("foo".to_string()), Some("bar".to_string())]; - - let (file_io, schema, table_location, _temp_dir) = - setup_kleene_logic(data_for_col_a, DataType::LargeUtf8); - let reader = ArrowReaderBuilder::new(file_io).build(); - - for (predicate, expected) in predicates { - println!("testing predicate {predicate}"); - let result_data = test_perform_read( - predicate.clone(), - schema.clone(), - table_location.clone(), - reader.clone(), - ) - .await; - - assert_eq!(result_data, expected, "predicate={predicate}"); - } - } - - async fn test_perform_read( - predicate: Predicate, - schema: SchemaRef, - table_location: String, - reader: ArrowReader, - ) -> Vec> { - let tasks = Box::pin(futures::stream::iter( - vec![Ok(FileScanTask { - start: 0, - length: 0, - record_count: None, - data_file_path: format!("{}/1.parquet", table_location), - data_file_format: DataFileFormat::Parquet, - schema: schema.clone(), - project_field_ids: vec![1], - predicate: Some(predicate.bind(schema, true).unwrap()), - deletes: vec![], - })] - .into_iter(), - )) as FileScanTaskStream; - - let result = reader - .read(tasks) - .unwrap() - .try_collect::>() - .await - .unwrap(); - - result[0].columns()[0] - .as_string_opt::() - .unwrap() - .iter() - .map(|v| v.map(ToOwned::to_owned)) - .collect::>() - } - - fn setup_kleene_logic( - data_for_col_a: Vec>, - col_a_type: DataType, - ) -> (FileIO, SchemaRef, String, TempDir) { - let schema = Arc::new( - Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::optional(1, "a", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - .unwrap(), - ); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("a", col_a_type.clone(), true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - ])); - - let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().to_str().unwrap().to_string(); - - let file_io = FileIO::from_path(&table_location).unwrap().build().unwrap(); - - let col = match col_a_type { - DataType::Utf8 => Arc::new(StringArray::from(data_for_col_a)) as ArrayRef, - DataType::LargeUtf8 => Arc::new(LargeStringArray::from(data_for_col_a)) as ArrayRef, - _ => panic!("unexpected col_a_type"), - }; - - let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![col]).unwrap(); - - // Write the Parquet files - let props = WriterProperties::builder() - .set_compression(Compression::SNAPPY) - .build(); - - let file = File::create(format!("{}/1.parquet", &table_location)).unwrap(); - let mut writer = - ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap(); - - writer.write(&to_write).expect("Writing batch"); - - // writer must be closed to write footer - writer.close().unwrap(); - - (file_io, schema, table_location, tmp_dir) - } - - #[test] - fn test_build_deletes_row_selection() { - let schema_descr = get_test_schema_descr(); - - let mut columns = vec![]; - for ptr in schema_descr.columns() { - let column = ColumnChunkMetaData::builder(ptr.clone()).build().unwrap(); - columns.push(column); - } - - let row_groups_metadata = vec![ - build_test_row_group_meta(schema_descr.clone(), columns.clone(), 1000, 0), - build_test_row_group_meta(schema_descr.clone(), columns.clone(), 500, 1), - build_test_row_group_meta(schema_descr.clone(), columns.clone(), 500, 2), - build_test_row_group_meta(schema_descr.clone(), columns.clone(), 1000, 3), - build_test_row_group_meta(schema_descr.clone(), columns.clone(), 500, 4), - ]; - - let selected_row_groups = Some(vec![1, 3]); - - /* cases to cover: - * {skip|select} {first|intermediate|last} {one row|multiple rows} in - {first|intermediate|last} {skipped|selected} row group - * row group selection disabled - */ - - let positional_deletes = RoaringTreemap::from_iter(&[ - 1, // in skipped rg 0, should be ignored - 3, // run of three consecutive items in skipped rg0 - 4, 5, 998, // two consecutive items at end of skipped rg0 - 999, 1000, // solitary row at start of selected rg1 (1, 9) - 1010, // run of 3 rows in selected rg1 - 1011, 1012, // (3, 485) - 1498, // run of two items at end of selected rg1 - 1499, 1500, // run of two items at start of skipped rg2 - 1501, 1600, // should ignore, in skipped rg2 - 1999, // single row at end of skipped rg2 - 2000, // run of two items at start of selected rg3 - 2001, // (4, 98) - 2100, // single row in selected row group 3 (1, 99) - 2200, // run of 3 consecutive rows in selected row group 3 - 2201, 2202, // (3, 796) - 2999, // single item at end of selected rg3 (1) - 3000, // single item at start of skipped rg4 - ]); - - let positional_deletes = DeleteVector::new(positional_deletes); - - // using selected row groups 1 and 3 - let result = ArrowReader::build_deletes_row_selection( - &row_groups_metadata, - &selected_row_groups, - &positional_deletes, - ) - .unwrap(); - - let expected = RowSelection::from(vec![ - RowSelector::skip(1), - RowSelector::select(9), - RowSelector::skip(3), - RowSelector::select(485), - RowSelector::skip(4), - RowSelector::select(98), - RowSelector::skip(1), - RowSelector::select(99), - RowSelector::skip(3), - RowSelector::select(796), - RowSelector::skip(1), - ]); - - assert_eq!(result, expected); - - // selecting all row groups - let result = ArrowReader::build_deletes_row_selection( - &row_groups_metadata, - &None, - &positional_deletes, - ) - .unwrap(); - - let expected = RowSelection::from(vec![ - RowSelector::select(1), - RowSelector::skip(1), - RowSelector::select(1), - RowSelector::skip(3), - RowSelector::select(992), - RowSelector::skip(3), - RowSelector::select(9), - RowSelector::skip(3), - RowSelector::select(485), - RowSelector::skip(4), - RowSelector::select(98), - RowSelector::skip(1), - RowSelector::select(398), - RowSelector::skip(3), - RowSelector::select(98), - RowSelector::skip(1), - RowSelector::select(99), - RowSelector::skip(3), - RowSelector::select(796), - RowSelector::skip(2), - RowSelector::select(499), - ]); - - assert_eq!(result, expected); - } - - fn build_test_row_group_meta( - schema_descr: SchemaDescPtr, - columns: Vec, - num_rows: i64, - ordinal: i16, - ) -> RowGroupMetaData { - RowGroupMetaData::builder(schema_descr.clone()) - .set_num_rows(num_rows) - .set_total_byte_size(2000) - .set_column_metadata(columns) - .set_ordinal(ordinal) - .build() - .unwrap() - } - - fn get_test_schema_descr() -> SchemaDescPtr { - use parquet::schema::types::Type as SchemaType; - - let schema = SchemaType::group_type_builder("schema") - .with_fields(vec![ - Arc::new( - SchemaType::primitive_type_builder("a", parquet::basic::Type::INT32) - .build() - .unwrap(), - ), - Arc::new( - SchemaType::primitive_type_builder("b", parquet::basic::Type::INT32) - .build() - .unwrap(), - ), - ]) - .build() - .unwrap(); - - Arc::new(SchemaDescriptor::new(Arc::new(schema))) - } -} diff --git a/crates/iceberg/src/arrow/reader/file_reader.rs b/crates/iceberg/src/arrow/reader/file_reader.rs new file mode 100644 index 0000000000..22355737a1 --- /dev/null +++ b/crates/iceberg/src/arrow/reader/file_reader.rs @@ -0,0 +1,374 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Async Parquet file reader that adapts an Iceberg `FileRead` to parquet's `AsyncFileReader`. + +use std::ops::Range; +use std::sync::Arc; + +use bytes::Bytes; +use futures::future::BoxFuture; +use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; +use parquet::arrow::arrow_reader::ArrowReaderOptions; +use parquet::arrow::async_reader::AsyncFileReader; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader}; + +use super::ParquetReadOptions; +use crate::io::{FileMetadata, FileRead}; + +/// ArrowFileReader is a wrapper around a FileRead that impls parquets AsyncFileReader. +pub struct ArrowFileReader { + meta: FileMetadata, + parquet_read_options: ParquetReadOptions, + r: Box, +} + +impl ArrowFileReader { + /// Create a new ArrowFileReader + pub fn new(meta: FileMetadata, r: Box) -> Self { + Self { + meta, + parquet_read_options: ParquetReadOptions::builder().build(), + r, + } + } + + /// Configure all Parquet read options. + pub(crate) fn with_parquet_read_options(mut self, options: ParquetReadOptions) -> Self { + self.parquet_read_options = options; + self + } +} + +impl AsyncFileReader for ArrowFileReader { + fn get_bytes(&mut self, range: Range) -> BoxFuture<'_, parquet::errors::Result> { + Box::pin( + self.r + .read(range.start..range.end) + .map_err(|err| parquet::errors::ParquetError::External(Box::new(err))), + ) + } + + /// Override the default `get_byte_ranges` which calls `get_bytes` sequentially. + /// The parquet reader calls this to fetch column chunks for a row group, so + /// without this override each column chunk is a serial round-trip to object storage. + /// Adapted from object_store's `coalesce_ranges` in `util.rs`. + fn get_byte_ranges( + &mut self, + ranges: Vec>, + ) -> BoxFuture<'_, parquet::errors::Result>> { + let coalesce_bytes = self.parquet_read_options.range_coalesce_bytes(); + let concurrency = self.parquet_read_options.range_fetch_concurrency().max(1); + + async move { + // Merge nearby ranges to reduce the number of object store requests. + let fetch_ranges = merge_ranges(&ranges, coalesce_bytes); + let r = &self.r; + + // Fetch merged ranges concurrently. + let fetched: Vec = futures::stream::iter(fetch_ranges.iter().cloned()) + .map(|range| async move { + r.read(range) + .await + .map_err(|e| parquet::errors::ParquetError::External(Box::new(e))) + }) + .buffered(concurrency) + .try_collect() + .await?; + + // Slice the fetched data back into the originally requested ranges. + Ok(ranges + .iter() + .map(|range| { + let idx = fetch_ranges.partition_point(|v| v.start <= range.start) - 1; + let fetch_range = &fetch_ranges[idx]; + let fetch_bytes = &fetched[idx]; + let start = (range.start - fetch_range.start) as usize; + let end = (range.end - fetch_range.start) as usize; + fetch_bytes.slice(start..end.min(fetch_bytes.len())) + }) + .collect()) + } + .boxed() + } + + fn get_metadata( + &mut self, + options: Option<&'_ ArrowReaderOptions>, + ) -> BoxFuture<'_, parquet::errors::Result>> { + let decryption_properties = options + .and_then(|opts| opts.file_decryption_properties()) + .cloned(); + + let metadata_options = options.map(|opts| opts.metadata_options().clone()); + + async move { + let reader = ParquetMetaDataReader::new() + .with_prefetch_hint(Some(self.parquet_read_options.metadata_size_hint())) + // Set the page policy first because it updates both column and offset policies. + .with_page_index_policy(PageIndexPolicy::from( + self.parquet_read_options.preload_page_index(), + )) + .with_column_index_policy(PageIndexPolicy::from( + self.parquet_read_options.preload_column_index(), + )) + .with_offset_index_policy(PageIndexPolicy::from( + self.parquet_read_options.preload_offset_index(), + )) + .with_metadata_options(metadata_options) + .with_decryption_properties(decryption_properties); + let size = self.meta.size; + let meta = reader.load_and_finish(self, size).await?; + + Ok(Arc::new(meta)) + } + .boxed() + } +} + +/// Merge overlapping or nearby byte ranges, combining ranges with gaps <= `coalesce` bytes. +/// Adapted from object_store's `merge_ranges` in `util.rs`. +fn merge_ranges(ranges: &[Range], coalesce: u64) -> Vec> { + if ranges.is_empty() { + return vec![]; + } + + let mut ranges = ranges.to_vec(); + ranges.sort_unstable_by_key(|r| r.start); + + let mut merged = Vec::with_capacity(ranges.len()); + let mut start_idx = 0; + let mut end_idx = 1; + + while start_idx != ranges.len() { + let mut range_end = ranges[start_idx].end; + + while end_idx != ranges.len() + && ranges[end_idx] + .start + .checked_sub(range_end) + .map(|delta| delta <= coalesce) + .unwrap_or(true) + { + range_end = range_end.max(ranges[end_idx].end); + end_idx += 1; + } + + merged.push(ranges[start_idx].start..range_end); + start_idx = end_idx; + end_idx += 1; + } + + merged +} + +#[cfg(test)] +mod tests { + use std::ops::Range; + + use parquet::arrow::async_reader::AsyncFileReader; + + use super::{ArrowFileReader, ParquetReadOptions, merge_ranges}; + use crate::io::{FileMetadata, FileRead}; + + #[test] + fn test_merge_ranges_empty() { + assert_eq!(merge_ranges(&[], 1024), Vec::>::new()); + } + + #[test] + fn test_merge_ranges_no_coalesce() { + // Ranges far apart should not be merged + let ranges = vec![0..100, 1_000_000..1_000_100]; + let merged = merge_ranges(&ranges, 1024); + assert_eq!(merged, vec![0..100, 1_000_000..1_000_100]); + } + + #[test] + fn test_merge_ranges_coalesce() { + // Ranges within the gap threshold should be merged + let ranges = vec![0..100, 200..300, 500..600]; + let merged = merge_ranges(&ranges, 1024); + assert_eq!(merged, vec![0..600]); + } + + #[test] + fn test_merge_ranges_overlapping() { + let ranges = vec![0..200, 100..300]; + let merged = merge_ranges(&ranges, 0); + assert_eq!(merged, vec![0..300]); + } + + #[test] + fn test_merge_ranges_unsorted() { + let ranges = vec![500..600, 0..100, 200..300]; + let merged = merge_ranges(&ranges, 1024); + assert_eq!(merged, vec![0..600]); + } + + /// Mock FileRead backed by a flat byte buffer. + struct MockFileRead { + data: bytes::Bytes, + } + + impl MockFileRead { + fn new(size: usize) -> Self { + // Fill with sequential byte values so slices are verifiable. + let data: Vec = (0..size).map(|i| (i % 256) as u8).collect(); + Self { + data: bytes::Bytes::from(data), + } + } + } + + #[async_trait::async_trait] + impl FileRead for MockFileRead { + async fn read(&self, range: Range) -> crate::Result { + Ok(self.data.slice(range.start as usize..range.end as usize)) + } + } + + #[tokio::test] + async fn test_get_byte_ranges_no_coalesce() { + let mock = MockFileRead::new(2048); + let expected_0 = mock.data.slice(0..100); + let expected_1 = mock.data.slice(1500..1600); + + let mut reader = ArrowFileReader::new(FileMetadata { size: 2048 }, Box::new(mock)) + .with_parquet_read_options( + ParquetReadOptions::builder() + .with_range_coalesce_bytes(0) + .build(), + ); + + let result = reader + .get_byte_ranges(vec![0..100, 1500..1600]) + .await + .unwrap(); + + assert_eq!(result.len(), 2); + assert_eq!(result[0], expected_0); + assert_eq!(result[1], expected_1); + } + + #[tokio::test] + async fn test_get_byte_ranges_with_coalesce() { + let mock = MockFileRead::new(1024); + let expected_0 = mock.data.slice(0..100); + let expected_1 = mock.data.slice(200..300); + let expected_2 = mock.data.slice(500..600); + + let mut reader = ArrowFileReader::new(FileMetadata { size: 1024 }, Box::new(mock)) + .with_parquet_read_options( + ParquetReadOptions::builder() + .with_range_coalesce_bytes(1024) + .build(), + ); + + // All ranges within coalesce threshold — should merge into one fetch. + let result = reader + .get_byte_ranges(vec![0..100, 200..300, 500..600]) + .await + .unwrap(); + + assert_eq!(result.len(), 3); + assert_eq!(result[0], expected_0); + assert_eq!(result[1], expected_1); + assert_eq!(result[2], expected_2); + } + + #[tokio::test] + async fn test_get_byte_ranges_empty() { + let mock = MockFileRead::new(1024); + let mut reader = ArrowFileReader::new(FileMetadata { size: 1024 }, Box::new(mock)); + + let result = reader.get_byte_ranges(vec![]).await.unwrap(); + assert!(result.is_empty()); + } + + #[tokio::test] + async fn test_get_byte_ranges_coalesce_max() { + let mock = MockFileRead::new(2048); + let expected_0 = mock.data.slice(0..100); + let expected_1 = mock.data.slice(1500..1600); + + let mut reader = ArrowFileReader::new(FileMetadata { size: 2048 }, Box::new(mock)) + .with_parquet_read_options( + ParquetReadOptions::builder() + .with_range_coalesce_bytes(u64::MAX) + .build(), + ); + + // u64::MAX coalesce — all ranges merge into a single fetch. + let result = reader + .get_byte_ranges(vec![0..100, 1500..1600]) + .await + .unwrap(); + + assert_eq!(result.len(), 2); + assert_eq!(result[0], expected_0); + assert_eq!(result[1], expected_1); + } + + #[tokio::test] + async fn test_get_byte_ranges_concurrency_zero() { + // concurrency=0 is clamped to 1, so this should not hang. + let mock = MockFileRead::new(1024); + let expected = mock.data.slice(0..100); + + let mut reader = ArrowFileReader::new(FileMetadata { size: 1024 }, Box::new(mock)) + .with_parquet_read_options( + ParquetReadOptions::builder() + .with_range_fetch_concurrency(0) + .build(), + ); + + let result = reader + .get_byte_ranges(vec![0..100, 200..300]) + .await + .unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0], expected); + } + + #[tokio::test] + async fn test_get_byte_ranges_concurrency_one() { + let mock = MockFileRead::new(2048); + let expected_0 = mock.data.slice(0..100); + let expected_1 = mock.data.slice(500..600); + let expected_2 = mock.data.slice(1500..1600); + + let mut reader = ArrowFileReader::new(FileMetadata { size: 2048 }, Box::new(mock)) + .with_parquet_read_options( + ParquetReadOptions::builder() + .with_range_coalesce_bytes(0) + .with_range_fetch_concurrency(1) + .build(), + ); + + // concurrency=1 with no coalescing — sequential fetches. + let result = reader + .get_byte_ranges(vec![0..100, 500..600, 1500..1600]) + .await + .unwrap(); + + assert_eq!(result.len(), 3); + assert_eq!(result[0], expected_0); + assert_eq!(result[1], expected_1); + assert_eq!(result[2], expected_2); + } +} diff --git a/crates/iceberg/src/arrow/reader/mod.rs b/crates/iceberg/src/arrow/reader/mod.rs new file mode 100644 index 0000000000..45982da94f --- /dev/null +++ b/crates/iceberg/src/arrow/reader/mod.rs @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Parquet file data reader + +use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader; +use crate::io::FileIO; +use crate::runtime::Runtime; +use crate::util::available_parallelism; + +/// Default gap between byte ranges below which they are coalesced into a +/// single request. Matches object_store's `OBJECT_STORE_COALESCE_DEFAULT`. +const DEFAULT_RANGE_COALESCE_BYTES: u64 = 1024 * 1024; + +/// Default maximum number of coalesced byte ranges fetched concurrently. +/// Matches object_store's `OBJECT_STORE_COALESCE_PARALLEL`. +const DEFAULT_RANGE_FETCH_CONCURRENCY: usize = 10; + +/// Default number of bytes to prefetch when parsing Parquet footer metadata. +/// Matches DataFusion's default `ParquetOptions::metadata_size_hint`. +const DEFAULT_METADATA_SIZE_HINT: usize = 512 * 1024; + +mod file_reader; +mod options; +mod pipeline; +mod positional_deletes; +mod predicate_visitor; +mod projection; +mod row_filter; +mod row_lineage; +pub use file_reader::ArrowFileReader; +pub use options::{ArrowReaderOptions, ParquetReadOptions}; +use predicate_visitor::{CollectFieldIdVisitor, PredicateConverter}; +use projection::{ + add_fallback_field_ids_to_arrow_schema, apply_name_mapping_to_arrow_schema, + coerce_fixed_size_lists, find_leaf_by_field_id, +}; + +/// Builder to create ArrowReader +pub struct ArrowReaderBuilder { + options: ArrowReaderOptions, + file_io: FileIO, + runtime: Runtime, +} + +impl ArrowReaderBuilder { + /// Create a new ArrowReaderBuilder + pub fn new(file_io: FileIO, runtime: Runtime) -> Self { + ArrowReaderBuilder { + options: ArrowReaderOptions::default(), + file_io, + runtime, + } + } + + /// Replaces all reader settings while preserving local I/O and runtime resources. + pub fn with_options(mut self, options: ArrowReaderOptions) -> Self { + self.options = options; + self + } + + /// Replaces Parquet I/O settings while preserving other reader settings. + pub fn with_parquet_options(mut self, options: ParquetReadOptions) -> Self { + self.options = self.options.with_parquet_options(options); + self + } + + /// Build the ArrowReader. + pub fn build(self) -> ArrowReader { + let options = self + .options + .with_data_file_concurrency_limit_if_unset(available_parallelism().get().max(1)); + let concurrency_limit_data_files = options + .data_file_concurrency_limit() + .expect("reader concurrency was set above") + .max(1); + ArrowReader { + options, + file_io: self.file_io.clone(), + delete_file_loader: CachingDeleteFileLoader::new( + self.file_io.clone(), + concurrency_limit_data_files, + self.runtime.clone(), + ), + runtime: self.runtime.clone(), + } + } +} + +/// Reads data from Parquet files +#[derive(Clone)] +pub struct ArrowReader { + options: ArrowReaderOptions, + file_io: FileIO, + delete_file_loader: CachingDeleteFileLoader, + + runtime: Runtime, +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use futures::{TryStreamExt, stream}; + + use super::{ArrowReaderBuilder, ArrowReaderOptions, ParquetReadOptions}; + use crate::io::FileIO; + use crate::test_utils::test_runtime; + use crate::util::available_parallelism; + + #[tokio::test] + async fn deserialized_zero_concurrency_completes_reads() { + let options: ArrowReaderOptions = + serde_json::from_str(r#"{"data_file_concurrency_limit":0}"#).unwrap(); + let reader = ArrowReaderBuilder::new(FileIO::new_with_memory(), test_runtime()) + .with_options(options) + .build(); + let batches = tokio::time::timeout( + Duration::from_secs(1), + reader + .read(Box::pin(stream::empty())) + .unwrap() + .stream() + .try_collect::>(), + ) + .await + .expect("zero concurrency must not stall the task stream") + .unwrap(); + assert!(batches.is_empty()); + } + + #[test] + fn default_options_restore_local_reader_defaults() { + let reader = ArrowReaderBuilder::new(FileIO::new_with_memory(), test_runtime()) + .with_options( + ArrowReaderOptions::builder() + .with_batch_size(1024) + .with_data_file_concurrency_limit(3) + .with_row_group_filtering_enabled(false) + .with_row_selection_enabled(true) + .with_bloom_filter_enabled(true) + .with_parquet_options( + ParquetReadOptions::builder() + .with_range_fetch_concurrency(99) + .build(), + ) + .build(), + ) + .with_options(ArrowReaderOptions::default()) + .build(); + assert_eq!( + reader.options, + ArrowReaderOptions::builder() + .with_data_file_concurrency_limit(available_parallelism().get()) + .build() + ); + } + + #[test] + fn explicit_options_configure_reader_settings() { + let parquet = ParquetReadOptions::builder() + .with_metadata_size_hint(4096) + .with_range_coalesce_bytes(8192) + .with_range_fetch_concurrency(2) + .build(); + let options = ArrowReaderOptions::builder() + .with_batch_size(17) + .with_data_file_concurrency_limit(3) + .with_row_group_filtering_enabled(false) + .with_row_selection_enabled(true) + .with_bloom_filter_enabled(true) + .with_parquet_options(parquet) + .build(); + let reader = ArrowReaderBuilder::new(FileIO::new_with_memory(), test_runtime()) + .with_options(options.clone()) + .build(); + assert_eq!(reader.options, options); + } +} diff --git a/crates/iceberg/src/arrow/reader/options.rs b/crates/iceberg/src/arrow/reader/options.rs new file mode 100644 index 0000000000..5add478405 --- /dev/null +++ b/crates/iceberg/src/arrow/reader/options.rs @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use super::{ + DEFAULT_METADATA_SIZE_HINT, DEFAULT_RANGE_COALESCE_BYTES, DEFAULT_RANGE_FETCH_CONCURRENCY, +}; + +/// Configures Arrow reads independently of file planning and local I/O resources. +/// +/// Serializable alongside planned file tasks so readers can be constructed later. +/// In self-describing map formats such as JSON, field names are part of the +/// serialized format: missing fields use defaults and unknown fields are ignored. +/// Compatibility across different crate versions is not otherwise guaranteed. +/// +/// ```no_run +/// # use iceberg::{Result, arrow::{ArrowReaderOptions, ParquetReadOptions}, table::Table}; +/// # async fn read(table: &Table) -> Result<()> { +/// let options = ArrowReaderOptions::builder() +/// .with_batch_size(1024) +/// .with_parquet_options( +/// ParquetReadOptions::builder() +/// .with_metadata_size_hint(512 * 1024) +/// .build(), +/// ) +/// .build(); +/// let stream = table.scan().build()?.to_arrow(options).await?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TypedBuilder)] +#[serde(default)] +#[builder(field_defaults(default, setter(prefix = "with_")))] +pub struct ArrowReaderOptions { + /// Desired batch size, or the reader default when unset. + #[builder(default = None, setter(strip_option))] + batch_size: Option, + /// Maximum in-flight data files, or the construction path's default when unset. + #[builder(default = None, setter(strip_option))] + data_file_concurrency_limit: Option, + /// Whether to prune row groups using their statistics. + #[builder(default = true)] + row_group_filtering_enabled: bool, + /// Whether to use page indexes for row selection. + row_selection_enabled: bool, + /// Whether to use bloom filters for row-group pruning. + bloom_filter_enabled: bool, + /// I/O tuning applied when reading Parquet files. + parquet_options: ParquetReadOptions, +} + +impl Default for ArrowReaderOptions { + fn default() -> Self { + Self::builder().build() + } +} + +impl ArrowReaderOptions { + /// Supplies a fallback limit without replacing an explicit reader limit. + pub(super) fn with_data_file_concurrency_limit_if_unset(mut self, limit: usize) -> Self { + self.data_file_concurrency_limit.get_or_insert(limit); + self + } + + /// Returns the configured batch size. + pub fn batch_size(&self) -> Option { + self.batch_size + } + + /// Returns the configured data file concurrency limit. + /// + /// When unset, reader construction uses execution-host parallelism. + /// Zero is treated as one. + pub fn data_file_concurrency_limit(&self) -> Option { + self.data_file_concurrency_limit + } + + /// Returns whether row-group statistics filtering is enabled. + /// + /// Enabled by default: statistics can skip groups that cannot match the + /// predicate, generally improving performance with little overhead. + pub fn row_group_filtering_enabled(&self) -> bool { + self.row_group_filtering_enabled + } + + /// Returns whether page-index row selection is enabled. + /// + /// Disabled by default: parsing page indexes can cost more than the reads + /// it avoids. Evaluate it against the table's sorting, partitioning, row-group + /// size, and page size before enabling it. + pub fn row_selection_enabled(&self) -> bool { + self.row_selection_enabled + } + + /// Returns whether bloom-filter pruning is enabled. + /// + /// Disabled by default: bloom filters require extra reads per relevant + /// column and row group. They can eliminate groups for equality and IN predicates. + pub fn bloom_filter_enabled(&self) -> bool { + self.bloom_filter_enabled + } + + /// Returns the configured parquet options. + pub fn parquet_options(&self) -> &ParquetReadOptions { + &self.parquet_options + } + + pub(super) fn with_parquet_options(mut self, value: ParquetReadOptions) -> Self { + self.parquet_options = value; + self + } +} + +/// Options for tuning Parquet file I/O. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TypedBuilder)] +#[serde(default)] +#[builder(field_defaults(setter(prefix = "with_")))] +pub struct ParquetReadOptions { + /// Number of bytes to prefetch for parsing the Parquet metadata. + /// + /// This hint can help reduce the number of fetch requests. For more details see the + /// [ParquetMetaDataReader documentation](https://docs.rs/parquet/latest/parquet/file/metadata/struct.ParquetMetaDataReader.html#method.with_prefetch_hint). + /// + /// Defaults to 512 KiB, matching DataFusion's default `ParquetOptions::metadata_size_hint`. + #[builder(default = DEFAULT_METADATA_SIZE_HINT)] + metadata_size_hint: usize, + /// Gap threshold for merging nearby byte ranges into a single request. + /// Ranges with gaps smaller than this value will be coalesced. + /// + /// Defaults to 1 MiB, matching object_store's `OBJECT_STORE_COALESCE_DEFAULT`. + #[builder(default = DEFAULT_RANGE_COALESCE_BYTES)] + range_coalesce_bytes: u64, + /// Maximum number of merged byte ranges to fetch concurrently. + /// + /// Defaults to 10, matching object_store's `OBJECT_STORE_COALESCE_PARALLEL`. + #[builder(default = DEFAULT_RANGE_FETCH_CONCURRENCY)] + range_fetch_concurrency: usize, + #[builder(default = true)] + preload_column_index: bool, + #[builder(default = true)] + preload_offset_index: bool, + /// Stored page-index flag; the reader derives preloading per data file from + /// the predicate and delete files, overriding this value. + #[builder(default = false)] + preload_page_index: bool, +} + +impl Default for ParquetReadOptions { + fn default() -> Self { + Self::builder().build() + } +} + +impl ParquetReadOptions { + pub(super) fn with_preload_page_index(mut self, enabled: bool) -> Self { + self.preload_page_index = enabled; + self + } + + /// Returns whether to preload the column index when reading Parquet metadata. + pub fn preload_column_index(&self) -> bool { + self.preload_column_index + } + + /// Returns whether to preload the offset index when reading Parquet metadata. + pub fn preload_offset_index(&self) -> bool { + self.preload_offset_index + } + + /// Returns the stored page-index flag. + /// + /// The reader ignores this value: page-index preloading is derived per data + /// file from the predicate and delete files. + pub fn preload_page_index(&self) -> bool { + self.preload_page_index + } + + /// Returns the configured metadata size hint. + pub fn metadata_size_hint(&self) -> usize { + self.metadata_size_hint + } + + /// Returns the byte-gap threshold for coalescing read requests. + pub fn range_coalesce_bytes(&self) -> u64 { + self.range_coalesce_bytes + } + + /// Returns the maximum number of concurrent range requests. + pub fn range_fetch_concurrency(&self) -> usize { + self.range_fetch_concurrency + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::unset(None, 7)] + #[case::explicit(Some(3), 3)] + #[case::zero(Some(0), 0)] + fn default_data_file_concurrency_preserves_explicit_options( + #[case] configured: Option, + #[case] expected: usize, + ) { + // Include unrelated settings to check that resolving concurrency preserves them. + let options: ArrowReaderOptions = serde_json::from_value(serde_json::json!({ + "batch_size": 17, + "data_file_concurrency_limit": configured, + "row_selection_enabled": true, + })) + .unwrap(); + // Resolve the reader fallback; explicit zero is normalized later by the reader. + let options = options.with_data_file_concurrency_limit_if_unset(7); + let expected = ArrowReaderOptions::builder() + .with_batch_size(17) + .with_data_file_concurrency_limit(expected) + .with_row_selection_enabled(true) + .build(); + assert_eq!(options, expected); + } + + #[test] + fn missing_options_use_defaults() { + let decoded: ArrowReaderOptions = serde_json::from_str(r#"{"batch_size":17}"#).unwrap(); + assert_eq!( + decoded, + ArrowReaderOptions::builder().with_batch_size(17).build() + ); + } + + #[test] + fn unknown_options_are_ignored() { + let decoded: ArrowReaderOptions = + serde_json::from_str(r#"{"future_option":true}"#).unwrap(); + assert_eq!(decoded, ArrowReaderOptions::default()); + } + + #[test] + fn unset_concurrency_serializes_as_null() { + assert_eq!( + serde_json::to_value(ArrowReaderOptions::default()).unwrap()["data_file_concurrency_limit"], + serde_json::Value::Null, + ); + } + + #[test] + fn parquet_serialization_preserves_preload_settings() { + let options = ParquetReadOptions { + preload_column_index: false, + preload_offset_index: false, + preload_page_index: true, + ..ParquetReadOptions::default() + }; + let encoded = serde_json::to_value(&options).unwrap(); + assert_eq!( + encoded, + serde_json::json!({ + "metadata_size_hint": DEFAULT_METADATA_SIZE_HINT, + "range_coalesce_bytes": DEFAULT_RANGE_COALESCE_BYTES, + "range_fetch_concurrency": DEFAULT_RANGE_FETCH_CONCURRENCY, + "preload_column_index": false, + "preload_offset_index": false, + "preload_page_index": true, + }), + ); + let decoded: ParquetReadOptions = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded, options); + } + + #[test] + fn partial_parquet_options_preserve_defaults() { + let options: ArrowReaderOptions = + serde_json::from_str(r#"{"parquet_options":{"range_fetch_concurrency":2}}"#).unwrap(); + assert_eq!( + options, + ArrowReaderOptions::builder() + .with_parquet_options( + ParquetReadOptions::builder() + .with_range_fetch_concurrency(2) + .build(), + ) + .build() + ); + } +} diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs new file mode 100644 index 0000000000..4e9a0756e4 --- /dev/null +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -0,0 +1,4412 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The main `ArrowReader` pipeline: reading a stream of `FileScanTask`s, +//! opening Parquet files and resolving schemas, then wiring projection, +//! predicates, row-group / row selection, and delete handling into a stream +//! of transformed Arrow `RecordBatch`es. + +use std::borrow::Cow; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; + +use arrow_schema::{DataType, Field}; +use futures::{StreamExt, TryStreamExt}; +use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions}; +use parquet::arrow::{ + PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder, ProjectionMask, RowNumber, +}; +use parquet::encryption::decrypt::FileDecryptionProperties; + +use super::row_lineage::synthesize_row_id_column; +use super::{ + ArrowFileReader, ArrowReader, ParquetReadOptions, add_fallback_field_ids_to_arrow_schema, + apply_name_mapping_to_arrow_schema, coerce_fixed_size_lists, find_leaf_by_field_id, +}; +use crate::arrow::build_partition_constant; +use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader; +use crate::arrow::int96::coerce_int96_timestamps; +use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder; +use crate::arrow::scan_metrics::{CountingFileRead, ScanMetrics, ScanResult}; +use crate::encryption::StandardKeyMetadata; +use crate::error::{Result, invalid_data}; +use crate::expr::BoundPredicate; +use crate::expr::visitors::bloom_filter_evaluator::{ + BloomFilterEvaluator, ColumnBloomFilter, collect_bloom_filter_field_ids, +}; +use crate::io::{FileIO, FileMetadata, FileRead}; +use crate::metadata_columns::{ + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_COL_NAME_POS, + RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_FILE, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_FIELD_ID_PARTITION, + RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, RESERVED_FIELD_ID_SPEC_ID, is_metadata_field, +}; +use crate::runtime::Runtime; +use crate::scan::{ArrowRecordBatchStream, FileScanTask, FileScanTaskStream}; +use crate::spec::{Datum, PartitionSpec, Struct}; +use crate::{Error, ErrorKind}; + +impl ArrowReader { + /// Take a stream of FileScanTasks and reads all the files. + /// Returns a [`ScanResult`] containing the record batch stream and scan metrics. + pub fn read(self, tasks: FileScanTaskStream) -> Result { + let concurrency_limit_data_files = self + .options + .data_file_concurrency_limit() + .expect("reader construction sets concurrency") + .max(1); + let scan_metrics = ScanMetrics::new(); + + let task_reader = FileScanTaskReader { + batch_size: self.options.batch_size(), + file_io: self.file_io, + delete_file_loader: self + .delete_file_loader + .with_scan_metrics(scan_metrics.clone()), + row_group_filtering_enabled: self.options.row_group_filtering_enabled(), + row_selection_enabled: self.options.row_selection_enabled(), + bloom_filter_enabled: self.options.bloom_filter_enabled(), + parquet_read_options: self.options.parquet_options().clone(), + scan_metrics: scan_metrics.clone(), + runtime: self.runtime, + }; + + // Fast-path for single concurrency to avoid overhead of try_flatten_unordered + let stream: ArrowRecordBatchStream = if concurrency_limit_data_files == 1 { + Box::pin( + tasks + .and_then(move |task| task_reader.clone().process(task)) + .map_err(|err| { + Error::new(ErrorKind::Unexpected, "file scan task generate failed") + .with_source(err) + }) + .try_flatten(), + ) + } else { + Box::pin( + tasks + .map_ok(move |task| task_reader.clone().process(task)) + .map_err(|err| { + Error::new(ErrorKind::Unexpected, "file scan task generate failed") + .with_source(err) + }) + .try_buffer_unordered(concurrency_limit_data_files) + .try_flatten_unordered(concurrency_limit_data_files), + ) + }; + + Ok(ScanResult::new(stream, scan_metrics)) + } +} + +// Metadata columns synthesized without reading any data column, so a projection of only +// these (including the empty projection, e.g. `COUNT(*)`) can be pruned to zero data +// columns. Narrower than `is_metadata_field`, which also matches `_deleted` -- excluded here +// because it has no synthesis handler. +const PRUNABLE_METADATA_FIELDS: &[i32] = &[ + RESERVED_FIELD_ID_FILE, + RESERVED_FIELD_ID_SPEC_ID, + RESERVED_FIELD_ID_PARTITION, + RESERVED_FIELD_ID_POS, + RESERVED_FIELD_ID_ROW_ID, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, +]; + +/// Per-scan state for processing [`FileScanTask`]s. Created once per +/// [`ArrowReader::read`] call and cloned per task. +#[derive(Clone)] +struct FileScanTaskReader { + batch_size: Option, + file_io: FileIO, + delete_file_loader: CachingDeleteFileLoader, + row_group_filtering_enabled: bool, + row_selection_enabled: bool, + bloom_filter_enabled: bool, + parquet_read_options: ParquetReadOptions, + scan_metrics: ScanMetrics, + runtime: Runtime, +} + +// Borrow an existing selection, or enumerate all groups for a whole-file scan. +fn candidate_row_groups( + selected_row_groups: Option<&[usize]>, + num_row_groups: usize, +) -> Cow<'_, [usize]> { + match selected_row_groups { + Some(indices) => Cow::Borrowed(indices), + None => Cow::Owned((0..num_row_groups).collect()), + } +} + +impl FileScanTaskReader { + async fn process(self, task: FileScanTask) -> Result { + // A known-empty data file contributes no rows, regardless of predicates or deletes. + if task.record_count() == Some(0) { + return Ok(Box::pin(futures::stream::empty())); + } + + let should_load_page_index = (self.row_selection_enabled && task.predicate().is_some()) + || !task.deletes().is_empty(); + + let delete_filter_rx = self + .delete_file_loader + .load_deletes(task.deletes(), task.schema_ref()); + + // `try_flatten_unordered` stops polling the buffered opens once it holds + // `concurrency_limit_data_files` streams, so spawn the open (#3276). + // Footer and page-index decode also run on the IO runtime. + // Aborts on drop. + let path = task.data_file_path().to_string(); + let file_io = self.file_io.clone(); + let file_size_in_bytes = task.file_size_in_bytes(); + let bytes_read = Arc::clone(self.scan_metrics.bytes_read_counter()); + let key_metadata = task.key_metadata().map(|km| km.to_vec()); + let parquet_read_options = self + .parquet_read_options + .clone() + .with_preload_page_index(should_load_page_index); + let opening = self + .runtime + .io() + .spawn(async move { + ArrowReader::open_parquet_file( + &path, + &file_io, + file_size_in_bytes, + parquet_read_options, + &bytes_read, + key_metadata.as_deref(), + ) + .await + }) + .abort_on_drop(); + let (parquet_file_reader, arrow_metadata) = opening.await??; + + // Check if Parquet file has embedded field IDs + // Corresponds to Java's ParquetSchemaUtil.hasIds() + // Reference: parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java:118 + let missing_field_ids = arrow_metadata + .schema() + .fields() + .iter() + .next() + .is_some_and(|f| f.metadata().get(PARQUET_FIELD_ID_META_KEY).is_none()); + + // Position-based fallback applies only when the file has no embedded field IDs + // AND no name mapping is available. With a name mapping, field IDs are assigned + // to the Arrow schema below, and projection/predicate planning must use them + // (see #2403). + let use_position_fallback = missing_field_ids && task.name_mapping().is_none(); + + let project_pos = task.project_field_ids().contains(&RESERVED_FIELD_ID_POS); + let project_row_id = task.project_field_ids().contains(&RESERVED_FIELD_ID_ROW_ID); + + // The RowNumber virtual column materializes `_pos`. It is also the per-row + // positional fallback for `_row_id` (`first_row_id + pos`), so add it whenever + // `_row_id` is synthesized. A null `first_row_id` nulls the whole `_row_id` + // column, so nothing is synthesized and the column is not needed. + let need_row_number = project_pos || (project_row_id && task.first_row_id().is_some()); + + let field_ids = task.project_field_ids(); + // Vacuously true for an empty projection (e.g. `COUNT(*)`), so no `is_empty()` guard. + let no_data_columns_projected = field_ids + .iter() + .all(|id| PRUNABLE_METADATA_FIELDS.contains(id)); + + let install_row_number = need_row_number || no_data_columns_projected; + + let arrow_metadata = Self::configure_arrow_reader_metadata( + arrow_metadata, + &task, + missing_field_ids, + install_row_number, + )?; + + // Build the stream reader, reusing the already-opened file reader + let mut record_batch_stream_builder = + ParquetRecordBatchStreamBuilder::new_with_metadata(parquet_file_reader, arrow_metadata); + + // Whether the file physically carries the `_last_updated_sequence_number` column + // (some engines, e.g. Iceberg Java on rewrite, write it per-row), resolved by its + // embedded field id against the Parquet schema. + let project_last_updated_seq = task + .project_field_ids() + .contains(&RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER); + + // Parquet leaf index of the physically-stored column, resolved by its embedded + // reserved field id. `find_leaf_by_field_id` tolerates id-less leaves (e.g. a + // Variant column's internal metadata/value leaves, which the spec requires to have + // no id), so an unprojected variant alongside a metadata column with correct ID does + // not hide it. + let phys_last_updated_seq_leaf = if project_last_updated_seq { + find_leaf_by_field_id( + record_batch_stream_builder.parquet_schema(), + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ) + } else { + None + }; + + // Present by name but not by the embedded id (only meaningful when no by-id column + // was found). An unthreadable shape we reject rather than coalesce incorrectly. + let last_updated_seq_present_by_name_only = project_last_updated_seq + && phys_last_updated_seq_leaf.is_none() + && record_batch_stream_builder + .schema() + .fields() + .iter() + .any(|f| f.name() == RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER); + + // Read the physical column only when first_row_id is set (with a data sequence + // number to fall back to). A null first_row_id drops the leaf and nulls the whole + // column below, discarding any per-row values the file carries -- matching Java + // (`ValueReaders.lastUpdated` nulls when the base row id is null). + let coalesce_last_updated_seq_leaf = phys_last_updated_seq_leaf + .filter(|_| task.first_row_id().is_some() && task.data_sequence_number().is_some()); + + let phys_row_id_leaf = if project_row_id { + find_leaf_by_field_id( + record_batch_stream_builder.parquet_schema(), + RESERVED_FIELD_ID_ROW_ID, + ) + } else { + None + }; + + // A column named `_row_id` that carries no embedded field id, in a file that DOES + // use embedded ids -- an unthreadable physically-stored `_row_id`, rejected below. + // Gated on `!use_position_fallback`: under positional fallback every column lacks an + // embedded id and synthetic ids are assigned by position, so a user column that + // happens to be named `_row_id` is real data, not a reserved metadata column. + let row_id_present_by_name_only = project_row_id + && !use_position_fallback + && phys_row_id_leaf.is_none() + && record_batch_stream_builder + .schema() + .fields() + .iter() + .any(|f| f.name() == RESERVED_COL_NAME_ROW_ID); + + // Read the physical column only when first_row_id is set. A null first_row_id + // nulls the whole column below (matching Java `ValueReaders.rowIds`). + let coalesce_row_id_leaf = phys_row_id_leaf.filter(|_| task.first_row_id().is_some()); + + // Filter out metadata fields for Parquet projection (they don't exist in files) + let project_field_ids_without_metadata: Vec = task + .project_field_ids() + .iter() + .filter(|&&id| !is_metadata_field(id)) + .copied() + .collect(); + + // Create projection mask based on field IDs + // - If file has embedded IDs: field-ID-based projection + // - If name mapping applied: field-ID-based projection using the IDs the name + // mapping assigned to the Arrow schema + // - Otherwise: position-based fallback projection + let mut projection_mask = ArrowReader::get_arrow_projection_mask( + &project_field_ids_without_metadata, + task.schema(), + record_batch_stream_builder.parquet_schema(), + record_batch_stream_builder.schema(), + use_position_fallback, // Whether to use position-based (true) or field-ID-based (false) projection + )?; + + // A projection with no data columns leaves `project_field_ids_without_metadata` + // empty, which `get_arrow_projection_mask` maps to "read all columns". Downgrade that + // to "read no data columns": `install_row_number` put the RowNumber virtual column on + // every such projection as a row-count source independent of the data columns, so the + // count survives with zero data columns read. + // + // This runs BEFORE the union so any physical metadata leaf is added onto a `none` + // base, pruning the read to just that leaf (`union` with an `all` base stays `all`). + if project_field_ids_without_metadata.is_empty() && install_row_number { + projection_mask = + ProjectionMask::none(record_batch_stream_builder.parquet_schema().num_columns()); + } + + // Union in the physical leaves of any metadata columns we will coalesce. Their + // reserved field ids are not in the task schema, so they can't be requested through + // `get_arrow_projection_mask` (which resolves ids against the task schema); add + // their Parquet leaves directly. + for leaf in [coalesce_last_updated_seq_leaf, coalesce_row_id_leaf] + .into_iter() + .flatten() + { + let phys_mask = + ProjectionMask::leaves(record_batch_stream_builder.parquet_schema(), vec![leaf]); + projection_mask.union(&phys_mask); + } + + record_batch_stream_builder = + record_batch_stream_builder.with_projection(projection_mask.clone()); + + // RecordBatchTransformer performs any transformations required on the RecordBatches + // that come back from the file, such as type promotion, default column insertion, + // column re-ordering, partition constants, and virtual field addition (like _file) + let mut record_batch_transformer_builder = + RecordBatchTransformerBuilder::new(task.schema_ref(), task.project_field_ids()); + + // Add the _file metadata column if it's in the projected fields + if task.project_field_ids().contains(&RESERVED_FIELD_ID_FILE) { + let file_datum = Datum::string(task.data_file_path().to_string()); + record_batch_transformer_builder = + record_batch_transformer_builder.with_constant(RESERVED_FIELD_ID_FILE, file_datum); + } + + if task + .project_field_ids() + .contains(&RESERVED_FIELD_ID_SPEC_ID) + { + let partition_spec = task + .partition_spec() + .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Partition spec is missing"))?; + + let spec_id_datum = Datum::int(partition_spec.spec_id()); + record_batch_transformer_builder = record_batch_transformer_builder + .with_constant(RESERVED_FIELD_ID_SPEC_ID, spec_id_datum); + } + + if project_last_updated_seq { + // Materialize the column, gated on the data file's `first_row_id`. Java gates + // it this way (`ValueReaders.lastUpdated` returns nulls when the base row id is + // null); the spec itself only says the column is assigned the manifest entry's + // sequence number on read. + record_batch_transformer_builder = + match (task.first_row_id(), task.data_sequence_number()) { + (Some(_), Some(seq)) => { + let datum = Datum::long(seq); + if coalesce_last_updated_seq_leaf.is_some() { + // The file physically carries the column: read the per-row value, + // falling back to the data sequence number only where null. + record_batch_transformer_builder.with_coalesced_last_updated_seq_column( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + datum, + ) + } else if last_updated_seq_present_by_name_only { + // Present by name but without the embedded field id (name mapping / + // positional fallback). The transformer keys the source column by + // field id, so we can't thread it; no real writer produces this, so + // reject loudly rather than silently overwrite with the constant. + // Arm-local by design: only this arm reads the physical column, so + // only here can a name-only column defeat us. The `(None, _)` arm + // nulls the column without reading it, so it needs no such guard. + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Reading a physically-stored _last_updated_sequence_number column \ + without an embedded field id is not supported", + )); + } else { + // Column absent: derive it from the data sequence number. + record_batch_transformer_builder.with_constant( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + datum, + ) + } + } + // Null first_row_id (v1/v2, or a pre-upgrade v3 snapshot): the column is null. + (None, _) => record_batch_transformer_builder.with_null_metadata_column( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + )?, + // first_row_id present but no data sequence number: after manifest + // inheritance a committed entry always has one, so this is a malformed + // manifest rather than a legitimate null. + (Some(_), None) => { + return Err(invalid_data!( + "Data file {} has a first_row_id but no data sequence number", + task.data_file_path() + )); + } + }; + } + + if project_row_id { + // A name-only physical `_row_id` can't be threaded (synthesis keys the leaf by + // its reserved field id). Reject only when `first_row_id` is set; otherwise the + // leaf is never read and `_row_id` is nulled out downstream (matching Java + // `ValueReaders.rowIds`), so a pre-v3 file with a user column named `_row_id` + // reads back as null rather than erroring. + if task.first_row_id().is_some() && row_id_present_by_name_only { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Reading a physically-stored _row_id column without an embedded field id \ + is not supported", + )); + } + + // `_row_id` is synthesized downstream over the record-batch stream (see + // `row_lineage::synthesize_row_id_column`); the transformer only passes the + // resulting column through, like `_pos`. + record_batch_transformer_builder = + record_batch_transformer_builder.with_virtual_field(RESERVED_FIELD_ID_ROW_ID); + } + + if let (Some(partition_spec), Some(partition_data)) = + (task.partition_spec(), task.partition()) + { + record_batch_transformer_builder = record_batch_transformer_builder + .with_partition(Arc::clone(partition_spec), partition_data.clone())?; + } + + if project_pos { + record_batch_transformer_builder = + record_batch_transformer_builder.with_virtual_field(RESERVED_FIELD_ID_POS); + } + + // Add the _partition metadata struct column if it's in the projected fields. + // Computed lazily here at read time from the unified partition type + task's spec + data. + if task + .project_field_ids() + .contains(&RESERVED_FIELD_ID_PARTITION) + && let Some(unified_type) = task.unified_partition_type() + { + let (spec, partition_data) = match (task.partition_spec(), task.partition()) { + (Some(spec), Some(data)) => (Arc::clone(spec), data.clone()), + // A missing spec/data is only acceptable when there are no partition + // fields to fill (unpartitioned table). If the unified type has fields + // but we lack a spec or data, the task is inconsistent and we cannot + // build the _partition column. + _ if unified_type.fields().is_empty() => { + (Arc::new(PartitionSpec::unpartition_spec()), Struct::empty()) + } + _ => { + return Err(Error::new( + ErrorKind::Unexpected, + "cannot build _partition column: unified partition type has fields \ + but the scan task is missing its partition spec or data", + )); + } + }; + let constant = build_partition_constant(unified_type, &spec, &partition_data)?; + record_batch_transformer_builder = + record_batch_transformer_builder.with_partition_constant(constant); + } + + let mut record_batch_transformer = record_batch_transformer_builder.build(); + + if let Some(batch_size) = self.batch_size { + record_batch_stream_builder = record_batch_stream_builder.with_batch_size(batch_size); + } + + let delete_filter = delete_filter_rx.await.unwrap()?; + let delete_predicate = delete_filter.build_equality_delete_predicate(&task).await?; + + // In addition to the optional predicate supplied in the `FileScanTask`, + // we also have an optional predicate resulting from equality delete files. + // If both are present, we logical-AND them together to form a single filter + // predicate that we can pass to the `RecordBatchStreamBuilder`. + let final_predicate = match (task.predicate(), delete_predicate) { + (None, None) => None, + (Some(predicate), None) => Some(predicate.clone()), + (None, Some(ref predicate)) => Some(predicate.clone()), + (Some(filter_predicate), Some(delete_predicate)) => { + Some(filter_predicate.clone().and(delete_predicate)) + } + }; + + // There are three possible sources for potential lists of selected RowGroup indices, + // and two for `RowSelection`s. + // Selected RowGroup index lists can come from three sources: + // * When task.start and task.length specify a byte range (file splitting); + // * When there are equality delete files that are applicable; + // * When there is a scan predicate and row_group_filtering_enabled = true. + // `RowSelection`s can be created in either or both of the following cases: + // * When there are positional delete files that are applicable; + // * When there is a scan predicate and row_selection_enabled = true + // Note that row group filtering from predicates only happens when + // there is a scan predicate AND row_group_filtering_enabled = true, + // but we perform row selection filtering if there are applicable + // equality delete files OR (there is a scan predicate AND row_selection_enabled), + // since the only implemented method of applying positional deletes is + // by using a `RowSelection`. + let mut selected_row_group_indices = None; + let mut row_selection = None; + + // Filter row groups based on byte range from task.start and task.length. + // If both start and length are 0, read the entire file (backwards compatibility). + if task.start() != 0 || task.length() != 0 { + let byte_range_filtered_row_groups = ArrowReader::filter_row_groups_by_byte_range( + record_batch_stream_builder.metadata(), + task.start(), + task.length(), + )?; + selected_row_group_indices = Some(byte_range_filtered_row_groups); + } + + if let Some(predicate) = final_predicate { + // Normalize NOT out of the predicate once for every consumer below. The + // conservative row-group and page-index evaluators cannot soundly negate + // a NOT node (negating a "might match" bound can prune matching rows, and + // the page-index evaluator rejects NOT outright). A FileScanTask built or + // deserialized outside TableScan carries an un-normalized predicate, so + // rewrite it here. Matches TableScan::with_filter. + let predicate = predicate.rewrite_not(); + + let (iceberg_field_ids, field_id_map) = ArrowReader::build_field_id_set_and_map( + record_batch_stream_builder.parquet_schema(), + record_batch_stream_builder.schema(), + &predicate, + use_position_fallback, + )?; + + let row_filter = ArrowReader::get_row_filter( + &predicate, + record_batch_stream_builder.parquet_schema(), + &iceberg_field_ids, + &field_id_map, + )?; + record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter); + + if self.row_group_filtering_enabled { + let candidates = candidate_row_groups( + selected_row_group_indices.as_deref(), + record_batch_stream_builder.metadata().num_row_groups(), + ); + + selected_row_group_indices = Some(ArrowReader::get_selected_row_group_indices( + &predicate, + record_batch_stream_builder.metadata(), + &field_id_map, + task.schema(), + candidates.as_ref(), + )?); + } + + if self.bloom_filter_enabled { + let candidates = candidate_row_groups( + selected_row_group_indices.as_deref(), + record_batch_stream_builder.metadata().num_row_groups(), + ); + + // Spawned for the same reason as the open (#3276). + let bloom_predicate = predicate.clone(); + let bloom_candidates = candidates.as_ref().to_vec(); + let bloom_field_ids = field_id_map.clone(); + let mut builder = record_batch_stream_builder; + let (builder, bloom_filtered) = self + .runtime + .io() + .spawn(async move { + let filtered = Self::filter_row_groups_by_bloom_filter( + &bloom_predicate, + &mut builder, + &bloom_candidates, + &bloom_field_ids, + ) + .await; + (builder, filtered) + }) + .abort_on_drop() + .await?; + record_batch_stream_builder = builder; + let bloom_filtered = bloom_filtered?; + + if bloom_filtered.len() < candidates.len() { + selected_row_group_indices = Some(bloom_filtered); + } + } + + if self.row_selection_enabled { + row_selection = ArrowReader::get_row_selection_for_filter_predicate( + &predicate, + record_batch_stream_builder.metadata(), + &selected_row_group_indices, + &field_id_map, + task.schema(), + )?; + } + } + + let positional_delete_indexes = delete_filter.get_delete_vector(&task); + + if let Some(positional_delete_indexes) = positional_delete_indexes { + let delete_row_selection = { + let positional_delete_indexes = positional_delete_indexes.lock().unwrap(); + + ArrowReader::build_deletes_row_selection( + record_batch_stream_builder.metadata().row_groups(), + &selected_row_group_indices, + &positional_delete_indexes, + ) + }?; + + // merge the row selection from the delete files with the row selection + // from the filter predicate, if there is one from the filter predicate + row_selection = match row_selection { + None => Some(delete_row_selection), + Some(filter_row_selection) => { + Some(filter_row_selection.intersection(&delete_row_selection)) + } + }; + } + + if let Some(row_selection) = row_selection { + record_batch_stream_builder = + record_batch_stream_builder.with_row_selection(row_selection); + } + + if let Some(selected_row_group_indices) = selected_row_group_indices { + record_batch_stream_builder = + record_batch_stream_builder.with_row_groups(selected_row_group_indices); + } + + // Build the batch stream and send all the RecordBatches that it generates + // to the requester. When `_row_id` is projected, synthesize it over the raw parquet + // batches (using the reader-produced `_pos` position) before the transformer, which + // then passes it through as a virtual field. + let first_row_id = task.first_row_id(); + let record_batch_stream = record_batch_stream_builder.build()?.map(move |batch| { + let mut batch = batch.map_err(|err| -> Error { err.into() })?; + if project_row_id { + batch = synthesize_row_id_column(batch, first_row_id)?; + } + // Process the record batch (type promotion, column reordering, virtual fields, etc.) + record_batch_transformer.process_record_batch(batch) + }); + + Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream) + } + + /// Applies all task-specific schema and virtual-column options, rebuilding the + /// Arrow reader metadata at most once. + fn configure_arrow_reader_metadata( + arrow_metadata: ArrowReaderMetadata, + task: &FileScanTask, + missing_field_ids: bool, + install_row_number: bool, + ) -> Result { + // Schema resolution follows the Iceberg Column Projection rule: + // "Columns in Iceberg data files are selected by field id." + // https://iceberg.apache.org/spec/#column-projection + // + // This mirrors Java's ReadConf strategy: use embedded IDs with pruneColumns(); + // otherwise, use applyNameMapping() followed by pruneColumns() when configured, or + // addFallbackIds() followed by pruneColumnsFallback() for position-based fallback. + // The fast path returns early without materializing an owned schema. + // + // Rewrite FixedSizeList hints to List before name mapping / fallback IDs so + // parquet-rs uses the 3-level LIST reader. Same placement as #3467's list-view rewrite. + let without_fixed_size_lists = coerce_fixed_size_lists(arrow_metadata.schema()); + let file_schema = without_fixed_size_lists + .as_ref() + .unwrap_or(arrow_metadata.schema()); + let arrow_schema = if missing_field_ids { + let schema = if let Some(name_mapping) = task.name_mapping() { + apply_name_mapping_to_arrow_schema(Arc::clone(file_schema), name_mapping)? + } else { + add_fallback_field_ids_to_arrow_schema(file_schema) + }; + // Coerce INT96 timestamp columns before building the stream reader to avoid + // i64 overflow in arrow-rs. Apply this after assigning any missing field IDs + // so the final schema contains both changes. + coerce_int96_timestamps(&schema, task.schema()).unwrap_or(schema) + } else if let Some(coerced) = coerce_int96_timestamps(file_schema, task.schema()) { + coerced + } else if without_fixed_size_lists.is_some() || install_row_number { + Arc::clone(file_schema) + } else { + return Ok(arrow_metadata); + }; + + let mut options = ArrowReaderOptions::new().with_schema(Arc::clone(&arrow_schema)); + if install_row_number { + let row_number_field = Arc::new( + Field::new(RESERVED_COL_NAME_POS, DataType::Int64, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_POS.to_string(), + )])) + .with_extension_type(RowNumber), + ); + options = options.with_virtual_columns(vec![row_number_field])?; + } + + ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!( + "Failed to create ArrowReaderMetadata with the configured reader options \ + (missing_field_ids: {missing_field_ids}, \ + install_row_number: {install_row_number}, schema: {arrow_schema})" + ), + ) + .with_source(e) + }) + } + + /// Reads bloom filters for relevant columns and evaluates the predicate + /// against them to filter out row groups that definitely don't match. + async fn filter_row_groups_by_bloom_filter( + predicate: &BoundPredicate, + builder: &mut ParquetRecordBatchStreamBuilder, + candidate_row_groups: &[usize], + field_id_map: &HashMap, + ) -> Result> { + // Only collect field IDs from eq/in predicates — the only types + // bloom filters can help with. Skip columns not in the parquet schema. + let bloom_filter_field_ids: Vec = collect_bloom_filter_field_ids(predicate)? + .into_iter() + .filter(|id| field_id_map.contains_key(id)) + .collect(); + + if bloom_filter_field_ids.is_empty() { + return Ok(candidate_row_groups.to_vec()); + } + + let mut result = Vec::with_capacity(candidate_row_groups.len()); + + for &rg_idx in candidate_row_groups { + let mut bloom_filters: HashMap = HashMap::new(); + + for &field_id in &bloom_filter_field_ids { + let col_idx = field_id_map[&field_id]; + let col_meta = builder.metadata().row_group(rg_idx).column(col_idx); + + // Only attempt to load if this column chunk actually has a bloom filter + if col_meta.bloom_filter_offset().is_none() { + continue; + } + + let physical_type = col_meta.column_type(); + let type_length = col_meta.column_descr().type_length(); + + match builder + .get_row_group_column_bloom_filter(rg_idx, col_idx) + .await + { + Ok(Some(sbbf)) => { + bloom_filters.insert( + field_id, + ColumnBloomFilter::new(sbbf, physical_type, type_length), + ); + } + Ok(None) => {} + Err(e) => { + // Left absent from the map, so the evaluator treats the column + // as might-match and the row group survives. + tracing::debug!( + "Bloom filter for field {field_id} in row group {rg_idx} could not be read: {e}" + ); + } + } + } + + match BloomFilterEvaluator::eval(predicate, &bloom_filters) { + Ok(true) => result.push(rg_idx), + Ok(false) => { /* Row group pruned by bloom filter */ } + Err(e) => { + tracing::debug!( + "Bloom filter evaluation failed for row group {rg_idx}, including it: {e}" + ); + result.push(rg_idx); + } + } + } + + Ok(result) + } +} + +impl ArrowReader { + /// Opens a Parquet file and loads its metadata, wrapping the reader with + /// [`CountingFileRead`] so all I/O is accumulated into `bytes_read`. + pub(crate) async fn open_parquet_file( + data_file_path: &str, + file_io: &FileIO, + file_size_in_bytes: u64, + parquet_read_options: ParquetReadOptions, + bytes_read: &Arc, + key_metadata: Option<&[u8]>, + ) -> Result<(ArrowFileReader, ArrowReaderMetadata)> { + let parquet_file = file_io.new_input(data_file_path)?; + let counting_reader = + CountingFileRead::new(parquet_file.reader().await?, Arc::clone(bytes_read)); + Self::build_parquet_reader( + Box::new(counting_reader), + file_size_in_bytes, + parquet_read_options, + key_metadata, + ) + .await + } + + async fn build_parquet_reader( + parquet_reader: Box, + file_size_in_bytes: u64, + parquet_read_options: ParquetReadOptions, + key_metadata: Option<&[u8]>, + ) -> Result<(ArrowFileReader, ArrowReaderMetadata)> { + let mut reader = ArrowFileReader::new( + FileMetadata { + size: file_size_in_bytes, + }, + parquet_reader, + ) + .with_parquet_read_options(parquet_read_options); + + let arrow_reader_options = Self::build_arrow_reader_options(key_metadata)?; + + let arrow_metadata = ArrowReaderMetadata::load_async(&mut reader, arrow_reader_options) + .await + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "Failed to load Parquet metadata").with_source(e) + })?; + + Ok((reader, arrow_metadata)) + } + + /// Builds `ArrowReaderOptions`, adding `FileDecryptionProperties` when + /// key metadata is present for Parquet Modular Encryption. + fn build_arrow_reader_options(key_metadata: Option<&[u8]>) -> Result { + match key_metadata { + Some(km) => { + let standard_key_metadata = StandardKeyMetadata::decode(km)?; + let mut builder = FileDecryptionProperties::builder( + standard_key_metadata.encryption_key().as_bytes().to_vec(), + ); + if let Some(aad) = standard_key_metadata.aad_prefix() { + builder = builder.with_aad_prefix(aad.to_vec()); + } + let decryption_properties = builder.build().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + "Failed to build Parquet file decryption properties", + ) + .with_source(e) + })?; + Ok( + ArrowReaderOptions::new() + .with_file_decryption_properties(decryption_properties), + ) + } + None => Ok(ArrowReaderOptions::default()), + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::fs::File; + use std::sync::Arc; + + use arrow_array::cast::AsArray; + use arrow_array::{Array, ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray}; + use arrow_cast::cast; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use futures::TryStreamExt; + use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; + use parquet::basic::Compression; + use parquet::file::properties::WriterProperties; + use tempfile::TempDir; + + use crate::Runtime; + use crate::arrow::test_utils::write_encrypted_parquet; + use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions as ReaderOptions}; + use crate::io::FileIO; + use crate::metadata_columns::{ + RESERVED_COL_NAME_POS, RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_FILE, + RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, + }; + use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; + use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Schema, SchemaRef, Type}; + + fn record_count_task(path: &str, file_size: u64, record_count: Option) -> FileScanTask { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + FileScanTask::builder() + .with_data_file_path(path.to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(file_size) + .with_record_count(record_count) + .with_schema(Arc::new(schema)) + .with_project_field_ids(vec![1]) + .with_case_sensitive(true) + .build() + .unwrap() + } + + async fn read_tasks( + tasks: Vec, + concurrency: usize, + ) -> crate::Result> { + let tasks = futures::stream::iter(tasks.into_iter().map(Ok)); + ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_data_file_concurrency_limit(concurrency) + .build(), + ) + .build() + .read(Box::pin(tasks))? + .stream() + .try_collect() + .await + } + + #[tokio::test] + async fn test_read_zero_record_files_without_opening_them() { + let temp_dir = TempDir::new().unwrap(); + let missing_path = temp_dir + .path() + .join("missing.parquet") + .display() + .to_string(); + + for concurrency in [1, 2] { + // The record count, not the byte size, determines whether the file is empty. + let tasks = vec![ + record_count_task(&missing_path, 0, Some(0)), + record_count_task(&missing_path, 128, Some(0)), + ]; + let batches = read_tasks(tasks, concurrency).await.unwrap(); + assert!(batches.is_empty()); + } + } + + #[tokio::test] + async fn test_read_zero_size_files_with_nonzero_or_unknown_record_count() { + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("empty.parquet").display().to_string(); + File::create(&path).unwrap(); + + for concurrency in [1, 2] { + for record_count in [Some(1), None] { + let task = record_count_task(&path, 0, record_count); + let error = read_tasks(vec![task], concurrency).await.unwrap_err(); + assert!( + error + .to_string() + .contains("file size of 0 is less than footer"), + "{error}" + ); + } + } + } + + #[tokio::test] + async fn test_read_nonempty_files_alongside_zero_record_files() { + use arrow_array::types::Int32Type; + + let temp_dir = TempDir::new().unwrap(); + let dir = temp_dir.path().to_str().unwrap(); + let path = write_plain_parquet(dir, "data.parquet", vec![], vec![]); + let file_size = std::fs::metadata(&path).unwrap().len(); + let missing_path = format!("{dir}/missing.parquet"); + + for concurrency in [1, 2] { + for record_count in [Some(3), None] { + let empty_task = record_count_task(&missing_path, 0, Some(0)); + let data_task = record_count_task(&path, file_size, record_count); + let tasks = vec![empty_task.clone(), data_task, empty_task]; + let batches = read_tasks(tasks, concurrency).await.unwrap(); + + assert_eq!(batches.len(), 1); + let ids = batches[0].column(0).as_primitive::(); + assert_eq!(ids.values(), &[1, 2, 3]); + } + } + } + + // INT96 encoding: [nanos_low_u32, nanos_high_u32, julian_day_u32] + // Julian day 2_440_588 = Unix epoch (1970-01-01) + const UNIX_EPOCH_JULIAN: i64 = 2_440_588; + const MICROS_PER_DAY: i64 = 86_400_000_000; + // Noon on 3333-01-01 (Julian day 2_953_529) — outside the i64 nanosecond range (~1677-2262). + const INT96_TEST_NANOS_WITHIN_DAY: u64 = 43_200_000_000_000; + const INT96_TEST_JULIAN_DAY: u32 = 2_953_529; + + fn make_int96_test_value() -> (parquet::data_type::Int96, i64) { + let mut val = parquet::data_type::Int96::new(); + val.set_data( + (INT96_TEST_NANOS_WITHIN_DAY & 0xFFFFFFFF) as u32, + (INT96_TEST_NANOS_WITHIN_DAY >> 32) as u32, + INT96_TEST_JULIAN_DAY, + ); + let expected_micros = (INT96_TEST_JULIAN_DAY as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY + + (INT96_TEST_NANOS_WITHIN_DAY / 1_000) as i64; + (val, expected_micros) + } + + async fn read_int96_batches( + file_path: &str, + schema: SchemaRef, + project_field_ids: Vec, + ) -> Vec { + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let file_size = std::fs::metadata(file_path).unwrap().len(); + let task = FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(file_size) + .with_data_file_path(file_path.to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(project_field_ids) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap() + } + + // ArrowWriter cannot write INT96, so we use SerializedFileWriter directly. + fn write_int96_parquet_file( + table_location: &str, + filename: &str, + with_field_ids: bool, + ) -> (String, Vec) { + use parquet::basic::{Repetition, Type as PhysicalType}; + use parquet::data_type::{Int32Type, Int96, Int96Type}; + use parquet::file::writer::SerializedFileWriter; + use parquet::schema::types::Type as SchemaType; + + let file_path = format!("{table_location}/{filename}"); + + let mut ts_builder = SchemaType::primitive_type_builder("ts", PhysicalType::INT96) + .with_repetition(Repetition::OPTIONAL); + let mut id_builder = SchemaType::primitive_type_builder("id", PhysicalType::INT32) + .with_repetition(Repetition::REQUIRED); + + if with_field_ids { + ts_builder = ts_builder.with_id(Some(1)); + id_builder = id_builder.with_id(Some(2)); + } + + let schema = SchemaType::group_type_builder("schema") + .with_fields(vec![ + Arc::new(ts_builder.build().unwrap()), + Arc::new(id_builder.build().unwrap()), + ]) + .build() + .unwrap(); + + // Dates outside the i64 nanosecond range (~1677-2262) overflow without coercion. + const NOON_NANOS: u64 = INT96_TEST_NANOS_WITHIN_DAY; + const JULIAN_3333: u32 = INT96_TEST_JULIAN_DAY; + const JULIAN_2100: u32 = 2_488_070; + + let test_data: Vec<(u32, u32, u32, i64)> = vec![ + // 3333-01-01 00:00:00 + ( + 0, + 0, + JULIAN_3333, + (JULIAN_3333 as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY, + ), + // 3333-01-01 12:00:00 + ( + (NOON_NANOS & 0xFFFFFFFF) as u32, + (NOON_NANOS >> 32) as u32, + JULIAN_3333, + (JULIAN_3333 as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY + + (NOON_NANOS / 1_000) as i64, + ), + // 2100-01-01 00:00:00 + ( + 0, + 0, + JULIAN_2100, + (JULIAN_2100 as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY, + ), + ]; + + let int96_values: Vec = test_data + .iter() + .map(|(lo, hi, day, _)| { + let mut v = Int96::new(); + v.set_data(*lo, *hi, *day); + v + }) + .collect(); + + let id_values: Vec = (0..test_data.len() as i32).collect(); + let expected_micros: Vec = test_data.iter().map(|(_, _, _, m)| *m).collect(); + + let file = File::create(&file_path).unwrap(); + let mut writer = + SerializedFileWriter::new(file, Arc::new(schema), Default::default()).unwrap(); + + let mut row_group = writer.next_row_group().unwrap(); + { + // def=1: ts is OPTIONAL and present. No repetition levels (top-level columns). + let mut col = row_group.next_column().unwrap().unwrap(); + col.typed::() + .write_batch(&int96_values, Some(&vec![1; test_data.len()]), None) + .unwrap(); + col.close().unwrap(); + } + { + let mut col = row_group.next_column().unwrap().unwrap(); + col.typed::() + .write_batch(&id_values, None, None) + .unwrap(); + col.close().unwrap(); + } + row_group.close().unwrap(); + writer.close().unwrap(); + + (file_path, expected_micros) + } + + async fn assert_int96_read_matches( + file_path: &str, + schema: SchemaRef, + project_field_ids: Vec, + expected_micros: &[i64], + ) { + use arrow_array::TimestampMicrosecondArray; + + let batches = read_int96_batches(file_path, schema, project_field_ids).await; + + assert_eq!(batches.len(), 1); + let ts_array = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .expect("Expected TimestampMicrosecondArray"); + + for (i, expected) in expected_micros.iter().enumerate() { + assert_eq!( + ts_array.value(i), + *expected, + "Row {i}: got {}, expected {expected}", + ts_array.value(i) + ); + } + } + + /// Writes a single-column Parquet file encrypted with `encryption_key`, then reads it + /// back through `ArrowReader` and asserts the round-tripped values. The key length + /// selects the AES-GCM variant in arrow-rs (16 -> AES-128, 32 -> AES-256). + async fn assert_encrypted_parquet_roundtrip(encryption_key: &[u8]) { + let aad_prefix = b"aad_prefix"; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let id_data = Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_data]).unwrap(); + + let file_path = format!("{table_location}/encrypted.parquet"); + write_encrypted_parquet(&file_path, &batch, encryption_key, Some(aad_prefix)); + + let key_metadata = crate::encryption::StandardKeyMetadata::try_new(encryption_key) + .unwrap() + .with_aad_prefix(aad_prefix) + .encode() + .unwrap(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1]) + .with_case_sensitive(false) + .with_key_metadata(Some(key_metadata)) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_eq!(batches.len(), 1); + let ids = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ids.values(), &[10, 20, 30]); + } + + #[tokio::test] + async fn test_read_encrypted_parquet_aes_128() { + assert_encrypted_parquet_roundtrip(b"0123456789abcdef").await; + } + + #[tokio::test] + async fn test_read_encrypted_parquet_aes_256() { + assert_encrypted_parquet_roundtrip(b"0123456789abcdef0123456789abcdef").await; + } + + #[tokio::test] + async fn test_read_encrypted_parquet_without_key_metadata_fails() { + let encryption_key = b"0123456789abcdef"; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let id_data = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_data]).unwrap(); + + let file_path = format!("{table_location}/encrypted_no_key.parquet"); + write_encrypted_parquet(&file_path, &batch, encryption_key, None); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result: Result, _> = + reader.read(tasks).unwrap().stream().try_collect().await; + + let err = result.unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::Unexpected); + let err_str = format!("{err}"); + assert!( + err_str.contains("encrypted footer"), + "Expected error about encrypted footer, got: {err_str}" + ); + assert!( + err_str.contains("decryption properties were not provided"), + "Expected error about missing decryption properties, got: {err_str}" + ); + } + + /// Writes a plain (unencrypted) single-column Int32 "id" parquet file with the + /// given extra Arrow fields/columns appended, returning the file path. + fn write_plain_parquet( + dir: &str, + name: &str, + extra_fields: Vec, + extra_columns: Vec, + ) -> String { + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + write_parquet_with(dir, name, extra_fields, extra_columns, props) + } + + fn write_parquet_with( + dir: &str, + name: &str, + extra_fields: Vec, + extra_columns: Vec, + props: WriterProperties, + ) -> String { + let mut fields = + vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ]; + fields.extend(extra_fields); + let arrow_schema = Arc::new(ArrowSchema::new(fields)); + + let mut columns: Vec = vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]; + columns.extend(extra_columns); + let batch = RecordBatch::try_new(arrow_schema.clone(), columns).unwrap(); + + let file_path = format!("{dir}/{name}"); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + file_path + } + + fn last_updated_seq_task( + file_path: String, + first_row_id: Option, + data_sequence_number: Option, + ) -> FileScanTask { + use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER]) + .with_first_row_id(first_row_id) + .with_data_sequence_number(data_sequence_number) + .with_case_sensitive(false) + .build() + .unwrap() + } + + /// Asserts the logical per-row values of the `_last_updated_sequence_number` + /// column across all batches, independent of the physical (run-end) encoding. + fn assert_last_updated_seq_column(batches: &[RecordBatch], expected: &[Option]) { + use arrow_array::cast::AsArray; + use arrow_cast::cast; + use arrow_schema::DataType; + + use crate::metadata_columns::RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER; + + let mut actual = Vec::new(); + for batch in batches { + let col = batch + .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER) + .expect("_last_updated_sequence_number column should be present"); + let logical = cast(col, &DataType::Int64).unwrap(); + let values = logical.as_primitive::(); + for i in 0..values.len() { + actual.push((!values.is_null(i)).then(|| values.value(i))); + } + } + assert_eq!(actual, expected); + } + + #[tokio::test] + async fn test_last_updated_sequence_number_null_when_no_first_row_id() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let file_path = write_plain_parquet(dir, "no_first_row_id.parquet", vec![], vec![]); + + // A file with a null first_row_id (v1/v2, or a pre-upgrade v3 snapshot) produces + // a null _last_updated_sequence_number column, even though it has a data + // sequence number; the spec gates both lineage columns on first_row_id. + let task = last_updated_seq_task(file_path, None, Some(9)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_last_updated_seq_column(&batches, &[None, None, None]); + } + + #[tokio::test] + async fn test_last_updated_sequence_number_error_when_no_data_seq() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let file_path = write_plain_parquet(dir, "no_data_seq.parquet", vec![], vec![]); + + // first_row_id present but data_sequence_number absent: after manifest + // inheritance a committed entry always has one, so this is a malformed + // manifest and must error rather than fabricate or null the column. + let task = last_updated_seq_task(file_path, Some(42), None); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result: Result, _> = + reader.read(tasks).unwrap().stream().try_collect().await; + + let err = result.unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::DataInvalid); + assert!( + format!("{err}").contains("no data sequence number"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_last_updated_sequence_number_derived_from_data_seq() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let file_path = write_plain_parquet(dir, "with_first_row_id.parquet", vec![], vec![]); + + // Non-null first_row_id + data sequence number -> the derived value (the data + // sequence number) for every row. This is the only value-producing arm. + let task = last_updated_seq_task(file_path, Some(42), Some(7)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_last_updated_seq_column(&batches, &[Some(7), Some(7), Some(7)]); + } + + #[tokio::test] + async fn test_last_updated_sequence_number_mixed_files_share_schema() { + use arrow_select::concat::concat_batches; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // Three files in one scan exercising all three column paths, which must all + // produce the SAME Arrow type (run-end-encoded) or concatenation fails: + // - constant: first_row_id set, no physical column -> derived constant + // - null gate: no first_row_id -> null column + // - coalesce: first_row_id set, physical column present -> per-row + fallback + let constant = last_updated_seq_task( + write_plain_parquet(dir, "constant.parquet", vec![], vec![]), + Some(42), + Some(7), + ); + let nulled = last_updated_seq_task( + write_plain_parquet(dir, "nulled.parquet", vec![], vec![]), + None, + Some(7), + ); + let coalesced = last_updated_seq_task( + write_plain_parquet( + dir, + "coalesced.parquet", + vec![physical_last_updated_seq_field()], + vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef], + ), + Some(50), + Some(7), + ); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![ + Ok(constant), + Ok(nulled), + Ok(coalesced), + ])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_eq!(batches.len(), 3); + // Identical schema across all three paths -> concat succeeds. + let schema = batches[0].schema(); + concat_batches(&schema, &batches) + .expect("constant, null and coalesce files must share one column type"); + } + + /// A parquet field carrying the embedded `_last_updated_sequence_number` field id. + fn physical_last_updated_seq_field() -> Field { + use crate::metadata_columns::{ + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + }; + Field::new( + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + DataType::Int64, + true, + ) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER.to_string(), + )])) + } + + #[tokio::test] + async fn test_last_updated_sequence_number_physical_column_coalesced() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // A file that physically carries the column, as Iceberg Java writes when + // carrying rows forward across a rewrite: some rows have a stored value, some + // are null (added/modified rows, inherited on read). + let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "with_seq.parquet", + vec![physical_last_updated_seq_field()], + vec![seq_col], + ); + + let task = last_updated_seq_task(file_path, Some(100), Some(9)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Per-row value where non-null; the data sequence number (9) where null. + assert_last_updated_seq_column(&batches, &[Some(5), Some(9), Some(8)]); + } + + #[tokio::test] + async fn test_last_updated_sequence_number_coalesced_with_pos_column() { + use crate::metadata_columns::{ + RESERVED_COL_NAME_POS, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + RESERVED_FIELD_ID_POS, + }; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "with_seq_and_pos.parquet", + vec![physical_last_updated_seq_field()], + vec![seq_col], + ); + + // Co-project `_pos` (a virtual column appended to the Arrow output schema) with the + // physical coalesce column. This guards that the physical column's index is + // resolved in the Parquet schema, not the Arrow schema (whose indices shift once + // virtual columns are appended). + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![ + 1, + RESERVED_FIELD_ID_POS, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]) + .with_first_row_id(Some(100)) + .with_data_sequence_number(Some(9)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // The seq column still coalesces correctly... + assert_last_updated_seq_column(&batches, &[Some(5), Some(9), Some(8)]); + // ...and `_pos` is the row position, unaffected by the physical-column union. + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + assert_eq!(pos_col.values(), &[0, 1, 2]); + } + + #[tokio::test] + async fn test_last_updated_sequence_number_physical_column_nulled_without_first_row_id() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "with_seq_no_first_row_id.parquet", + vec![physical_last_updated_seq_field()], + vec![seq_col], + ); + + // Null first_row_id: the whole column is null even though the file physically + // carries per-row values -- the gate wins, and the physical column is not read. + let task = last_updated_seq_task(file_path, None, Some(9)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_last_updated_seq_column(&batches, &[None, None, None]); + } + + #[tokio::test] + async fn test_last_updated_sequence_number_present_by_name_without_id_unsupported() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // Column present by name but WITHOUT the embedded field id (e.g. name mapping / + // positional fallback). The transformer keys the source column by field id, so + // this shape can't be threaded and is rejected loudly. + let seq_field = Field::new( + crate::metadata_columns::RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + DataType::Int64, + true, + ); + let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = + write_plain_parquet(dir, "with_seq_by_name.parquet", vec![seq_field], vec![ + seq_col, + ]); + + let task = last_updated_seq_task(file_path, Some(100), Some(9)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result: Result, _> = + reader.read(tasks).unwrap().stream().try_collect().await; + + let err = result.unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + assert!( + format!("{err}").contains("without an embedded field id"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_last_updated_sequence_number_physical_column_first_row_id_without_data_seq() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "with_seq_no_data_seq.parquet", + vec![physical_last_updated_seq_field()], + vec![seq_col], + ); + + // first_row_id set but no data sequence number: after manifest inheritance a + // committed entry always has one, so this is a malformed manifest, rejected loudly + // rather than nulled. + let task = last_updated_seq_task(file_path, Some(100), None); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result: Result, _> = + reader.read(tasks).unwrap().stream().try_collect().await; + + let err = result.unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::DataInvalid); + assert!( + format!("{err}").contains("no data sequence number"), + "unexpected error: {err}" + ); + } + + /// A scan task projecting `id` + `_row_id`, with the given `first_row_id`. + fn row_id_task(file_path: String, first_row_id: Option) -> FileScanTask { + row_id_task_with_options(file_path, first_row_id, 0, 0, vec![]) + } + + fn row_id_task_with_options( + file_path: String, + first_row_id: Option, + start: u64, + length: u64, + deletes: Vec, + ) -> FileScanTask { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(start) + .with_length(length) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1, RESERVED_FIELD_ID_ROW_ID]) + .with_first_row_id(first_row_id) + .with_deletes(deletes) + .with_case_sensitive(false) + .build() + .unwrap() + } + + /// Asserts the logical per-row values of the `_row_id` column across all batches, + /// independent of the physical (run-end) encoding. + fn assert_row_id_column(batches: &[RecordBatch], expected: &[Option]) { + use arrow_array::cast::AsArray; + use arrow_cast::cast; + use arrow_schema::DataType; + + let mut actual = Vec::new(); + for batch in batches { + let col = batch + .column_by_name(RESERVED_COL_NAME_ROW_ID) + .expect("_row_id column should be present"); + let logical = cast(col, &DataType::Int64).unwrap(); + let values = logical.as_primitive::(); + for i in 0..values.len() { + actual.push((!values.is_null(i)).then(|| values.value(i))); + } + } + assert_eq!(actual, expected); + } + + /// A parquet field carrying the embedded `_row_id` field id. + fn physical_row_id_field() -> Field { + Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true).with_metadata(HashMap::from([ + ( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_ROW_ID.to_string(), + ), + ])) + } + + #[tokio::test] + async fn test_row_id_synthesized_from_first_row_id_and_pos() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let file_path = write_plain_parquet(dir, "row_id_synth.parquet", vec![], vec![]); + + // No physical column: every row is first_row_id + pos. + let task = row_id_task(file_path, Some(100)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]); + } + + #[tokio::test] + async fn test_row_id_physical_column_coalesced() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // A file that physically carries `_row_id`, as written when carrying rows forward + // across a rewrite: some rows have a stored value, some are null. + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_phys.parquet", + vec![physical_row_id_field()], + vec![id_col], + ); + + let task = row_id_task(file_path, Some(100)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Per-row value where non-null; first_row_id + pos (101) where null. + assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]); + } + + #[tokio::test] + async fn test_row_id_only_synthesis_reads_no_data_columns() { + // The common v3 case: a new-row file with `first_row_id` set and NO physically + // stored `_row_id`, projecting only `_row_id`. `_row_id` synthesis installs the + // RowNumber virtual column (via `need_row_number`), so the row count comes from it + // -- the scan must read no data columns, not fall back to reading everything. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + let meta_only = metadata_projection_task_with_first_row_id( + write_parquet_with_wide_column(dir, "row_id_only.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![RESERVED_FIELD_ID_ROW_ID], + Some(100), + ); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]); + + // A scan that also projects the wide data column must read materially more. + let with_data = metadata_projection_task_with_first_row_id( + write_parquet_with_wide_column(dir, "row_id_only_ref.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![2, RESERVED_FIELD_ID_ROW_ID], + Some(100), + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_row_id-only synthesis should read fewer bytes than a scan of the wide column: \ + {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_row_id_only_null_first_row_id_reads_no_data_columns() { + // A null `first_row_id` (v1/v2, or a pre-upgrade v3 snapshot) nulls the whole + // `_row_id` column, so nothing is synthesized -- but the column still has a length. + // The RowNumber counter installed for the metadata-only projection supplies it, so + // the scan reads no data columns instead of reading everything just for the count. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + let meta_only = metadata_projection_task_with_first_row_id( + write_parquet_with_wide_column(dir, "row_id_null.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![RESERVED_FIELD_ID_ROW_ID], + None, + ); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + assert_row_id_column(&batches, &[None, None, None]); + + let with_data = metadata_projection_task_with_first_row_id( + write_parquet_with_wide_column(dir, "row_id_null_ref.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![2, RESERVED_FIELD_ID_ROW_ID], + None, + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_row_id-only scan with a null first_row_id should read fewer bytes than a scan \ + of the wide column: {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_last_updated_seq_only_reads_no_data_columns() { + use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER; + + // `SELECT _last_updated_sequence_number` derives a per-row constant (the data + // sequence number) with no physical column to read. The metadata-only RowNumber + // counter supplies the row count, so the scan prunes the data columns rather than + // reading them all just to size the constant. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + let lusn_only_task = |file_path: String, project_field_ids: Vec| { + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(id_and_wide_schema()) + .with_project_field_ids(project_field_ids) + .with_first_row_id(Some(42)) + .with_data_sequence_number(Some(7)) + .with_case_sensitive(false) + .build() + .unwrap() + }; + + let meta_only = lusn_only_task( + write_parquet_with_wide_column(dir, "lusn_only.parquet", vec![], vec![]), + vec![RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER], + ); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + assert_last_updated_seq_column(&batches, &[Some(7), Some(7), Some(7)]); + + let with_data = lusn_only_task( + write_parquet_with_wide_column(dir, "lusn_only_ref.parquet", vec![], vec![]), + vec![2, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_last_updated_sequence_number-only scan should read fewer bytes than a scan of \ + the wide column: {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_partition_only_reads_no_data_columns() { + use arrow_array::StructArray; + + use crate::metadata_columns::{RESERVED_COL_NAME_PARTITION, RESERVED_FIELD_ID_PARTITION}; + use crate::spec::{Literal, PartitionSpec, Struct, Transform}; + + // `SELECT _partition` materializes a struct constant from the task's partition + // metadata, with no physical column to read. It is the only struct-constant metadata + // column; the RowNumber counter sizes it, so the scan prunes the data columns. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let schema = id_and_wide_schema(); + let spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .with_spec_id(7) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let unified_type = Arc::new(spec.partition_type(&schema).unwrap()); + let partition_data = Struct::from_iter(vec![Some(Literal::int(42))]); + + let partition_task = |file_path: String, project_field_ids: Vec| { + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids) + .with_partition_spec(Some(spec.clone())) + .with_partition(Some(partition_data.clone())) + .with_unified_partition_type(Some(unified_type.clone())) + .with_case_sensitive(false) + .build() + .unwrap() + }; + + let meta_only = partition_task( + write_parquet_with_wide_column(dir, "partition_only.parquet", vec![], vec![]), + vec![RESERVED_FIELD_ID_PARTITION], + ); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + let partition_col = batches[0] + .column_by_name(RESERVED_COL_NAME_PARTITION) + .expect("_partition column should be present") + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(partition_col.len(), 3); + let inner = partition_col + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(inner.values(), &[42, 42, 42]); + + let with_data = partition_task( + write_parquet_with_wide_column(dir, "partition_only_ref.parquet", vec![], vec![]), + vec![2, RESERVED_FIELD_ID_PARTITION], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_partition-only scan should read fewer bytes than a scan of the wide column: \ + {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_row_id_resolves_alongside_id_less_leaf() { + // A file with an id-less leaf (mimicking a Variant column's internal metadata/value + // leaves, which the spec requires to have no field id) plus a physical `_row_id` + // that carries its embedded id. The reserved id must still resolve -- an + // all-or-nothing field map would bail on the id-less leaf and wrongly reject the file. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let idless_field = Field::new("variant_internal", DataType::Utf8, true); + let idless_col = Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef; + let row_id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_with_idless_leaf.parquet", + vec![idless_field, physical_row_id_field()], + vec![idless_col, row_id_col], + ); + + let task = row_id_task(file_path, Some(100)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Physical value where non-null; first_row_id + pos (101) where null. + assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]); + } + + #[tokio::test] + async fn test_row_id_and_last_updated_seq_co_projected() { + use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER; + + // Both lineage columns projected together over a file carrying both physical + // leaves. Each must materialize independently -- neither leaf's mask clobbers the + // other, and the two synthesized columns keep their own values. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let row_id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let seq_col = Arc::new(Int64Array::from(vec![Some(50), None, Some(70)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_and_seq.parquet", + vec![physical_row_id_field(), physical_last_updated_seq_field()], + vec![row_id_col, seq_col], + ); + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![ + 1, + RESERVED_FIELD_ID_ROW_ID, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]) + .with_first_row_id(Some(100)) + .with_data_sequence_number(Some(9)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // _row_id: physical value where non-null, else first_row_id + pos (101). + assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]); + // _last_updated_sequence_number: physical value where non-null, else data seq (9). + assert_last_updated_seq_column(&batches, &[Some(50), Some(9), Some(70)]); + } + + #[tokio::test] + async fn test_row_id_null_when_no_first_row_id() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // Physically carries `_row_id`, but the file has a null first_row_id. + let id_col = Arc::new(Int64Array::from(vec![Some(5), Some(6), Some(7)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_no_first.parquet", + vec![physical_row_id_field()], + vec![id_col], + ); + + // Null first_row_id: the whole column is null; the physical values are not read. + let task = row_id_task(file_path, None); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_row_id_column(&batches, &[None, None, None]); + } + + #[tokio::test] + async fn test_row_id_with_pos_column() { + use crate::metadata_columns::RESERVED_COL_NAME_POS; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_and_pos.parquet", + vec![physical_row_id_field()], + vec![id_col], + ); + + // Co-project `_pos` and `_row_id`. `_row_id` synthesis consumes the position, and + // `_pos` is also emitted -- the RowNumber column must be added once and the two + // must not interfere. + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1, RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID]) + .with_first_row_id(Some(100)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // `_row_id` coalesces correctly... + assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]); + // ...and `_pos` is the row position, not double-counted. + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + assert_eq!(pos_col.values(), &[0, 1, 2]); + } + + #[tokio::test] + async fn test_row_id_mixed_files_share_schema() { + use arrow_select::concat::concat_batches; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // Three files in one scan exercising all three column paths, which must all + // produce the SAME Arrow type (plain Int64) or concatenation fails: + // - synthesis: first_row_id set, no physical column -> first_row_id + pos + // - null gate: no first_row_id -> null column + // - coalesce: first_row_id set, physical column present -> per-row + fallback + let synth = row_id_task( + write_plain_parquet(dir, "row_id_synth2.parquet", vec![], vec![]), + Some(42), + ); + let nulled = row_id_task( + write_plain_parquet(dir, "row_id_null2.parquet", vec![], vec![]), + None, + ); + let coalesced = row_id_task( + write_plain_parquet( + dir, + "row_id_coalesced2.parquet", + vec![physical_row_id_field()], + vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef], + ), + Some(50), + ); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![ + Ok(synth), + Ok(nulled), + Ok(coalesced), + ])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_eq!(batches.len(), 3); + let schema = batches[0].schema(); + concat_batches(&schema, &batches) + .expect("synthesis, null and coalesce files must share one column type"); + } + + #[tokio::test] + async fn test_row_id_present_by_name_without_id_unsupported() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // Column present by name but WITHOUT the embedded field id. The transformer keys + // the source column by field id, so this shape can't be threaded and is rejected. + let id_field = Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true); + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = + write_plain_parquet(dir, "row_id_by_name.parquet", vec![id_field], vec![id_col]); + + let task = row_id_task(file_path, Some(100)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result: Result, _> = + reader.read(tasks).unwrap().stream().try_collect().await; + + let err = result.unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported); + assert!( + format!("{err}").contains("without an embedded field id"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_row_id_present_by_name_without_id_nulls_when_no_lineage() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // Same name-only shape as the reject test above, but the file carries no row lineage + // (`first_row_id = None`), as a migrated pre-v3 file with a user column named + // `_row_id` would. The physical leaf is never read, so `_row_id` is nulled out + // rather than rejected (matching Java `ValueReaders.rowIds`). + let id_field = Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true); + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_by_name_no_lineage.parquet", + vec![id_field], + vec![id_col], + ); + + let task = row_id_task(file_path, None); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_row_id_column(&batches, &[None, None, None]); + } + + /// Builds a `row_id_task` (see above) that additionally carries a bound predicate, + /// so a `RowSelection` is applied when the reader has row selection enabled. + fn row_id_task_with_predicate( + file_path: String, + first_row_id: Option, + extra_project_field_ids: Vec, + predicate: crate::expr::Predicate, + ) -> FileScanTask { + use crate::expr::Bind; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let bound = predicate.bind(Arc::clone(&schema), false).unwrap(); + + let mut project_field_ids = vec![1]; + project_field_ids.extend(extra_project_field_ids); + project_field_ids.push(RESERVED_FIELD_ID_ROW_ID); + + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(project_field_ids) + .with_predicate(Some(bound)) + .with_first_row_id(first_row_id) + .with_case_sensitive(false) + .build() + .unwrap() + } + + #[tokio::test] + async fn test_row_id_stable_under_row_selection() { + use crate::expr::Reference; + use crate::spec::Datum; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // id = [1, 2, 3]; drop the middle physical row via a predicate + row selection. + let file_path = write_plain_parquet(dir, "row_id_selection.parquet", vec![], vec![]); + + let task = row_id_task_with_predicate( + file_path, + Some(100), + vec![], + Reference::new("id").not_equal_to(Datum::int(2)), + ); + + // Row selection must be enabled for the predicate to produce a RowSelection. + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // The survivors are physical rows 0 and 2, so their _row_id is first_row_id + the + // PHYSICAL position: [100, 102]. A dense output index would wrongly give [100, 101]. + assert_row_id_column(&batches, &[Some(100), Some(102)]); + } + + #[tokio::test] + async fn test_row_id_coalesce_stable_under_row_selection() { + use crate::expr::Reference; + use crate::spec::Datum; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // Physical _row_id = [Some(5), None, Some(8)] over id = [1, 2, 3]. Dropping the + // middle row must keep the physical column and the RowNumber fallback row-aligned. + let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef; + let file_path = write_plain_parquet( + dir, + "row_id_coalesce_selection.parquet", + vec![physical_row_id_field()], + vec![id_col], + ); + + let task = row_id_task_with_predicate( + file_path, + Some(100), + vec![], + Reference::new("id").not_equal_to(Datum::int(2)), + ); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Rows 0 and 2 survive: their stored values (5, 8) pass through. The dropped + // row's null (which would have fallen back to 100 + 1) is gone -- proving the + // physical column and the positional fallback are filtered by the same selection. + assert_row_id_column(&batches, &[Some(5), Some(8)]); + } + + #[tokio::test] + async fn test_row_id_global_across_row_groups() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // 5 rows written with max_row_group_size = 2 -> 3 row groups. `_pos` must be the + // GLOBAL file position, so `_row_id` continues across row-group boundaries rather + // than restarting per group (which would silently duplicate ids). + let file_path = format!("{dir}/row_id_multi_rg.parquet"); + let field = Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from( + vec![1, 2, 3, 4, 5], + ))]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(2)) + .build(); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let task = row_id_task(file_path, Some(0)); + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_row_id_column(&batches, &[Some(0), Some(1), Some(2), Some(3), Some(4)]); + } + + #[tokio::test] + async fn test_row_id_global_when_first_row_group_pruned() { + use parquet::file::reader::{FileReader, SerializedFileReader}; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // 6 rows with max_row_group_size = 2 -> 3 row groups. A byte-range split that prunes + // row group 0 reaches the reader via `with_row_groups()` -- a different path than the + // `RowSelection` cases above. The survivors must keep their GLOBAL positions (starting + // at 2), so a per-group RowNumber restart would surface as duplicate ids here. + let file_path = format!("{dir}/row_id_prune_rg0.parquet"); + let field = Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from( + vec![1, 2, 3, 4, 5, 6], + ))]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(2)) + .build(); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // A byte range starting just past row group 0 prunes it (its midpoint falls below + // `start`) while keeping groups 1 and 2 (physical rows 2..6). + let metadata = SerializedFileReader::new(File::open(&file_path).unwrap()) + .unwrap() + .metadata() + .clone(); + assert_eq!(metadata.num_row_groups(), 3); + let start = 4 + metadata.row_group(0).compressed_size() as u64; + let file_size = std::fs::metadata(&file_path).unwrap().len(); + + let task = row_id_task_with_options(file_path, Some(100), start, file_size - start, vec![]); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Groups 1 and 2 survive at physical positions 2..6 -> _row_id 102..106, not a + // per-group restart at 100. + assert_row_id_column(&batches, &[Some(102), Some(103), Some(104), Some(105)]); + } + + /// Writes a positional delete file (`file_path` + `pos` reserved columns) marking the + /// given `positions` of `data_file_path` as deleted. + fn write_positional_delete( + dir: &str, + name: &str, + data_file_path: &str, + positions: &[i64], + ) -> String { + use arrow_array::StringArray; + + let file_path_field = + Field::new("file_path", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2147483546".to_string(), + )])); + let pos_field = Field::new("pos", DataType::Int64, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2147483545".to_string(), + )])); + let arrow_schema = Arc::new(ArrowSchema::new(vec![file_path_field, pos_field])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(StringArray::from(vec![data_file_path; positions.len()])), + Arc::new(Int64Array::from(positions.to_vec())), + ]) + .unwrap(); + + let path = format!("{dir}/{name}"); + let file = File::create(&path).unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + path + } + + #[tokio::test] + async fn test_row_id_survives_positional_delete() { + use crate::spec::DataContentType; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // id = [1, 2, 3]; a positional delete drops the middle physical row (pos = 1). + let data_path = write_plain_parquet(dir, "row_id_posdel_data.parquet", vec![], vec![]); + let del_path = write_positional_delete(dir, "row_id_posdel.parquet", &data_path, &[1]); + + let delete = FileScanTaskDeleteFile::builder() + .with_file_path(del_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(); + let task = row_id_task_with_options(data_path, Some(100), 0, 0, vec![delete]); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Survivors keep their ABSOLUTE positions -- [100, 102], not renumbered [100, 101]. + // Positional-delete selection reaches the reader via a different path than predicate + // selection, so this covers it explicitly. + assert_row_id_column(&batches, &[Some(100), Some(102)]); + } + + #[tokio::test] + async fn test_row_id_name_collision_under_positional_fallback() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // A file with NO embedded field ids (positional fallback) whose columns include one + // literally named `_row_id` (user data). Projecting `_row_id` must NOT be rejected as + // an unthreadable physical metadata column -- under fallback the reserved column is + // synthesized and the same-named user column is just data. + let file_path = format!("{dir}/fallback_row_id_name.parquet"); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true), + ])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Int64Array::from(vec![7i64, 8, 9])), + ]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let task = row_id_task(file_path, Some(100)); + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + // Not rejected; `_row_id` is synthesized as first_row_id + pos. + assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]); + } + + #[tokio::test] + async fn test_read_encrypted_parquet_with_wrong_key_fails() { + let encryption_key = b"0123456789abcdef"; + let wrong_key = b"fedcba9876543210"; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let id_data = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_data]).unwrap(); + + let file_path = format!("{table_location}/encrypted_wrong_key.parquet"); + write_encrypted_parquet(&file_path, &batch, encryption_key, None); + + let wrong_key_metadata = crate::encryption::StandardKeyMetadata::try_new(wrong_key) + .unwrap() + .encode() + .unwrap(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![1]) + .with_case_sensitive(false) + .with_key_metadata(Some(wrong_key_metadata)) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result: Result, _> = + reader.read(tasks).unwrap().stream().try_collect().await; + + let err = result.unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::Unexpected); + let err_str = format!("{err}"); + assert!( + err_str.contains("unable to decrypt parquet footer"), + "Expected error about decryption failure, got: {err_str}" + ); + } + + /// Test that concurrency=1 reads all files correctly and in deterministic order. + /// This verifies the fast-path optimization for single concurrency. + #[tokio::test] + async fn test_read_with_concurrency_one() { + use arrow_array::Int32Array; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "file_num", Type::Primitive(PrimitiveType::Int)) + .into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("file_num", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + // Create 3 parquet files with different data + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + for file_num in 0..3 { + let id_data = Arc::new(Int32Array::from_iter_values( + file_num * 10..(file_num + 1) * 10, + )) as ArrayRef; + let file_num_data = Arc::new(Int32Array::from(vec![file_num; 10])) as ArrayRef; + + let to_write = + RecordBatch::try_new(arrow_schema.clone(), vec![id_data, file_num_data]).unwrap(); + + let file = File::create(format!("{table_location}/file_{file_num}.parquet")).unwrap(); + let mut writer = + ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap(); + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + } + + // Read with concurrency=1 (fast-path) + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_data_file_concurrency_limit(1) + .build(), + ) + .build(); + + // Create tasks in a specific order: file_0, file_1, file_2 + let tasks = vec![ + Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/file_0.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/file_0.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .build() + .unwrap()), + Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/file_1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/file_1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .build() + .unwrap()), + Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/file_2.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/file_2.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .build() + .unwrap()), + ]; + + let tasks_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream; + + let result = reader + .read(tasks_stream) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + // Verify we got all 30 rows (10 from each file) + let total_rows: usize = result.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 30, "Should have 30 total rows"); + + // Collect all ids and file_nums to verify data + let mut all_ids = Vec::new(); + let mut all_file_nums = Vec::new(); + + for batch in &result { + let id_col = batch + .column(0) + .as_primitive::(); + let file_num_col = batch + .column(1) + .as_primitive::(); + + for i in 0..batch.num_rows() { + all_ids.push(id_col.value(i)); + all_file_nums.push(file_num_col.value(i)); + } + } + + assert_eq!(all_ids.len(), 30); + assert_eq!(all_file_nums.len(), 30); + + // With concurrency=1 and sequential processing, files should be processed in order + // file_0: ids 0-9, file_num=0 + // file_1: ids 10-19, file_num=1 + // file_2: ids 20-29, file_num=2 + for i in 0..10 { + assert_eq!(all_file_nums[i], 0, "First 10 rows should be from file_0"); + assert_eq!(all_ids[i], i as i32, "IDs should be 0-9"); + } + for i in 10..20 { + assert_eq!(all_file_nums[i], 1, "Next 10 rows should be from file_1"); + assert_eq!(all_ids[i], i as i32, "IDs should be 10-19"); + } + for i in 20..30 { + assert_eq!(all_file_nums[i], 2, "Last 10 rows should be from file_2"); + assert_eq!(all_ids[i], i as i32, "IDs should be 20-29"); + } + } + + #[tokio::test] + async fn test_read_int96_timestamps_with_field_ids() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let (file_path, expected_micros) = + write_int96_parquet_file(&table_location, "with_ids.parquet", true); + + assert_int96_read_matches(&file_path, schema, vec![1, 2], &expected_micros).await; + } + + #[tokio::test] + async fn test_read_int96_timestamps_without_field_ids() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let (file_path, expected_micros) = + write_int96_parquet_file(&table_location, "no_ids.parquet", false); + + assert_int96_read_matches(&file_path, schema, vec![1, 2], &expected_micros).await; + } + + #[tokio::test] + async fn test_read_int96_timestamps_with_fallback_ids_and_pos() { + use arrow_array::TimestampMicrosecondArray; + + // Regression test for the combined path this refactor introduced: a field-id-less + // file (positional fallback IDs) with an INT96 column and a `_pos` projection. + // All three transforms -- field-ID assignment, INT96 coercion, and the row-number + // virtual column -- apply in the single ArrowReaderMetadata rebuild. + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let (file_path, expected_micros) = + write_int96_parquet_file(&table_location, "no_ids_with_pos.parquet", false); + + let batches = + read_int96_batches(&file_path, schema, vec![1, 2, RESERVED_FIELD_ID_POS]).await; + + assert_eq!(batches.len(), 1); + // The INT96 timestamps are coerced to micros... + let ts_col = batches[0] + .column_by_name("ts") + .expect("ts column should be present") + .as_any() + .downcast_ref::() + .expect("Expected TimestampMicrosecondArray"); + for (i, expected) in expected_micros.iter().enumerate() { + assert_eq!(ts_col.value(i), *expected, "Row {i}"); + } + // ...and `_pos`, materialized by the RowNumber virtual column, counts rows 0,1,2. + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + assert_eq!(pos_col.values(), &[0, 1, 2]); + } + + #[tokio::test] + async fn test_read_int96_timestamps_with_field_ids_and_pos() { + use arrow_array::TimestampMicrosecondArray; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let (file_path, expected_micros) = + write_int96_parquet_file(&table_location, "with_ids_with_pos.parquet", true); + + let batches = + read_int96_batches(&file_path, schema, vec![1, 2, RESERVED_FIELD_ID_POS]).await; + + assert_eq!(batches.len(), 1); + let ts_col = batches[0] + .column_by_name("ts") + .expect("ts column should be present") + .as_any() + .downcast_ref::() + .expect("Expected TimestampMicrosecondArray"); + + for (i, expected) in expected_micros.iter().enumerate() { + assert_eq!(ts_col.value(i), *expected, "Row {i}"); + } + + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + + assert_eq!(pos_col.values(), &[0, 1, 2]); + } + + #[tokio::test] + async fn test_read_int96_timestamps_in_struct() { + use arrow_array::{StructArray, TimestampMicrosecondArray}; + use parquet::basic::{Repetition, Type as PhysicalType}; + use parquet::data_type::Int96Type; + use parquet::file::writer::SerializedFileWriter; + use parquet::schema::types::Type as SchemaType; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_path = format!("{table_location}/struct_int96.parquet"); + + let ts_type = SchemaType::primitive_type_builder("ts", PhysicalType::INT96) + .with_repetition(Repetition::OPTIONAL) + .with_id(Some(2)) + .build() + .unwrap(); + + let struct_type = SchemaType::group_type_builder("data") + .with_repetition(Repetition::REQUIRED) + .with_id(Some(1)) + .with_fields(vec![Arc::new(ts_type)]) + .build() + .unwrap(); + + let parquet_schema = SchemaType::group_type_builder("schema") + .with_fields(vec![Arc::new(struct_type)]) + .build() + .unwrap(); + + let (int96_val, expected_micros) = make_int96_test_value(); + + let file = File::create(&file_path).unwrap(); + let mut writer = + SerializedFileWriter::new(file, Arc::new(parquet_schema), Default::default()).unwrap(); + + // def=1: struct is REQUIRED so no level, ts is OPTIONAL and present (1). + // No repetition levels needed (no repeated groups). + let mut row_group = writer.next_row_group().unwrap(); + { + let mut col = row_group.next_column().unwrap().unwrap(); + col.typed::() + .write_batch(&[int96_val], Some(&[1]), None) + .unwrap(); + col.close().unwrap(); + } + row_group.close().unwrap(); + writer.close().unwrap(); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "data", + Type::Struct(crate::spec::StructType::new(vec![ + NestedField::optional( + 2, + "ts", + Type::Primitive(PrimitiveType::Timestamp), + ) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + + let batches = read_int96_batches(&file_path, iceberg_schema, vec![1]).await; + + assert_eq!(batches.len(), 1); + let struct_array = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .expect("Expected StructArray"); + let ts_array = struct_array + .column(0) + .as_any() + .downcast_ref::() + .expect("Expected TimestampMicrosecondArray inside struct"); + + assert_eq!( + ts_array.value(0), + expected_micros, + "INT96 in struct: got {}, expected {expected_micros}", + ts_array.value(0) + ); + } + + #[tokio::test] + async fn test_read_int96_timestamps_in_list() { + use arrow_array::{ListArray, TimestampMicrosecondArray}; + use parquet::basic::{Repetition, Type as PhysicalType}; + use parquet::data_type::Int96Type; + use parquet::file::writer::SerializedFileWriter; + use parquet::schema::types::Type as SchemaType; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_path = format!("{table_location}/list_int96.parquet"); + + // 3-level LIST encoding: + // optional group timestamps (LIST) { + // repeated group list { + // optional int96 element; + // } + // } + let element_type = SchemaType::primitive_type_builder("element", PhysicalType::INT96) + .with_repetition(Repetition::OPTIONAL) + .with_id(Some(2)) + .build() + .unwrap(); + + let list_group = SchemaType::group_type_builder("list") + .with_repetition(Repetition::REPEATED) + .with_fields(vec![Arc::new(element_type)]) + .build() + .unwrap(); + + let list_type = SchemaType::group_type_builder("timestamps") + .with_repetition(Repetition::OPTIONAL) + .with_id(Some(1)) + .with_logical_type(Some(parquet::basic::LogicalType::List)) + .with_fields(vec![Arc::new(list_group)]) + .build() + .unwrap(); + + let parquet_schema = SchemaType::group_type_builder("schema") + .with_fields(vec![Arc::new(list_type)]) + .build() + .unwrap(); + + let (int96_val, expected_micros) = make_int96_test_value(); + + let file = File::create(&file_path).unwrap(); + let mut writer = + SerializedFileWriter::new(file, Arc::new(parquet_schema), Default::default()).unwrap(); + + // Write a single row with a list containing one INT96 element. + // def=3: list present (1) + repeated group (2) + element present (3) + // rep=0: start of a new list + let mut row_group = writer.next_row_group().unwrap(); + { + let mut col = row_group.next_column().unwrap().unwrap(); + col.typed::() + .write_batch(&[int96_val], Some(&[3]), Some(&[0])) + .unwrap(); + col.close().unwrap(); + } + row_group.close().unwrap(); + writer.close().unwrap(); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional( + 1, + "timestamps", + Type::List(crate::spec::ListType { + element_field: NestedField::optional( + 2, + "element", + Type::Primitive(PrimitiveType::Timestamp), + ) + .into(), + }), + ) + .into(), + ]) + .build() + .unwrap(), + ); + + let batches = read_int96_batches(&file_path, iceberg_schema, vec![1]).await; + + assert_eq!(batches.len(), 1); + let list_array = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .expect("Expected ListArray"); + let ts_array = list_array + .values() + .as_any() + .downcast_ref::() + .expect("Expected TimestampMicrosecondArray inside list"); + + assert_eq!( + ts_array.value(0), + expected_micros, + "INT96 in list: got {}, expected {expected_micros}", + ts_array.value(0) + ); + } + + #[tokio::test] + async fn test_read_int96_timestamps_in_map() { + use arrow_array::{MapArray, TimestampMicrosecondArray}; + use parquet::basic::{Repetition, Type as PhysicalType}; + use parquet::data_type::{ByteArrayType, Int96Type}; + use parquet::file::writer::SerializedFileWriter; + use parquet::schema::types::Type as SchemaType; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_path = format!("{table_location}/map_int96.parquet"); + + // MAP encoding: + // optional group ts_map (MAP) { + // repeated group key_value { + // required binary key (UTF8); + // optional int96 value; + // } + // } + let key_type = SchemaType::primitive_type_builder("key", PhysicalType::BYTE_ARRAY) + .with_repetition(Repetition::REQUIRED) + .with_logical_type(Some(parquet::basic::LogicalType::String)) + .with_id(Some(2)) + .build() + .unwrap(); + + let value_type = SchemaType::primitive_type_builder("value", PhysicalType::INT96) + .with_repetition(Repetition::OPTIONAL) + .with_id(Some(3)) + .build() + .unwrap(); + + let key_value_group = SchemaType::group_type_builder("key_value") + .with_repetition(Repetition::REPEATED) + .with_fields(vec![Arc::new(key_type), Arc::new(value_type)]) + .build() + .unwrap(); + + let map_type = SchemaType::group_type_builder("ts_map") + .with_repetition(Repetition::OPTIONAL) + .with_id(Some(1)) + .with_logical_type(Some(parquet::basic::LogicalType::Map)) + .with_fields(vec![Arc::new(key_value_group)]) + .build() + .unwrap(); + + let parquet_schema = SchemaType::group_type_builder("schema") + .with_fields(vec![Arc::new(map_type)]) + .build() + .unwrap(); + + let (int96_val, expected_micros) = make_int96_test_value(); + + let file = File::create(&file_path).unwrap(); + let mut writer = + SerializedFileWriter::new(file, Arc::new(parquet_schema), Default::default()).unwrap(); + + // Write a single row with a map containing one key-value pair. + // rep=0 for both columns: start of a new map. + // key def=2: map present (1) + key_value entry present (2), key is REQUIRED. + // value def=3: map present (1) + key_value entry present (2) + value present (3). + let mut row_group = writer.next_row_group().unwrap(); + { + let mut col = row_group.next_column().unwrap().unwrap(); + col.typed::() + .write_batch( + &[parquet::data_type::ByteArray::from("event_time")], + Some(&[2]), + Some(&[0]), + ) + .unwrap(); + col.close().unwrap(); + } + { + let mut col = row_group.next_column().unwrap().unwrap(); + col.typed::() + .write_batch(&[int96_val], Some(&[3]), Some(&[0])) + .unwrap(); + col.close().unwrap(); + } + row_group.close().unwrap(); + writer.close().unwrap(); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional( + 1, + "ts_map", + Type::Map(crate::spec::MapType { + key_field: NestedField::required( + 2, + "key", + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::optional( + 3, + "value", + Type::Primitive(PrimitiveType::Timestamp), + ) + .into(), + }), + ) + .into(), + ]) + .build() + .unwrap(), + ); + + let batches = read_int96_batches(&file_path, iceberg_schema, vec![1]).await; + + assert_eq!(batches.len(), 1); + let map_array = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .expect("Expected MapArray"); + let ts_array = map_array + .values() + .as_any() + .downcast_ref::() + .expect("Expected TimestampMicrosecondArray as map values"); + + assert_eq!( + ts_array.value(0), + expected_micros, + "INT96 in map: got {}, expected {expected_micros}", + ts_array.value(0) + ); + } + + /// Writes `id` (Int32) plus a wide string column (field id 2) whose bytes dominate + /// the file, so that reading it is visible in `bytes_read`. + /// + /// `extra_fields`/`extra_columns` (e.g. a physical metadata leaf) are appended after + /// the `id` and wide columns, mirroring `write_plain_parquet`'s shape. + fn write_parquet_with_wide_column( + dir: &str, + name: &str, + extra_fields: Vec, + extra_columns: Vec, + ) -> String { + let wide_field = + Field::new("wide", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])); + // Varied bytes so the column chunk does not compress away under SNAPPY, keeping + // the `bytes_read` difference between projecting it and not unambiguous. + let wide_values: Vec = (0..3) + .map(|i| { + (0..2048) + .map(|j| ((i * 2048 + j) % 251) as u8 as char) + .collect() + }) + .collect(); + + let mut fields = vec![wide_field]; + fields.extend(extra_fields); + let mut columns: Vec = vec![Arc::new(StringArray::from(wide_values))]; + columns.extend(extra_columns); + write_plain_parquet(dir, name, fields, columns) + } + + /// Schema with `id` (field 1, Int) and `wide` (field 2, String), matching + /// `write_parquet_with_wide_column`. + fn id_and_wide_schema() -> SchemaRef { + Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "wide", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ) + } + + /// Builds a scan task over `file_path` projecting `project_field_ids`. + fn metadata_projection_task( + file_path: String, + schema: SchemaRef, + project_field_ids: Vec, + ) -> FileScanTask { + metadata_projection_task_with_first_row_id(file_path, schema, project_field_ids, None) + } + + fn metadata_projection_task_with_first_row_id( + file_path: String, + schema: SchemaRef, + project_field_ids: Vec, + first_row_id: Option, + ) -> FileScanTask { + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(project_field_ids) + .with_first_row_id(first_row_id) + .with_case_sensitive(false) + .build() + .unwrap() + } + + /// Runs a single-task scan and returns the batches plus the bytes read from storage. + async fn scan_task(task: FileScanTask) -> (Vec, u64) { + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let scan = reader.read(tasks).unwrap(); + let metrics = scan.metrics().clone(); + let batches = scan.stream().try_collect().await.unwrap(); + (batches, metrics.bytes_read()) + } + + #[tokio::test] + async fn test_pos_only_projection_reads_no_data_columns() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + let pos_only = metadata_projection_task( + write_parquet_with_wide_column(dir, "pos_only.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![RESERVED_FIELD_ID_POS], + ); + let (batches, pos_only_bytes) = scan_task(pos_only).await; + + // Only `_pos` is materialized -- no data columns. + assert_eq!(batches[0].num_columns(), 1); + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + assert_eq!(pos_col.values(), &[0, 1, 2]); + + // A scan of the same-shaped file that also projects the wide data column must read + // materially more, proving the wide column chunk was not fetched above. + let with_data = metadata_projection_task( + write_parquet_with_wide_column(dir, "pos_only_ref.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![2, RESERVED_FIELD_ID_POS], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + pos_only_bytes < with_data_bytes, + "_pos-only scan should read fewer bytes than a scan of the wide column: \ + {pos_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_empty_projection_reads_no_data_columns() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + let empty = metadata_projection_task( + write_parquet_with_wide_column(dir, "empty.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![], + ); + let (batches, empty_bytes) = scan_task(empty).await; + + // No data columns come out, but the fixture's 3 rows are still counted. + assert!(batches.iter().all(|b| b.num_columns() == 0)); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 3); + + let with_data = metadata_projection_task( + write_parquet_with_wide_column(dir, "empty_ref.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![2], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + empty_bytes < with_data_bytes, + "empty projection should read fewer bytes than the wide column: \ + {empty_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_empty_projection_counts_rows_after_positional_delete() { + use crate::spec::DataContentType; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let data_path = write_plain_parquet(dir, "empty_posdel_data.parquet", vec![], vec![]); + let del_path = write_positional_delete(dir, "empty_posdel.parquet", &data_path, &[1]); + let delete = FileScanTaskDeleteFile::builder() + .with_file_path(del_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(); + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&data_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(data_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(id_and_wide_schema()) + .with_project_field_ids(vec![]) + .with_deletes(vec![delete]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let (batches, _) = scan_task(task).await; + + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 2); + } + + #[tokio::test] + async fn test_empty_projection_counts_rows_after_predicate() { + use crate::expr::{Bind, Reference}; + use crate::spec::Datum; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let file_path = write_plain_parquet(dir, "empty_predicate.parquet", vec![], vec![]); + let schema = id_and_wide_schema(); + // `id` is filtered on but not projected, so the row filter reads it on its own mask. + let bound = Reference::new("id") + .not_equal_to(Datum::int(2)) + .bind(Arc::clone(&schema), false) + .unwrap(); + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![]) + .with_predicate(Some(bound)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let (batches, _) = scan_task(task).await; + + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 2); + } + + #[tokio::test] + async fn test_pos_only_projection_keeps_absolute_pos_under_predicate() { + use crate::expr::{Bind, Reference}; + use crate::spec::Datum; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // id = [1, 2, 3]; drop the middle physical row via a predicate + row selection. + let file_path = write_plain_parquet(dir, "pos_only_predicate.parquet", vec![], vec![]); + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let bound = Reference::new("id") + .not_equal_to(Datum::int(2)) + .bind(Arc::clone(&schema), false) + .unwrap(); + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![RESERVED_FIELD_ID_POS]) + .with_predicate(Some(bound)) + .with_case_sensitive(false) + .build() + .unwrap(); + + // Row selection must be enabled for the predicate to filter rows. The row filter + // reads `id` for its own evaluation even though `id` is not projected; the surviving + // rows must keep their ABSOLUTE positions (0 and 2), not renumbered (0 and 1). + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + let pos: Vec = batches + .iter() + .flat_map(|b| { + b.column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::() + .values() + .to_vec() + }) + .collect(); + assert_eq!(pos, vec![0, 2]); + } + + #[tokio::test] + async fn test_pos_and_file_projection() { + use crate::metadata_columns::RESERVED_COL_NAME_FILE; + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + // The motivating row-lineage shape: a synthesized position column (mask -> none) + // alongside a materialized per-file constant. + let file_path = write_parquet_with_wide_column(dir, "pos_and_file.parquet", vec![], vec![]); + let task = metadata_projection_task(file_path.clone(), id_and_wide_schema(), vec![ + RESERVED_FIELD_ID_POS, + RESERVED_FIELD_ID_FILE, + ]); + let (batches, _) = scan_task(task).await; + + // Both metadata columns materialize; no data column is read. + assert_eq!(batches[0].num_columns(), 2); + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + assert_eq!(pos_col.values(), &[0, 1, 2]); + let file_col = batches[0] + .column_by_name(RESERVED_COL_NAME_FILE) + .expect("_file column should be present"); + let file_col = cast(file_col, &DataType::Utf8).unwrap(); + let file_col = file_col.as_any().downcast_ref::().unwrap(); + assert_eq!(file_col.value(0), file_path); + } + + #[tokio::test] + async fn test_pos_and_physical_seq_projection_reads_only_the_leaf() { + use crate::metadata_columns::{ + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + }; + + // A v3 rewrite that carried rows forward stores `_last_updated_sequence_number` + // per-row. Projecting only `_pos` + the sequence column must read just that one + // physical leaf, not every data column. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + // File: id (1), wide data column (2), physical _last_updated_sequence_number. + let write = |name: &str| { + write_parquet_with_wide_column( + dir, + name, + vec![physical_last_updated_seq_field()], + vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef], + ) + }; + + let seq_task = |path: String, ids: Vec| { + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(id_and_wide_schema()) + .with_project_field_ids(ids) + .with_first_row_id(Some(100)) + .with_data_sequence_number(Some(9)) + .with_case_sensitive(false) + .build() + .unwrap() + }; + + let meta_only = seq_task(write("pos_seq.parquet"), vec![ + RESERVED_FIELD_ID_POS, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + // `_pos` and the coalesced sequence column materialize; the wide column does not. + let pos_col = batches[0] + .column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_primitive::(); + assert_eq!(pos_col.values(), &[0, 1, 2]); + let seq_col = batches[0] + .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER) + .expect("_last_updated_sequence_number column should be present"); + let seq_col = cast(seq_col, &DataType::Int64).unwrap(); + let seq_col = seq_col.as_any().downcast_ref::().unwrap(); + // Per-row stored value where non-null, else the data sequence number (9). + assert_eq!(seq_col.value(0), 5); + assert_eq!(seq_col.value(1), 9); + assert_eq!(seq_col.value(2), 8); + assert!(batches[0].column_by_name("wide").is_none()); + + // A scan that also projects the wide data column must read materially more, + // proving the metadata-only scan pruned to just the sequence leaf. + let with_data = seq_task(write("pos_seq_ref.parquet"), vec![ + 2, + RESERVED_FIELD_ID_POS, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_pos + physical sequence scan should read fewer bytes than one that also \ + reads the wide column: {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_seq_only_projection_reads_only_the_leaf() { + use crate::metadata_columns::{ + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + }; + + // `_last_updated_sequence_number` alone (no `_pos`, no data column). This is a + // metadata-only projection, so RowNumber drives the `none()` downgrade and the + // physical leaf is unioned back in, pruning the read to just that leaf. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let write = |name: &str| { + write_parquet_with_wide_column( + dir, + name, + vec![physical_last_updated_seq_field()], + vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef], + ) + }; + let seq_task = |path: String, ids: Vec| { + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(id_and_wide_schema()) + .with_project_field_ids(ids) + .with_first_row_id(Some(100)) + .with_data_sequence_number(Some(9)) + .with_case_sensitive(false) + .build() + .unwrap() + }; + + let meta_only = seq_task(write("seq_only.parquet"), vec![ + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + let seq_col = batches[0] + .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER) + .expect("_last_updated_sequence_number column should be present"); + let seq_col = cast(seq_col, &DataType::Int64).unwrap(); + let seq_col = seq_col.as_any().downcast_ref::().unwrap(); + assert_eq!(seq_col.value(0), 5); + assert_eq!(seq_col.value(1), 9); + assert_eq!(seq_col.value(2), 8); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 3); + assert!(batches[0].column_by_name("wide").is_none()); + + let with_data = seq_task(write("seq_only_ref.parquet"), vec![ + 2, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "seq-only scan should read fewer bytes than one that also reads the wide \ + column: {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_seq_only_null_first_row_id_reads_no_data_columns() { + use crate::metadata_columns::{ + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + }; + + // Seq-only projection with a null first_row_id: the column is nulled and the + // physical leaf is NOT read (the gated `coalesce_last_updated_seq_leaf` is None). + // This is a metadata-only projection, so RowNumber supplies the row count and the + // data columns are pruned; the row count is still 3 and the values all null. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let seq_task = |name: &str, ids: Vec| { + let file_path = write_parquet_with_wide_column( + dir, + name, + vec![physical_last_updated_seq_field()], + vec![Arc::new(Int64Array::from(vec![Some(5), Some(6), Some(7)])) as ArrayRef], + ); + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(id_and_wide_schema()) + .with_project_field_ids(ids) + .with_first_row_id(None) + .with_data_sequence_number(Some(9)) + .with_case_sensitive(false) + .build() + .unwrap() + }; + + let meta_only = seq_task("seq_only_null_first.parquet", vec![ + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 3); + let seq_col = batches[0] + .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER) + .expect("_last_updated_sequence_number column should be present"); + let seq_col = cast(seq_col, &DataType::Int64).unwrap(); + let seq_col = seq_col.as_any().downcast_ref::().unwrap(); + assert!((0..3).all(|i| seq_col.is_null(i))); + + let with_data = seq_task("seq_only_null_first_ref.parquet", vec![ + 2, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ]); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "seq-only scan with a null first_row_id should read fewer bytes than a scan of \ + the wide column: {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_file_only_reads_no_data_columns() { + use crate::metadata_columns::RESERVED_COL_NAME_FILE; + + // `SELECT _file` materializes a per-file string constant with no physical column to + // read. The metadata-only RowNumber counter sizes it, so the scan prunes the data + // columns rather than reading them all just to recover the row count. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + + let file_path = write_parquet_with_wide_column(dir, "file_only.parquet", vec![], vec![]); + let meta_only = metadata_projection_task(file_path.clone(), id_and_wide_schema(), vec![ + RESERVED_FIELD_ID_FILE, + ]); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 3); + let file_col = batches[0] + .column_by_name(RESERVED_COL_NAME_FILE) + .expect("_file column should be present"); + let file_col = cast(file_col, &DataType::Utf8).unwrap(); + let file_col = file_col.as_any().downcast_ref::().unwrap(); + assert_eq!(file_col.value(0), file_path); + + let with_data = metadata_projection_task( + write_parquet_with_wide_column(dir, "file_only_ref.parquet", vec![], vec![]), + id_and_wide_schema(), + vec![2, RESERVED_FIELD_ID_FILE], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_file-only scan should read fewer bytes than a scan of the wide column: \ + {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_spec_id_only_reads_no_data_columns() { + use crate::metadata_columns::{RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_SPEC_ID}; + use crate::spec::{Literal, PartitionSpec, Struct, Transform}; + + // `SELECT _spec_id` materializes an int constant from the task's partition spec id, + // with no physical column to read. The RowNumber counter sizes it, so the scan + // prunes the data columns rather than reading them all to recover the row count. + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let schema = id_and_wide_schema(); + let spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .with_spec_id(7) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + + let spec_id_task = |file_path: String, project_field_ids: Vec| { + FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids) + .with_partition(Some(Struct::from_iter([Some(Literal::int(42))]))) + .with_partition_spec(Some(spec.clone())) + .with_case_sensitive(false) + .build() + .unwrap() + }; + + let meta_only = spec_id_task( + write_parquet_with_wide_column(dir, "spec_id_only.parquet", vec![], vec![]), + vec![RESERVED_FIELD_ID_SPEC_ID], + ); + let (batches, meta_only_bytes) = scan_task(meta_only).await; + + assert_eq!(batches[0].num_columns(), 1); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 3); + let spec_id_col = batches[0] + .column_by_name(RESERVED_COL_NAME_SPEC_ID) + .expect("_spec_id column should be present"); + let spec_id_col = cast(spec_id_col, &DataType::Int32).unwrap(); + let spec_id_col = spec_id_col.as_any().downcast_ref::().unwrap(); + assert_eq!(spec_id_col.values(), &[7, 7, 7]); + + let with_data = spec_id_task( + write_parquet_with_wide_column(dir, "spec_id_only_ref.parquet", vec![], vec![]), + vec![2, RESERVED_FIELD_ID_SPEC_ID], + ); + let (_, with_data_bytes) = scan_task(with_data).await; + + assert!( + meta_only_bytes < with_data_bytes, + "_spec_id-only scan should read fewer bytes than a scan of the wide column: \ + {meta_only_bytes} vs {with_data_bytes}" + ); + } + + #[tokio::test] + async fn test_empty_projection_preserves_row_count() { + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.path().to_str().unwrap(); + let file_path = write_plain_parquet(dir, "empty_projection.parquet", vec![], vec![]); + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let task = metadata_projection_task(file_path, schema, vec![]); + let (batches, _) = scan_task(task).await; + + // A bare COUNT(*)-style empty projection must still report the row count. + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 3); + } + + /// The reads a data file needs must finish even when nothing polls the scan, + /// and a dropped scan must stop reading (#3276). + mod open_progress { + use std::ops::Range; + use std::sync::Mutex; + + use async_trait::async_trait; + use bytes::Bytes; + use futures::StreamExt; + use futures::stream::BoxStream; + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + use serde::{Deserialize, Serialize}; + + use super::*; + use crate::Result; + use crate::arrow::ParquetReadOptions; + use crate::arrow::scan_metrics::ScanMetrics; + use crate::expr::{Bind, Predicate, Reference}; + use crate::io::{ + FileIOBuilder, FileMetadata, FileRead, FileWrite, InputFile, MemoryStorage, OutputFile, + Storage, StorageConfig, StorageFactory, + }; + use crate::scan::ArrowRecordBatchStream; + use crate::spec::Datum; + + const CONCURRENCY: usize = 2; + /// Smaller than a footer, so one open takes more than one request. + const METADATA_SIZE_HINT: usize = 64; + /// Bytes one open reads: the hint, then the footer and its indexes. + /// The footer stores the writer's `created_by` version, so a `parquet` + /// upgrade can change this. + const OPEN_BYTES: u64 = 547; + /// Scheduler turns, far more than a request needs. A bound only because + /// `#[tokio::test]` runs the test and the spawned reads on one thread. + const YIELDS: usize = 100; + + #[derive(Debug, Default)] + struct Reads { + last_start: Option, + started: usize, + finished: usize, + } + + /// What the storage was asked for, and how much of it came back. + /// `ScanMetrics` counts a read when the request goes out, so it cannot + /// tell a read that hangs from one that answers. + #[derive(Debug, Clone, Default)] + struct ReadLog(Arc>); + + impl ReadLog { + /// Where the newest read began, which says which read it is. + fn last_start(&self) -> Option { + self.0.lock().unwrap().last_start + } + + fn in_flight(&self) -> usize { + let reads = self.0.lock().unwrap(); + reads.started - reads.finished + } + } + + /// In-memory files whose reads take a scheduler turn to answer. + #[derive(Debug, Clone, Default, Serialize, Deserialize)] + struct DeferredStorage { + #[serde(skip)] + files: MemoryStorage, + #[serde(skip)] + log: ReadLog, + } + + struct DeferredRead { + contents: Bytes, + log: ReadLog, + } + + #[async_trait] + impl FileRead for DeferredRead { + async fn read(&self, range: Range) -> Result { + { + let mut reads = self.log.0.lock().unwrap(); + reads.last_start = Some(range.start); + reads.started += 1; + } + // The yield stands in for a request in flight: it finishes only + // if something still drives the read. + tokio::task::yield_now().await; + self.log.0.lock().unwrap().finished += 1; + Ok(self + .contents + .slice(range.start as usize..range.end as usize)) + } + } + + #[async_trait] + #[typetag::serde] + impl Storage for DeferredStorage { + async fn reader(&self, path: &str) -> Result> { + Ok(Box::new(DeferredRead { + contents: self.files.read(path).await?, + log: self.log.clone(), + })) + } + + fn new_input(&self, path: &str) -> Result { + Ok(InputFile::new(Arc::new(self.clone()), path.to_string())) + } + + async fn exists(&self, path: &str) -> Result { + self.files.exists(path).await + } + + async fn metadata(&self, path: &str) -> Result { + self.files.metadata(path).await + } + + async fn read(&self, path: &str) -> Result { + self.files.read(path).await + } + + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { + self.files.write(path, bs).await + } + + async fn writer(&self, path: &str) -> Result> { + self.files.writer(path).await + } + + async fn delete(&self, path: &str) -> Result<()> { + self.files.delete(path).await + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + self.files.delete_prefix(path).await + } + + async fn delete_stream(&self, paths: BoxStream<'static, String>) -> Result<()> { + self.files.delete_stream(paths).await + } + + fn new_output(&self, path: &str) -> Result { + Ok(OutputFile::new(Arc::new(self.clone()), path.to_string())) + } + } + + #[derive(Debug, Default, Serialize, Deserialize)] + struct DeferredStorageFactory { + #[serde(skip)] + storage: DeferredStorage, + } + + #[typetag::serde] + impl StorageFactory for DeferredStorageFactory { + fn build(&self, _config: &StorageConfig) -> Result> { + Ok(Arc::new(self.storage.clone())) + } + } + + /// A scan of `files` copies of `contents`, the metrics that count the bytes + /// it has read, and the log of what the storage saw. + async fn deferred_scan_of( + contents: Bytes, + files: usize, + predicate: Option, + ) -> (ScanMetrics, ReadLog, ArrowRecordBatchStream) { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let bound = predicate.map(|p| p.bind(Arc::clone(&schema), false).unwrap()); + + let storage = DeferredStorage::default(); + let log = storage.log.clone(); + let mut tasks = Vec::with_capacity(files); + for file_num in 0..files { + let path = format!("memory:/data/file_{file_num}.parquet"); + storage.write(&path, contents.clone()).await.unwrap(); + tasks.push( + FileScanTask::builder() + .with_data_file_path(path) + .with_file_size_in_bytes(contents.len() as u64) + .with_start(0) + .with_length(0) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(Arc::clone(&schema)) + .with_project_field_ids(vec![1]) + .with_predicate(bound.clone()) + .with_case_sensitive(false) + .build() + .unwrap(), + ); + } + + let file_io = FileIOBuilder::new(Arc::new(DeferredStorageFactory { storage })).build(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_data_file_concurrency_limit(CONCURRENCY) + .with_bloom_filter_enabled(bound.is_some()) + .with_parquet_options( + ParquetReadOptions::builder() + .with_metadata_size_hint(METADATA_SIZE_HINT) + .build(), + ) + .build(), + ) + .build(); + let tasks = + Box::pin(futures::stream::iter(tasks.into_iter().map(Ok))) as FileScanTaskStream; + let result = reader.read(tasks).unwrap(); + (result.metrics().clone(), log, result.stream()) + } + + /// `CONCURRENCY` identical files and no predicate, so the open is the only read. + async fn deferred_scan() -> (ScanMetrics, ArrowRecordBatchStream) { + let tmp_dir = TempDir::new().unwrap(); + let on_disk = write_plain_parquet( + tmp_dir.path().to_str().unwrap(), + "data.parquet", + vec![], + vec![], + ); + let contents = Bytes::from(std::fs::read(on_disk).unwrap()); + let (metrics, _, scan) = deferred_scan_of(contents, CONCURRENCY, None).await; + (metrics, scan) + } + + /// Gives the scheduler turns until `done` is true, or `YIELDS` turns pass. + async fn spin(done: impl Fn() -> bool) { + for _ in 0..YIELDS { + if done() { + return; + } + tokio::task::yield_now().await; + } + } + + /// An open must finish its requests even when the scan is never polled again. + #[tokio::test] + async fn test_open_proceeds_without_the_scan_being_polled() { + let (metrics, scan) = deferred_scan().await; + let mut scan = Box::pin(scan); + let every_open = OPEN_BYTES * CONCURRENCY as u64; + + // One poll starts an open per concurrency slot. Nothing polls the + // scan after this, so only a spawned open can read any further. + let _ = futures::poll!(scan.next()); + spin(|| metrics.bytes_read() >= every_open).await; + assert_eq!( + metrics.bytes_read(), + every_open, + "the opens stopped where the scan stopped polling" + ); + } + + /// A dropped scan must stop opening files. + #[tokio::test] + async fn test_dropped_scan_stops_opening_files() { + let (metrics, scan) = deferred_scan().await; + let mut scan = Box::pin(scan); + let every_open = OPEN_BYTES * CONCURRENCY as u64; + + // Drop while the opens are part way through, or there is nothing to abort. + let _ = futures::poll!(scan.next()); + spin(|| metrics.bytes_read() > 0).await; + let in_flight = metrics.bytes_read(); + assert!(in_flight > 0, "no open reached the storage"); + assert!(in_flight < every_open, "the opens finished before the drop"); + + drop(scan); + spin(|| false).await; + assert_eq!( + metrics.bytes_read(), + in_flight, + "a dropped scan read {} more bytes", + metrics.bytes_read() - in_flight + ); + } + + /// The bloom filter fetch is the next read after the open, and runs in the + /// same buffered future, so it must finish on its own too. + #[tokio::test] + async fn test_bloom_filter_fetch_proceeds_without_the_scan_being_polled() { + let tmp_dir = TempDir::new().unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_bloom_filter_enabled(true) + .build(); + let on_disk = write_parquet_with( + tmp_dir.path().to_str().unwrap(), + "bloom.parquet", + vec![], + vec![], + props, + ); + let bloom_offset = + ParquetRecordBatchReaderBuilder::try_new(File::open(&on_disk).unwrap()) + .unwrap() + .metadata() + .row_group(0) + .column(0) + .bloom_filter_offset() + .expect("the writer put a bloom filter on `id`") as u64; + + let contents = Bytes::from(std::fs::read(&on_disk).unwrap()); + let predicate = Reference::new("id").equal_to(Datum::int(2)); + let (_, log, scan) = deferred_scan_of(contents, 1, Some(predicate)).await; + let mut scan = Box::pin(scan); + + // Poll only until the open is done and the bloom filter fetch is the + // read in flight. Any later poll would drive that fetch itself. + let at_the_bloom_filter = || log.last_start() == Some(bloom_offset); + for _ in 0..YIELDS { + if at_the_bloom_filter() { + break; + } + let _ = futures::poll!(scan.next()); + tokio::task::yield_now().await; + } + assert!(at_the_bloom_filter(), "the bloom filter was never read"); + + // Nothing polls the scan from here, so only a spawned fetch can finish. + spin(|| log.in_flight() == 0).await; + assert_eq!( + log.in_flight(), + 0, + "a read was left in flight where the scan stopped polling" + ); + } + } +} diff --git a/crates/iceberg/src/arrow/reader/positional_deletes.rs b/crates/iceberg/src/arrow/reader/positional_deletes.rs new file mode 100644 index 0000000000..2046da5a29 --- /dev/null +++ b/crates/iceberg/src/arrow/reader/positional_deletes.rs @@ -0,0 +1,1294 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Positional delete handling for `ArrowReader`: converting a `DeleteVector` +//! into a Parquet `RowSelection` that skips the deleted rows, while respecting +//! any row-group selection made by the predicate evaluator. + +use parquet::arrow::arrow_reader::{RowSelection, RowSelector}; +use parquet::file::metadata::RowGroupMetaData; + +use super::ArrowReader; +use crate::delete_vector::DeleteVector; +use crate::error::Result; + +impl ArrowReader { + /// computes a `RowSelection` from positional delete indices. + /// + /// Using the Parquet page index, we build a `RowSelection` that rejects rows that are indicated + /// as having been deleted by a positional delete, taking into account any row groups that have + /// been skipped entirely by the filter predicate + pub(super) fn build_deletes_row_selection( + row_group_metadata_list: &[RowGroupMetaData], + selected_row_groups: &Option>, + positional_deletes: &DeleteVector, + ) -> Result { + let mut results: Vec = Vec::new(); + let mut selected_row_groups_idx = 0; + let mut current_row_group_base_idx: u64 = 0; + // Deleted positions as half-open ranges, so a run of deletes costs one step rather than + // one per row. + let mut deleted_ranges = positional_deletes.ranges(); + let mut next_deleted_range = deleted_ranges.next(); + + for (idx, row_group_metadata) in row_group_metadata_list.iter().enumerate() { + let row_group_num_rows = row_group_metadata.num_rows() as u64; + let next_row_group_base_idx = current_row_group_base_idx + row_group_num_rows; + + // if row group selection is enabled, + if let Some(selected_row_groups) = selected_row_groups { + // if we've consumed all the selected row groups, we're done + if selected_row_groups_idx == selected_row_groups.len() { + break; + } + + if idx == selected_row_groups[selected_row_groups_idx] { + // we're in a selected row group. Increment selected_row_groups_idx + // so that next time around the for loop we're looking for the next + // selected row group + selected_row_groups_idx += 1; + } else { + // Skip the deletes in this row group without walking them. A range that + // continues into the next row group is cut to start there. + match next_deleted_range { + Some((_, end)) if end <= next_row_group_base_idx => { + deleted_ranges.skip_to(next_row_group_base_idx); + next_deleted_range = deleted_ranges.next(); + } + Some((start, end)) if start < next_row_group_base_idx => { + next_deleted_range = Some((next_row_group_base_idx, end)); + } + _ => {} + } + + // still increment the current page base index but then skip to the next row group + // in the file + current_row_group_base_idx += row_group_num_rows; + continue; + } + } + + let mut current_idx = current_row_group_base_idx; + while let Some((start, end)) = next_deleted_range { + if start >= next_row_group_base_idx { + break; + } + + // `select` the rows before the deleted range, then `skip` the part of the range + // that lies in this row group + let skip_start = start.max(current_idx); + let skip_end = end.min(next_row_group_base_idx); + if skip_start > current_idx { + results.push(RowSelector::select((skip_start - current_idx) as usize)); + } + if skip_end > skip_start { + results.push(RowSelector::skip((skip_end - skip_start) as usize)); + } + current_idx = current_idx.max(skip_end); + + if end > next_row_group_base_idx { + // the range continues into the next row group + break; + } + next_deleted_range = deleted_ranges.next(); + } + + if current_idx < next_row_group_base_idx { + results.push(RowSelector::select( + (next_row_group_base_idx - current_idx) as usize, + )); + } + + current_row_group_base_idx += row_group_num_rows; + } + + Ok(results.into()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::fs::File; + use std::sync::Arc; + + use arrow_array::cast::AsArray; + use arrow_array::{RecordBatch, StringArray}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use futures::TryStreamExt; + use parquet::arrow::arrow_reader::{RowSelection, RowSelector}; + use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; + use parquet::basic::Compression; + use parquet::file::metadata::{ColumnChunkMetaData, RowGroupMetaData}; + use parquet::file::properties::WriterProperties; + use parquet::schema::types::{SchemaDescPtr, SchemaDescriptor}; + use roaring::RoaringTreemap; + use tempfile::TempDir; + + use crate::Runtime; + use crate::arrow::{ArrowReader, ArrowReaderBuilder}; + use crate::delete_vector::{DeleteVector, mixed_delete_vectors}; + use crate::io::FileIO; + use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; + use crate::spec::{DataContentType, DataFileFormat, NestedField, PrimitiveType, Schema, Type}; + use crate::test_utils::encode_dv_blob; + + fn build_test_row_group_meta( + schema_descr: SchemaDescPtr, + columns: Vec, + num_rows: i64, + ordinal: i32, + ) -> RowGroupMetaData { + RowGroupMetaData::builder(schema_descr.clone()) + .set_num_rows(num_rows) + .set_total_byte_size(2000) + .set_column_metadata(columns) + .set_ordinal(ordinal) + .build() + .unwrap() + } + + fn get_test_schema_descr() -> SchemaDescPtr { + use parquet::schema::types::Type as SchemaType; + + let schema = SchemaType::group_type_builder("schema") + .with_fields(vec![ + Arc::new( + SchemaType::primitive_type_builder("a", parquet::basic::Type::INT32) + .build() + .unwrap(), + ), + Arc::new( + SchemaType::primitive_type_builder("b", parquet::basic::Type::INT32) + .build() + .unwrap(), + ), + ]) + .build() + .unwrap(); + + Arc::new(SchemaDescriptor::new(Arc::new(schema))) + } + + /// Row groups and deleted runs that cross the 2^32 position boundary, with and without row + /// group selection. + #[test] + fn test_build_deletes_row_selection_across_2_pow_32() { + let schema_descr = get_test_schema_descr(); + let columns: Vec<_> = schema_descr + .columns() + .iter() + .map(|ptr| ColumnChunkMetaData::builder(ptr.clone()).build().unwrap()) + .collect(); + let row_groups = |sizes: &[i64]| -> Vec { + sizes + .iter() + .enumerate() + .map(|(i, &n)| { + build_test_row_group_meta(schema_descr.clone(), columns.clone(), n, i as i32) + }) + .collect() + }; + let select = RowSelector::select; + let skip = RowSelector::skip; + let high = 1u64 << 32; + let big = high as usize; + + // Row group 1 starts at 2^32 - 2. The run 2^32 - 3 ..= 2^32 crosses both boundaries. + let meta = row_groups(&[(1 << 32) - 2, 8]); + let deletes = DeleteVector::new(RoaringTreemap::from_iter([ + 1, + high - 3, + high - 2, + high - 1, + high, + high + 3, + ])); + let build = |selected: Option>| { + ArrowReader::build_deletes_row_selection(&meta, &selected, &deletes).unwrap() + }; + assert_eq!( + build(None), + RowSelection::from(vec![ + select(1), + skip(1), + select(big - 5), + skip(4), + select(2), + skip(1), + select(2), + ]) + ); + assert_eq!( + build(Some(vec![0])), + RowSelection::from(vec![select(1), skip(1), select(big - 5), skip(1)]) + ); + assert_eq!( + build(Some(vec![1])), + RowSelection::from(vec![skip(3), select(2), skip(1), select(2)]) + ); + + // Skipping row group 0 lands past 2^32, in the next bitmap. + let meta = row_groups(&[(1 << 32) + 1, 4]); + let deletes = DeleteVector::new(RoaringTreemap::from_iter([5, high - 1, high, high + 2])); + assert_eq!( + ArrowReader::build_deletes_row_selection(&meta, &Some(vec![1]), &deletes).unwrap(), + RowSelection::from(vec![select(1), skip(1), select(2)]) + ); + } + + /// The selection for `deleted` built one deleted position at a time. + fn expected_deletes_row_selection( + row_group_sizes: &[u64], + selected_row_groups: &Option>, + deleted: &[u64], + ) -> RowSelection { + let mut selectors = Vec::new(); + let mut base = 0; + for (idx, &num_rows) in row_group_sizes.iter().enumerate() { + let end = base + num_rows; + if selected_row_groups + .as_ref() + .is_none_or(|selected| selected.contains(&idx)) + { + let mut row = base; + let first = deleted.partition_point(|&p| p < base); + for &pos in deleted[first..].iter().take_while(|&&p| p < end) { + selectors.push(RowSelector::select((pos - row) as usize)); + selectors.push(RowSelector::skip(1)); + row = pos + 1; + } + selectors.push(RowSelector::select((end - row) as usize)); + } + base = end; + } + selectors.into() + } + + /// An empty row group inside a deleted run, once selected and once skipped. + #[test] + fn test_build_deletes_row_selection_empty_row_group_inside_run() { + let schema_descr = get_test_schema_descr(); + let columns: Vec<_> = schema_descr + .columns() + .iter() + .map(|ptr| ColumnChunkMetaData::builder(ptr.clone()).build().unwrap()) + .collect(); + let row_groups: Vec = [3, 0, 3] + .iter() + .enumerate() + .map(|(i, &n)| { + build_test_row_group_meta(schema_descr.clone(), columns.clone(), n, i as i32) + }) + .collect(); + // Rows 1..5 are deleted: two in row group 0 and two in row group 2. + let deletes = DeleteVector::new(RoaringTreemap::from_iter(1..5)); + let select = RowSelector::select; + let skip = RowSelector::skip; + for (selected, expected) in [ + (None, vec![select(1), skip(4), select(1)]), + (Some(vec![0, 1, 2]), vec![select(1), skip(4), select(1)]), + (Some(vec![0, 2]), vec![select(1), skip(4), select(1)]), + (Some(vec![1, 2]), vec![skip(2), select(1)]), + ] { + assert_eq!( + ArrowReader::build_deletes_row_selection(&row_groups, &selected, &deletes).unwrap(), + RowSelection::from(expected), + "selected {selected:?}" + ); + } + } + + /// Random row group layouts and selections over the delete vectors the iterator tests use, + /// compared with a selection built one deleted position at a time. + #[test] + fn test_build_deletes_row_selection_matches_model() { + use rand::rngs::StdRng; + use rand::{Rng, SeedableRng}; + + let mut rng = StdRng::seed_from_u64(3); + let schema_descr = get_test_schema_descr(); + let columns: Vec<_> = schema_descr + .columns() + .iter() + .map(|ptr| ColumnChunkMetaData::builder(ptr.clone()).build().unwrap()) + .collect(); + + for deletes in mixed_delete_vectors() { + let deleted: Vec = deletes.iter().collect(); + for _ in 0..8 { + // Half the time the file ends at or before a deleted position, so the deletes + // past its end must be ignored. + let num_rows = match deleted.len() { + n if n > 0 && rng.random_bool(0.5) => { + deleted[rng.random_range(0..n)].saturating_add(rng.random_range(0..2)) + } + _ => deleted.last().map_or(0, |&last| last + 1) + rng.random_range(0..100), + } + .max(1); + // Row group boundaries, half of them next to a deleted position so that they + // fall inside runs. Repeated cuts leave empty row groups. + let mut cuts: Vec = (0..rng.random_range(0..8)) + .map(|_| match deleted.len() { + n if n > 0 && rng.random_bool(0.5) => (deleted[rng.random_range(0..n)] + + rng.random_range(0..3)) + .saturating_sub(1), + _ => rng.random_range(0..num_rows), + }) + .filter(|&cut| cut > 0 && cut < num_rows) + .collect(); + cuts.extend([0, num_rows]); + cuts.sort_unstable(); + let sizes: Vec = cuts.windows(2).map(|w| w[1] - w[0]).collect(); + let row_groups: Vec = sizes + .iter() + .enumerate() + .map(|(i, &n)| { + build_test_row_group_meta( + schema_descr.clone(), + columns.clone(), + n as i64, + i as i32, + ) + }) + .collect(); + let selected = rng + .random_bool(0.75) + .then(|| (0..sizes.len()).filter(|_| rng.random_bool(0.5)).collect()); + + assert_eq!( + ArrowReader::build_deletes_row_selection(&row_groups, &selected, &deletes) + .unwrap(), + expected_deletes_row_selection(&sizes, &selected, &deleted), + "row groups {sizes:?}, selected {selected:?}" + ); + } + } + } + + #[test] + fn test_build_deletes_row_selection() { + let schema_descr = get_test_schema_descr(); + + let mut columns = vec![]; + for ptr in schema_descr.columns() { + let column = ColumnChunkMetaData::builder(ptr.clone()).build().unwrap(); + columns.push(column); + } + + let row_groups_metadata = vec![ + build_test_row_group_meta(schema_descr.clone(), columns.clone(), 1000, 0), + build_test_row_group_meta(schema_descr.clone(), columns.clone(), 500, 1), + build_test_row_group_meta(schema_descr.clone(), columns.clone(), 500, 2), + build_test_row_group_meta(schema_descr.clone(), columns.clone(), 1000, 3), + build_test_row_group_meta(schema_descr.clone(), columns.clone(), 500, 4), + ]; + + let selected_row_groups = Some(vec![1, 3]); + + /* cases to cover: + * {skip|select} {first|intermediate|last} {one row|multiple rows} in + {first|intermediate|last} {skipped|selected} row group + * row group selection disabled + */ + + let positional_deletes = RoaringTreemap::from_iter(&[ + 1, // in skipped rg 0, should be ignored + 3, // run of three consecutive items in skipped rg0 + 4, 5, 998, // two consecutive items at end of skipped rg0 + 999, 1000, // solitary row at start of selected rg1 (1, 9) + 1010, // run of 3 rows in selected rg1 + 1011, 1012, // (3, 485) + 1498, // run of two items at end of selected rg1 + 1499, 1500, // run of two items at start of skipped rg2 + 1501, 1600, // should ignore, in skipped rg2 + 1999, // single row at end of skipped rg2 + 2000, // run of two items at start of selected rg3 + 2001, // (4, 98) + 2100, // single row in selected row group 3 (1, 99) + 2200, // run of 3 consecutive rows in selected row group 3 + 2201, 2202, // (3, 796) + 2999, // single item at end of selected rg3 (1) + 3000, // single item at start of skipped rg4 + ]); + + let positional_deletes = DeleteVector::new(positional_deletes); + + // using selected row groups 1 and 3 + let result = ArrowReader::build_deletes_row_selection( + &row_groups_metadata, + &selected_row_groups, + &positional_deletes, + ) + .unwrap(); + + let expected = RowSelection::from(vec![ + RowSelector::skip(1), + RowSelector::select(9), + RowSelector::skip(3), + RowSelector::select(485), + RowSelector::skip(4), + RowSelector::select(98), + RowSelector::skip(1), + RowSelector::select(99), + RowSelector::skip(3), + RowSelector::select(796), + RowSelector::skip(1), + ]); + + assert_eq!(result, expected); + + // selecting all row groups + let result = ArrowReader::build_deletes_row_selection( + &row_groups_metadata, + &None, + &positional_deletes, + ) + .unwrap(); + + let expected = RowSelection::from(vec![ + RowSelector::select(1), + RowSelector::skip(1), + RowSelector::select(1), + RowSelector::skip(3), + RowSelector::select(992), + RowSelector::skip(3), + RowSelector::select(9), + RowSelector::skip(3), + RowSelector::select(485), + RowSelector::skip(4), + RowSelector::select(98), + RowSelector::skip(1), + RowSelector::select(398), + RowSelector::skip(3), + RowSelector::select(98), + RowSelector::skip(1), + RowSelector::select(99), + RowSelector::skip(3), + RowSelector::select(796), + RowSelector::skip(2), + RowSelector::select(499), + ]); + + assert_eq!(result, expected); + } + + /// Test for bug where position deletes in later row groups are not applied correctly. + /// + /// When a file has multiple row groups and a position delete targets a row in a later + /// row group, the `build_deletes_row_selection` function had a bug where it would + /// fail to increment `current_row_group_base_idx` when skipping row groups. + /// + /// This test creates: + /// - A data file with 200 rows split into 2 row groups (0-99, 100-199) + /// - A position delete file that deletes row 199 (last row in second row group) + /// + /// Expected behavior: Should return 199 rows (with id=200 deleted) + /// Bug behavior: Returns 200 rows (delete is not applied) + /// + /// This bug was discovered while running Apache Spark + Apache Iceberg integration tests + /// through DataFusion Comet. The following Iceberg Java tests failed due to this bug: + /// - `org.apache.iceberg.spark.extensions.TestMergeOnReadDelete::testDeleteWithMultipleRowGroupsParquet` + /// - `org.apache.iceberg.spark.extensions.TestMergeOnReadUpdate::testUpdateWithMultipleRowGroupsParquet` + #[tokio::test] + async fn test_position_delete_across_multiple_row_groups() { + use arrow_array::{Int32Array, Int64Array}; + use parquet::file::reader::{FileReader, SerializedFileReader}; + + // Field IDs for positional delete schema + const FIELD_ID_POSITIONAL_DELETE_FILE_PATH: u64 = 2147483546; + const FIELD_ID_POSITIONAL_DELETE_POS: u64 = 2147483545; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + + // Create table schema with a single 'id' column + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + // Step 1: Create data file with 200 rows in 2 row groups + // Row group 0: rows 0-99 (ids 1-100) + // Row group 1: rows 100-199 (ids 101-200) + let data_file_path = format!("{table_location}/data.parquet"); + + let batch1 = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(1..=100), + )]) + .unwrap(); + + let batch2 = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(101..=200), + )]) + .unwrap(); + + // Force each batch into its own row group + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(100)) + .build(); + + let file = File::create(&data_file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + writer.write(&batch1).expect("Writing batch 1"); + writer.write(&batch2).expect("Writing batch 2"); + writer.close().unwrap(); + + // Verify we created 2 row groups + let verify_file = File::open(&data_file_path).unwrap(); + let verify_reader = SerializedFileReader::new(verify_file).unwrap(); + assert_eq!( + verify_reader.metadata().num_row_groups(), + 2, + "Should have 2 row groups" + ); + + // Step 2: Create position delete file that deletes row 199 (id=200, last row in row group 1) + let delete_file_path = format!("{table_location}/deletes.parquet"); + + let delete_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("file_path", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + FIELD_ID_POSITIONAL_DELETE_FILE_PATH.to_string(), + )])), + Field::new("pos", DataType::Int64, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + FIELD_ID_POSITIONAL_DELETE_POS.to_string(), + )])), + ])); + + // Delete row at position 199 (0-indexed, so it's the last row: id=200) + let delete_batch = RecordBatch::try_new(delete_schema.clone(), vec![ + Arc::new(StringArray::from_iter_values(vec![data_file_path.clone()])), + Arc::new(Int64Array::from_iter_values(vec![199i64])), + ]) + .unwrap(); + + let delete_props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let delete_file = File::create(&delete_file_path).unwrap(); + let mut delete_writer = + ArrowWriter::try_new(delete_file, delete_schema, Some(delete_props)).unwrap(); + delete_writer.write(&delete_batch).unwrap(); + delete_writer.close().unwrap(); + + // Step 3: Read the data file with the delete applied + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&data_file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_record_count(Some(200)) + .with_data_file_path(data_file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema.clone()) + .with_project_field_ids(vec![1]) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_size_in_bytes(std::fs::metadata(&delete_file_path).unwrap().len()) + .with_file_path(delete_file_path) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(), + ]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + // Step 4: Verify we got 199 rows (not 200) + let total_rows: usize = result.iter().map(|b| b.num_rows()).sum(); + + println!("Total rows read: {total_rows}"); + println!("Expected: 199 rows (deleted row 199 which had id=200)"); + + // This assertion will FAIL before the fix and PASS after the fix + assert_eq!( + total_rows, 199, + "Expected 199 rows after deleting row 199, but got {total_rows} rows. \ + The bug causes position deletes in later row groups to be ignored." + ); + + // Verify the deleted row (id=200) is not present + let all_ids: Vec = result + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .iter() + .copied() + }) + .collect(); + + assert!( + !all_ids.contains(&200), + "Row with id=200 should be deleted but was found in results" + ); + + // Verify we have all other ids (1-199) + let expected_ids: Vec = (1..=199).collect(); + assert_eq!( + all_ids, expected_ids, + "Should have ids 1-199 but got different values" + ); + } + + /// Test for bug where position deletes are lost when skipping unselected row groups. + /// + /// This is a variant of `test_position_delete_across_multiple_row_groups` that exercises + /// the row group selection code path (`selected_row_groups: Some([...])`). + /// + /// When a file has multiple row groups and only some are selected for reading, + /// the `build_deletes_row_selection` function must correctly skip over deletes in + /// unselected row groups WITHOUT consuming deletes that belong to selected row groups. + /// + /// This test creates: + /// - A data file with 200 rows split into 2 row groups (0-99, 100-199) + /// - A position delete file that deletes row 199 (last row in second row group) + /// - Row group selection that reads ONLY row group 1 (rows 100-199) + /// + /// Expected behavior: Should return 99 rows (with row 199 deleted) + /// Bug behavior: Returns 100 rows (delete is lost when skipping row group 0) + /// + /// The bug occurs when processing row group 0 (unselected): + /// ```rust + /// delete_vector_iter.advance_to(next_row_group_base_idx); // Position at first delete >= 100 + /// next_deleted_row_idx_opt = delete_vector_iter.next(); // BUG: Consumes delete at 199! + /// ``` + /// + /// The fix is to NOT call `next()` after `advance_to()` when skipping unselected row groups, + /// because `advance_to()` already positions the iterator correctly without consuming elements. + #[tokio::test] + async fn test_position_delete_with_row_group_selection() { + use arrow_array::{Int32Array, Int64Array}; + use parquet::file::reader::{FileReader, SerializedFileReader}; + + // Field IDs for positional delete schema + const FIELD_ID_POSITIONAL_DELETE_FILE_PATH: u64 = 2147483546; + const FIELD_ID_POSITIONAL_DELETE_POS: u64 = 2147483545; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + + // Create table schema with a single 'id' column + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + // Step 1: Create data file with 200 rows in 2 row groups + // Row group 0: rows 0-99 (ids 1-100) + // Row group 1: rows 100-199 (ids 101-200) + let data_file_path = format!("{table_location}/data.parquet"); + + let batch1 = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(1..=100), + )]) + .unwrap(); + + let batch2 = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(101..=200), + )]) + .unwrap(); + + // Force each batch into its own row group + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(100)) + .build(); + + let file = File::create(&data_file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + writer.write(&batch1).expect("Writing batch 1"); + writer.write(&batch2).expect("Writing batch 2"); + writer.close().unwrap(); + + // Verify we created 2 row groups + let verify_file = File::open(&data_file_path).unwrap(); + let verify_reader = SerializedFileReader::new(verify_file).unwrap(); + assert_eq!( + verify_reader.metadata().num_row_groups(), + 2, + "Should have 2 row groups" + ); + + // Step 2: Create position delete file that deletes row 199 (id=200, last row in row group 1) + let delete_file_path = format!("{table_location}/deletes.parquet"); + + let delete_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("file_path", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + FIELD_ID_POSITIONAL_DELETE_FILE_PATH.to_string(), + )])), + Field::new("pos", DataType::Int64, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + FIELD_ID_POSITIONAL_DELETE_POS.to_string(), + )])), + ])); + + // Delete row at position 199 (0-indexed, so it's the last row: id=200) + let delete_batch = RecordBatch::try_new(delete_schema.clone(), vec![ + Arc::new(StringArray::from_iter_values(vec![data_file_path.clone()])), + Arc::new(Int64Array::from_iter_values(vec![199i64])), + ]) + .unwrap(); + + let delete_props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let delete_file = File::create(&delete_file_path).unwrap(); + let mut delete_writer = + ArrowWriter::try_new(delete_file, delete_schema, Some(delete_props)).unwrap(); + delete_writer.write(&delete_batch).unwrap(); + delete_writer.close().unwrap(); + + // Step 3: Get byte ranges to read ONLY row group 1 (rows 100-199) + // This exercises the row group selection code path where row group 0 is skipped + let metadata_file = File::open(&data_file_path).unwrap(); + let metadata_reader = SerializedFileReader::new(metadata_file).unwrap(); + let metadata = metadata_reader.metadata(); + + let row_group_0 = metadata.row_group(0); + let row_group_1 = metadata.row_group(1); + + let rg0_start = 4u64; // Parquet files start with 4-byte magic "PAR1" + let rg1_start = rg0_start + row_group_0.compressed_size() as u64; + let rg1_length = row_group_1.compressed_size() as u64; + + println!( + "Row group 0: starts at byte {}, {} bytes compressed", + rg0_start, + row_group_0.compressed_size() + ); + println!( + "Row group 1: starts at byte {}, {} bytes compressed", + rg1_start, + row_group_1.compressed_size() + ); + + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + // Create FileScanTask that reads ONLY row group 1 via byte range filtering + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&data_file_path).unwrap().len()) + .with_start(rg1_start) + .with_length(rg1_length) + .with_record_count(Some(100)) // Row group 1 has 100 rows + .with_data_file_path(data_file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema.clone()) + .with_project_field_ids(vec![1]) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_size_in_bytes(std::fs::metadata(&delete_file_path).unwrap().len()) + .with_file_path(delete_file_path) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(), + ]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + // Step 4: Verify we got 99 rows (not 100) + // Row group 1 has 100 rows (ids 101-200), minus 1 delete (id=200) = 99 rows + let total_rows: usize = result.iter().map(|b| b.num_rows()).sum(); + + println!("Total rows read from row group 1: {total_rows}"); + println!("Expected: 99 rows (row group 1 has 100 rows, 1 delete at position 199)"); + + // This assertion will FAIL before the fix and PASS after the fix + assert_eq!( + total_rows, 99, + "Expected 99 rows from row group 1 after deleting position 199, but got {total_rows} rows. \ + The bug causes position deletes to be lost when advance_to() is followed by next() \ + when skipping unselected row groups." + ); + + // Verify the deleted row (id=200) is not present + let all_ids: Vec = result + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .iter() + .copied() + }) + .collect(); + + assert!( + !all_ids.contains(&200), + "Row with id=200 should be deleted but was found in results" + ); + + // Verify we have ids 101-199 (not 101-200) + let expected_ids: Vec = (101..=199).collect(); + assert_eq!( + all_ids, expected_ids, + "Should have ids 101-199 but got different values" + ); + } + + /// Test for bug where stale cached delete causes infinite loop when skipping row groups. + /// + /// This test exposes the inverse scenario of `test_position_delete_with_row_group_selection`: + /// - Position delete targets a row in the SKIPPED row group (not the selected one) + /// - After calling advance_to(), the cached delete index is stale + /// - Without updating the cache, the code enters an infinite loop + /// + /// This test creates: + /// - A data file with 200 rows split into 2 row groups (0-99, 100-199) + /// - A position delete file that deletes row 0 (first row in SKIPPED row group 0) + /// - Row group selection that reads ONLY row group 1 (rows 100-199) + /// + /// The bug occurs when skipping row group 0: + /// ```rust + /// let mut next_deleted_row_idx_opt = delete_vector_iter.next(); // Some(0) + /// // ... skip to row group 1 ... + /// delete_vector_iter.advance_to(100); // Iterator advances past delete at 0 + /// // BUG: next_deleted_row_idx_opt is still Some(0) - STALE! + /// // When processing row group 1: + /// // current_idx = 100, next_deleted_row_idx = 0, next_row_group_base_idx = 200 + /// // Loop condition: 0 < 200 (true) + /// // But: current_idx (100) > next_deleted_row_idx (0) + /// // And: current_idx (100) != next_deleted_row_idx (0) + /// // Neither branch executes -> INFINITE LOOP! + /// ``` + /// + /// Expected behavior: Should return 100 rows (delete at 0 doesn't affect row group 1) + /// Bug behavior: Infinite loop in build_deletes_row_selection + #[tokio::test] + async fn test_position_delete_in_skipped_row_group() { + use arrow_array::{Int32Array, Int64Array}; + use parquet::file::reader::{FileReader, SerializedFileReader}; + + // Field IDs for positional delete schema + const FIELD_ID_POSITIONAL_DELETE_FILE_PATH: u64 = 2147483546; + const FIELD_ID_POSITIONAL_DELETE_POS: u64 = 2147483545; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + + // Create table schema with a single 'id' column + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + // Step 1: Create data file with 200 rows in 2 row groups + // Row group 0: rows 0-99 (ids 1-100) + // Row group 1: rows 100-199 (ids 101-200) + let data_file_path = format!("{table_location}/data.parquet"); + + let batch1 = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(1..=100), + )]) + .unwrap(); + + let batch2 = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(101..=200), + )]) + .unwrap(); + + // Force each batch into its own row group + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(100)) + .build(); + + let file = File::create(&data_file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + writer.write(&batch1).expect("Writing batch 1"); + writer.write(&batch2).expect("Writing batch 2"); + writer.close().unwrap(); + + // Verify we created 2 row groups + let verify_file = File::open(&data_file_path).unwrap(); + let verify_reader = SerializedFileReader::new(verify_file).unwrap(); + assert_eq!( + verify_reader.metadata().num_row_groups(), + 2, + "Should have 2 row groups" + ); + + // Step 2: Create position delete file that deletes row 0 (id=1, first row in row group 0) + let delete_file_path = format!("{table_location}/deletes.parquet"); + + let delete_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("file_path", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + FIELD_ID_POSITIONAL_DELETE_FILE_PATH.to_string(), + )])), + Field::new("pos", DataType::Int64, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + FIELD_ID_POSITIONAL_DELETE_POS.to_string(), + )])), + ])); + + // Delete row at position 0 (0-indexed, so it's the first row: id=1) + let delete_batch = RecordBatch::try_new(delete_schema.clone(), vec![ + Arc::new(StringArray::from_iter_values(vec![data_file_path.clone()])), + Arc::new(Int64Array::from_iter_values(vec![0i64])), + ]) + .unwrap(); + + let delete_props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let delete_file = File::create(&delete_file_path).unwrap(); + let mut delete_writer = + ArrowWriter::try_new(delete_file, delete_schema, Some(delete_props)).unwrap(); + delete_writer.write(&delete_batch).unwrap(); + delete_writer.close().unwrap(); + + // Step 3: Get byte ranges to read ONLY row group 1 (rows 100-199) + // This exercises the row group selection code path where row group 0 is skipped + let metadata_file = File::open(&data_file_path).unwrap(); + let metadata_reader = SerializedFileReader::new(metadata_file).unwrap(); + let metadata = metadata_reader.metadata(); + + let row_group_0 = metadata.row_group(0); + let row_group_1 = metadata.row_group(1); + + let rg0_start = 4u64; // Parquet files start with 4-byte magic "PAR1" + let rg1_start = rg0_start + row_group_0.compressed_size() as u64; + let rg1_length = row_group_1.compressed_size() as u64; + + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + // Create FileScanTask that reads ONLY row group 1 via byte range filtering + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&data_file_path).unwrap().len()) + .with_start(rg1_start) + .with_length(rg1_length) + .with_record_count(Some(100)) // Row group 1 has 100 rows + .with_data_file_path(data_file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema.clone()) + .with_project_field_ids(vec![1]) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_size_in_bytes(std::fs::metadata(&delete_file_path).unwrap().len()) + .with_file_path(delete_file_path) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .build() + .unwrap(), + ]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + // Step 4: Verify we got 100 rows (all of row group 1) + // The delete at position 0 is in row group 0, which is skipped, so it doesn't affect us + let total_rows: usize = result.iter().map(|b| b.num_rows()).sum(); + + assert_eq!( + total_rows, 100, + "Expected 100 rows from row group 1 (delete at position 0 is in skipped row group 0). \ + If this hangs or fails, it indicates the cached delete index was not updated after advance_to()." + ); + + // Verify we have all ids from row group 1 (101-200) + let all_ids: Vec = result + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .iter() + .copied() + }) + .collect(); + + let expected_ids: Vec = (101..=200).collect(); + assert_eq!( + all_ids, expected_ids, + "Should have ids 101-200 (all of row group 1)" + ); + } + + /// End-to-end read of a data file with a V3 deletion vector applied. Exercises the whole + /// deletion-vector read path together: the DV coordinates on the scan task, the loader + /// reading the blob by range from its Puffin file, decoding it with DeleteVector::deserialize, + /// validating cardinality against record_count, and ArrowReader filtering the deleted rows. + #[tokio::test] + async fn test_deletion_vector_applied_end_to_end() { + use arrow_array::Int32Array; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + // Data file: ids 1..=5 at positions 0..=4. + let data_file_path = format!("{table_location}/data.parquet"); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(1..=5), + )]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(&data_file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // Deletion vector deleting positions 1 and 3 (ids 2 and 4), serialized as a + // deletion-vector-v1 blob and embedded in a Puffin-like file at a non-zero offset. + let blob = encode_dv_blob([1u64, 3]); + let content_offset = 12u64; + let content_size = blob.len() as u64; + let mut dv_file_bytes = vec![0u8; content_offset as usize]; + dv_file_bytes.extend_from_slice(&blob); + let dv_path = format!("{table_location}/deletes.puffin"); + std::fs::write(&dv_path, &dv_file_bytes).unwrap(); + + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&data_file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_record_count(Some(5)) + .with_data_file_path(data_file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema.clone()) + .with_project_field_ids(vec![1]) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_path(dv_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&dv_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some(data_file_path.clone())) + .with_content_offset(Some(content_offset)) + .with_content_size_in_bytes(Some(content_size)) + .with_record_count(Some(2)) + .build() + .unwrap(), + ]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let ids: Vec = result + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .iter() + .copied() + }) + .collect(); + + // Positions 1 and 3 (ids 2 and 4) are deleted; ids 1, 3, 5 remain. + assert_eq!(ids, vec![1, 3, 5]); + } + + /// A deletion vector whose decoded cardinality disagrees with the manifest entry's + /// `record_count` must fail the read rather than silently applying the wrong deletes, the + /// same invariant Iceberg-Java enforces in `BitmapPositionDeleteIndex.deserializeBitmap`. + #[tokio::test] + async fn test_deletion_vector_cardinality_mismatch_fails_read() { + use arrow_array::Int32Array; + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let data_file_path = format!("{table_location}/data.parquet"); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from_iter_values(1..=5), + )]) + .unwrap(); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(&data_file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // The blob decodes to 2 positions, but the manifest entry claims 5. + let blob = encode_dv_blob([1u64, 3]); + let dv_path = format!("{table_location}/deletes.puffin"); + std::fs::write(&dv_path, &blob).unwrap(); + + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&data_file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_record_count(Some(5)) + .with_data_file_path(data_file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema.clone()) + .with_project_field_ids(vec![1]) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_path(dv_path.clone()) + .with_file_size_in_bytes(std::fs::metadata(&dv_path).unwrap().len()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some(data_file_path.clone())) + .with_content_offset(Some(0)) + .with_content_size_in_bytes(Some(blob.len() as u64)) + .with_record_count(Some(5)) + .build() + .unwrap(), + ]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result: Result, _> = + reader.read(tasks).unwrap().stream().try_collect().await; + + let err = result.unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::DataInvalid); + assert!(err.message().contains("expected 5 from record_count")); + } +} diff --git a/crates/iceberg/src/arrow/reader/predicate_visitor.rs b/crates/iceberg/src/arrow/reader/predicate_visitor.rs new file mode 100644 index 0000000000..264c34a9a3 --- /dev/null +++ b/crates/iceberg/src/arrow/reader/predicate_visitor.rs @@ -0,0 +1,839 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Visitors that translate Iceberg bound predicates into the pieces needed for +//! Arrow-level evaluation: collecting referenced field IDs and producing +//! per-record-batch predicate closures. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use arrow_arith::boolean::{and, and_kleene, is_not_null, is_null, not, or, or_kleene}; +use arrow_array::cast::AsArray; +use arrow_array::types::{Float32Type, Float64Type}; +use arrow_array::{Array, ArrayRef, BooleanArray, Datum as ArrowDatum, RecordBatch, Scalar}; +use arrow_buffer::BooleanBuffer; +use arrow_cast::cast::cast; +use arrow_ord::cmp::{eq, gt, gt_eq, lt, lt_eq, neq}; +use arrow_schema::{ArrowError, DataType}; +use arrow_string::like::starts_with; +use fnv::FnvHashSet; +use parquet::schema::types::SchemaDescriptor; + +use crate::arrow::get_arrow_datum; +use crate::error::{Result, invalid_data}; +use crate::expr::visitors::bound_predicate_visitor::BoundPredicateVisitor; +use crate::expr::{BoundPredicate, BoundReference}; +use crate::spec::Datum; + +/// A visitor to collect field ids from bound predicates. +pub(super) struct CollectFieldIdVisitor { + pub(super) field_ids: HashSet, +} + +impl CollectFieldIdVisitor { + pub(super) fn field_ids(self) -> HashSet { + self.field_ids + } +} + +impl BoundPredicateVisitor for CollectFieldIdVisitor { + type T = (); + + fn always_true(&mut self) -> Result<()> { + Ok(()) + } + + fn always_false(&mut self) -> Result<()> { + Ok(()) + } + + fn and(&mut self, _lhs: (), _rhs: ()) -> Result<()> { + Ok(()) + } + + fn or(&mut self, _lhs: (), _rhs: ()) -> Result<()> { + Ok(()) + } + + fn not(&mut self, _inner: ()) -> Result<()> { + Ok(()) + } + + fn is_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn not_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn is_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn not_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn less_than( + &mut self, + reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn less_than_or_eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn greater_than( + &mut self, + reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn greater_than_or_eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn not_eq( + &mut self, + reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn starts_with( + &mut self, + reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn not_starts_with( + &mut self, + reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn r#in( + &mut self, + reference: &BoundReference, + _literals: &FnvHashSet, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } + + fn not_in( + &mut self, + reference: &BoundReference, + _literals: &FnvHashSet, + _predicate: &BoundPredicate, + ) -> Result<()> { + self.field_ids.insert(reference.field().id); + Ok(()) + } +} + +/// A visitor to convert Iceberg bound predicates to Arrow predicates. +pub(super) struct PredicateConverter<'a> { + /// The Parquet schema descriptor. + pub(super) parquet_schema: &'a SchemaDescriptor, + /// The map between field id and leaf column index in Parquet schema. + pub(super) column_map: &'a HashMap, + /// The required column indices in Parquet schema for the predicates. + pub(super) column_indices: &'a Vec, +} + +impl PredicateConverter<'_> { + /// When visiting a bound reference, we return index of the leaf column in the + /// required column indices which is used to project the column in the record batch. + /// Return None if the field id is not found in the column map, which is possible + /// due to schema evolution. + fn bound_reference(&mut self, reference: &BoundReference) -> Result> { + // The leaf column's index in Parquet schema. + if let Some(column_idx) = self.column_map.get(&reference.field().id) { + if self.parquet_schema.get_column_root(*column_idx).is_group() { + return Err(invalid_data!( + "Leaf column `{}` in predicates isn't a root column in Parquet schema.", + reference.field().name + )); + } + + // The leaf column's index in the required column indices. + let index = self + .column_indices + .iter() + .position(|&idx| idx == *column_idx) + .ok_or(invalid_data!( + "Leaf column `{}` in predicates cannot be found in the required column indices.", + reference.field().name + ))?; + + Ok(Some(index)) + } else { + Ok(None) + } + } + + /// Build an Arrow predicate that always returns true. + fn build_always_true(&self) -> Result> { + Ok(Box::new(|batch| { + Ok(constant_bool_array(true, batch.num_rows())) + })) + } + + /// Build an Arrow predicate that always returns false. + fn build_always_false(&self) -> Result> { + Ok(Box::new(|batch| { + Ok(constant_bool_array(false, batch.num_rows())) + })) + } +} + +/// Builds a non-null `BooleanArray` of `len` elements all set to `value`. +fn constant_bool_array(value: bool, len: usize) -> BooleanArray { + let buffer = if value { + BooleanBuffer::new_set(len) + } else { + BooleanBuffer::new_unset(len) + }; + + BooleanArray::new(buffer, None) +} + +/// Gets the leaf column from the record batch for the required column index. Only +/// supports top-level columns for now. +fn project_column( + batch: &RecordBatch, + column_idx: usize, +) -> std::result::Result { + let column = batch.column(column_idx); + + match column.data_type() { + DataType::Struct(_) => Err(ArrowError::SchemaError( + "Does not support struct column yet.".to_string(), + )), + _ => Ok(column.clone()), + } +} + +fn compute_is_nan(array: &ArrayRef) -> std::result::Result { + // Compute NaN over the contiguous values slice, then fold the null bitmap + // in with a single bitwise AND so that null slots become false. + let (is_nan, nulls) = match array.data_type() { + DataType::Float32 => { + let arr = array.as_primitive::(); + ( + BooleanBuffer::from_iter(arr.values().iter().map(|v| v.is_nan())), + arr.nulls(), + ) + } + DataType::Float64 => { + let arr = array.as_primitive::(); + ( + BooleanBuffer::from_iter(arr.values().iter().map(|v| v.is_nan())), + arr.nulls(), + ) + } + _ => unreachable!("is_nan is only valid for float types"), + }; + + let values = match nulls { + Some(nulls) => &is_nan & nulls.inner(), + None => is_nan, + }; + + Ok(BooleanArray::new(values, None)) +} + +pub(super) type PredicateResult = + dyn FnMut(RecordBatch) -> std::result::Result + Send + 'static; + +impl BoundPredicateVisitor for PredicateConverter<'_> { + type T = Box; + + fn always_true(&mut self) -> Result> { + self.build_always_true() + } + + fn always_false(&mut self) -> Result> { + self.build_always_false() + } + + fn and( + &mut self, + mut lhs: Box, + mut rhs: Box, + ) -> Result> { + Ok(Box::new(move |batch| { + let left = lhs(batch.clone())?; + let right = rhs(batch)?; + and_kleene(&left, &right) + })) + } + + fn or( + &mut self, + mut lhs: Box, + mut rhs: Box, + ) -> Result> { + Ok(Box::new(move |batch| { + let left = lhs(batch.clone())?; + let right = rhs(batch)?; + or_kleene(&left, &right) + })) + } + + fn not(&mut self, mut inner: Box) -> Result> { + Ok(Box::new(move |batch| { + let pred_ret = inner(batch)?; + not(&pred_ret) + })) + } + + fn is_null( + &mut self, + reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + Ok(Box::new(move |batch| { + let column = project_column(&batch, idx)?; + is_null(&column) + })) + } else { + // A missing column, treating it as null. + self.build_always_true() + } + } + + fn not_null( + &mut self, + reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + Ok(Box::new(move |batch| { + let column = project_column(&batch, idx)?; + is_not_null(&column) + })) + } else { + // A missing column, treating it as null. + self.build_always_false() + } + } + + fn is_nan( + &mut self, + reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + Ok(Box::new(move |batch| { + let column = project_column(&batch, idx)?; + compute_is_nan(&column) + })) + } else { + // A missing column, treating it as null. + self.build_always_false() + } + } + + fn not_nan( + &mut self, + reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + Ok(Box::new(move |batch| { + let column = project_column(&batch, idx)?; + let is_nan = compute_is_nan(&column)?; + not(&is_nan) + })) + } else { + // A missing column, treating it as null. + self.build_always_true() + } + } + + fn less_than( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literal = get_arrow_datum(literal)?; + + Ok(Box::new(move |batch| { + let left = project_column(&batch, idx)?; + let literal = try_cast_literal(&literal, left.data_type())?; + lt(&left, literal.as_ref()) + })) + } else { + // A missing column, treating it as null. + self.build_always_true() + } + } + + fn less_than_or_eq( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literal = get_arrow_datum(literal)?; + + Ok(Box::new(move |batch| { + let left = project_column(&batch, idx)?; + let literal = try_cast_literal(&literal, left.data_type())?; + lt_eq(&left, literal.as_ref()) + })) + } else { + // A missing column, treating it as null. + self.build_always_true() + } + } + + fn greater_than( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literal = get_arrow_datum(literal)?; + + Ok(Box::new(move |batch| { + let left = project_column(&batch, idx)?; + let literal = try_cast_literal(&literal, left.data_type())?; + gt(&left, literal.as_ref()) + })) + } else { + // A missing column, treating it as null. + self.build_always_false() + } + } + + fn greater_than_or_eq( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literal = get_arrow_datum(literal)?; + + Ok(Box::new(move |batch| { + let left = project_column(&batch, idx)?; + let literal = try_cast_literal(&literal, left.data_type())?; + gt_eq(&left, literal.as_ref()) + })) + } else { + // A missing column, treating it as null. + self.build_always_false() + } + } + + fn eq( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literal = get_arrow_datum(literal)?; + + Ok(Box::new(move |batch| { + let left = project_column(&batch, idx)?; + let literal = try_cast_literal(&literal, left.data_type())?; + eq(&left, literal.as_ref()) + })) + } else { + // A missing column, treating it as null. + self.build_always_false() + } + } + + fn not_eq( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literal = get_arrow_datum(literal)?; + + Ok(Box::new(move |batch| { + let left = project_column(&batch, idx)?; + let literal = try_cast_literal(&literal, left.data_type())?; + neq(&left, literal.as_ref()) + })) + } else { + // A missing column, treating it as null. + self.build_always_false() + } + } + + fn starts_with( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literal = get_arrow_datum(literal)?; + + Ok(Box::new(move |batch| { + let left = project_column(&batch, idx)?; + let literal = try_cast_literal(&literal, left.data_type())?; + starts_with(&left, literal.as_ref()) + })) + } else { + // A missing column, treating it as null. + self.build_always_false() + } + } + + fn not_starts_with( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literal = get_arrow_datum(literal)?; + + Ok(Box::new(move |batch| { + let left = project_column(&batch, idx)?; + let literal = try_cast_literal(&literal, left.data_type())?; + // update here if arrow ever adds a native not_starts_with + not(&starts_with(&left, literal.as_ref())?) + })) + } else { + // A missing column, treating it as null. + self.build_always_true() + } + } + + fn r#in( + &mut self, + reference: &BoundReference, + literals: &FnvHashSet, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literals: Vec<_> = literals + .iter() + .map(|lit| get_arrow_datum(lit).unwrap()) + .collect(); + + Ok(Box::new(move |batch| { + // update this if arrow ever adds a native is_in kernel + let left = project_column(&batch, idx)?; + + let mut acc = constant_bool_array(false, batch.num_rows()); + for literal in &literals { + let literal = try_cast_literal(literal, left.data_type())?; + acc = or(&acc, &eq(&left, literal.as_ref())?)? + } + + Ok(acc) + })) + } else { + // A missing column, treating it as null. + self.build_always_false() + } + } + + fn not_in( + &mut self, + reference: &BoundReference, + literals: &FnvHashSet, + _predicate: &BoundPredicate, + ) -> Result> { + if let Some(idx) = self.bound_reference(reference)? { + let literals: Vec<_> = literals + .iter() + .map(|lit| get_arrow_datum(lit).unwrap()) + .collect(); + + Ok(Box::new(move |batch| { + // update this if arrow ever adds a native not_in kernel + let left = project_column(&batch, idx)?; + let mut acc = constant_bool_array(true, batch.num_rows()); + for literal in &literals { + let literal = try_cast_literal(literal, left.data_type())?; + acc = and(&acc, &neq(&left, literal.as_ref())?)? + } + + Ok(acc) + })) + } else { + // A missing column, treating it as null. + self.build_always_true() + } + } +} + +/// The Arrow type of an array that the Parquet reader reads may not match the exact Arrow type +/// that Iceberg uses for literals - but they are effectively the same logical type, +/// i.e. LargeUtf8 and Utf8 or Utf8View and Utf8 or Utf8View and LargeUtf8. +/// +/// The Arrow compute kernels that we use must match the type exactly, so first cast the literal +/// into the type of the batch we read from Parquet before sending it to the compute kernel. +fn try_cast_literal( + literal: &Arc, + column_type: &DataType, +) -> std::result::Result, ArrowError> { + let literal_array = literal.get().0; + + // No cast required + if literal_array.data_type() == column_type { + return Ok(Arc::clone(literal)); + } + + let literal_array = cast(literal_array, column_type)?; + Ok(Arc::new(Scalar::new(literal_array))) +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::sync::Arc; + + use arrow_array::{Array, BooleanArray, RecordBatch}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use parquet::schema::parser::parse_message_type; + use parquet::schema::types::SchemaDescriptor; + + use super::{CollectFieldIdVisitor, PredicateConverter, constant_bool_array}; + use crate::expr::visitors::bound_predicate_visitor::visit; + use crate::expr::{Bind, Predicate, Reference}; + use crate::spec::{NestedField, PrimitiveType, Schema, SchemaRef, Type}; + + fn table_schema_simple() -> SchemaRef { + Arc::new( + Schema::builder() + .with_schema_id(1) + .with_identifier_field_ids(vec![2]) + .with_fields(vec![ + NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), + NestedField::optional(4, "qux", Type::Primitive(PrimitiveType::Float)).into(), + ]) + .build() + .unwrap(), + ) + } + + #[test] + fn test_collect_field_id() { + let schema = table_schema_simple(); + let expr = Reference::new("qux").is_null(); + let bound_expr = expr.bind(schema, true).unwrap(); + + let mut visitor = CollectFieldIdVisitor { + field_ids: HashSet::default(), + }; + visit(&mut visitor, &bound_expr).unwrap(); + + let mut expected = HashSet::default(); + expected.insert(4_i32); + + assert_eq!(visitor.field_ids, expected); + } + + #[test] + fn test_collect_field_id_with_and() { + let schema = table_schema_simple(); + let expr = Reference::new("qux") + .is_null() + .and(Reference::new("baz").is_null()); + let bound_expr = expr.bind(schema, true).unwrap(); + + let mut visitor = CollectFieldIdVisitor { + field_ids: HashSet::default(), + }; + visit(&mut visitor, &bound_expr).unwrap(); + + let mut expected = HashSet::default(); + expected.insert(4_i32); + expected.insert(3); + + assert_eq!(visitor.field_ids, expected); + } + + #[test] + fn test_collect_field_id_with_or() { + let schema = table_schema_simple(); + let expr = Reference::new("qux") + .is_null() + .or(Reference::new("baz").is_null()); + let bound_expr = expr.bind(schema, true).unwrap(); + + let mut visitor = CollectFieldIdVisitor { + field_ids: HashSet::default(), + }; + visit(&mut visitor, &bound_expr).unwrap(); + + let mut expected = HashSet::default(); + expected.insert(4_i32); + expected.insert(3); + + assert_eq!(visitor.field_ids, expected); + } + + #[test] + fn test_constant_bool_array() { + for len in [0, 8192] { + let all_true = constant_bool_array(true, len); + assert_eq!(all_true.len(), len); + assert_eq!(all_true.null_count(), 0); + assert!(all_true.iter().all(|v| v == Some(true))); + + let all_false = constant_bool_array(false, len); + assert_eq!(all_false.len(), len); + assert_eq!(all_false.null_count(), 0); + assert!(all_false.iter().all(|v| v == Some(false))); + } + } + + fn apply_predicate_to_batch( + predicate: Predicate, + schema: SchemaRef, + batch: RecordBatch, + ) -> BooleanArray { + let bound = predicate.bind(schema, true).unwrap(); + + // Build a trivial Parquet schema with one float column at field id 4 + let message_type = " + message schema { + optional float qux = 4; + } + "; + let parquet_type = parse_message_type(message_type).expect("parse schema"); + let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); + + let column_map = HashMap::from([(4i32, 0usize)]); + let column_indices = vec![0usize]; + + let mut converter = PredicateConverter { + parquet_schema: &parquet_schema, + column_map: &column_map, + column_indices: &column_indices, + }; + + let mut predicate_fn = visit(&mut converter, &bound).unwrap(); + predicate_fn(batch).unwrap() + } + + #[test] + fn test_predicate_converter_nan() { + use arrow_array::Float32Array; + + let schema = table_schema_simple(); + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "qux", + DataType::Float32, + true, + )])); + let values = vec![Some(1.0f32), Some(f32::NAN), None, Some(0.0f32)]; + + // is_nan: non-null-propagating per Java's implementation - NULL → false + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Float32Array::from( + values.clone(), + ))]) + .unwrap(); + let result = + apply_predicate_to_batch(Reference::new("qux").is_nan(), schema.clone(), batch); + assert_eq!( + [ + result.value(0), + result.value(1), + result.value(2), + result.value(3) + ], + [false, true, false, false] + ); + assert!(!result.is_null(2)); + + // not_nan: non-null-propagating per Java's implementation - NULL → true + let batch = + RecordBatch::try_new(arrow_schema, vec![Arc::new(Float32Array::from(values))]).unwrap(); + let result = apply_predicate_to_batch(Reference::new("qux").is_not_nan(), schema, batch); + assert_eq!( + [ + result.value(0), + result.value(1), + result.value(2), + result.value(3) + ], + [true, false, true, true] + ); + assert!(!result.is_null(2)); + } +} diff --git a/crates/iceberg/src/arrow/reader/projection.rs b/crates/iceberg/src/arrow/reader/projection.rs new file mode 100644 index 0000000000..676d8498d7 --- /dev/null +++ b/crates/iceberg/src/arrow/reader/projection.rs @@ -0,0 +1,5076 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Column projection for `ArrowReader`: building the Parquet projection mask +//! from Iceberg field IDs, and mapping field IDs between Iceberg and Parquet +//! (including fallback handling for files without embedded IDs). + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::ops::Range; +use std::str::FromStr; +use std::sync::Arc; + +use arrow_schema::{ + DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, +}; +use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ProjectionMask}; +use parquet::schema::types::{SchemaDescriptor, Type as ParquetType}; + +use super::{ArrowReader, CollectFieldIdVisitor}; +use crate::arrow::schema::validate_unshredded_variant; +use crate::arrow::{VariantExtensionType, arrow_schema_to_schema, arrow_type_to_type}; +use crate::error::{Result, invalid_data}; +use crate::expr::BoundPredicate; +use crate::expr::visitors::bound_predicate_visitor::visit; +use crate::spec::{ + LIST_FIELD_NAME, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MappedField, NameMapping, + NestedField, PrimitiveType, Schema, Type, +}; +use crate::{Error, ErrorKind}; + +impl ArrowReader { + pub(super) fn build_field_id_set_and_map( + parquet_schema: &SchemaDescriptor, + arrow_schema: &ArrowSchemaRef, + predicate: &BoundPredicate, + use_position_fallback: bool, + ) -> Result<(HashSet, HashMap)> { + // Collects all Iceberg field IDs referenced in the filter predicate + let mut collector = CollectFieldIdVisitor { + field_ids: HashSet::default(), + }; + visit(&mut collector, predicate)?; + + let iceberg_field_ids = collector.field_ids(); + + let field_id_map = match build_field_id_map(parquet_schema)? { + Some(map) => map, + // No embedded field IDs and no name mapping: position-based fallback + None if use_position_fallback => build_fallback_field_id_map(parquet_schema), + // No embedded field IDs, but a name mapping assigned them to the Arrow + // schema: resolve columns through the mapped Arrow field-id metadata + None => build_field_id_map_from_arrow_schema(arrow_schema), + }; + + Ok((iceberg_field_ids, field_id_map)) + } + + pub(super) fn get_arrow_projection_mask( + field_ids: &[i32], + iceberg_schema_of_task: &Schema, + parquet_schema: &SchemaDescriptor, + arrow_schema: &ArrowSchemaRef, + use_fallback: bool, // Position-based fallback: file lacks embedded field IDs and no name mapping assigned any + ) -> Result { + fn type_promotion_is_valid( + file_type: Option<&PrimitiveType>, + projected_type: Option<&PrimitiveType>, + ) -> bool { + match (file_type, projected_type) { + (Some(lhs), Some(rhs)) if lhs == rhs => true, + (Some(PrimitiveType::Int), Some(PrimitiveType::Long)) => true, + (Some(PrimitiveType::Float), Some(PrimitiveType::Double)) => true, + ( + Some(PrimitiveType::Decimal { + precision: file_precision, + scale: file_scale, + }), + Some(PrimitiveType::Decimal { + precision: requested_precision, + scale: requested_scale, + }), + ) if requested_precision >= file_precision && file_scale == requested_scale => true, + // Uuid will be store as Fixed(16) in parquet file, so the read back type will be Fixed(16). + (Some(PrimitiveType::Fixed(16)), Some(PrimitiveType::Uuid)) => true, + _ => false, + } + } + + if field_ids.is_empty() { + return Ok(ProjectionMask::all()); + } + + if use_fallback { + // Position-based projection selects whole top-level columns without checking a + // variant's storage layout. + for field_id in field_ids { + if let Some(field) = iceberg_schema_of_task.field_by_id(*field_id) + && type_contains_variant(&field.field_type) + { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Reading variant columns from Parquet files without field ids is not supported yet", + )); + } + } + // Position-based projection necessary because file lacks embedded field IDs + Self::get_arrow_projection_mask_fallback(field_ids, parquet_schema) + } else { + // Field-ID-based projection using embedded field IDs from Parquet metadata + + // Parquet's columnar format requires leaf-level (not top-level struct/list/map) projection + let mut projected = ProjectedFields::default(); + for field_id in field_ids { + let field = iceberg_schema_of_task.field_by_id(*field_id); + if let Some(field) = field { + projected.include(field); + } + } + + Self::get_arrow_projection_mask_with_field_ids( + &projected, + iceberg_schema_of_task, + parquet_schema, + arrow_schema, + type_promotion_is_valid, + ) + } + } + + /// Standard projection using embedded field IDs from Parquet metadata. + /// For iceberg-java compatibility with ParquetSchemaUtil.pruneColumns(). + fn get_arrow_projection_mask_with_field_ids( + projected: &ProjectedFields, + iceberg_schema_of_task: &Schema, + parquet_schema: &SchemaDescriptor, + arrow_schema: &ArrowSchemaRef, + type_promotion_is_valid: fn(Option<&PrimitiveType>, Option<&PrimitiveType>) -> bool, + ) -> Result { + let leaf_field_ids = &projected.leaf_ids; + // Maps field_id → leaf column indices. `Vec` because a variant contributes two + // leaves (metadata + value) under a single field id. + let mut column_map: HashMap> = HashMap::new(); + let fields = arrow_schema.fields(); + // HashSet for O(1) membership checks instead of O(n) slice scans. + let leaf_field_id_set: HashSet = leaf_field_ids.iter().copied().collect(); + // Keep presence leaves here too, or a map whose value struct lost every child + // keeps only its key and fails conversion. + let presence: BTreeSet = + presence_leaves(fields, &projected.struct_ids, &leaf_field_id_set) + .into_iter() + .flat_map(|candidate| candidate.leaves) + .collect(); + + let projected_variant_ids: HashSet = leaf_field_ids + .iter() + .copied() + .filter(|id| { + iceberg_schema_of_task + .field_by_id(*id) + .is_some_and(|f| matches!(f.field_type.as_ref(), Type::Variant(_))) + }) + .collect(); + + // A variant's metadata/value leaves carry no field id, so the field-id scan below + // never finds them. Like iceberg-java's `PruneColumns`, the whole variant group is + // projected: record, for every Arrow leaf inside a projected variant, the variant's + // field id (numbering matches `filter_leaves`). + // + // Tagging the projected variants with the `arrow.parquet.variant` extension lets + // `arrow_schema_to_schema` fold each storage struct into `Type::Variant` without + // descending into its id-less sub-fields. + let (variant_leaves, tagged_fields) = if projected_variant_ids.is_empty() { + (HashMap::new(), fields.clone()) + } else { + let mut leaf_idx_to_variant_field_id = HashMap::new(); + let mut leaf_idx = 0usize; + Self::collect_variant_leaves( + fields, + &mut leaf_idx, + None, + &projected_variant_ids, + &mut leaf_idx_to_variant_field_id, + )?; + ( + leaf_idx_to_variant_field_id, + Self::attach_variant_extensions(fields, &projected_variant_ids), + ) + }; + + // Pre-project only the fields that have been selected, possibly avoiding converting + // some Arrow types that are not yet supported. Variant leaves are included here too + // (they carry no field id) so nested containers — e.g. a map's key_value struct — + // stay well-formed for `arrow_schema_to_schema` below. + let projected_arrow_schema = ArrowSchema::new_with_metadata( + tagged_fields.filter_leaves(|idx, f| { + variant_leaves.contains_key(&idx) + || presence.contains(&idx) + || parquet_field_id(f) + .is_some_and(|field_id| leaf_field_id_set.contains(&field_id)) + }), + arrow_schema.metadata().clone(), + ); + let iceberg_schema = arrow_schema_to_schema(&projected_arrow_schema)?; + + tagged_fields.filter_leaves(|idx, field| { + // Variant sub-fields: the parent group carries the field id, not the leaf, and + // `Type::Variant` is not a primitive, so skip the type-promotion check (matching + // iceberg-java, which projects the variant group whole with no type check). + if let Some(&variant_field_id) = variant_leaves.get(&idx) { + column_map.entry(variant_field_id).or_default().push(idx); + return true; + } + + let Some(field_id) = parquet_field_id(field) else { + return false; + }; + + let iceberg_field = iceberg_schema_of_task.field_by_id(field_id); + let parquet_iceberg_field = iceberg_schema.field_by_id(field_id); + + if iceberg_field.is_none() || parquet_iceberg_field.is_none() { + return false; + } + + if !type_promotion_is_valid( + parquet_iceberg_field + .unwrap() + .field_type + .as_primitive_type(), + iceberg_field.unwrap().field_type.as_primitive_type(), + ) { + return false; + } + + column_map.entry(field_id).or_default().push(idx); + true + }); + + // Schema evolution: New columns may not exist in old Parquet files. + // We only project existing columns; RecordBatchTransformer adds default/NULL values. + let mut indices = Vec::with_capacity(leaf_field_ids.len() + presence.len()); + for field_id in leaf_field_ids { + if let Some(col_indices) = column_map.get(field_id) { + indices.extend_from_slice(col_indices); + } + } + + // Read only for struct validity; `RecordBatchTransformer` drops these leaves. + indices.extend(presence); + + if indices.is_empty() { + // Edge case: All requested columns are new (don't exist in file). + // Project all columns so RecordBatchTransformer has a batch to transform. + Ok(ProjectionMask::all()) + } else { + Ok(ProjectionMask::leaves(parquet_schema, indices)) + } + } + + /// Walks an Arrow `Fields` tree, recording `leaf_idx → variant_field_id` for every + /// leaf that sits inside a projected variant column (top-level or nested in a + /// struct/list/map). + /// + /// The leaf numbering must match [`arrow_schema::Fields::filter_leaves`], since the + /// resulting map is consulted by index in the `filter_leaves` passes above. In + /// particular it mirrors `filter_leaves`' handling of `Dictionary`/`RunEndEncoded` + /// (unwrapped to their value type) and `Union` (each member counted) — otherwise a + /// mismatch would silently shift every variant leaf after it onto the wrong column. + fn collect_variant_leaves( + fields: &Fields, + leaf_idx: &mut usize, + variant_parent: Option, + projected_variant_ids: &HashSet, + leaf_idx_to_variant_field_id: &mut HashMap, + ) -> Result<()> { + for field in fields { + Self::collect_variant_leaves_in_field( + field, + leaf_idx, + variant_parent, + projected_variant_ids, + leaf_idx_to_variant_field_id, + )?; + } + Ok(()) + } + + fn collect_variant_leaves_in_field( + field: &FieldRef, + leaf_idx: &mut usize, + variant_parent: Option, + projected_variant_ids: &HashSet, + leaf_idx_to_variant_field_id: &mut HashMap, + ) -> Result<()> { + // Once inside a variant, stay inside; otherwise check whether this field is itself + // a projected variant column. + let effective_variant = match variant_parent { + Some(variant_field_id) => Some(variant_field_id), + None => { + let entered = Self::variant_field_id(field, projected_variant_ids); + if let Some(variant_field_id) = entered { + validate_unshredded_variant(field, variant_field_id)?; + } + entered + } + }; + + // Mirror `Fields::filter_leaves`: unwrap `Dictionary`/`RunEndEncoded` to their value + // type before deciding whether this is a leaf or a nested type to descend into. + let data_type = match field.data_type() { + DataType::Dictionary(_, value) => value.as_ref(), + DataType::RunEndEncoded(_, value) => value.data_type(), + other => other, + }; + + match data_type { + DataType::Struct(subfields) => { + Self::collect_variant_leaves( + subfields, + leaf_idx, + effective_variant, + projected_variant_ids, + leaf_idx_to_variant_field_id, + )?; + } + DataType::List(inner_field) + | DataType::LargeList(inner_field) + | DataType::FixedSizeList(inner_field, _) + | DataType::Map(inner_field, _) => { + Self::collect_variant_leaves_in_field( + inner_field, + leaf_idx, + effective_variant, + projected_variant_ids, + leaf_idx_to_variant_field_id, + )?; + } + DataType::Union(union_fields, _) => { + for (_, inner_field) in union_fields.iter() { + Self::collect_variant_leaves_in_field( + inner_field, + leaf_idx, + effective_variant, + projected_variant_ids, + leaf_idx_to_variant_field_id, + )?; + } + } + _ => { + if let Some(variant_field_id) = effective_variant { + leaf_idx_to_variant_field_id.insert(*leaf_idx, variant_field_id); + } + *leaf_idx += 1; + } + } + Ok(()) + } + + /// Returns `field`'s field id, read from the file or assigned by name mapping, if it is one + /// of the projected variants. + fn variant_field_id(field: &Field, projected_variant_ids: &HashSet) -> Option { + parquet_field_id(field).filter(|id| projected_variant_ids.contains(id)) + } + + /// Returns `fields` with the canonical `arrow.parquet.variant` extension attached to every + /// projected variant (recursing into struct/list/map to reach nested variants). + /// + /// A group is a variant when its field id resolves to `Type::Variant` in the Iceberg schema, + /// so files without the Parquet `VARIANT` annotation are read too. + fn attach_variant_extensions(fields: &Fields, projected_variant_ids: &HashSet) -> Fields { + fields + .iter() + .map(|field| Self::attach_variant_extension(field, projected_variant_ids)) + .collect() + } + + fn attach_variant_extension( + field: &FieldRef, + projected_variant_ids: &HashSet, + ) -> FieldRef { + // A variant's storage is a struct; tag it and don't descend — its metadata/value + // children are not themselves variants. + if Self::variant_field_id(field, projected_variant_ids).is_some() + && matches!(field.data_type(), DataType::Struct(_)) + { + return Arc::new( + field + .as_ref() + .clone() + .with_extension_type(VariantExtensionType), + ); + } + // Otherwise recurse into containers to reach nested variants. + let data_type = match field.data_type() { + DataType::Struct(children) => DataType::Struct(Self::attach_variant_extensions( + children, + projected_variant_ids, + )), + DataType::List(child) => { + DataType::List(Self::attach_variant_extension(child, projected_variant_ids)) + } + DataType::LargeList(child) => { + DataType::LargeList(Self::attach_variant_extension(child, projected_variant_ids)) + } + DataType::FixedSizeList(child, len) => DataType::FixedSizeList( + Self::attach_variant_extension(child, projected_variant_ids), + *len, + ), + DataType::Map(child, sorted) => DataType::Map( + Self::attach_variant_extension(child, projected_variant_ids), + *sorted, + ), + // `collect_variant_leaves` descends Dictionary, RunEndEncoded and Union only to keep + // leaf numbering in step with `filter_leaves`; Parquet-derived schemas never place a + // variant under them. + _ => return field.clone(), + }; + Arc::new(field.as_ref().clone().with_data_type(data_type)) + } + + /// Fallback projection for Parquet files without field IDs. + /// Uses position-based matching: field ID N → column position N-1. + /// Projects entire top-level columns (including nested content) for iceberg-java compatibility. + fn get_arrow_projection_mask_fallback( + field_ids: &[i32], + parquet_schema: &SchemaDescriptor, + ) -> Result { + // Position-based: field_id N → column N-1 (field IDs are 1-indexed) + let parquet_root_fields = parquet_schema.root_schema().get_fields(); + let mut root_indices = vec![]; + + for field_id in field_ids.iter() { + let parquet_pos = (*field_id - 1) as usize; + + if parquet_pos < parquet_root_fields.len() { + root_indices.push(parquet_pos); + } + // RecordBatchTransformer adds missing columns with NULL values + } + + if root_indices.is_empty() { + Ok(ProjectionMask::all()) + } else { + Ok(ProjectionMask::roots(parquet_schema, root_indices)) + } + } +} + +/// Whether `field_type` is, or transitively contains, a variant type. +fn type_contains_variant(field_type: &Type) -> bool { + match field_type { + Type::Variant(_) => true, + Type::Struct(s) => s + .fields() + .iter() + .any(|f| type_contains_variant(&f.field_type)), + Type::List(l) => type_contains_variant(&l.element_field.field_type), + Type::Map(m) => { + type_contains_variant(&m.key_field.field_type) + || type_contains_variant(&m.value_field.field_type) + } + Type::Primitive(_) => false, + } +} + +/// Build the map of parquet field id to Parquet column index in the schema. +/// Returns None if the Parquet file doesn't have field IDs embedded (e.g., migrated tables). +pub(super) fn build_field_id_map( + parquet_schema: &SchemaDescriptor, +) -> Result>> { + let mut column_map = HashMap::new(); + + for (idx, field) in parquet_schema.columns().iter().enumerate() { + let field_type = field.self_type(); + match field_type { + ParquetType::PrimitiveType { basic_info, .. } => { + if !basic_info.has_id() { + return Ok(None); + } + column_map.insert(basic_info.id(), idx); + } + ParquetType::GroupType { .. } => { + return Err(invalid_data!( + "Leaf column in schema should be primitive type but got {field_type:?}" + )); + } + }; + } + + Ok(Some(column_map)) +} + +/// Finds the Parquet leaf column index carrying `field_id` by its embedded id. +/// +/// Unlike [`build_field_id_map`], a leaf without an embedded id does not abort the +/// search -- it is simply skipped. This tolerates files that legitimately mix id-bearing +/// and id-less leaves, e.g. a Variant column whose internal metadata/value leaves are +/// required by the spec to have no field id, alongside a reserved metadata column that +/// does carry its id. +pub(super) fn find_leaf_by_field_id( + parquet_schema: &SchemaDescriptor, + field_id: i32, +) -> Option { + parquet_schema.columns().iter().position(|col| { + matches!( + col.self_type(), + ParquetType::PrimitiveType { basic_info, .. } + if basic_info.has_id() && basic_info.id() == field_id + ) + }) +} + +/// Build a fallback field ID map for Parquet files without embedded field IDs. +/// +/// Returns the number of primitive (leaf) columns in a Parquet type, recursing into groups. +fn leaf_count(ty: &parquet::schema::types::Type) -> usize { + if ty.is_primitive() { + 1 + } else { + ty.get_fields().iter().map(|f| leaf_count(f)).sum() + } +} + +/// Builds a mapping from fallback field IDs to leaf column indices for Parquet files +/// without embedded field IDs. Returns entries only for primitive top-level fields. +/// +/// Must use top-level field positions (not leaf column positions) to stay consistent +/// with `add_fallback_field_ids_to_arrow_schema`, which assigns ordinal IDs to +/// top-level Arrow fields. Using leaf positions instead would produce wrong indices +/// when nested types (struct/list/map) expand into multiple leaf columns. +/// +/// Mirrors iceberg-java's ParquetSchemaUtil.addFallbackIds() which iterates +/// fileSchema.getFields() assigning ordinal IDs to top-level fields. +pub(super) fn build_fallback_field_id_map( + parquet_schema: &SchemaDescriptor, +) -> HashMap { + let mut column_map = HashMap::new(); + let mut leaf_idx = 0; + + for (top_pos, field) in parquet_schema.root_schema().get_fields().iter().enumerate() { + let field_id = (top_pos + 1) as i32; + if field.is_primitive() { + column_map.insert(field_id, leaf_idx); + } + leaf_idx += leaf_count(field); + } + + column_map +} + +/// Builds a mapping from field IDs to leaf column indices using the field-id metadata +/// carried by the Arrow schema. +/// +/// Used for Parquet files without embedded field IDs when a name mapping has assigned +/// IDs to the Arrow schema (see [`apply_name_mapping_to_arrow_schema`]): the Parquet +/// schema descriptor itself still has no IDs, but the Arrow leaves are flattened in the +/// same depth-first order as Parquet leaf columns, so the Arrow leaf index lines up with +/// the Parquet column index. Columns the mapping did not match carry no field-id +/// metadata and are simply absent from the map. +fn build_field_id_map_from_arrow_schema(arrow_schema: &ArrowSchemaRef) -> HashMap { + let mut column_map = HashMap::new(); + arrow_schema.fields().filter_leaves(|idx, field| { + if let Some(field_id) = parquet_field_id(field) { + column_map.insert(field_id, idx); + } + false + }); + column_map +} + +/// Returns the field ID in `field`'s `PARQUET:field_id` metadata, if it has a valid one. +fn parquet_field_id(field: &Field) -> Option { + field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .and_then(|field_id| i32::from_str(field_id).ok()) +} + +/// Field ids a projection reads from the file. +#[derive(Default)] +struct ProjectedFields { + leaf_ids: Vec, + /// Non-empty structs, which may need a presence leaf. + struct_ids: HashSet, +} + +impl ProjectedFields { + /// Recursively extract leaf field IDs because Parquet projection works at the leaf column level. + /// Nested types (struct/list/map) are flattened in Parquet's columnar format. + fn include(&mut self, field: &NestedField) { + match field.field_type.as_ref() { + Type::Primitive(_) => { + self.leaf_ids.push(field.id); + } + Type::Struct(struct_type) => { + if !struct_type.fields().is_empty() { + self.struct_ids.insert(field.id); + } + for nested_field in struct_type.fields() { + self.include(nested_field); + } + } + Type::List(list_type) => { + self.include(&list_type.element_field); + } + Type::Map(map_type) => { + self.include(&map_type.key_field); + self.include(&map_type.value_field); + } + // A variant is an Iceberg leaf type but a Parquet group: only the parent + // group carries the field id, so we project it like a primitive. Its + // metadata/value leaves are resolved later via `collect_variant_leaves`. + Type::Variant(_) => { + self.leaf_ids.push(field.id); + } + } + } +} + +struct PresenceCandidate { + repetition: u32, + /// Leaf indexes to read: one leaf, or a whole map. + leaves: Range, +} + +/// One file leaf under each projected struct that no selected leaf covers, so the +/// struct keeps its definition level. Like Java's `PresenceColumnSelector`, inner +/// structs go first, and the leaf with the lowest repetition level wins, then the first. +fn presence_leaves( + fields: &Fields, + projected_structs: &HashSet, + leaf_field_ids: &HashSet, +) -> Vec { + if projected_structs.is_empty() { + return Vec::new(); + } + let mut walk = PresenceWalk { + projected_structs, + leaf_field_ids, + index: 0, + chosen: Vec::new(), + pick: true, + }; + for field in fields { + walk.visit(field, 0, false); + } + walk.chosen +} + +struct PresenceWalk<'a> { + projected_structs: &'a HashSet, + leaf_field_ids: &'a HashSet, + index: usize, + chosen: Vec, + /// Whether a struct here may record a presence leaf. + pick: bool, +} + +impl PresenceWalk<'_> { + /// Returns whether a selected leaf is under `field`, and if not, the best presence + /// candidate. A candidate also goes through the projection's type check, so what it + /// reads must convert and every field on its path needs a field id. Candidates are + /// only built under a projected struct, since nothing else can use them. + fn visit( + &mut self, + field: &Field, + repetition: u32, + in_projected: bool, + ) -> (bool, Option) { + // Unwrap once, like `Fields::filter_leaves`, to count leaves the same way. + let data_type = match field.data_type() { + DataType::Dictionary(_, value) => value.as_ref(), + DataType::RunEndEncoded(_, values) => values.data_type(), + other => other, + }; + let id = parquet_field_id(field); + let (covered, best) = match data_type { + DataType::Struct(children) => { + let projected = id.is_some_and(|id| self.projected_structs.contains(&id)); + let mut covered = false; + let mut best: Option = None; + for child in children { + let (child_covered, child_best) = + self.visit(child, repetition, in_projected || projected); + covered |= child_covered; + if let Some(c) = child_best + && best.as_ref().is_none_or(|b| c.repetition < b.repetition) + { + best = Some(c); + } + } + if self.pick + && !covered + && projected + && let Some(leaf) = best.take() + { + self.chosen.push(leaf); + covered = true; + } + (covered, best) + } + DataType::List(child) + | DataType::LargeList(child) + | DataType::FixedSizeList(child, _) => self.visit(child, repetition + 1, in_projected), + // Iceberg can't convert a list view. Count its leaves, but don't record + // presence under it, including for a projected struct nested inside. + DataType::ListView(child) | DataType::LargeListView(child) => { + let pick = self.pick; + self.pick = false; + let (covered, _) = self.visit(child, repetition + 1, false); + self.pick = pick; + (covered, None) + } + DataType::Map(entries, _) => { + let start = self.index; + // Parquet drops a map unless both its key and value are read, so the + // candidate is the whole map and its entries need none of their own. + let (covered, _) = self.visit(entries, repetition + 1, false); + let readable = + in_projected && !covered && arrow_type_to_type(field.data_type()).is_ok(); + let leaf = readable.then_some(PresenceCandidate { + repetition: repetition + 1, + leaves: start..self.index, + }); + (covered, leaf) + } + _ => { + let start = self.index; + self.index += 1; + let covered = id.is_some_and(|id| self.leaf_field_ids.contains(&id)); + let readable = + in_projected && !covered && arrow_type_to_type(field.data_type()).is_ok(); + let leaf = readable.then_some(PresenceCandidate { + repetition, + leaves: start..self.index, + }); + (covered, leaf) + } + }; + let best = best.filter(|_| !covered && id.is_some()); + (covered, best) + } +} + +/// Apply name mapping to Arrow schema for Parquet files lacking field IDs. +/// +/// Assigns Iceberg field IDs based on names at every depth using the name mapping, +/// enabling correct projection on migrated files (e.g., from Hive/Spark via add_files). +/// +/// Per Iceberg spec Column Projection rule #2: +/// "Use schema.name-mapping.default metadata to map field id to columns without field id" +/// https://iceberg.apache.org/spec/#column-projection +/// +/// Corresponds to Java's ParquetSchemaUtil.applyNameMapping() and ApplyNameMapping visitor. +/// The key difference is Java operates on Parquet MessageType, while we operate on Arrow Schema. +/// +/// # Arguments +/// * `arrow_schema` - Arrow schema from Parquet file (without field IDs) +/// * `name_mapping` - Name mapping from table metadata (TableProperties.DEFAULT_NAME_MAPPING) +/// +/// # Returns +/// Arrow schema with field IDs assigned based on name mapping +pub(super) fn apply_name_mapping_to_arrow_schema( + arrow_schema: ArrowSchemaRef, + name_mapping: &NameMapping, +) -> Result> { + debug_assert!( + arrow_schema + .fields() + .iter() + .next() + .is_none_or(|f| f.metadata().get(PARQUET_FIELD_ID_META_KEY).is_none()), + "Schema already has field IDs - name mapping should not be applied" + ); + + let mut mapped_fields_by_name = HashMap::with_capacity(name_mapping.fields().len()); + for mapped_field in name_mapping.fields() { + for name in mapped_field.names() { + // Preserve the existing first-match behavior for duplicate aliases. + mapped_fields_by_name + .entry(name.as_str()) + .or_insert(mapped_field); + } + } + + let fields_with_mapped_ids = try_map_fields(arrow_schema.fields(), |field| { + let mapped_field = mapped_fields_by_name.get(field.name().as_str()).copied(); + apply_name_mapping_to_field(field, mapped_field) + })?; + + Ok(Arc::new(ArrowSchema::new_with_metadata( + fields_with_mapped_ids, + arrow_schema.metadata().clone(), + ))) +} + +/// Assigns IDs from `mapped_field` to `field` and its nested fields, as Java's ApplyNameMapping +/// does. List elements match `element`, and map entries match `key` and `value`. A field +/// without an entry, or whose entry has no field ID, keeps no ID, and so do its nested fields. +fn apply_name_mapping_to_field( + field: &FieldRef, + mapped_field: Option<&MappedField>, +) -> Result { + let Some((mapped_field, field_id)) = + mapped_field.zip(mapped_field.and_then(MappedField::field_id)) + else { + return Ok(Arc::clone(field)); + }; + let map_nested = |nested: &FieldRef, name: &str| { + let mapped_nested = find_mapped_field(mapped_field.fields().iter().map(Arc::as_ref), name); + apply_name_mapping_to_field(nested, mapped_nested) + }; + + let data_type = match field.data_type() { + DataType::Struct(fields) => DataType::Struct(try_map_fields(fields, |nested| { + map_nested(nested, nested.name()) + })?), + DataType::List(element) => DataType::List(map_nested(element, LIST_FIELD_NAME)?), + DataType::LargeList(element) => DataType::LargeList(map_nested(element, LIST_FIELD_NAME)?), + DataType::FixedSizeList(element, size) => { + DataType::FixedSizeList(map_nested(element, LIST_FIELD_NAME)?, *size) + } + // The mapping has no entry for Arrow's entries struct. The shape checks are defensive, + // because parquet-rs builds every map as a struct of key and value. + DataType::Map(entries, sorted) => { + let DataType::Struct(entry_fields) = entries.data_type() else { + return Err(invalid_data!( + "Map field `{}` must have struct type", + field.name() + )); + }; + let [key, value] = &entry_fields[..] else { + return Err(invalid_data!( + "Map field `{}` must have exactly 2 fields, found {}", + field.name(), + entry_fields.len() + )); + }; + let entry_fields = Fields::from([ + map_nested(key, MAP_KEY_FIELD_NAME)?, + map_nested(value, MAP_VALUE_FIELD_NAME)?, + ]); + let entries = entries + .as_ref() + .clone() + .with_data_type(DataType::Struct(entry_fields)); + DataType::Map(Arc::new(entries), *sorted) + } + // `Fields::filter_leaves` treats `ListView` and `LargeListView` as leaves, so projection + // cannot select their elements even with IDs. + other => other.clone(), + }; + + let mut field = field.as_ref().clone().with_data_type(data_type); + field + .metadata_mut() + .insert(PARQUET_FIELD_ID_META_KEY.to_string(), field_id.to_string()); + Ok(Arc::new(field)) +} + +/// Returns the first entry in `mapped_fields` that lists `name`. +fn find_mapped_field<'a>( + mapped_fields: impl IntoIterator, + name: &str, +) -> Option<&'a MappedField> { + mapped_fields + .into_iter() + .find(|mapped_field| mapped_field.names().iter().any(|n| n == name)) +} + +/// Maps `fields` into a `Fields` sized up front, which `collect::>()` does not do. +fn try_map_fields( + fields: &Fields, + mut map_field: impl FnMut(&FieldRef) -> Result, +) -> Result { + let mut mapped = Vec::with_capacity(fields.len()); + for field in fields { + mapped.push(map_field(field)?); + } + Ok(Fields::from(mapped)) +} + +/// Rewrites file-side Arrow `FixedSizeList` hints to `List`. +/// +/// parquet-rs picks its list reader from the embedded Arrow schema. Iceberg has no +/// fixed-size list type, so a `FixedSizeList` hint must become `List` before the stream +/// reader is built. That uses the 3-level LIST reader (no null-row padding, no +/// `PromotePlan` conversion). Returns `None` when the schema has no `FixedSizeList`. +/// Unchanged fields are shared; nothing is allocated before the first change. +/// +/// Same placement as the list-view rewrite in #3467: call this before name mapping and +/// fallback IDs so nested IDs land on the rewritten `List` fields. +pub(super) fn coerce_fixed_size_lists(arrow_schema: &ArrowSchemaRef) -> Option { + let fields = coerce_fixed_size_lists_in_fields(arrow_schema.fields())?; + Some(Arc::new(ArrowSchema::new_with_metadata( + fields, + arrow_schema.metadata().clone(), + ))) +} + +/// Returns `fields` with `FixedSizeList` rewritten to `List`, or `None` if they have none. +fn coerce_fixed_size_lists_in_fields(fields: &Fields) -> Option { + let (index, first) = fields + .iter() + .enumerate() + .find_map(|(index, field)| Some((index, coerce_fixed_size_lists_in_field(field)?)))?; + let rest = fields + .iter() + .skip(index + 1) + .map(coerce_fixed_size_lists_or_share); + Some( + fields + .iter() + .take(index) + .cloned() + .chain([first]) + .chain(rest) + .collect(), + ) +} + +/// Returns `field` with `FixedSizeList` rewritten to `List`, or `None` if it has none. +fn coerce_fixed_size_lists_in_field(field: &FieldRef) -> Option { + let data_type = match field.data_type() { + DataType::FixedSizeList(element, _) => { + DataType::List(coerce_fixed_size_lists_or_share(element)) + } + DataType::List(element) => DataType::List(coerce_fixed_size_lists_in_field(element)?), + DataType::LargeList(element) => { + DataType::LargeList(coerce_fixed_size_lists_in_field(element)?) + } + DataType::Map(entries, sorted) => { + DataType::Map(coerce_fixed_size_lists_in_field(entries)?, *sorted) + } + DataType::Struct(fields) => DataType::Struct(coerce_fixed_size_lists_in_fields(fields)?), + _ => return None, + }; + Some(Arc::new(field.as_ref().clone().with_data_type(data_type))) +} + +fn coerce_fixed_size_lists_or_share(field: &FieldRef) -> FieldRef { + coerce_fixed_size_lists_in_field(field).unwrap_or_else(|| Arc::clone(field)) +} + +/// Add position-based fallback field IDs to Arrow schema for Parquet files lacking them. +/// Enables projection on migrated files (e.g., from Hive/Spark). +/// +/// Why at schema level (not per-batch): Efficiency - avoids repeated schema modification. +/// Why only top-level: Nested projection uses leaf column indices, not parent struct IDs. +/// Why 1-indexed: Compatibility with iceberg-java's ParquetSchemaUtil.addFallbackIds(). +pub(super) fn add_fallback_field_ids_to_arrow_schema( + arrow_schema: &ArrowSchemaRef, +) -> Arc { + debug_assert!( + arrow_schema + .fields() + .iter() + .next() + .is_none_or(|f| f.metadata().get(PARQUET_FIELD_ID_META_KEY).is_none()), + "Schema already has field IDs" + ); + + let fields_with_fallback_ids: Vec<_> = arrow_schema + .fields() + .iter() + .enumerate() + .map(|(pos, field)| { + let mut metadata = field.metadata().clone(); + let field_id = (pos + 1) as i32; // 1-indexed for Java compatibility + metadata.insert(PARQUET_FIELD_ID_META_KEY.to_string(), field_id.to_string()); + + Field::new(field.name(), field.data_type().clone(), field.is_nullable()) + .with_metadata(metadata) + }) + .collect(); + + Arc::new(ArrowSchema::new_with_metadata( + fields_with_fallback_ids, + arrow_schema.metadata().clone(), + )) +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::fs::File; + use std::sync::Arc; + + use arrow_array::cast::AsArray; + use arrow_array::{Array, ArrayRef, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema, TimeUnit}; + use futures::TryStreamExt; + use parquet::arrow::{ + ArrowWriter, PARQUET_FIELD_ID_META_KEY, ProjectionMask, parquet_to_arrow_schema, + }; + use parquet::basic::Compression; + use parquet::file::properties::WriterProperties; + use parquet::schema::parser::parse_message_type; + use parquet::schema::types::SchemaDescriptor; + use tempfile::TempDir; + + use crate::arrow::{ArrowReader, ArrowReaderBuilder, ArrowReaderOptions as ReaderOptions}; + use crate::expr::{Bind, Reference}; + use crate::io::FileIO; + use crate::scan::{FileScanTask, FileScanTaskStream}; + use crate::spec::{ + DataFileFormat, Datum, MappedField, NameMapping, NestedField, PrimitiveType, Schema, + StructType, Type, VariantType, + }; + use crate::{ErrorKind, Runtime}; + + #[test] + fn test_name_mapping_duplicate_alias_preserves_first_match() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "name", + DataType::Utf8, + true, + )])); + for (first_id, expected) in [(Some(2), Some("2")), (None, None)] { + let name_mapping = NameMapping::new(vec![ + MappedField::new(first_id, vec!["name".to_string()], vec![]), + MappedField::new(Some(4), vec!["name".to_string()], vec![]), + ]); + + let mapped = + super::apply_name_mapping_to_arrow_schema(arrow_schema.clone(), &name_mapping) + .unwrap(); + + assert_eq!( + mapped.fields()[0] + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .map(String::as_str), + expected, + "first matching field ID: {first_id:?}" + ); + } + } + + #[test] + fn test_arrow_projection_mask() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_identifier_field_ids(vec![1]) + .with_fields(vec![ + NestedField::required(1, "c1", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(2, "c2", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional( + 3, + "c3", + Type::Primitive(PrimitiveType::Decimal { + precision: 38, + scale: 3, + }), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("c1", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + // Type not supported + Field::new("c2", DataType::Duration(TimeUnit::Microsecond), true).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())]), + ), + // Precision is beyond the supported range + Field::new("c3", DataType::Decimal128(39, 3), true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "3".to_string(), + )])), + ])); + + let message_type = " +message schema { + required binary c1 (STRING) = 1; + optional int32 c2 (INTEGER(8,true)) = 2; + optional fixed_len_byte_array(17) c3 (DECIMAL(39,3)) = 3; +} + "; + let parquet_type = parse_message_type(message_type).expect("should parse schema"); + let parquet_schema = SchemaDescriptor::new(Arc::new(parquet_type)); + + // Try projecting the fields c2 and c3 with the unsupported data types + let err = ArrowReader::get_arrow_projection_mask( + &[1, 2, 3], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.to_string(), + "DataInvalid => Unsupported Arrow data type: Duration(µs)".to_string() + ); + + // Omitting field c2, we still get an error due to c3 being selected + let err = ArrowReader::get_arrow_projection_mask( + &[1, 3], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.to_string(), + "DataInvalid => Failed to create decimal type, source: DataInvalid => Decimals with precision larger than 38 are not supported: 39".to_string() + ); + + // Finally avoid selecting fields with unsupported data types + let mask = ArrowReader::get_arrow_projection_mask( + &[1], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .expect("Some ProjectionMask"); + assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0])); + } + + fn field_id_meta(id: i32) -> HashMap { + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string())]) + } + + /// Arrow `Struct(metadata: Binary, value: Binary)` — the unshredded variant layout. + fn variant_arrow_fields() -> Fields { + Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, false), + ]) + } + + /// A variant storage group as it arrives from Parquet: `Struct(metadata, value)` with the + /// field id on the group and no field ids on the sub-fields. The `arrow.parquet.variant` + /// extension is intentionally ABSENT — the read path recovers variant identity from the + /// Iceberg schema and self-attaches it, so this exercises files that lack the annotation. + fn variant_arrow_field(name: &str, id: i32) -> Field { + Field::new(name, DataType::Struct(variant_arrow_fields()), false) + .with_metadata(field_id_meta(id)) + } + + /// A variant is a Parquet group whose leaves carry no field id, so it is projected via + /// its enclosing group's field id and all of its leaves are read together. + #[test] + fn test_arrow_projection_mask_variant() { + // c1 (String, id 1) + v (Variant, id 2). + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "c1", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("c1", DataType::Utf8, false).with_metadata(field_id_meta(1)), + variant_arrow_field("v", 2), + ])); + let message_type = " +message schema { + required binary c1 (STRING) = 1; + required group v = 2 { + required binary metadata; + required binary value; + } +} +"; + let parquet_schema = + SchemaDescriptor::new(Arc::new(parse_message_type(message_type).unwrap())); + + // Both fields: all three leaves. + let mask = ArrowReader::get_arrow_projection_mask( + &[1, 2], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap(); + assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0, 1, 2])); + + // Variant only: its two leaves (metadata, value). + let mask_variant = ArrowReader::get_arrow_projection_mask( + &[2], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap(); + assert_eq!( + mask_variant, + ProjectionMask::leaves(&parquet_schema, vec![1, 2]) + ); + + // Primitive only: variant leaves must not leak in. + let mask_primitive = ArrowReader::get_arrow_projection_mask( + &[1], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap(); + assert_eq!( + mask_primitive, + ProjectionMask::leaves(&parquet_schema, vec![0]) + ); + } + + /// A shredded variant (with a `typed_value` sub-field) must be rejected, not silently + /// projected as metadata/value only. + #[test] + fn test_arrow_projection_mask_variant_shredded_is_rejected() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "c1", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(), + ); + let shredded = Field::new( + "v", + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, true), + Field::new("typed_value", DataType::Int64, true), + ])), + false, + ) + .with_metadata(field_id_meta(2)); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("c1", DataType::Utf8, false).with_metadata(field_id_meta(1)), + shredded, + ])); + let message_type = " +message schema { + required binary c1 (STRING) = 1; + required group v = 2 { + required binary metadata; + optional binary value; + optional int64 typed_value; + } +} +"; + let parquet_schema = + SchemaDescriptor::new(Arc::new(parse_message_type(message_type).unwrap())); + + let err = ArrowReader::get_arrow_projection_mask( + &[1, 2], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap_err(); + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::FeatureUnsupported, + "Reading shredded variant column `v` (field id 2) is not supported yet, see https://github.com/apache/iceberg-rust/issues/3606" + ) + ); + + // An unprojected shredded sibling does not affect other columns. + let mask = ArrowReader::get_arrow_projection_mask( + &[1], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap(); + assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0])); + } + + #[test] + fn test_arrow_projection_mask_validates_variant_storage() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "c1", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(); + let binary = |name: &str| Field::new(name, DataType::Binary, true); + let metadata = || Field::new("metadata", DataType::Binary, false); + let mask_for = |storage: DataType| { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("c1", DataType::Utf8, false).with_metadata(field_id_meta(1)), + Field::new("v", storage, false).with_metadata(field_id_meta(2)), + ])); + // Only the leaf count matters to the mask; two variant leaves throughout. + let parquet_schema = SchemaDescriptor::new(Arc::new( + parse_message_type( + "message schema { + required binary c1 (STRING) = 1; + required group v = 2 { required binary a; required binary b; } + }", + ) + .unwrap(), + )); + ArrowReader::get_arrow_projection_mask( + &[1, 2], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .map(|mask| (mask, parquet_schema)) + }; + let error_of = |storage: DataType| { + let err = mask_for(storage).unwrap_err(); + (err.kind(), err.message().to_string()) + }; + let invalid = |problem: &str| { + ( + ErrorKind::DataInvalid, + format!("Variant column `v` (field id 2) {problem}"), + ) + }; + + assert_eq!( + error_of(DataType::Struct(Fields::from(vec![ + metadata(), + binary("value"), + binary("foo"), + ]))), + invalid("has fields other than `metadata` and `value`") + ); + assert_eq!( + error_of(DataType::Struct(Fields::from(vec![ + metadata(), + binary("foo"), + ]))), + invalid("is missing the `value` field") + ); + assert_eq!( + error_of(DataType::Struct(Fields::from(vec![ + binary("metadata"), + binary("value"), + ]))), + invalid("has an optional `metadata` field, which must be required") + ); + assert_eq!( + error_of(DataType::Struct(Fields::from(vec![binary("value")]))), + invalid("is missing the `metadata` field") + ); + assert_eq!( + error_of(DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Utf8, false), + binary("value"), + ]))), + invalid("has a non-binary `metadata` field: Utf8") + ); + assert_eq!( + error_of(DataType::Binary), + invalid("must be stored as a group, found Binary") + ); + assert_eq!( + error_of(DataType::Struct(Fields::from(vec![ + binary("metadata"), + Field::new("typed_value", DataType::Int64, true), + ]))), + ( + ErrorKind::FeatureUnsupported, + "Reading shredded variant column `v` (field id 2) is not supported yet, see https://github.com/apache/iceberg-rust/issues/3606".to_string() + ) + ); + + // Children are matched by name, and any Arrow binary type holds the bytes. + let (mask, parquet_schema) = mask_for(DataType::Struct(Fields::from(vec![ + Field::new("value", DataType::LargeBinary, true), + Field::new("metadata", DataType::BinaryView, false), + ]))) + .unwrap(); + assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0, 1, 2])); + } + + /// A variant nested inside a struct must have both of its sub-leaves projected. + #[test] + fn test_arrow_projection_mask_variant_nested_in_struct() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "parent", + Type::Struct(StructType::new(vec![ + NestedField::required(2, "c2", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::required(3, "v", Type::Variant(VariantType)).into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let parent_fields = Fields::from(vec![ + Field::new("c2", DataType::Utf8, false).with_metadata(field_id_meta(2)), + variant_arrow_field("v", 3), + ]); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("parent", DataType::Struct(parent_fields), false) + .with_metadata(field_id_meta(1)), + ])); + let message_type = " +message schema { + required group parent = 1 { + required binary c2 (STRING) = 2; + required group v = 3 { + required binary metadata; + required binary value; + } + } +} +"; + let parquet_schema = + SchemaDescriptor::new(Arc::new(parse_message_type(message_type).unwrap())); + + // Nested variant: both of its leaves. + let mask_variant = ArrowReader::get_arrow_projection_mask( + &[3], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap(); + assert_eq!( + mask_variant, + ProjectionMask::leaves(&parquet_schema, vec![1, 2]), + "variant nested in a struct was dropped from projection" + ); + + // Sibling primitive: no variant leaves. + let mask_primitive = ArrowReader::get_arrow_projection_mask( + &[2], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap(); + assert_eq!( + mask_primitive, + ProjectionMask::leaves(&parquet_schema, vec![0]) + ); + + // Both. + let mask_both = ArrowReader::get_arrow_projection_mask( + &[2, 3], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap(); + assert_eq!( + mask_both, + ProjectionMask::leaves(&parquet_schema, vec![0, 1, 2]) + ); + } + + /// `map`: the map's key leaf and the variant value's metadata/value + /// leaves must all be projected, and the pruned schema must stay a well-formed 2-field + /// map (regression test for the projected-schema path). + #[test] + fn test_arrow_projection_mask_map_with_variant_value() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "m", + Type::Map(crate::spec::MapType::required( + 2, + Type::Primitive(PrimitiveType::String), + 3, + Type::Variant(VariantType), + )), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let entries = Field::new( + "key_value", + DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Utf8, false).with_metadata(field_id_meta(2)), + variant_arrow_field("value", 3), + ])), + false, + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("m", DataType::Map(Arc::new(entries), false), false) + .with_metadata(field_id_meta(1)), + ])); + let message_type = " +message schema { + required group m (MAP) = 1 { + repeated group key_value { + required binary key (STRING) = 2; + required group value = 3 { + required binary metadata; + required binary value; + } + } + } +} +"; + let parquet_schema = + SchemaDescriptor::new(Arc::new(parse_message_type(message_type).unwrap())); + + let mask = ArrowReader::get_arrow_projection_mask( + &[1], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap(); + assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0, 1, 2])); + } + + /// `list`: the variant element's metadata/value leaves must both be projected. + #[test] + fn test_arrow_projection_mask_list_with_variant() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "l", + Type::List(crate::spec::ListType::new( + NestedField::list_element(2, Type::Variant(VariantType), true).into(), + )), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + "l", + DataType::List(Arc::new(variant_arrow_field("element", 2))), + false, + ) + .with_metadata(field_id_meta(1)), + ])); + let message_type = " +message schema { + required group l (LIST) = 1 { + repeated group list { + required group element = 2 { + required binary metadata; + required binary value; + } + } + } +} +"; + let parquet_schema = + SchemaDescriptor::new(Arc::new(parse_message_type(message_type).unwrap())); + + let mask = ArrowReader::get_arrow_projection_mask( + &[1], + &schema, + &parquet_schema, + &arrow_schema, + false, + ) + .unwrap(); + assert_eq!(mask, ProjectionMask::leaves(&parquet_schema, vec![0, 1])); + } + + /// `collect_variant_leaves` must number leaves exactly like `Fields::filter_leaves`, + /// including `filter_leaves`' `Dictionary`/`RunEndEncoded` unwrapping and `Union` descent. + /// Otherwise a sibling of those kinds shifts every following variant leaf onto the wrong + /// column. + #[test] + fn test_collect_variant_leaves_numbering_matches_filter_leaves() { + use arrow_schema::{UnionFields, UnionMode}; + + let union_fields: UnionFields = [ + (0i8, Arc::new(Field::new("i", DataType::Int32, false))), + (1i8, Arc::new(Field::new("l", DataType::Int64, false))), + ] + .into_iter() + .collect(); + + // Siblings that each expand to multiple leaves under `filter_leaves`, placed before the + // variant: a Dictionary(Struct{a,b}), a Union{i32,i64}, and a RunEndEncoded(Struct{x,y}) + // — 2 leaves each. `filter_leaves` unwraps Dictionary/RunEndEncoded to their value type + // and descends Union members; our walk must match. + let dict_struct = DataType::Dictionary( + Box::new(DataType::Int32), + Box::new(DataType::Struct(Fields::from(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Int32, false), + ]))), + ); + let union = DataType::Union(union_fields, UnionMode::Sparse); + let run_end_encoded = DataType::RunEndEncoded( + Arc::new(Field::new("run_ends", DataType::Int32, false)), + Arc::new(Field::new( + "values", + DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int32, false), + Field::new("y", DataType::Int32, false), + ])), + true, + )), + ); + let int_item = || Arc::new(Field::new("item", DataType::Int32, false)); + let list_of_struct = DataType::List(Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![ + Field::new("p", DataType::Int32, false), + Field::new("q", DataType::Int32, false), + ])), + false, + ))); + let map = DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("keys", DataType::Utf8, false), + Field::new("values", DataType::Int32, true), + ])), + false, + )), + false, + ); + let fields = Fields::from(vec![ + Field::new("d", dict_struct, false), + Field::new("u", union, false), + Field::new("r", run_end_encoded, false), + Field::new("ls", list_of_struct, false), + Field::new("ll", DataType::LargeList(int_item()), false), + Field::new("fl", DataType::FixedSizeList(int_item(), 2), false), + Field::new("m", map, false), + variant_arrow_field("v", 2), + Field::new( + "w", + DataType::List(Arc::new(variant_arrow_field("element", 3))), + false, + ), + ]); + + let mut leaf_idx_to_variant_field_id = HashMap::new(); + let mut leaf_idx = 0usize; + ArrowReader::collect_variant_leaves( + &fields, + &mut leaf_idx, + None, + &HashSet::from([2, 3]), + &mut leaf_idx_to_variant_field_id, + ) + .unwrap(); + + let mut arrow_leaf_names = vec![]; + let _ = fields.filter_leaves(|idx, field| { + assert_eq!(idx, arrow_leaf_names.len()); + arrow_leaf_names.push(field.name().clone()); + false + }); + assert_eq!(arrow_leaf_names, [ + "a", "b", "i", "l", "x", "y", "p", "q", "item", "item", "keys", "values", "metadata", + "value", "metadata", "value", + ]); + assert_eq!(leaf_idx, arrow_leaf_names.len()); + assert_eq!( + leaf_idx_to_variant_field_id, + HashMap::from([(12, 2), (13, 2), (14, 3), (15, 3)]) + ); + } + + /// Test schema evolution: reading old Parquet file (with only column 'a') + /// using a newer table schema (with columns 'a' and 'b'). + /// This tests that: + /// 1. get_arrow_projection_mask allows missing columns + /// 2. RecordBatchTransformer adds missing column 'b' with NULL values + #[tokio::test] + async fn test_schema_evolution_add_column() { + use arrow_array::{Array, Int32Array}; + + // New table schema: columns 'a' and 'b' (b was added later, file only has 'a') + let new_schema = Arc::new( + Schema::builder() + .with_schema_id(2) + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "b", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + // Create Arrow schema for old Parquet file (only has column 'a') + let arrow_schema_old = Arc::new(ArrowSchema::new(vec![ + Field::new("a", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + // Write old Parquet file with only column 'a' + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let data_a = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; + let to_write = RecordBatch::try_new(arrow_schema_old.clone(), vec![data_a]).unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(format!("{table_location}/old_file.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + // Read the old Parquet file using the NEW schema (with column 'b') + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/old_file.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/old_file.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(new_schema.clone()) + .with_project_field_ids(vec![1, 2]) // Request both columns 'a' and 'b' + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + // Verify we got the correct data + assert_eq!(result.len(), 1); + let batch = &result[0]; + + // Should have 2 columns now + assert_eq!(batch.num_columns(), 2); + assert_eq!(batch.num_rows(), 3); + + // Column 'a' should have the original data + let col_a = batch + .column(0) + .as_primitive::(); + assert_eq!(col_a.values(), &[1, 2, 3]); + + // Column 'b' should be all NULLs (it didn't exist in the old file) + let col_b = batch + .column(1) + .as_primitive::(); + assert_eq!(col_b.null_count(), 3); + assert!(col_b.is_null(0)); + assert!(col_b.is_null(1)); + assert!(col_b.is_null(2)); + } + + /// Test reading Parquet files without field ID metadata (e.g., migrated tables). + /// This exercises the position-based fallback path. + /// + /// Corresponds to Java's ParquetSchemaUtil.addFallbackIds() + pruneColumnsFallback() + /// in /parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java + #[tokio::test] + async fn test_read_parquet_file_without_field_ids() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "age", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + // Parquet file from a migrated table - no field ID metadata + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, false), + Field::new("age", DataType::Int32, false), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let name_data = vec!["Alice", "Bob", "Charlie"]; + let age_data = vec![30, 25, 35]; + + use arrow_array::Int32Array; + let name_col = Arc::new(StringArray::from(name_data.clone())) as ArrayRef; + let age_col = Arc::new(Int32Array::from(age_data.clone())) as ArrayRef; + + let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![name_col, age_col]).unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + assert_eq!(result.len(), 1); + let batch = &result[0]; + assert_eq!(batch.num_rows(), 3); + assert_eq!(batch.num_columns(), 2); + + // Verify position-based mapping: field_id 1 → position 0, field_id 2 → position 1 + let name_array = batch.column(0).as_string::(); + assert_eq!(name_array.value(0), "Alice"); + assert_eq!(name_array.value(1), "Bob"); + assert_eq!(name_array.value(2), "Charlie"); + + let age_array = batch + .column(1) + .as_primitive::(); + assert_eq!(age_array.value(0), 30); + assert_eq!(age_array.value(1), 25); + assert_eq!(age_array.value(2), 35); + } + + /// Regression test for #2403: when a Parquet file lacks embedded field IDs but a + /// name mapping is present, projection must use the field IDs assigned by the name + /// mapping — not the position-based fallback (field_id N → column N-1). + /// + /// The scenario uses a file whose physical column order does not line up with the + /// Iceberg field IDs: physical columns are `[name, subdept]` while the mapping + /// assigns `name → 2` and `subdept → 4`. The position fallback would project + /// field 2 from physical column 1 (`subdept`) and drop field 4 (column 3 is out of + /// range), silently misreading data. + #[tokio::test] + async fn test_read_parquet_with_name_mapping_uses_mapped_field_ids() { + // Iceberg schema: physical file order does NOT match field-id order. + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "dept", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(4, "subdept", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(), + ); + + // Migrated Parquet file: no field-id metadata, physical columns [name, subdept]. + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, true), + Field::new("subdept", DataType::Utf8, true), + ])); + + let name_mapping = Arc::new(NameMapping::new(vec![ + MappedField::new(Some(2), vec!["name".to_string()], vec![]), + MappedField::new(Some(4), vec!["subdept".to_string()], vec![]), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let name_col = Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])) as ArrayRef; + let subdept_col = Arc::new(StringArray::from(vec!["comms", "tax", "audit"])) as ArrayRef; + + let to_write = + RecordBatch::try_new(arrow_schema.clone(), vec![name_col, subdept_col]).unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![2, 4]) + .with_case_sensitive(false) + .with_name_mapping(Some(name_mapping)) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + assert_eq!(result.len(), 1); + let batch = &result[0]; + assert_eq!(batch.num_rows(), 3); + assert_eq!(batch.num_columns(), 2); + + // field 2 (`name`) must come from physical column 0, not be NULL-filled. + let name_array = batch.column(0).as_string::(); + assert_eq!( + name_array.null_count(), + 0, + "`name` was NULL-filled: name mapping was ignored and position fallback was used" + ); + assert_eq!(name_array.value(0), "Alice"); + assert_eq!(name_array.value(1), "Bob"); + assert_eq!(name_array.value(2), "Charlie"); + + // field 4 (`subdept`) must come from physical column 1. + let subdept_array = batch.column(1).as_string::(); + assert_eq!(subdept_array.null_count(), 0); + assert_eq!(subdept_array.value(0), "comms"); + assert_eq!(subdept_array.value(1), "tax"); + assert_eq!(subdept_array.value(2), "audit"); + } + + /// Regression test for #2403, predicate side: with a name mapping present, predicate + /// pushdown must resolve field IDs via the mapping rather than the position fallback, + /// which would evaluate the filter against the wrong physical column. + #[tokio::test] + async fn test_predicate_on_name_mapped_file_uses_mapped_field_ids() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "dept", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(4, "subdept", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, true), + Field::new("subdept", DataType::Utf8, true), + ])); + + let name_mapping = Arc::new(NameMapping::new(vec![ + MappedField::new(Some(2), vec!["name".to_string()], vec![]), + MappedField::new(Some(4), vec!["subdept".to_string()], vec![]), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + // Engineered so that filtering the wrong physical column yields different rows: + // `name` and `subdept` both contain the value "Alice", on different rows. + let name_col = Arc::new(StringArray::from(vec!["Alice", "Bob", "Sue"])) as ArrayRef; + let subdept_col = Arc::new(StringArray::from(vec!["Bob", "Alice", "Alice"])) as ArrayRef; + + let to_write = + RecordBatch::try_new(arrow_schema.clone(), vec![name_col, subdept_col]).unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + let predicate = Reference::new("name").equal_to(Datum::string("Alice")); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_group_filtering_enabled(true) + .with_row_selection_enabled(true) + .build(), + ) + .build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![2, 4]) + .with_case_sensitive(false) + .with_name_mapping(Some(name_mapping)) + .with_predicate(Some(predicate.bind(schema, true).unwrap())) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let total_rows: usize = result.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + total_rows, 1, + "filter `name = \"Alice\"` matched the wrong rows: predicate was evaluated \ + against the wrong physical column" + ); + + let batch = &result[0]; + let name_array = batch.column(0).as_string::(); + assert_eq!(name_array.value(0), "Alice"); + let subdept_array = batch.column(1).as_string::(); + assert_eq!(subdept_array.value(0), "Bob"); + } + + /// Regression test for #3374: a struct in a name-mapped file reads correctly next to a + /// top-level column, and predicates resolve its fields. + #[tokio::test] + async fn test_read_parquet_with_name_mapping_maps_nested_fields() { + use arrow_array::types::Int32Type; + use arrow_array::{Int32Array, StructArray}; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(3, "a", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let name_mapping = Arc::new(NameMapping::new(vec![ + MappedField::new(Some(1), vec!["id".to_string()], vec![]), + MappedField::new(Some(2), vec!["s".to_string()], vec![MappedField::new( + Some(3), + vec!["a".to_string()], + vec![], + )]), + ])); + + // No field IDs at any level. + let s_col = StructArray::from(vec![( + Arc::new(Field::new("a", DataType::Int32, true)), + Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef, + )]); + let to_write = RecordBatch::try_from_iter_with_nullable(vec![ + ( + "id", + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + true, + ), + ("s", Arc::new(s_col) as ArrayRef, true), + ]) + .unwrap(); + let tmp_dir = TempDir::new().unwrap(); + let file_path = format!("{}/1.parquet", tmp_dir.path().to_str().unwrap()); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), None).unwrap(); + writer.write(&to_write).unwrap(); + writer.close().unwrap(); + + let file_size = std::fs::metadata(&file_path).unwrap().len(); + let tasks = |predicate| { + Box::pin(futures::stream::iter([Ok(FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .with_name_mapping(Some(name_mapping.clone())) + .with_predicate(predicate) + .build() + .unwrap())])) as FileScanTaskStream + }; + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + + let result = reader + .clone() + .read(tasks(None)) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + let s = result[0].column(1).as_struct(); + assert_eq!(s.null_count(), 0); + assert_eq!( + s.column(0).as_primitive::(), + &Int32Array::from(vec![10, 20, 30]) + ); + + // `s.a` resolves, so the row filter rejects the nested predicate as it does for files + // with embedded IDs, rather than matching no rows. + let predicate = Reference::new("s.a") + .greater_than(Datum::int(15)) + .bind(schema.clone(), true) + .unwrap(); + let err = reader + .read(tasks(Some(predicate))) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + /// Every nested Arrow type gets IDs, with list and map entries matched by their spec names. + #[test] + fn test_apply_name_mapping_to_nested_arrow_types() { + use arrow_schema::Fields; + + use super::apply_name_mapping_to_arrow_schema; + use crate::arrow::arrow_schema_to_schema; + + let field = |name: &str, data_type| Field::new(name, data_type, true); + let item = |data_type| Arc::new(field("item", data_type)); + let entries = Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("keys", DataType::Utf8, false), + field("values", DataType::Int32), + ])), + false, + ); + let x = Fields::from(vec![field("x", DataType::Int32)]); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + field("id", DataType::Int32), + // `s.id` must take its ID from the entry for `s`, not from the top-level `id`. + field( + "s", + DataType::Struct(Fields::from(vec![field("id", DataType::Int32)])), + ), + field("list", DataType::List(item(DataType::Int32))), + field("fixed", DataType::FixedSizeList(item(DataType::Int32), 2)), + field("large", DataType::LargeList(item(DataType::Struct(x)))), + field("map", DataType::Map(Arc::new(entries), true)), + ])); + let entry = + |id, name: &str, fields| MappedField::new(Some(id), vec![name.to_string()], fields); + let name_mapping = NameMapping::new(vec![ + entry(1, "id", vec![]), + entry(2, "s", vec![entry(3, "id", vec![])]), + entry(4, "list", vec![entry(5, "element", vec![])]), + entry(6, "fixed", vec![entry(7, "element", vec![])]), + entry(8, "large", vec![entry(9, "element", vec![entry( + 10, + "x", + vec![], + )])]), + entry(11, "map", vec![ + entry(12, "key", vec![]), + entry(13, "value", vec![]), + ]), + ]); + + let mapped = apply_name_mapping_to_arrow_schema(arrow_schema, &name_mapping).unwrap(); + + // The list size and the map ordering carry over. + assert!(matches!( + mapped.field(3).data_type(), + DataType::FixedSizeList(_, 2) + )); + assert!(matches!( + mapped.field(5).data_type(), + DataType::Map(_, true) + )); + // The conversion fails on any field without an ID. + let schema = arrow_schema_to_schema(&mapped).unwrap(); + let names: Vec<_> = (1..=13) + .map(|id| schema.name_by_field_id(id).unwrap_or_default()) + .collect(); + assert_eq!(names, [ + "id", + "s", + "s.id", + "list", + "list.element", + "fixed", + "fixed.element", + "large", + "large.element", + "large.element.x", + "map", + "map.key", + "map.value", + ]); + } + + /// A field without an entry, or whose entry has no field ID, keeps no ID, and so do its + /// nested fields. + #[test] + fn test_apply_name_mapping_skips_unmapped_fields() { + use arrow_schema::Fields; + + use super::{apply_name_mapping_to_arrow_schema, build_field_id_map_from_arrow_schema}; + + let int = |name: &str| Field::new(name, DataType::Int32, true); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new( + "s", + DataType::Struct(Fields::from(vec![int("a"), int("b")])), + true, + ), + Field::new("t", DataType::Struct(Fields::from(vec![int("c")])), true), + ])); + let entry = |id, name: &str, fields| MappedField::new(id, vec![name.to_string()], fields); + let name_mapping = NameMapping::new(vec![ + // No entry for `s.b`. + entry(Some(1), "s", vec![entry(Some(2), "a", vec![])]), + entry(None, "t", vec![entry(Some(3), "c", vec![])]), + ]); + + let mapped = apply_name_mapping_to_arrow_schema(arrow_schema, &name_mapping).unwrap(); + + // Only `s.a`, leaf 0, resolves to an ID. + assert_eq!( + build_field_id_map_from_arrow_schema(&mapped), + HashMap::from([(2, 0)]) + ); + } + + /// Test reading Parquet files without field IDs with partial projection. + /// Only a subset of columns are requested, verifying position-based fallback + /// handles column selection correctly. + #[tokio::test] + async fn test_read_parquet_without_field_ids_partial_projection() { + use arrow_array::Int32Array; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "col1", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "col2", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(3, "col3", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(4, "col4", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("col1", DataType::Utf8, false), + Field::new("col2", DataType::Int32, false), + Field::new("col3", DataType::Utf8, false), + Field::new("col4", DataType::Int32, false), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let col1_data = Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef; + let col2_data = Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef; + let col3_data = Arc::new(StringArray::from(vec!["c", "d"])) as ArrayRef; + let col4_data = Arc::new(Int32Array::from(vec![30, 40])) as ArrayRef; + + let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![ + col1_data, col2_data, col3_data, col4_data, + ]) + .unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 3]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + assert_eq!(result.len(), 1); + let batch = &result[0]; + assert_eq!(batch.num_rows(), 2); + assert_eq!(batch.num_columns(), 2); + + let col1_array = batch.column(0).as_string::(); + assert_eq!(col1_array.value(0), "a"); + assert_eq!(col1_array.value(1), "b"); + + let col3_array = batch.column(1).as_string::(); + assert_eq!(col3_array.value(0), "c"); + assert_eq!(col3_array.value(1), "d"); + } + + /// Test reading Parquet files without field IDs with schema evolution. + /// The Iceberg schema has more fields than the Parquet file, testing that + /// missing columns are filled with NULLs. + #[tokio::test] + async fn test_read_parquet_without_field_ids_schema_evolution() { + use arrow_array::{Array, Int32Array}; + + // Schema with field 3 added after the file was written + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "age", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "city", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, false), + Field::new("age", DataType::Int32, false), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let name_data = Arc::new(StringArray::from(vec!["Alice", "Bob"])) as ArrayRef; + let age_data = Arc::new(Int32Array::from(vec![30, 25])) as ArrayRef; + + let to_write = + RecordBatch::try_new(arrow_schema.clone(), vec![name_data, age_data]).unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2, 3]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + assert_eq!(result.len(), 1); + let batch = &result[0]; + assert_eq!(batch.num_rows(), 2); + assert_eq!(batch.num_columns(), 3); + + let name_array = batch.column(0).as_string::(); + assert_eq!(name_array.value(0), "Alice"); + assert_eq!(name_array.value(1), "Bob"); + + let age_array = batch + .column(1) + .as_primitive::(); + assert_eq!(age_array.value(0), 30); + assert_eq!(age_array.value(1), 25); + + // Verify missing column filled with NULLs + let city_array = batch.column(2).as_string::(); + assert_eq!(city_array.null_count(), 2); + assert!(city_array.is_null(0)); + assert!(city_array.is_null(1)); + } + + /// Test reading Parquet files without field IDs that have multiple row groups. + /// This ensures the position-based fallback works correctly across row group boundaries. + #[tokio::test] + async fn test_read_parquet_without_field_ids_multiple_row_groups() { + use arrow_array::Int32Array; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "value", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, false), + Field::new("value", DataType::Int32, false), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + // Small row group size to create multiple row groups + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_write_batch_size(2) + .set_max_row_group_row_count(Some(2)) + .build(); + + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + // Write 6 rows in 3 batches (will create 3 row groups) + for batch_num in 0..3 { + let name_data = Arc::new(StringArray::from(vec![ + format!("name_{}", batch_num * 2), + format!("name_{}", batch_num * 2 + 1), + ])) as ArrayRef; + let value_data = + Arc::new(Int32Array::from(vec![batch_num * 2, batch_num * 2 + 1])) as ArrayRef; + + let batch = + RecordBatch::try_new(arrow_schema.clone(), vec![name_data, value_data]).unwrap(); + writer.write(&batch).expect("Writing batch"); + } + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + assert!(!result.is_empty()); + + let mut all_names = Vec::new(); + let mut all_values = Vec::new(); + + for batch in &result { + let name_array = batch.column(0).as_string::(); + let value_array = batch + .column(1) + .as_primitive::(); + + for i in 0..batch.num_rows() { + all_names.push(name_array.value(i).to_string()); + all_values.push(value_array.value(i)); + } + } + + assert_eq!(all_names.len(), 6); + assert_eq!(all_values.len(), 6); + + for i in 0..6 { + assert_eq!(all_names[i], format!("name_{i}")); + assert_eq!(all_values[i], i as i32); + } + } + + /// Test reading Parquet files without field IDs with nested types (struct). + /// Java's pruneColumnsFallback() projects entire top-level columns including nested content, + /// but only top-level fields get fallback ids, so struct fields read as null. + #[tokio::test] + async fn test_read_parquet_without_field_ids_reads_struct_fields_as_null() { + use arrow_array::{Int32Array, StructArray}; + use arrow_schema::Fields; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required( + 2, + "person", + Type::Struct(StructType::new(vec![ + NestedField::optional( + 3, + "name", + Type::Primitive(PrimitiveType::String), + ) + .into(), + NestedField::optional(4, "age", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "person", + DataType::Struct(Fields::from(vec![ + Field::new("name", DataType::Utf8, false), + Field::new("age", DataType::Int32, false), + ])), + false, + ), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let id_data = Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef; + let name_data = Arc::new(StringArray::from(vec!["Alice", "Bob"])) as ArrayRef; + let age_data = Arc::new(Int32Array::from(vec![30, 25])) as ArrayRef; + let person_data = Arc::new(StructArray::from(vec![ + ( + Arc::new(Field::new("name", DataType::Utf8, false)), + name_data, + ), + ( + Arc::new(Field::new("age", DataType::Int32, false)), + age_data, + ), + ])) as ArrayRef; + + let to_write = + RecordBatch::try_new(arrow_schema.clone(), vec![id_data, person_data]).unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + assert_eq!(result.len(), 1); + let batch = &result[0]; + assert_eq!(batch.num_rows(), 2); + assert_eq!(batch.num_columns(), 2); + + let id_array = batch + .column(0) + .as_primitive::(); + assert_eq!(id_array.value(0), 1); + assert_eq!(id_array.value(1), 2); + + let person_array = batch.column(1).as_struct(); + assert_eq!(person_array.num_columns(), 2); + assert_eq!(person_array.null_count(), 0); + assert_eq!(person_array.column(0).null_count(), 2); + assert_eq!(person_array.column(1).null_count(), 2); + } + + /// Test reading Parquet files without field IDs with schema evolution - column added in the middle. + /// When a new column is inserted between existing columns in the schema order, + /// the fallback projection must correctly map field IDs to output positions. + #[tokio::test] + async fn test_read_parquet_without_field_ids_schema_evolution_add_column_in_middle() { + use arrow_array::{Array, Int32Array}; + + let arrow_schema_old = Arc::new(ArrowSchema::new(vec![ + Field::new("col0", DataType::Int32, true), + Field::new("col1", DataType::Int32, true), + ])); + + // New column added between existing columns: col0 (id=1), newCol (id=5), col1 (id=2) + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "col0", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(5, "newCol", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "col1", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let col0_data = Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef; + let col1_data = Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef; + + let to_write = + RecordBatch::try_new(arrow_schema_old.clone(), vec![col0_data, col1_data]).unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 5, 2]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + assert_eq!(result.len(), 1); + let batch = &result[0]; + assert_eq!(batch.num_rows(), 2); + assert_eq!(batch.num_columns(), 3); + + let result_col0 = batch + .column(0) + .as_primitive::(); + assert_eq!(result_col0.value(0), 1); + assert_eq!(result_col0.value(1), 2); + + // New column should be NULL (doesn't exist in old file) + let result_newcol = batch + .column(1) + .as_primitive::(); + assert_eq!(result_newcol.null_count(), 2); + assert!(result_newcol.is_null(0)); + assert!(result_newcol.is_null(1)); + + let result_col1 = batch + .column(2) + .as_primitive::(); + assert_eq!(result_col1.value(0), 10); + assert_eq!(result_col1.value(1), 20); + } + + /// Test reading Parquet files without field IDs with a filter that eliminates all row groups. + /// During development of field ID mapping, we saw a panic when row_selection_enabled=true and + /// all row groups are filtered out. + #[tokio::test] + async fn test_read_parquet_without_field_ids_filter_eliminates_all_rows() { + use arrow_array::{Float64Array, Int32Array}; + + // Schema with fields that will use fallback IDs 1, 2, 3 + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "value", Type::Primitive(PrimitiveType::Double)) + .into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new("value", DataType::Float64, false), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + // Write data where all ids are >= 10 + let id_data = Arc::new(Int32Array::from(vec![10, 11, 12])) as ArrayRef; + let name_data = Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef; + let value_data = Arc::new(Float64Array::from(vec![100.0, 200.0, 300.0])) as ArrayRef; + + let to_write = + RecordBatch::try_new(arrow_schema.clone(), vec![id_data, name_data, value_data]) + .unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + // Filter that eliminates all row groups: id < 5 + let predicate = Reference::new("id").less_than(Datum::int(5)); + + // Enable both row_group_filtering and row_selection - triggered the panic + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_group_filtering_enabled(true) + .with_row_selection_enabled(true) + .build(), + ) + .build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2, 3]) + .with_case_sensitive(false) + .with_predicate(Some(predicate.bind(schema, true).unwrap())) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + // Should no longer panic + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + // Should return empty results + assert!(result.is_empty() || result.iter().all(|batch| batch.num_rows() == 0)); + } + + /// Test bucket partitioning reads source column from data file (not partition metadata). + /// + /// This is an integration test verifying the complete ArrowReader pipeline with bucket partitioning. + /// It corresponds to TestRuntimeFiltering tests in Iceberg Java (e.g., testRenamedSourceColumnTable). + /// + /// # Iceberg Spec Requirements + /// + /// Per the Iceberg spec "Column Projection" section: + /// > "Return the value from partition metadata if an **Identity Transform** exists for the field" + /// + /// This means: + /// - Identity transforms (e.g., `identity(dept)`) use constants from partition metadata + /// - Non-identity transforms (e.g., `bucket(4, id)`) must read source columns from data files + /// - Partition metadata for bucket transforms stores bucket numbers (0-3), NOT source values + /// + /// Java's PartitionUtil.constantsMap() implements this via: + /// ```java + /// if (field.transform().isIdentity()) { + /// idToConstant.put(field.sourceId(), converted); + /// } + /// ``` + /// + /// # What This Test Verifies + /// + /// This test ensures the full ArrowReader → RecordBatchTransformer pipeline correctly handles + /// bucket partitioning when FileScanTask provides partition_spec and partition_data: + /// + /// - Parquet file has field_id=1 named "id" with actual data [1, 5, 9, 13] + /// - FileScanTask specifies partition_spec with bucket(4, id) and partition_data with bucket=1 + /// - RecordBatchTransformer.constants_map() excludes bucket-partitioned field from constants + /// - ArrowReader correctly reads [1, 5, 9, 13] from the data file + /// - Values are NOT replaced with constant 1 from partition metadata + /// + /// # Why This Matters + /// + /// Without correct handling: + /// - Runtime filtering would break (e.g., `WHERE id = 5` would fail) + /// - Query results would be incorrect (all rows would have id=1) + /// - Bucket partitioning would be unusable for query optimization + /// + /// # References + /// - Iceberg spec: format/spec.md "Column Projection" + "Partition Transforms" + /// - Java test: spark/src/test/java/.../TestRuntimeFiltering.java + /// - Java impl: core/src/main/java/org/apache/iceberg/util/PartitionUtil.java + #[tokio::test] + async fn test_bucket_partitioning_reads_source_column_from_file() { + use arrow_array::Int32Array; + + use crate::spec::{Literal, PartitionSpec, Struct, Transform}; + + // Iceberg schema with id and name columns + let schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + // Partition spec: bucket(4, id) + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("id", "id_bucket", Transform::Bucket(4)) + .unwrap() + .build() + .unwrap(), + ); + + // Partition data: bucket value is 1 + let partition_data = Struct::from_iter(vec![Some(Literal::int(1))]); + + // Create Arrow schema with field IDs for Parquet file + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("name", DataType::Utf8, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + + // Write Parquet file with data + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_io = FileIO::new_with_fs(); + + let id_data = Arc::new(Int32Array::from(vec![1, 5, 9, 13])) as ArrayRef; + let name_data = + Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie", "Dave"])) as ArrayRef; + + let to_write = + RecordBatch::try_new(arrow_schema.clone(), vec![id_data, name_data]).unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(format!("{}/data.parquet", &table_location)).unwrap(); + let mut writer = ArrowWriter::try_new(file, to_write.schema(), Some(props)).unwrap(); + writer.write(&to_write).expect("Writing batch"); + writer.close().unwrap(); + + // Read the Parquet file with partition spec and data + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/data.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/data.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .with_partition(Some(partition_data)) + .with_partition_spec(Some(partition_spec)) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + // Verify we got the correct data + assert_eq!(result.len(), 1); + let batch = &result[0]; + + assert_eq!(batch.num_columns(), 2); + assert_eq!(batch.num_rows(), 4); + + // The id column MUST contain actual values from the Parquet file [1, 5, 9, 13], + // NOT the constant partition value 1 + let id_col = batch + .column(0) + .as_primitive::(); + assert_eq!(id_col.value(0), 1); + assert_eq!(id_col.value(1), 5); + assert_eq!(id_col.value(2), 9); + assert_eq!(id_col.value(3), 13); + + let name_col = batch.column(1).as_string::(); + assert_eq!(name_col.value(0), "Alice"); + assert_eq!(name_col.value(1), "Bob"); + assert_eq!(name_col.value(2), "Charlie"); + assert_eq!(name_col.value(3), "Dave"); + } + + /// Regression for : + /// predicate on a column after nested types in a migrated file (no field IDs). + /// Schema has struct, list, and map columns before the predicate target (`id`), + /// exercising the fallback field ID mapping across all nested type variants. + #[tokio::test] + async fn test_predicate_on_migrated_file_with_nested_types() { + use serde::{Deserialize, Serialize}; + use serde_arrow::schema::{SchemaLike, TracingOptions}; + + #[derive(Serialize, Deserialize)] + struct Person { + name: String, + age: i32, + } + + #[derive(Serialize, Deserialize)] + struct Row { + person: Person, + people: Vec, + props: std::collections::BTreeMap, + id: i32, + } + + let rows = vec![ + Row { + person: Person { + name: "Alice".into(), + age: 30, + }, + people: vec![Person { + name: "Alice".into(), + age: 30, + }], + props: [("k1".into(), "v1".into())].into(), + id: 1, + }, + Row { + person: Person { + name: "Bob".into(), + age: 25, + }, + people: vec![Person { + name: "Bob".into(), + age: 25, + }], + props: [("k2".into(), "v2".into())].into(), + id: 2, + }, + Row { + person: Person { + name: "Carol".into(), + age: 40, + }, + people: vec![Person { + name: "Carol".into(), + age: 40, + }], + props: [("k3".into(), "v3".into())].into(), + id: 3, + }, + ]; + + let tracing_options = TracingOptions::default() + .map_as_struct(false) + .strings_as_large_utf8(false) + .sequence_as_large_list(false); + let fields = Vec::::from_type::(tracing_options).unwrap(); + let arrow_schema = Arc::new(ArrowSchema::new(fields.clone())); + let batch = serde_arrow::to_record_batch(&fields, &rows).unwrap(); + + // Fallback field IDs: person=1, people=2, props=3, id=4 + let iceberg_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "person", + Type::Struct(StructType::new(vec![ + NestedField::required( + 5, + "name", + Type::Primitive(PrimitiveType::String), + ) + .into(), + NestedField::required(6, "age", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + NestedField::required( + 2, + "people", + Type::List(crate::spec::ListType { + element_field: NestedField::required( + 7, + "element", + Type::Struct(StructType::new(vec![ + NestedField::required( + 8, + "name", + Type::Primitive(PrimitiveType::String), + ) + .into(), + NestedField::required( + 9, + "age", + Type::Primitive(PrimitiveType::Int), + ) + .into(), + ])), + ) + .into(), + }), + ) + .into(), + NestedField::required( + 3, + "props", + Type::Map(crate::spec::MapType { + key_field: NestedField::required( + 10, + "key", + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::required( + 11, + "value", + Type::Primitive(PrimitiveType::String), + ) + .into(), + }), + ) + .into(), + NestedField::required(4, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_path = format!("{table_location}/1.parquet"); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap(); + writer.write(&batch).expect("Writing batch"); + writer.close().unwrap(); + + let predicate = Reference::new("id").greater_than(Datum::int(1)); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_group_filtering_enabled(true) + .with_row_selection_enabled(true) + .build(), + ) + .build(); + + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(iceberg_schema.clone()) + .with_project_field_ids(vec![4]) + .with_case_sensitive(false) + .with_predicate(Some(predicate.bind(iceberg_schema, true).unwrap())) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let ids: Vec = result + .iter() + .flat_map(|b| { + b.column(0) + .as_primitive::() + .values() + .iter() + .copied() + }) + .collect(); + assert_eq!(ids, vec![2, 3]); + } + + /// Reads a file written with an older nested schema through an evolved one. This relies + /// on parquet-rs attaching `PARQUET:field_id` to struct children, list elements, and map + /// keys and values, which the hand-built fixtures in `record_batch_transformer` assume. + #[tokio::test] + async fn test_read_evolved_nested_fields_by_field_id() { + use serde_json::{Value, json}; + + use crate::arrow::schema_to_arrow_schema; + use crate::spec::PrimitiveType::{Int, Long}; + use crate::spec::{ListType, MapType}; + + fn struct_of(fields: &[(i32, &str, PrimitiveType)]) -> Type { + Type::Struct(StructType::new( + fields + .iter() + .map(|(id, name, ty)| { + NestedField::optional(*id, *name, Type::Primitive(ty.clone())).into() + }) + .collect(), + )) + } + fn schema(s: Type, element: Type, value: Type) -> Schema { + let string = Type::Primitive(PrimitiveType::String); + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(Int)).into(), + NestedField::optional(2, "s", s).into(), + NestedField::optional( + 3, + "l", + Type::List(ListType::new( + NestedField::list_element(7, element, true).into(), + )), + ) + .into(), + NestedField::optional( + 4, + "m", + Type::Map(MapType::required(9, string, 10, value)), + ) + .into(), + ]) + .build() + .unwrap() + } + fn to_batch(schema: &Schema, rows: Value) -> RecordBatch { + let fields = schema_to_arrow_schema(schema).unwrap().fields().to_vec(); + serde_arrow::to_record_batch(&fields, &rows).unwrap() + } + + let old_schema = schema( + struct_of(&[(5, "a", Int), (6, "b", PrimitiveType::String)]), + struct_of(&[(8, "x", Int)]), + struct_of(&[(11, "x", Int)]), + ); + // Reorders `s`, adds children before, between, and after existing ones, and + // promotes int to long. + let new_schema = Arc::new(schema( + struct_of(&[ + (6, "b", PrimitiveType::String), + (12, "c", Int), + (5, "a", Long), + ]), + struct_of(&[(13, "y", Int), (8, "x", Int)]), + struct_of(&[(11, "x", Long), (14, "y", Int)]), + )); + let old_batch = to_batch( + &old_schema, + json!([ + {"id": 1, "s": {"a": 1, "b": "p"}, "l": [{"x": 10}, {"x": 11}], "m": {"k1": {"x": 100}}}, + {"id": 2, "s": null, "l": null, "m": {"k2": {"x": 200}}}, + {"id": 3, "s": {"a": 3, "b": null}, "l": [{"x": 30}], "m": null}, + ]), + ); + let expected = to_batch( + &new_schema, + json!([ + { + "id": 1, + "s": {"b": "p", "c": null, "a": 1}, + "l": [{"y": null, "x": 10}, {"y": null, "x": 11}], + "m": {"k1": {"x": 100, "y": null}}, + }, + {"id": 2, "s": null, "l": null, "m": {"k2": {"x": 200, "y": null}}}, + {"id": 3, "s": {"b": null, "c": null, "a": 3}, "l": [{"y": null, "x": 30}], "m": null}, + ]), + ); + + let tmp_dir = TempDir::new().unwrap(); + let file_path = format!("{}/old.parquet", tmp_dir.path().to_str().unwrap()); + let mut writer = + ArrowWriter::try_new(File::create(&file_path).unwrap(), old_batch.schema(), None) + .unwrap(); + writer.write(&old_batch).unwrap(); + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(new_schema) + .with_project_field_ids(vec![1, 2, 3, 4]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + assert_eq!(result, vec![expected]); + } + + // --------------------------------------------------------------------- + // End-to-end variant read tests (synthetic Parquet). + // + // Unless a test says otherwise, the files carry a field id on the variant group, none on + // its leaves, and no Parquet VARIANT annotation. `value` is required, as in iceberg-java's + // unshredded layout. + // --------------------------------------------------------------------- + + // Known variant bytes the reader must round-trip unchanged. `value` is distinct per row, + // so a dropped or reordered row is caught, not just the struct shape. + const VARIANT_METADATA: [&[u8]; 3] = [b"\x01", b"\x01", b"\x01"]; + const VARIANT_VALUE: [&[u8]; 3] = [b"\x0a", b"\x0b", b"\x0c"]; + + /// The 3-row variant `StructArray` for [`VARIANT_METADATA`]/[`VARIANT_VALUE`]. + fn variant_struct_array() -> ArrayRef { + use arrow_array::{BinaryArray, StructArray}; + let metadata = Arc::new(BinaryArray::from(VARIANT_METADATA.to_vec())) as ArrayRef; + let value = Arc::new(BinaryArray::from(VARIANT_VALUE.to_vec())) as ArrayRef; + Arc::new(StructArray::new( + variant_arrow_fields(), + vec![metadata, value], + None, + )) as ArrayRef + } + + /// Asserts `field` is a variant and `column` holds the bytes of [`variant_struct_array`] + /// in the table schema's Arrow type. + fn assert_variant(field: &Field, column: &ArrayRef) { + use arrow_array::StructArray; + let DataType::Struct(table_fields) = + crate::arrow::type_to_arrow_type(&Type::Variant(VariantType)).unwrap() + else { + unreachable!() + }; + let expected = StructArray::new( + table_fields, + variant_struct_array().as_struct().columns().to_vec(), + None, + ); + assert_eq!(field.extension_type_name(), Some("arrow.parquet.variant")); + assert_eq!(column.as_struct(), &expected); + } + + /// Asserts the top-level variant `v` of `batch` round-tripped. + fn assert_top_level_variant(batch: &RecordBatch) { + let schema = batch.schema(); + assert_variant( + schema.field_with_name("v").unwrap(), + batch.column_by_name("v").unwrap(), + ); + } + + /// Asserts the `id` column holds exactly `[1, 2, 3]`. + fn assert_id_values(col: &ArrayRef) { + let ids = col.as_primitive::(); + assert_eq!(ids.values(), &[1, 2, 3]); + } + + fn write_variant_parquet( + arrow_schema: Arc, + columns: Vec, + dir: &str, + ) -> String { + let batch = RecordBatch::try_new(arrow_schema.clone(), columns).unwrap(); + let path = format!("{dir}/variant.parquet"); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let mut writer = + ArrowWriter::try_new(File::create(&path).unwrap(), arrow_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + path + } + + async fn try_read_variant( + schema: Arc, + path: &str, + project_field_ids: Vec, + name_mapping: Option>, + ) -> crate::Result> { + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path.to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(project_field_ids) + .with_name_mapping(name_mapping) + .with_case_sensitive(false) + .build()?; + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + reader.read(tasks)?.stream().try_collect().await + } + + /// Reads `path`, checking that each batch's schema matches its columns, nested field + /// metadata included. + async fn read_variant( + schema: Arc, + path: &str, + project_field_ids: Vec, + ) -> Vec { + let batches = try_read_variant(schema, path, project_field_ids, None) + .await + .unwrap(); + for batch in &batches { + RecordBatch::try_new(batch.schema(), batch.columns().to_vec()).unwrap(); + } + batches + } + + /// `id` (int, id 1) + optional `v` (variant, id 2). + fn top_level_variant_schema() -> Arc { + Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(), + ) + } + + /// Writes [`top_level_variant_schema`] rows with `v` stored as `variant_field`. + fn write_top_level_variant_as(dir: &str, id_field: Field, variant_field: Field) -> String { + use arrow_array::{Int32Array, StructArray}; + let DataType::Struct(storage) = variant_field.data_type() else { + unreachable!() + }; + let variant = Arc::new(StructArray::new( + storage.clone(), + variant_struct_array().as_struct().columns().to_vec(), + None, + )) as ArrayRef; + let arrow_schema = Arc::new(ArrowSchema::new(vec![id_field, variant_field])); + let id = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; + write_variant_parquet(arrow_schema, vec![id, variant], dir) + } + + fn write_top_level_variant(dir: &str) -> Arc { + write_top_level_variant_as( + dir, + Field::new("id", DataType::Int32, false).with_metadata(field_id_meta(1)), + variant_arrow_field("v", 2), + ); + top_level_variant_schema() + } + + #[tokio::test] + async fn test_read_variant_full_scan() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let schema = write_top_level_variant(dir); + + let batches = read_variant(schema, &format!("{dir}/variant.parquet"), vec![1, 2]).await; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 3); + assert_id_values(batches[0].column_by_name("id").unwrap()); + assert_top_level_variant(&batches[0]); + } + + #[tokio::test] + async fn test_read_variant_only() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let schema = write_top_level_variant(dir); + + let batches = read_variant(schema, &format!("{dir}/variant.parquet"), vec![2]).await; + assert_eq!(batches[0].num_rows(), 3); + assert_top_level_variant(&batches[0]); + assert!(batches[0].column_by_name("id").is_none()); + } + + #[tokio::test] + async fn test_read_variant_sibling_only_excludes_variant() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let schema = write_top_level_variant(dir); + + let batches = read_variant(schema, &format!("{dir}/variant.parquet"), vec![1]).await; + assert_eq!(batches[0].num_rows(), 3); + assert_id_values(batches[0].column_by_name("id").unwrap()); + assert!(batches[0].column_by_name("v").is_none()); + } + + /// Null variant rows stay null when the variant is rebuilt into the table type, here because + /// the file's `value` is required and the table type's is nullable. + #[tokio::test] + async fn test_read_variant_with_null_rows() { + use arrow_array::StructArray; + use arrow_buffer::NullBuffer; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let with_null_row = |fields: Fields| { + Arc::new(StructArray::new( + fields, + variant_struct_array().as_struct().columns().to_vec(), + Some(NullBuffer::from(vec![true, false, true])), + )) as ArrayRef + }; + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("v", DataType::Struct(variant_arrow_fields()), true) + .with_metadata(field_id_meta(2)), + ])); + let path = write_variant_parquet( + arrow_schema, + vec![with_null_row(variant_arrow_fields())], + dir, + ); + + let batches = read_variant(top_level_variant_schema(), &path, vec![2]).await; + let DataType::Struct(table_fields) = + crate::arrow::type_to_arrow_type(&Type::Variant(VariantType)).unwrap() + else { + unreachable!() + }; + let schema = batches[0].schema(); + assert_eq!( + schema.field_with_name("v").unwrap().extension_type_name(), + Some("arrow.parquet.variant") + ); + assert_eq!( + batches[0].column_by_name("v").unwrap(), + &with_null_row(table_fields) + ); + } + + /// A file written before `v` was added to the table reads `v` as null in every row. + #[tokio::test] + async fn test_read_variant_missing_from_file() { + use arrow_array::{Int32Array, new_null_array}; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(field_id_meta(1)), + ])); + let id = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; + let path = write_variant_parquet(arrow_schema, vec![id], dir); + let table_type = crate::arrow::type_to_arrow_type(&Type::Variant(VariantType)).unwrap(); + + for project_field_ids in [vec![1, 2], vec![2]] { + let batches = read_variant(top_level_variant_schema(), &path, project_field_ids).await; + let schema = batches[0].schema(); + assert_eq!( + schema.field_with_name("v").unwrap().extension_type_name(), + Some("arrow.parquet.variant") + ); + assert_eq!( + batches[0].column_by_name("v").unwrap(), + &new_null_array(&table_type, 3) + ); + } + } + + /// The file stores the variant exactly as the table schema's Arrow type, so only the + /// extension has to be added. + #[tokio::test] + async fn test_read_variant_stored_as_table_type_gets_extension() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let storage = Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, true), + ]); + let path = write_top_level_variant_as( + dir, + Field::new("id", DataType::Int32, false).with_metadata(field_id_meta(1)), + Field::new("v", DataType::Struct(storage), true).with_metadata(field_id_meta(2)), + ); + + let batches = read_variant(top_level_variant_schema(), &path, vec![1, 2]).await; + assert_id_values(batches[0].column_by_name("id").unwrap()); + assert_top_level_variant(&batches[0]); + } + + #[tokio::test] + async fn test_read_variant_matches_storage_fields_by_name() { + use arrow_array::{LargeBinaryArray, StructArray}; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let storage = Fields::from(vec![ + Field::new("value", DataType::LargeBinary, false), + Field::new("metadata", DataType::LargeBinary, false), + ]); + let variant = Arc::new(StructArray::new( + storage.clone(), + vec![ + Arc::new(LargeBinaryArray::from(VARIANT_VALUE.to_vec())) as ArrayRef, + Arc::new(LargeBinaryArray::from(VARIANT_METADATA.to_vec())) as ArrayRef, + ], + None, + )) as ArrayRef; + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("v", DataType::Struct(storage), false).with_metadata(field_id_meta(2)), + ])); + let path = write_variant_parquet(arrow_schema, vec![variant], dir); + + let batches = read_variant(top_level_variant_schema(), &path, vec![2]).await; + assert_top_level_variant(&batches[0]); + } + + #[tokio::test] + async fn test_read_variant_with_dictionary_storage() { + use arrow_array::BinaryArray; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let dictionary = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Binary)); + let storage = Fields::from(vec![ + Field::new("metadata", dictionary.clone(), false), + Field::new("value", dictionary.clone(), false), + ]); + let encode = |bytes: [&[u8]; 3]| { + arrow_cast::cast(&BinaryArray::from(bytes.to_vec()), &dictionary).unwrap() + }; + let variant = Arc::new(arrow_array::StructArray::new( + storage.clone(), + vec![encode(VARIANT_METADATA), encode(VARIANT_VALUE)], + None, + )) as ArrayRef; + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("v", DataType::Struct(storage), false).with_metadata(field_id_meta(2)), + ])); + let path = write_variant_parquet(arrow_schema, vec![variant], dir); + + let batches = read_variant(top_level_variant_schema(), &path, vec![2]).await; + assert_top_level_variant(&batches[0]); + } + + /// The variant group carries the Parquet VARIANT annotation, as iceberg-java writes it. + #[tokio::test] + async fn test_read_variant_with_parquet_annotation() { + use parquet::basic::{LogicalType, Repetition, Type as PhysicalType}; + use parquet::data_type::{ByteArray, ByteArrayType, Int32Type}; + use parquet::file::writer::SerializedFileWriter; + use parquet::schema::types::Type as ParquetType; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let binary = |name: &str| { + Arc::new( + ParquetType::primitive_type_builder(name, PhysicalType::BYTE_ARRAY) + .with_repetition(Repetition::REQUIRED) + .build() + .unwrap(), + ) + }; + let id = ParquetType::primitive_type_builder("id", PhysicalType::INT32) + .with_repetition(Repetition::REQUIRED) + .with_id(Some(1)) + .build() + .unwrap(); + let variant = ParquetType::group_type_builder("v") + .with_repetition(Repetition::OPTIONAL) + .with_logical_type(Some(LogicalType::variant(Some(1)))) + .with_id(Some(2)) + .with_fields(vec![binary("metadata"), binary("value")]) + .build() + .unwrap(); + let parquet_schema = ParquetType::group_type_builder("schema") + .with_fields(vec![Arc::new(id), Arc::new(variant)]) + .build() + .unwrap(); + let path = format!("{dir}/annotated.parquet"); + let mut writer = SerializedFileWriter::new( + File::create(&path).unwrap(), + Arc::new(parquet_schema), + Arc::new(WriterProperties::builder().build()), + ) + .unwrap(); + let mut row_group = writer.next_row_group().unwrap(); + let mut column = row_group.next_column().unwrap().unwrap(); + column + .typed::() + .write_batch(&[1, 2, 3], None, None) + .unwrap(); + column.close().unwrap(); + for bytes in [VARIANT_METADATA, VARIANT_VALUE] { + let values: Vec = + bytes.iter().map(|b| ByteArray::from(b.to_vec())).collect(); + let mut column = row_group.next_column().unwrap().unwrap(); + column + .typed::() + .write_batch(&values, Some(&[1, 1, 1]), None) + .unwrap(); + column.close().unwrap(); + } + row_group.close().unwrap(); + writer.close().unwrap(); + + let batches = read_variant(top_level_variant_schema(), &path, vec![1, 2]).await; + assert_id_values(batches[0].column_by_name("id").unwrap()); + assert_top_level_variant(&batches[0]); + } + + #[tokio::test] + async fn test_read_variant_with_name_mapping() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let path = write_top_level_variant_as( + dir, + Field::new("id", DataType::Int32, false), + Field::new("v", DataType::Struct(variant_arrow_fields()), false), + ); + let name_mapping = Arc::new(NameMapping::new(vec![ + MappedField::new(Some(1), vec!["id".to_string()], vec![]), + MappedField::new(Some(2), vec!["v".to_string()], vec![]), + ])); + + let batches = try_read_variant( + top_level_variant_schema(), + &path, + vec![1, 2], + Some(name_mapping), + ) + .await + .unwrap(); + assert_id_values(batches[0].column_by_name("id").unwrap()); + assert_top_level_variant(&batches[0]); + } + + #[tokio::test] + async fn test_read_variant_without_field_ids_is_rejected() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let path = write_top_level_variant_as( + dir, + Field::new("id", DataType::Int32, false), + Field::new("v", DataType::Struct(variant_arrow_fields()), false), + ); + + let err = try_read_variant(top_level_variant_schema(), &path, vec![1, 2], None) + .await + .unwrap_err(); + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::FeatureUnsupported, + "Reading variant columns from Parquet files without field ids is not supported yet" + ) + ); + } + + #[tokio::test] + async fn test_read_variant_nested_in_struct() { + use arrow_array::{Int32Array, StructArray}; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + + // id (int, 1) + nested struct (2) { payload variant (3) }. + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional( + 2, + "nested", + Type::Struct(StructType::new(vec![ + NestedField::optional(3, "payload", Type::Variant(VariantType)).into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let nested_fields = Fields::from(vec![variant_arrow_field("payload", 3)]); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(field_id_meta(1)), + Field::new("nested", DataType::Struct(nested_fields.clone()), false) + .with_metadata(field_id_meta(2)), + ])); + let id = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; + let nested = Arc::new(StructArray::new( + nested_fields, + vec![variant_struct_array()], + None, + )) as ArrayRef; + let path = write_variant_parquet(arrow_schema, vec![id, nested], dir); + + let batches = read_variant(schema, &path, vec![2]).await; + assert_eq!(batches[0].num_rows(), 3); + let nested = batches[0].column_by_name("nested").unwrap().as_struct(); + let DataType::Struct(nested_fields) = nested.data_type() else { + unreachable!() + }; + assert_variant(&nested_fields[0], nested.column(0)); + } + + /// A null `nested` row reads as null with `payload` stored optional, and stored required as + /// iceberg-java writes a required variant, where `payload` has no null buffer of its own. + #[tokio::test] + async fn test_read_variant_nested_in_null_struct_row() { + use arrow_array::StructArray; + use arrow_buffer::NullBuffer; + let nested_nulls = NullBuffer::from(vec![true, false, true]); + for payload_required in [false, true] { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let payload = if payload_required { + NestedField::required(3, "payload", Type::Variant(VariantType)) + } else { + NestedField::optional(3, "payload", Type::Variant(VariantType)) + }; + let nested_type = Type::Struct(StructType::new(vec![payload.into()])); + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(2, "nested", nested_type.clone()).into(), + ]) + .build() + .unwrap(), + ); + let payload_nulls = (!payload_required).then(|| nested_nulls.clone()); + let nested_with = |nested_fields: Fields, variant_fields: Fields| { + Arc::new(StructArray::new( + nested_fields, + vec![Arc::new(StructArray::new( + variant_fields, + variant_struct_array().as_struct().columns().to_vec(), + payload_nulls.clone(), + )) as ArrayRef], + Some(nested_nulls.clone()), + )) as ArrayRef + }; + let file_nested_fields = Fields::from(vec![ + Field::new( + "payload", + DataType::Struct(variant_arrow_fields()), + !payload_required, + ) + .with_metadata(field_id_meta(3)), + ]); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("nested", DataType::Struct(file_nested_fields.clone()), true) + .with_metadata(field_id_meta(2)), + ])); + let path = write_variant_parquet( + arrow_schema, + vec![nested_with(file_nested_fields, variant_arrow_fields())], + dir, + ); + + let batches = read_variant(schema, &path, vec![2]).await; + let DataType::Struct(table_nested_fields) = + crate::arrow::type_to_arrow_type(&nested_type).unwrap() + else { + unreachable!() + }; + let DataType::Struct(table_variant_fields) = + crate::arrow::type_to_arrow_type(&Type::Variant(VariantType)).unwrap() + else { + unreachable!() + }; + assert_eq!( + table_nested_fields[0].extension_type_name(), + Some("arrow.parquet.variant") + ); + let nested = batches[0].column_by_name("nested").unwrap(); + assert_eq!( + nested, + &nested_with(table_nested_fields, table_variant_fields) + ); + // Struct equality skips children under a null parent, so pin `payload`'s own nulls. + assert_eq!(nested.as_struct().column(0).nulls(), payload_nulls.as_ref()); + } + } + + #[tokio::test] + async fn test_read_variant_in_list() { + use arrow_array::{Int32Array, ListArray}; + use arrow_buffer::OffsetBuffer; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + + // id (int, 1) + l list (2, element 3); 3 rows, each list holds one variant. + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional( + 2, + "l", + Type::List(crate::spec::ListType::new( + NestedField::list_element(3, Type::Variant(VariantType), true).into(), + )), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let element_field = Arc::new(variant_arrow_field("element", 3)); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(field_id_meta(1)), + Field::new("l", DataType::List(element_field.clone()), false) + .with_metadata(field_id_meta(2)), + ])); + let id = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; + let list = Arc::new(ListArray::new( + element_field, + OffsetBuffer::new(vec![0, 1, 2, 3].into()), + variant_struct_array(), + None, + )) as ArrayRef; + let path = write_variant_parquet(arrow_schema, vec![id, list], dir); + + let batches = read_variant(schema, &path, vec![2]).await; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 3); + let list = batches[0].column_by_name("l").unwrap().as_list::(); + let DataType::List(element_field) = list.data_type() else { + unreachable!() + }; + // Three single-element lists → the flattened element values are the 3 variants. + assert_variant(element_field, list.values()); + } + + #[tokio::test] + async fn test_read_variant_in_map() { + use arrow_array::{Int32Array, MapArray, StringArray, StructArray}; + use arrow_buffer::OffsetBuffer; + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + + // id (int, 1) + m map (2; key 3, value 4); 3 rows, each map one entry. + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional( + 2, + "m", + Type::Map(crate::spec::MapType::new( + NestedField::map_key_element(3, Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::map_value_element(4, Type::Variant(VariantType), true) + .into(), + )), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let key_field = Field::new("key", DataType::Utf8, false).with_metadata(field_id_meta(3)); + let value_field = variant_arrow_field("value", 4); + let entry_fields = Fields::from(vec![key_field, value_field]); + let entries_field = Arc::new(Field::new( + "key_value", + DataType::Struct(entry_fields.clone()), + false, + )); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(field_id_meta(1)), + Field::new("m", DataType::Map(entries_field.clone(), false), false) + .with_metadata(field_id_meta(2)), + ])); + let id = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; + let entries = StructArray::new( + entry_fields, + vec![ + Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef, + variant_struct_array(), + ], + None, + ); + let map = Arc::new(MapArray::new( + entries_field, + OffsetBuffer::new(vec![0, 1, 2, 3].into()), + entries, + None, + false, + )) as ArrayRef; + let path = write_variant_parquet(arrow_schema, vec![id, map], dir); + + let batches = read_variant(schema, &path, vec![2]).await; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 3); + let map = batches[0].column_by_name("m").unwrap().as_map(); + // The map's value column holds the 3 variants. + assert_variant(&map.entries().fields()[1], map.values()); + } + + #[test] + fn test_projection_mask_reads_one_presence_leaf() { + // `s` is struct<6: b list<7: int>, w struct<14: int>, 8: u struct<16: uint64, 5: a int>, + // 3: c int>, and the table replaced every child. Presence must be `u.a`: the first + // leaf with the lowest repetition level that the type check accepts. `v` holds only + // a map, which is read whole, so it must convert. + let parquet_schema = SchemaDescriptor::new(Arc::new( + parse_message_type( + " + message schema { + required int32 id = 1; + optional group s = 2 { + optional group b (LIST) = 6 { + repeated group list { + optional int32 element = 7; + } + } + optional group w { + optional int32 a2 = 14; + } + optional group u = 8 { + optional int64 bad (INTEGER(64, false)) = 16; + optional int32 a = 5; + } + optional int32 c = 3; + } + optional group v = 4 { + optional group m (MAP) = 10 { + repeated group key_value { + required binary key (STRING) = 11; + optional int64 value (INTEGER(64, false)) = 12; + } + } + } + } + ", + ) + .unwrap(), + )); + let file_arrow = Arc::new(parquet_to_arrow_schema(&parquet_schema, None).unwrap()); + let table = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(9, "n", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + NestedField::optional( + 4, + "v", + Type::Struct(StructType::new(vec![ + NestedField::optional(13, "n", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let mask = |ids: &[i32]| { + ArrowReader::get_arrow_projection_mask(ids, &table, &parquet_schema, &file_arrow, false) + .unwrap() + }; + assert_eq!( + mask(&[1, 2]), + ProjectionMask::leaves(&parquet_schema, vec![0, 4]) + ); + assert_eq!(mask(&[2]), ProjectionMask::leaves(&parquet_schema, vec![4])); + assert_eq!(mask(&[1]), ProjectionMask::leaves(&parquet_schema, vec![0])); + assert_eq!( + mask(&[1, 4]), + ProjectionMask::leaves(&parquet_schema, vec![0]) + ); + + // An `ARROW:schema` hint can wrap leaves in types `filter_leaves` descends through, + // so the walk must count leaves the same way. Presence for `h` is `d` at leaf 4. + // For `g`, `lv2` would win on order, but Iceberg can't convert a list view, so + // presence is `e` at leaf 6. + let parquet_schema = SchemaDescriptor::new(Arc::new( + parse_message_type( + " + message schema { + optional group h = 20 { + optional group fl (LIST) = 21 { + repeated group list { + optional group element = 22 { + optional int32 p = 23; + optional int32 q = 24; + } + } + } + optional group lv (LIST) = 25 { + repeated group list { + optional group element = 26 { + optional int32 p = 27; + optional int32 q = 28; + } + } + } + optional binary d (STRING) = 29; + } + optional group g = 31 { + optional group lv2 (LIST) = 32 { + repeated group list { + optional int32 element = 33; + } + } + optional group e (LIST) = 34 { + repeated group list { + optional int32 element = 35; + } + } + } + } + ", + ) + .unwrap(), + )); + // With a hint, parquet-rs takes field ids from the hint's metadata. + let field = |name: &str, data_type, id: i32| { + Field::new(name, data_type, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + id.to_string(), + )])) + }; + let element = |id: i32| { + let pq = vec![ + field("p", DataType::Int32, id + 1), + field("q", DataType::Int32, id + 2), + ]; + Arc::new(field("element", DataType::Struct(pq.into()), id)) + }; + let int_element = |id: i32| Arc::new(field("element", DataType::Int32, id)); + let hint = ArrowSchema::new(vec![ + field( + "h", + DataType::Struct(Fields::from(vec![ + field("fl", DataType::FixedSizeList(element(22), 2), 21), + field("lv", DataType::ListView(element(26)), 25), + field( + "d", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + 29, + ), + ])), + 20, + ), + field( + "g", + DataType::Struct(Fields::from(vec![ + field("lv2", DataType::ListView(int_element(33)), 32), + field("e", DataType::List(int_element(35)), 34), + ])), + 31, + ), + ]); + let metadata = vec![parquet::file::metadata::KeyValue::new( + parquet::arrow::ARROW_SCHEMA_META_KEY.to_string(), + parquet::arrow::encode_arrow_schema(&hint), + )]; + let file_arrow = + Arc::new(parquet_to_arrow_schema(&parquet_schema, Some(&metadata)).unwrap()); + let replaced = |id, name, child_id| { + NestedField::optional( + id, + name, + Type::Struct(StructType::new(vec![ + NestedField::optional(child_id, "n", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into() + }; + let table = Schema::builder() + .with_fields(vec![replaced(20, "h", 30), replaced(31, "g", 36)]) + .build() + .unwrap(); + assert_eq!( + ArrowReader::get_arrow_projection_mask( + &[20, 31], + &table, + &parquet_schema, + &file_arrow, + false + ) + .unwrap(), + ProjectionMask::leaves(&parquet_schema, vec![4, 6]) + ); + + // `element` is still projected, and the file nests it under a list view. + // Recording `p` would keep the list view in the projected schema and fail + // conversion, so presence for `h` is `d` at leaf 2. + let parquet_schema = SchemaDescriptor::new(Arc::new( + parse_message_type( + " + message schema { + optional group h = 20 { + optional group lv (LIST) = 25 { + repeated group list { + optional group element = 26 { + optional int32 p = 27; + optional int32 q = 28; + } + } + } + optional binary d (STRING) = 29; + } + } + ", + ) + .unwrap(), + )); + let hint = ArrowSchema::new(vec![field( + "h", + DataType::Struct(Fields::from(vec![ + field("lv", DataType::ListView(element(26)), 25), + field("d", DataType::Utf8, 29), + ])), + 20, + )]); + let metadata = vec![parquet::file::metadata::KeyValue::new( + parquet::arrow::ARROW_SCHEMA_META_KEY.to_string(), + parquet::arrow::encode_arrow_schema(&hint), + )]; + let file_arrow = + Arc::new(parquet_to_arrow_schema(&parquet_schema, Some(&metadata)).unwrap()); + let table = Schema::builder() + .with_fields(vec![ + NestedField::optional( + 20, + "h", + Type::Struct(StructType::new(vec![ + NestedField::optional( + 26, + "element", + Type::Struct(StructType::new(vec![ + NestedField::optional(30, "n", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(); + assert_eq!( + ArrowReader::get_arrow_projection_mask( + &[20], + &table, + &parquet_schema, + &file_arrow, + false + ) + .unwrap(), + ProjectionMask::leaves(&parquet_schema, vec![2]) + ); + } + + /// Cases from Java's `ReadFormatModelTests` for #17320, without initial-defaults: + /// every requested child of a struct is missing from the file, so its validity must + /// come from another leaf under it. + #[tokio::test] + async fn test_struct_presence_when_requested_children_missing() { + use serde_json::{Value, json}; + + use crate::arrow::schema_to_arrow_schema; + use crate::spec::{ListType, MapType, NestedFieldRef}; + + fn int(id: i32, name: &str) -> NestedFieldRef { + NestedField::optional(id, name, Type::Primitive(PrimitiveType::Int)).into() + } + fn int_struct(id: i32) -> Type { + Type::Struct(StructType::new(vec![int(id, "x")])) + } + // The table schema re-adds every struct child under a new id. `t` and `u` replace + // their only child, a list and a map, so presence comes from a repeated leaf. + fn schema(table: bool) -> Schema { + let id = |file_id: i32| if table { file_id + 100 } else { file_id }; + let t = if table { + Type::Struct(StructType::new(vec![int(110, "n")])) + } else { + Type::Struct(StructType::new(vec![ + NestedField::optional( + 10, + "tags", + Type::List(ListType::new( + NestedField::list_element( + 11, + Type::Primitive(PrimitiveType::Int), + false, + ) + .into(), + )), + ) + .into(), + ])) + }; + let u = if table { + Type::Struct(StructType::new(vec![int(123, "n")])) + } else { + Type::Struct(StructType::new(vec![ + NestedField::optional( + 23, + "attrs", + Type::Map(MapType::optional( + 24, + Type::Primitive(PrimitiveType::String), + 25, + Type::Primitive(PrimitiveType::Int), + )), + ) + .into(), + ])) + }; + let s = Type::Struct(StructType::new(vec![ + int(id(5), "x"), + NestedField::optional(7, "inner", int_struct(id(8))).into(), + ])); + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "s", s).into(), + NestedField::optional(3, "t", t).into(), + NestedField::optional(6, "u", u).into(), + NestedField::optional( + 4, + "m", + Type::Map(MapType::optional( + 13, + Type::Primitive(PrimitiveType::String), + 14, + int_struct(id(15)), + )), + ) + .into(), + NestedField::optional( + 17, + "l", + Type::List(ListType::new( + NestedField::list_element(18, int_struct(id(19)), false).into(), + )), + ) + .into(), + // Java skips presence for a struct that can never be null. Here the + // struct still needs a column, or it reads as a missing required field. + NestedField::required(21, "r", int_struct(id(22))).into(), + ]) + .build() + .unwrap() + } + fn to_batch(schema: &Schema, rows: Value) -> RecordBatch { + let fields = schema_to_arrow_schema(schema).unwrap().fields().to_vec(); + serde_arrow::to_record_batch(&fields, &rows).unwrap() + } + + let file_batch = to_batch( + &schema(false), + json!([ + {"id": 1, "s": {"x": null, "inner": {"x": 5}}, "t": {"tags": [1]}, "u": {"attrs": {"a": 1}}, + "m": {"k": {"x": 7}, "k2": null}, "l": [{"x": 1}, null], "r": {"x": 1}}, + {"id": 2, "s": {"x": 1, "inner": null}, "t": null, "u": null, + "m": null, "l": [], "r": {"x": 2}}, + {"id": 3, "s": null, "t": {"tags": null}, "u": {"attrs": null}, + "m": {}, "l": null, "r": {"x": null}}, + {"id": 4, "s": {"x": 2, "inner": {"x": null}}, "t": {"tags": [2, 3, 4]}, "u": {"attrs": {}}, + "m": {"k": null}, "l": [null], "r": {"x": 4}}, + ]), + ); + let table_schema = Arc::new(schema(true)); + let expected = to_batch( + &table_schema, + json!([ + {"id": 1, "s": {"x": null, "inner": {"x": null}}, "t": {"n": null}, "u": {"n": null}, + "m": {"k": {"x": null}, "k2": null}, "l": [{"x": null}, null], "r": {"x": null}}, + {"id": 2, "s": {"x": null, "inner": null}, "t": null, "u": null, + "m": null, "l": [], "r": {"x": null}}, + {"id": 3, "s": null, "t": {"n": null}, "u": {"n": null}, + "m": {}, "l": null, "r": {"x": null}}, + {"id": 4, "s": {"x": null, "inner": {"x": null}}, "t": {"n": null}, "u": {"n": null}, + "m": {"k": null}, "l": [null], "r": {"x": null}}, + ]), + ); + + let tmp_dir = TempDir::new().unwrap(); + let file_path = format!("{}/old.parquet", tmp_dir.path().to_str().unwrap()); + let mut writer = + ArrowWriter::try_new(File::create(&file_path).unwrap(), file_batch.schema(), None) + .unwrap(); + writer.write(&file_batch).unwrap(); + writer.close().unwrap(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema) + .with_project_field_ids(vec![1, 2, 3, 6, 4, 17, 21]) + .with_case_sensitive(false) + .build() + .unwrap())])) as FileScanTaskStream; + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + assert_eq!(result, vec![expected]); + } + + #[test] + fn test_coerce_fixed_size_lists_rewrites_nested() { + let item = |ty| Arc::new(Field::new("item", ty, true)); + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("plain", DataType::Int32, true), + Field::new( + "fsl", + DataType::FixedSizeList(item(DataType::Int32), 2), + true, + ), + Field::new( + "nested", + DataType::Struct(Fields::from(vec![Field::new( + "inner", + DataType::FixedSizeList(item(DataType::Int32), 3), + true, + )])), + true, + ), + Field::new( + "list_of_fsl", + DataType::List(Arc::new(Field::new( + "element", + DataType::FixedSizeList(item(DataType::Int32), 2), + true, + ))), + true, + ), + ])); + + let coerced = super::coerce_fixed_size_lists(&schema).expect("should rewrite"); + assert!(matches!( + coerced.field_with_name("fsl").unwrap().data_type(), + DataType::List(_) + )); + let DataType::Struct(fields) = coerced.field_with_name("nested").unwrap().data_type() + else { + panic!("nested should stay Struct"); + }; + assert!(matches!(fields[0].data_type(), DataType::List(_))); + let DataType::List(element) = coerced.field_with_name("list_of_fsl").unwrap().data_type() + else { + panic!("list_of_fsl should stay List"); + }; + assert!(matches!(element.data_type(), DataType::List(_))); + assert!(super::coerce_fixed_size_lists(&coerced).is_none()); + } + + /// parquet-rs can return a LIST as `FixedSizeList` when the file's embedded Arrow schema + /// says so. Rewrite the hint to `List` before reading so the scan never sees FixedSizeList. + #[tokio::test] + async fn test_read_fixed_size_list_column_as_list() { + use arrow_array::{FixedSizeListArray, Int32Array}; + use arrow_buffer::{NullBuffer, OffsetBuffer}; + + use crate::spec::ListType; + + let with_id = |field: Field, id: i32| { + field.with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + id.to_string(), + )])) + }; + + // Row 0: [1, 2]; row 1: null; row 2: [5, 6]. + let values = Arc::new(Int32Array::from(vec![ + Some(1), + Some(2), + None, + None, + Some(5), + Some(6), + ])) as ArrayRef; + let row_nulls = NullBuffer::from(vec![true, false, true]); + let optional_element = Arc::new(with_id(Field::new("element", DataType::Int32, true), 3)); + let required_element = Arc::new(with_id(Field::new("element", DataType::Int32, false), 5)); + let optional_fsl = FixedSizeListArray::new( + Arc::clone(&optional_element), + 2, + Arc::clone(&values), + Some(row_nulls.clone()), + ); + let required_fsl = + FixedSizeListArray::new(Arc::clone(&required_element), 2, values, Some(row_nulls)); + let file_schema = Arc::new(ArrowSchema::new(vec![ + with_id( + Field::new( + "optional", + DataType::FixedSizeList(optional_element, 2), + true, + ), + 2, + ), + with_id( + Field::new( + "required", + DataType::FixedSizeList(required_element, 2), + true, + ), + 4, + ), + with_id(Field::new("id", DataType::Int32, true), 1), + ])); + let batch = RecordBatch::try_new(file_schema.clone(), vec![ + Arc::new(optional_fsl) as ArrayRef, + Arc::new(required_fsl) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + ]) + .unwrap(); + + let tmp_dir = TempDir::new().unwrap(); + let path = format!("{}/fsl.parquet", tmp_dir.path().to_str().unwrap()); + let mut writer = + ArrowWriter::try_new(File::create(&path).unwrap(), file_schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let table_schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional( + 2, + "optional", + Type::List(ListType::new( + NestedField::list_element( + 3, + Type::Primitive(PrimitiveType::Int), + false, + ) + .into(), + )), + ) + .into(), + NestedField::optional( + 4, + "required", + Type::List(ListType::new( + NestedField::list_element(5, Type::Primitive(PrimitiveType::Int), true) + .into(), + )), + ) + .into(), + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema) + .with_project_field_ids(vec![1, 2, 4]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + assert_eq!(result.len(), 1); + let out = &result[0]; + assert_eq!(out.num_rows(), 3); + + let expected_offsets = OffsetBuffer::new(vec![0, 2, 2, 4].into()); + let expected_values = Arc::new(Int32Array::from(vec![1, 2, 5, 6])) as ArrayRef; + let expected_nulls = NullBuffer::from(vec![true, false, true]); + let schema = out.schema(); + for (name, element_nullable) in [("optional", true), ("required", false)] { + let col = schema.index_of(name).unwrap(); + let list = out.column(col).as_list::(); + assert_eq!(list.offsets(), &expected_offsets, "{name} offsets"); + assert_eq!( + list.values().as_ref(), + expected_values.as_ref(), + "{name} values" + ); + assert_eq!(list.nulls(), Some(&expected_nulls), "{name} nulls"); + let DataType::List(element) = schema.field(col).data_type() else { + panic!("{name} should be List"); + }; + assert_eq!( + element.is_nullable(), + element_nullable, + "{name} element nullability" + ); + } + let id_col = schema.index_of("id").unwrap(); + assert_eq!( + out.column(id_col).as_ref(), + &Int32Array::from(vec![1, 2, 3]) as &dyn Array + ); + } + + /// Migrated files have no field IDs. Cover name mapping and fallback IDs with a null row, + /// nested FixedSizeList shapes, and a required top-level element. + #[tokio::test] + async fn test_read_fixed_size_list_without_field_ids() { + use arrow_array::{FixedSizeListArray, Int32Array, ListArray, MapArray, StructArray}; + use arrow_buffer::{NullBuffer, OffsetBuffer}; + + use crate::spec::{ListType, MapType}; + + let item = |nullable| Arc::new(Field::new("item", DataType::Int32, nullable)); + let values = Arc::new(Int32Array::from(vec![ + Some(1), + Some(2), + None, + None, + Some(5), + Some(6), + ])) as ArrayRef; + let row_nulls = NullBuffer::from(vec![true, false, true]); + let fsl = |nullable_element| { + FixedSizeListArray::new( + item(nullable_element), + 2, + Arc::clone(&values), + Some(row_nulls.clone()), + ) + }; + + let top = fsl(false); + let in_struct = StructArray::new( + Fields::from(vec![Field::new( + "inner", + DataType::FixedSizeList(item(true), 2), + true, + )]), + vec![Arc::new(fsl(true)) as ArrayRef], + None, + ); + let map_entries = StructArray::new( + Fields::from(vec![ + Field::new("keys", DataType::Utf8, false), + Field::new("values", DataType::FixedSizeList(item(true), 2), true), + ]), + vec![ + Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef, + Arc::new(FixedSizeListArray::new( + item(true), + 2, + Arc::new(Int32Array::from(vec![ + Some(10), + Some(11), + Some(20), + Some(21), + ])) as ArrayRef, + None, + )) as ArrayRef, + ], + None, + ); + let as_map_value = MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(map_entries.fields().clone()), + false, + )), + OffsetBuffer::new(vec![0, 1, 1, 2].into()), + map_entries, + Some(row_nulls.clone()), + false, + ); + let list_of_fsl = ListArray::new( + Arc::new(Field::new( + "element", + DataType::FixedSizeList(item(true), 2), + true, + )), + OffsetBuffer::new(vec![0, 1, 1, 2].into()), + Arc::new(FixedSizeListArray::new( + item(true), + 2, + Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(5), Some(6)])) as ArrayRef, + None, + )) as ArrayRef, + Some(row_nulls.clone()), + ); + let nested_fsl = FixedSizeListArray::new( + Arc::new(Field::new( + "item", + DataType::FixedSizeList(item(true), 2), + true, + )), + 1, + Arc::new(FixedSizeListArray::new( + item(true), + 2, + Arc::clone(&values), + None, + )) as ArrayRef, + Some(row_nulls), + ); + + let file_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("top", DataType::FixedSizeList(item(false), 2), true), + Field::new( + "s", + DataType::Struct(Fields::from(vec![Field::new( + "inner", + DataType::FixedSizeList(item(true), 2), + true, + )])), + true, + ), + Field::new( + "m", + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("keys", DataType::Utf8, false), + Field::new("values", DataType::FixedSizeList(item(true), 2), true), + ])), + false, + )), + false, + ), + true, + ), + Field::new( + "list_of_fsl", + DataType::List(Arc::new(Field::new( + "element", + DataType::FixedSizeList(item(true), 2), + true, + ))), + true, + ), + Field::new( + "nested_fsl", + DataType::FixedSizeList( + Arc::new(Field::new( + "item", + DataType::FixedSizeList(item(true), 2), + true, + )), + 1, + ), + true, + ), + ])); + let batch = RecordBatch::try_new(file_schema.clone(), vec![ + Arc::new(top) as ArrayRef, + Arc::new(in_struct) as ArrayRef, + Arc::new(as_map_value) as ArrayRef, + Arc::new(list_of_fsl) as ArrayRef, + Arc::new(nested_fsl) as ArrayRef, + ]) + .unwrap(); + + let tmp_dir = TempDir::new().unwrap(); + let path = format!("{}/fsl_no_ids.parquet", tmp_dir.path().to_str().unwrap()); + let mut writer = + ArrowWriter::try_new(File::create(&path).unwrap(), file_schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + let file_size = std::fs::metadata(&path).unwrap().len(); + + let int_ty = Type::Primitive(PrimitiveType::Int); + let list_of = |id, required| { + Type::List(ListType::new( + NestedField::list_element(id, int_ty.clone(), required).into(), + )) + }; + let table_schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "top", list_of(6, true)).into(), + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(7, "inner", list_of(8, false)).into(), + ])), + ) + .into(), + NestedField::optional( + 3, + "m", + Type::Map(MapType::optional( + 9, + Type::Primitive(PrimitiveType::String), + 10, + list_of(11, false), + )), + ) + .into(), + NestedField::optional( + 4, + "list_of_fsl", + Type::List(ListType::new( + NestedField::list_element(12, list_of(13, false), false).into(), + )), + ) + .into(), + NestedField::optional( + 5, + "nested_fsl", + Type::List(ListType::new( + NestedField::list_element(14, list_of(15, false), false).into(), + )), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let name_mapping = Arc::new(NameMapping::new(vec![ + MappedField::new(Some(1), vec!["top".to_string()], vec![MappedField::new( + Some(6), + vec!["element".to_string()], + vec![], + )]), + MappedField::new(Some(2), vec!["s".to_string()], vec![MappedField::new( + Some(7), + vec!["inner".to_string()], + vec![MappedField::new( + Some(8), + vec!["element".to_string()], + vec![], + )], + )]), + MappedField::new(Some(3), vec!["m".to_string()], vec![ + MappedField::new(Some(9), vec!["key".to_string()], vec![]), + MappedField::new(Some(10), vec!["value".to_string()], vec![MappedField::new( + Some(11), + vec!["element".to_string()], + vec![], + )]), + ]), + MappedField::new(Some(4), vec!["list_of_fsl".to_string()], vec![ + MappedField::new(Some(12), vec!["element".to_string()], vec![ + MappedField::new(Some(13), vec!["element".to_string()], vec![]), + ]), + ]), + MappedField::new(Some(5), vec!["nested_fsl".to_string()], vec![ + MappedField::new(Some(14), vec!["element".to_string()], vec![ + MappedField::new(Some(15), vec!["element".to_string()], vec![]), + ]), + ]), + ])); + + let assert_top_list = |out: &RecordBatch| { + assert_eq!(out.num_rows(), 3); + let top = out + .column(out.schema().index_of("top").unwrap()) + .as_list::(); + assert_eq!( + top.nulls(), + Some(&NullBuffer::from(vec![true, false, true])) + ); + assert_eq!( + top.values().as_ref(), + &Int32Array::from(vec![1, 2, 5, 6]) as &dyn Array + ); + let schema = out.schema(); + for name in ["s", "m", "list_of_fsl", "nested_fsl"] { + let field = schema.field_with_name(name).unwrap(); + assert!( + !matches!(field.data_type(), DataType::FixedSizeList(_, _)), + "{name} should not stay FixedSizeList: {:?}", + field.data_type() + ); + } + }; + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let mapped_tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(Arc::clone(&table_schema)) + .with_project_field_ids(vec![1, 2, 3, 4, 5]) + .with_case_sensitive(false) + .with_name_mapping(Some(Arc::clone(&name_mapping))) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + let mapped = reader + .read(mapped_tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + assert_eq!(mapped.len(), 1); + assert_top_list(&mapped[0]); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build(); + let fallback_tasks = Box::pin(futures::stream::iter( + vec![Ok(FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(table_schema) + .with_project_field_ids(vec![1, 2, 3, 4, 5]) + .with_case_sensitive(false) + .build() + .unwrap())] + .into_iter(), + )) as FileScanTaskStream; + let fallback = reader + .read(fallback_tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + assert_eq!(fallback.len(), 1); + assert_top_list(&fallback[0]); + } +} diff --git a/crates/iceberg/src/arrow/reader/row_filter.rs b/crates/iceberg/src/arrow/reader/row_filter.rs new file mode 100644 index 0000000000..5f1fe5efed --- /dev/null +++ b/crates/iceberg/src/arrow/reader/row_filter.rs @@ -0,0 +1,2070 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Predicate-driven row filtering for `ArrowReader`: constructing Arrow `RowFilter`s +//! from Iceberg predicates, row-group selection based on column statistics, and +//! row-selection via the Parquet page index. Also includes byte-range row-group +//! filtering used for file splitting. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use parquet::arrow::ProjectionMask; +use parquet::arrow::arrow_reader::{ArrowPredicateFn, RowFilter, RowSelection}; +use parquet::file::metadata::ParquetMetaData; +use parquet::schema::types::SchemaDescriptor; + +use super::{ArrowReader, PredicateConverter}; +use crate::error::Result; +use crate::expr::BoundPredicate; +use crate::expr::visitors::bound_predicate_visitor::visit; +use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator; +use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator; +use crate::spec::Schema; +use crate::{Error, ErrorKind}; + +impl ArrowReader { + pub(super) fn get_row_filter( + predicates: &BoundPredicate, + parquet_schema: &SchemaDescriptor, + iceberg_field_ids: &HashSet, + field_id_map: &HashMap, + ) -> Result { + // Collect Parquet column indices from field ids. + // If the field id is not found in Parquet schema, it will be ignored due to schema evolution. + let mut column_indices = iceberg_field_ids + .iter() + .filter_map(|field_id| field_id_map.get(field_id).cloned()) + .collect::>(); + column_indices.sort(); + + // The converter that converts `BoundPredicates` to `ArrowPredicates` + let mut converter = PredicateConverter { + parquet_schema, + column_map: field_id_map, + column_indices: &column_indices, + }; + + // After collecting required leaf column indices used in the predicate, + // creates the projection mask for the Arrow predicates. + let projection_mask = ProjectionMask::leaves(parquet_schema, column_indices.clone()); + let predicate_func = visit(&mut converter, predicates)?; + let arrow_predicate = ArrowPredicateFn::new(projection_mask, predicate_func); + Ok(RowFilter::new(vec![Box::new(arrow_predicate)])) + } + + pub(super) fn get_selected_row_group_indices( + predicate: &BoundPredicate, + parquet_metadata: &Arc, + field_id_map: &HashMap, + snapshot_schema: &Schema, + candidate_row_groups: &[usize], + ) -> Result> { + let row_groups_metadata = parquet_metadata.row_groups(); + let mut results = Vec::with_capacity(candidate_row_groups.len()); + + for &idx in candidate_row_groups { + let row_group_metadata = row_groups_metadata.get(idx).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "row group index {idx} out of range ({} row groups)", + row_groups_metadata.len() + ), + ) + })?; + if RowGroupMetricsEvaluator::eval( + predicate, + row_group_metadata, + field_id_map, + snapshot_schema, + )? { + results.push(idx); + } + } + + Ok(results) + } + + /// Computes a [`RowSelection`] by evaluating the filter predicate against + /// the Parquet page index (column index + offset index). + /// + /// Returns `Ok(None)` when the Parquet file lacks column or offset index + /// metadata (common with older files written before page indexes became + /// standard). In that case page-level pruning is simply skipped; row-group + /// filtering and the Arrow row filter still apply the predicate. + /// + /// `Ok(Some(empty))` case means that all rows were filtered by the predicate - returning zero rows + pub(super) fn get_row_selection_for_filter_predicate( + predicate: &BoundPredicate, + parquet_metadata: &Arc, + selected_row_groups: &Option>, + field_id_map: &HashMap, + snapshot_schema: &Schema, + ) -> Result> { + let Some(page_index) = parquet_metadata.page_index() else { + tracing::debug!("PageIndex was absent while reading this file"); + return Ok(None); + }; + + if !page_index.has_column_indexes() { + tracing::debug!("ColumnIndex was absent while reading this file"); + return Ok(None); + } + + if !page_index.has_offset_indexes() { + tracing::debug!("OffsetIndex was absent while reading this file"); + return Ok(None); + } + + // If all row groups were filtered out, return an empty RowSelection (select no rows) + // + if let Some(selected_row_groups) = selected_row_groups + && selected_row_groups.is_empty() + { + return Ok(Some(RowSelection::from(Vec::new()))); + } + + let mut selected_row_groups_idx = 0; + + let mut results = Vec::new(); + for (idx, row_group_metadata) in parquet_metadata.row_groups().iter().enumerate() { + if let Some(selected_row_groups) = selected_row_groups { + // skip row groups that aren't present in selected_row_groups + if idx == selected_row_groups[selected_row_groups_idx] { + selected_row_groups_idx += 1; + } else { + continue; + } + } + + let selections_for_page = PageIndexEvaluator::eval( + predicate, + &parquet_metadata.page_index_for_row_group(idx), + row_group_metadata, + field_id_map, + snapshot_schema, + )?; + + results.push(selections_for_page); + + if let Some(selected_row_groups) = selected_row_groups + && selected_row_groups_idx == selected_row_groups.len() + { + break; + } + } + + Ok(Some( + results.into_iter().flatten().collect::>().into(), + )) + } + + /// Filters row groups by byte range to support Iceberg's file splitting. + /// + /// Engines split a data file into multiple scan tasks, each covering a byte range + /// `[start, start+length)`. Normally Iceberg planning aligns these splits to row group + /// boundaries using the data file's `split_offsets` metadata, so a task's range never + /// bisects a row group. But when `split_offsets` is missing (e.g. a manually written or + /// non-conforming file), planning falls back to tiling the file at the requested split + /// size, and a task's range can land in the middle of a row group. + /// + /// A row group must be read by exactly one task, otherwise its rows are duplicated. We + /// assign ownership by the row group's midpoint: a task owns a row group only if its range + /// contains that midpoint. Because the tasks tile the file contiguously and disjointly, + /// each midpoint falls in exactly one task. This matches parquet-mr's `BlockMetaData` + /// midpoint semantics. For a whole-file task (`start=0, length=fileSize`, as iceberg-rust's + /// own planner emits) every midpoint lies in range, so all row groups are selected. + pub(super) fn filter_row_groups_by_byte_range( + parquet_metadata: &Arc, + start: u64, + length: u64, + ) -> Result> { + let row_groups = parquet_metadata.row_groups(); + let mut selected = Vec::new(); + let end = start + length; + + // Row groups are stored sequentially after the 4-byte magic header. + let mut current_byte_offset = 4u64; + + for (idx, row_group) in row_groups.iter().enumerate() { + let row_group_size = row_group.compressed_size() as u64; + let row_group_midpoint = current_byte_offset + row_group_size / 2; + + // Half-open ownership: a midpoint on a task boundary belongs to the upper task, + // so exactly one task ever claims a given row group. + if start <= row_group_midpoint && row_group_midpoint < end { + selected.push(idx); + } + + current_byte_offset += row_group_size; + } + + Ok(selected) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::fs::File; + use std::sync::Arc; + + use arrow_array::cast::AsArray; + use arrow_array::{ + ArrayRef, Decimal128Array, Float32Array, Int32Array, Int64Array, LargeStringArray, + RecordBatch, StringArray, + }; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use futures::TryStreamExt; + use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; + use parquet::basic::Compression; + use parquet::file::metadata::page_index::PageIndexProvider; + use parquet::file::metadata::{FileMetaData, ParquetMetaData, ParquetMetaDataBuilder}; + use parquet::file::properties::{EnabledStatistics, WriterProperties}; + use parquet::schema::parser::parse_message_type; + use parquet::schema::types::SchemaDescriptor; + use tempfile::TempDir; + + use crate::Runtime; + use crate::arrow::{ + ArrowReader, ArrowReaderBuilder, ArrowReaderOptions as ReaderOptions, ParquetReadOptions, + }; + use crate::expr::{Bind, BoundPredicate, Predicate, Reference}; + use crate::io::FileIO; + use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream}; + use crate::spec::{ + DataContentType, DataFileFormat, Datum, NestedField, PrimitiveType, Schema, SchemaRef, Type, + }; + + async fn test_perform_read( + predicate: Predicate, + schema: SchemaRef, + table_location: String, + reader: ArrowReader, + ) -> Vec> { + let tasks = { + let task = FileScanTask::builder() + .with_file_size_in_bytes( + std::fs::metadata(format!("{table_location}/1.parquet")) + .unwrap() + .len(), + ) + .with_start(0) + .with_length(0) + .with_data_file_path(format!("{table_location}/1.parquet")) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_predicate(Some(predicate.bind(schema, true).unwrap())) + .with_case_sensitive(false) + .build() + .unwrap(); + Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream + }; + + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + result[0].columns()[0] + .as_string_opt::() + .unwrap() + .iter() + .map(|v| v.map(ToOwned::to_owned)) + .collect::>() + } + + fn setup_kleene_logic( + data_for_col_a: Vec>, + col_a_type: DataType, + ) -> (FileIO, SchemaRef, String, TempDir) { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "a", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("a", col_a_type.clone(), true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + + let file_io = FileIO::new_with_fs(); + + let col = match col_a_type { + DataType::Utf8 => Arc::new(StringArray::from(data_for_col_a)) as ArrayRef, + DataType::LargeUtf8 => Arc::new(LargeStringArray::from(data_for_col_a)) as ArrayRef, + _ => panic!("unexpected col_a_type"), + }; + + let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![col]).unwrap(); + + // Write the Parquet files + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + let file = File::create(format!("{table_location}/1.parquet")).unwrap(); + let mut writer = + ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap(); + + writer.write(&to_write).expect("Writing batch"); + + // writer must be closed to write footer + writer.close().unwrap(); + + (file_io, schema, table_location, tmp_dir) + } + + #[tokio::test] + async fn test_kleene_logic_or_behaviour() { + // a IS NULL OR a = 'foo' + let predicate = Reference::new("a") + .is_null() + .or(Reference::new("a").equal_to(Datum::string("foo"))); + + // Table data: [NULL, "foo", "bar"] + let data_for_col_a = vec![None, Some("foo".to_string()), Some("bar".to_string())]; + + // Expected: [NULL, "foo"]. + let expected = vec![None, Some("foo".to_string())]; + + let (file_io, schema, table_location, _temp_dir) = + setup_kleene_logic(data_for_col_a, DataType::Utf8); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let result_data = test_perform_read(predicate, schema, table_location, reader).await; + + assert_eq!(result_data, expected); + } + + #[tokio::test] + async fn test_kleene_logic_and_behaviour() { + // a IS NOT NULL AND a != 'foo' + let predicate = Reference::new("a") + .is_not_null() + .and(Reference::new("a").not_equal_to(Datum::string("foo"))); + + // Table data: [NULL, "foo", "bar"] + let data_for_col_a = vec![None, Some("foo".to_string()), Some("bar".to_string())]; + + // Expected: ["bar"]. + let expected = vec![Some("bar".to_string())]; + + let (file_io, schema, table_location, _temp_dir) = + setup_kleene_logic(data_for_col_a, DataType::Utf8); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + let result_data = test_perform_read(predicate, schema, table_location, reader).await; + + assert_eq!(result_data, expected); + } + + #[tokio::test] + async fn test_predicate_cast_literal() { + let predicates = vec![ + // a == 'foo' + (Reference::new("a").equal_to(Datum::string("foo")), vec![ + Some("foo".to_string()), + ]), + // a != 'foo' + ( + Reference::new("a").not_equal_to(Datum::string("foo")), + vec![Some("bar".to_string())], + ), + // STARTS_WITH(a, 'foo') + (Reference::new("a").starts_with(Datum::string("f")), vec![ + Some("foo".to_string()), + ]), + // NOT STARTS_WITH(a, 'foo') + ( + Reference::new("a").not_starts_with(Datum::string("f")), + vec![Some("bar".to_string())], + ), + // a < 'foo' + (Reference::new("a").less_than(Datum::string("foo")), vec![ + Some("bar".to_string()), + ]), + // a <= 'foo' + ( + Reference::new("a").less_than_or_equal_to(Datum::string("foo")), + vec![Some("foo".to_string()), Some("bar".to_string())], + ), + // a > 'foo' + ( + Reference::new("a").greater_than(Datum::string("bar")), + vec![Some("foo".to_string())], + ), + // a >= 'foo' + ( + Reference::new("a").greater_than_or_equal_to(Datum::string("foo")), + vec![Some("foo".to_string())], + ), + // a IN ('foo', 'bar') + ( + Reference::new("a").is_in([Datum::string("foo"), Datum::string("baz")]), + vec![Some("foo".to_string())], + ), + // a NOT IN ('foo', 'bar') + ( + Reference::new("a").is_not_in([Datum::string("foo"), Datum::string("baz")]), + vec![Some("bar".to_string())], + ), + ]; + + // Table data: ["foo", "bar"] + let data_for_col_a = vec![Some("foo".to_string()), Some("bar".to_string())]; + + let (file_io, schema, table_location, _temp_dir) = + setup_kleene_logic(data_for_col_a, DataType::LargeUtf8); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + for (predicate, expected) in predicates { + println!("testing predicate {predicate}"); + let result_data = test_perform_read( + predicate.clone(), + schema.clone(), + table_location.clone(), + reader.clone(), + ) + .await; + + assert_eq!(result_data, expected, "predicate={predicate}"); + } + } + + /// Verifies that file splits respect byte ranges and only read specific row groups. + #[tokio::test] + async fn test_file_splits_respect_byte_ranges() { + use arrow_array::Int32Array; + use parquet::file::reader::{FileReader, SerializedFileReader}; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_path = format!("{table_location}/multi_row_group.parquet"); + + // Force each batch into its own row group for testing byte range filtering. + let batch1 = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from( + (0..100).collect::>(), + ))]) + .unwrap(); + let batch2 = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from( + (100..200).collect::>(), + ))]) + .unwrap(); + let batch3 = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from( + (200..300).collect::>(), + ))]) + .unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(100)) + .build(); + + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + writer.write(&batch1).expect("Writing batch 1"); + writer.write(&batch2).expect("Writing batch 2"); + writer.write(&batch3).expect("Writing batch 3"); + writer.close().unwrap(); + + // Read the file metadata to get row group byte positions + let file = File::open(&file_path).unwrap(); + let reader = SerializedFileReader::new(file).unwrap(); + let metadata = reader.metadata(); + + println!("File has {} row groups", metadata.num_row_groups()); + assert_eq!(metadata.num_row_groups(), 3, "Expected 3 row groups"); + + // Get byte positions for each row group + let row_group_0 = metadata.row_group(0); + let row_group_1 = metadata.row_group(1); + let row_group_2 = metadata.row_group(2); + + let rg0_start = 4u64; // Parquet files start with 4-byte magic "PAR1" + let rg1_start = rg0_start + row_group_0.compressed_size() as u64; + let rg2_start = rg1_start + row_group_1.compressed_size() as u64; + let file_end = rg2_start + row_group_2.compressed_size() as u64; + + println!( + "Row group 0: {} rows, starts at byte {}, {} bytes compressed", + row_group_0.num_rows(), + rg0_start, + row_group_0.compressed_size() + ); + println!( + "Row group 1: {} rows, starts at byte {}, {} bytes compressed", + row_group_1.num_rows(), + rg1_start, + row_group_1.compressed_size() + ); + println!( + "Row group 2: {} rows, starts at byte {}, {} bytes compressed", + row_group_2.num_rows(), + rg2_start, + row_group_2.compressed_size() + ); + + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + // Task 1: read only the first row group + let task1 = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(rg0_start) + .with_length(row_group_0.compressed_size() as u64) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_record_count(Some(100)) + .with_case_sensitive(false) + .build() + .unwrap(); + + // Task 2: read the second and third row groups + let task2 = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(rg1_start) + .with_length(file_end - rg1_start) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_record_count(Some(200)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks1 = Box::pin(futures::stream::iter(vec![Ok(task1)])) as FileScanTaskStream; + let result1 = reader + .clone() + .read(tasks1) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let total_rows_task1: usize = result1.iter().map(|b| b.num_rows()).sum(); + println!( + "Task 1 (bytes {}-{}) returned {} rows", + rg0_start, + rg0_start + row_group_0.compressed_size() as u64, + total_rows_task1 + ); + + let tasks2 = Box::pin(futures::stream::iter(vec![Ok(task2)])) as FileScanTaskStream; + let result2 = reader + .read(tasks2) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let total_rows_task2: usize = result2.iter().map(|b| b.num_rows()).sum(); + println!("Task 2 (bytes {rg1_start}-{file_end}) returned {total_rows_task2} rows"); + + assert_eq!( + total_rows_task1, 100, + "Task 1 should read only the first row group (100 rows), but got {total_rows_task1} rows" + ); + + assert_eq!( + total_rows_task2, 200, + "Task 2 should read only the second+third row groups (200 rows), but got {total_rows_task2} rows" + ); + + // Verify the actual data values are correct (not just the row count) + if total_rows_task1 > 0 { + let first_batch = &result1[0]; + let id_col = first_batch + .column(0) + .as_primitive::(); + let first_val = id_col.value(0); + let last_val = id_col.value(id_col.len() - 1); + println!("Task 1 data range: {first_val} to {last_val}"); + + assert_eq!(first_val, 0, "Task 1 should start with id=0"); + assert_eq!(last_val, 99, "Task 1 should end with id=99"); + } + + if total_rows_task2 > 0 { + let first_batch = &result2[0]; + let id_col = first_batch + .column(0) + .as_primitive::(); + let first_val = id_col.value(0); + println!("Task 2 first value: {first_val}"); + + assert_eq!(first_val, 100, "Task 2 should start with id=100, not id=0"); + } + + // Combining byte-range and predicate row-group filtering should narrow the + // split's candidates instead of evaluating and intersecting independent lists. + // This task owns row groups 1 and 2, while the predicate can match row groups + // 0 and 2. Only row group 2 should be read. + let predicate = Reference::new("id") + .less_than(Datum::int(100)) + .or(Reference::new("id").greater_than_or_equal_to(Datum::int(200))) + .bind(schema.clone(), true) + .unwrap(); + + // Test the pruning decision directly, not just the final filtered rows. + let parquet_metadata = Arc::new((*metadata).clone()); + let field_id_map = HashMap::from([(1, 0)]); + for (candidates, expected) in [(vec![1, 2], vec![2]), (vec![0, 1], vec![0])] { + let selected = ArrowReader::get_selected_row_group_indices( + &predicate, + &parquet_metadata, + &field_id_map, + schema.as_ref(), + &candidates, + ) + .unwrap(); + assert_eq!(selected, expected); + } + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(rg1_start) + .with_length(file_end - rg1_start) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_predicate(Some(predicate)) + .with_record_count(Some(200)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_group_filtering_enabled(true) + .build(), + ) + .build(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let mut ids = Vec::new(); + for batch in &result { + let id_col = batch + .column(0) + .as_primitive::(); + ids.extend((0..id_col.len()).map(|index| id_col.value(index))); + } + + assert_eq!(ids, (200..300).collect::>()); + } + + /// A single data file split into multiple sub-row-group byte ranges (as Spark/Iceberg + /// planning produces when split-size is smaller than a row group) must still yield each + /// row exactly once. The previous overlap-based selection let every split whose byte range + /// touched a row group read it, duplicating rows; ownership by midpoint reads each row group + /// from exactly one split. + #[tokio::test] + async fn test_sub_row_group_splits_do_not_duplicate_rows() { + use arrow_array::Int32Array; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_path = format!("{table_location}/sub_split.parquet"); + + // Three row groups of 100 rows each (ids 0..300). + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(100)) + .build(); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + for chunk in [0..100, 100..200, 200..300] { + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from(chunk.collect::>()), + )]) + .unwrap(); + writer.write(&batch).expect("Writing batch"); + } + writer.close().unwrap(); + + let file_size = std::fs::metadata(&file_path).unwrap().len(); + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + // Tile the whole file into 64-byte splits, mirroring Spark's split-size planning, and + // read every split. A 64-byte split is far smaller than a row group, so each row group + // is touched by several splits but must be owned (read) by exactly one. + let mut ids = Vec::new(); + let split_size = 64u64; + let mut start = 0u64; + while start < file_size { + let length = split_size.min(file_size - start); + let task = FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(start) + .with_length(length) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches = reader + .clone() + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + for batch in &batches { + let col = batch + .column(0) + .as_primitive::(); + ids.extend(col.values().iter().copied()); + } + + start += length; + } + + ids.sort_unstable(); + assert_eq!( + ids, + (0..300).collect::>(), + "each row must be read exactly once across all splits, got {} rows", + ids.len() + ); + } + + /// When a split boundary lands exactly on a row group's midpoint, half-open ownership + /// (`start <= midpoint < end`) must hand that row group to the upper split only: the lower + /// split ends at the midpoint and so excludes it, the upper split starts at the midpoint and + /// so claims it. Two splits meeting exactly at the middle row group's midpoint must therefore + /// read every row once, with the middle row group going to the upper split. + #[tokio::test] + async fn test_split_boundary_on_row_group_midpoint() { + use arrow_array::Int32Array; + use parquet::file::reader::{FileReader, SerializedFileReader}; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_path = format!("{table_location}/midpoint.parquet"); + + // Three row groups of 100 rows each (ids 0..300). + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(100)) + .build(); + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + for chunk in [0..100, 100..200, 200..300] { + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new( + Int32Array::from(chunk.collect::>()), + )]) + .unwrap(); + writer.write(&batch).expect("Writing batch"); + } + writer.close().unwrap(); + + // Locate the middle row group's exact midpoint. Row groups are stored back to back after + // the 4-byte magic header. + let metadata = SerializedFileReader::new(File::open(&file_path).unwrap()) + .unwrap() + .metadata() + .clone(); + assert_eq!(metadata.num_row_groups(), 3); + let rg1_start = 4 + metadata.row_group(0).compressed_size() as u64; + let rg1_size = metadata.row_group(1).compressed_size() as u64; + let rg1_midpoint = rg1_start + rg1_size / 2; + let file_end = rg1_start + rg1_size + metadata.row_group(2).compressed_size() as u64; + + let file_size = std::fs::metadata(&file_path).unwrap().len(); + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + + // Two splits meeting exactly at rg1's midpoint. The lower split ends there, the upper + // starts there; the middle row group must fall to the upper split alone. + let mut per_split = Vec::new(); + for (start, end) in [(0, rg1_midpoint), (rg1_midpoint, file_end)] { + let task = FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(start) + .with_length(end - start) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches = reader + .clone() + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let mut ids = Vec::new(); + for batch in &batches { + let col = batch + .column(0) + .as_primitive::(); + ids.extend(col.values().iter().copied()); + } + per_split.push(ids); + } + + assert_eq!( + per_split[0], + (0..100).collect::>(), + "lower split, ending at rg1's midpoint, must read only rg0" + ); + assert_eq!( + per_split[1], + (100..300).collect::>(), + "upper split, starting at rg1's midpoint, must read rg1 and rg2" + ); + } + + fn int_schema() -> SchemaRef { + Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![Arc::new(NestedField::required( + 1, + "x", + Type::Primitive(PrimitiveType::Int), + ))]) + .build() + .unwrap(), + ) + } + + fn simple_predicate(schema: SchemaRef) -> BoundPredicate { + Reference::new("x") + .greater_than(Datum::int(0)) + .bind(schema.clone(), false) + .unwrap() + } + + fn field_id_map() -> HashMap { + let mut m = HashMap::new(); + m.insert(1_i32, 0_usize); + m + } + + fn metadata_no_page_indexes() -> Arc { + let msg_type = parse_message_type("message schema { REQUIRED INT32 x; }").unwrap(); + let schema_desc = Arc::new(SchemaDescriptor::new(Arc::new(msg_type))); + let file_meta = FileMetaData::new(2, 0, None, None, schema_desc.clone(), None); + Arc::new(ParquetMetaDataBuilder::new(file_meta).build()) + } + + #[derive(Debug)] + struct TestPageIndexProvider { + has_column_indexes: bool, + has_offset_indexes: bool, + } + + impl PageIndexProvider for TestPageIndexProvider { + fn has_offset_indexes(&self) -> bool { + self.has_offset_indexes + } + + fn has_column_indexes(&self) -> bool { + self.has_column_indexes + } + + fn column_index( + &self, + _: usize, + _: usize, + ) -> Option<&parquet::file::page_index::column_index::ColumnIndexMetaData> { + None + } + + fn offset_index( + &self, + _: usize, + _: usize, + ) -> Option<&parquet::file::page_index::offset_index::OffsetIndexMetaData> { + None + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + } + + fn metadata_column_index_only() -> Arc { + let msg_type = parse_message_type("message schema { REQUIRED INT32 x; }").unwrap(); + let schema_desc = Arc::new(SchemaDescriptor::new(Arc::new(msg_type))); + let file_meta = FileMetaData::new(2, 0, None, None, schema_desc, None); + Arc::new( + ParquetMetaDataBuilder::new(file_meta) + .set_page_index(Some(Arc::new(TestPageIndexProvider { + has_column_indexes: true, + has_offset_indexes: false, + }))) + .build(), + ) + } + + fn metadata_with_both_indexes() -> Arc { + let msg_type = parse_message_type("message schema { REQUIRED INT32 x; }").unwrap(); + let schema_desc = Arc::new(SchemaDescriptor::new(Arc::new(msg_type))); + let file_meta = FileMetaData::new(2, 0, None, None, schema_desc, None); + Arc::new( + ParquetMetaDataBuilder::new(file_meta) + .set_page_index(Some(Arc::new(TestPageIndexProvider { + has_column_indexes: true, + has_offset_indexes: true, + }))) + .build(), + ) + } + + /// A `FileScanTask` built or deserialized outside `TableScan` carries an + /// un-normalized predicate, so a raw `Not(...)` can reach the reader. + #[tokio::test] + async fn test_not_predicate_read_returns_matching_rows() { + use arrow_array::Int32Array; + + let schema = int_schema(); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("x", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_path = format!("{table_location}/not_pred.parquet"); + + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), None).unwrap(); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from( + (0..=10).collect::>(), + ))]) + .unwrap(); + writer.write(&batch).expect("Writing batch"); + writer.close().unwrap(); + + // NOT(x > 5) == x <= 5, bound without rewriting, as a non-TableScan caller + // would supply it. + let predicate = (!Reference::new("x").greater_than(Datum::int(5))) + .bind(schema.clone(), false) + .unwrap(); + assert!( + matches!(predicate, BoundPredicate::Not(_)), + "test must drive a raw Not(...) node" + ); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_group_filtering_enabled(true) + .build(), + ) + .build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_predicate(Some(predicate)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let mut values = Vec::new(); + for batch in &batches { + let col = batch + .column(0) + .as_primitive::(); + values.extend(col.values().iter().copied()); + } + + assert_eq!(values, (0..=5).collect::>()); + } + + #[tokio::test] + async fn test_not_predicate_read_through_page_index() { + use arrow_array::Int32Array; + use parquet::file::reader::{FileReader, SerializedFileReader}; + use parquet::file::serialized_reader::ReadOptionsBuilder; + + let schema = int_schema(); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("x", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().to_str().unwrap().to_string(); + let file_path = format!("{table_location}/not_pred_pages.parquet"); + + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), None).unwrap(); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from( + (0..100).collect::>(), + ))]) + .unwrap(); + writer.write(&batch).expect("Writing batch"); + writer.close().unwrap(); + + { + let f = File::open(&file_path).unwrap(); + let opts = ReadOptionsBuilder::new().with_page_index().build(); + let rdr = SerializedFileReader::new_with_options(f, opts).unwrap(); + assert!( + rdr.metadata() + .page_index() + .is_some_and(|page_index| page_index.has_column_indexes()), + "fixture must carry a column index so the page-index path runs" + ); + } + + // NOT(x >= 50) == x < 50, as a non-TableScan caller would supply it. + let predicate = (!Reference::new("x").greater_than_or_equal_to(Datum::int(50))) + .bind(schema.clone(), false) + .unwrap(); + assert!(matches!(predicate, BoundPredicate::Not(_))); + + let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_group_filtering_enabled(false) + .with_row_selection_enabled(true) + .build(), + ) + .build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(vec![1]) + .with_predicate(Some(predicate)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches = reader + .read(tasks) + .unwrap() + .stream() + .try_collect::>() + .await + .unwrap(); + + let mut values = Vec::new(); + for batch in &batches { + let col = batch + .column(0) + .as_primitive::(); + values.extend(col.values().iter().copied()); + } + + assert_eq!(values, (0..50).collect::>()); + } + + #[test] + fn test_row_group_pruning_rejects_invalid_candidate() { + let schema = int_schema(); + let predicate = simple_predicate(schema.clone()); + let metadata = metadata_no_page_indexes(); + + let err = ArrowReader::get_selected_row_group_indices( + &predicate, + &metadata, + &field_id_map(), + schema.as_ref(), + &[0], + ) + .unwrap_err(); + + assert_eq!(err.kind(), crate::ErrorKind::DataInvalid); + } + + /// Testing suite regarding: https://github.com/apache/iceberg-rust/issues/2452 + /// Testing when: both indices are absent, some present, both present + #[test] + fn test_absent_column_index_returns_ok_none() { + let schema = int_schema(); + let predicate = simple_predicate(schema.clone()); + let metadata = metadata_no_page_indexes(); + let field_id_map = field_id_map(); + + let result = ArrowReader::get_row_selection_for_filter_predicate( + &predicate, + &metadata, + &None, + &field_id_map, + schema.as_ref(), + ); + + assert!( + result.is_ok(), + "expected Ok(_), got Err: {:?}", + result.unwrap_err() + ); + assert!( + result.unwrap().is_none(), + "expected Ok(None) when column index is absent" + ); + } + + #[test] + fn test_absent_offset_index_returns_ok_none() { + let schema = int_schema(); + let predicate = simple_predicate(schema.clone()); + let metadata = metadata_column_index_only(); + let field_id_map = field_id_map(); + + let result = ArrowReader::get_row_selection_for_filter_predicate( + &predicate, + &metadata, + &None, + &field_id_map, + schema.as_ref(), + ); + + assert!( + result.is_ok(), + "expected Ok(_), got Err: {:?}", + result.unwrap_err() + ); + assert!( + result.unwrap().is_none(), + "expected Ok(None) when offset index is absent" + ); + } + + #[test] + fn test_absent_column_index_with_selected_row_groups_returns_ok_none() { + let schema = int_schema(); + let predicate = simple_predicate(schema.clone()); + let metadata = metadata_no_page_indexes(); + let field_id_map = field_id_map(); + let selected = Some(vec![0usize, 1]); + + let result = ArrowReader::get_row_selection_for_filter_predicate( + &predicate, + &metadata, + &selected, + &field_id_map, + schema.as_ref(), + ); + + assert!(result.is_ok()); + assert!( + result.unwrap().is_none(), + "absent column index must short-circuit before selected_row_groups is inspected" + ); + } + + #[test] + fn test_absent_offset_index_with_selected_row_groups_returns_ok_none() { + let schema = int_schema(); + let predicate = simple_predicate(schema.clone()); + let metadata = metadata_column_index_only(); + let field_id_map = field_id_map(); + let selected = Some(vec![0usize]); + + let result = ArrowReader::get_row_selection_for_filter_predicate( + &predicate, + &metadata, + &selected, + &field_id_map, + schema.as_ref(), + ); + + assert!(result.is_ok()); + assert!( + result.unwrap().is_none(), + "absent offset index must short-circuit before selected_row_groups is inspected" + ); + } + + #[test] + fn test_absent_column_index_with_empty_selected_row_groups_returns_ok_none() { + let schema = int_schema(); + let predicate = simple_predicate(schema.clone()); + let metadata = metadata_no_page_indexes(); + let field_id_map = field_id_map(); + let selected: Option> = Some(vec![]); + + let result = ArrowReader::get_row_selection_for_filter_predicate( + &predicate, + &metadata, + &selected, + &field_id_map, + schema.as_ref(), + ); + + assert!(result.is_ok()); + assert!( + result.unwrap().is_none(), + "column index check must fire before the empty-selected-row-groups branch" + ); + } + + #[test] + fn test_both_indexes_present_empty_selected_row_groups_returns_ok_some_empty() { + let schema = int_schema(); + let predicate = simple_predicate(schema.clone()); + let metadata = metadata_with_both_indexes(); + let field_id_map = field_id_map(); + let selected: Option> = Some(vec![]); + + let result = ArrowReader::get_row_selection_for_filter_predicate( + &predicate, + &metadata, + &selected, + &field_id_map, + schema.as_ref(), + ); + + assert!( + result.is_ok(), + "expected Ok(_), got Err: {:?}", + result.unwrap_err() + ); + + let row_selection = result.unwrap().expect( + "expected Ok(Some(_)) when both indexes are present and all row groups are filtered", + ); + + assert_eq!( + row_selection.row_count(), + 0, + "RowSelection must be empty (zero rows selected) when selected_row_groups is empty" + ); + } + + /// Full-suite regression test for issue: https://github.com/apache/iceberg-rust/issues/2452 + #[tokio::test] + async fn test_scan_without_page_indexes_does_not_error() { + // Building schema (both iceberg and arrow for RecordBatch) + let iceberg_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("name", DataType::Utf8, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + + let tmp_dir = TempDir::new().unwrap(); + let file_path = format!("{}/data.parquet", tmp_dir.path().to_str().unwrap()); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + // Disabling page statistics + .set_statistics_enabled(EnabledStatistics::None) + .build(); + + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])) as ArrayRef, + Arc::new(StringArray::from(vec![ + Some("alice"), + Some("bob"), + None, + Some("dana"), + Some("eve"), + ])) as ArrayRef, + ]) + .unwrap(); + + let file = File::create(&file_path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // Truly exercising a file without column/offset index + { + use parquet::file::reader::{FileReader, SerializedFileReader}; + let f = File::open(&file_path).unwrap(); + let rdr = SerializedFileReader::new(f).unwrap(); + assert!( + rdr.metadata().page_index().is_none(), + "test fixture must produce a file without a column index" + ); + } + + // Predicate: id > 2 + let predicate = Reference::new("id") + .greater_than(Datum::int(2)) + .bind(iceberg_schema.clone(), false) + .unwrap(); + + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io.clone(), Runtime::current()) + // Enabling row selection () + .with_options( + ReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .build(); + + let file_size = std::fs::metadata(&file_path).unwrap().len(); + let task = FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(iceberg_schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_predicate(Some(predicate)) + .with_case_sensitive(false) + .build() + .unwrap(); + + let stream = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(stream) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + let ids: Vec = batches + .iter() + .flat_map(|b| { + b.column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied() + }) + .collect(); + + assert_eq!( + ids, + vec![3, 4, 5], + "predicate must still be enforced via Arrow row filter even without page indexes" + ); + + // Absent index + position delete field present + let pos_del_path = format!("{}/pos-del.parquet", tmp_dir.path().to_str().unwrap()); + + // Build the position delete Arrow schema (standard Iceberg layout). + let pos_del_arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("file_path", DataType::Utf8, false), + Field::new("pos", DataType::Int64, false), + ])); + + let pos_del_batch = RecordBatch::try_new(pos_del_arrow_schema.clone(), vec![ + // Both deletions reference the same data file + Arc::new(StringArray::from(vec![ + file_path.as_str(), + file_path.as_str(), + ])) as ArrayRef, + // Delete by index - index-0 (`1` in test case) and index-2 (`3` in test case) + Arc::new(Int64Array::from(vec![0i64, 2i64])) as ArrayRef, + ]) + .unwrap(); + + // Write position delete file also without indices + let pos_del_props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_statistics_enabled(EnabledStatistics::None) + .build(); + + let pos_del_file = File::create(&pos_del_path).unwrap(); + let mut pos_del_writer = ArrowWriter::try_new( + pos_del_file, + pos_del_arrow_schema.clone(), + Some(pos_del_props), + ) + .unwrap(); + pos_del_writer.write(&pos_del_batch).unwrap(); + pos_del_writer.close().unwrap(); + + // Predicate: id > 1 (note that `3` was also removed by position delete) + let predicate_sub2 = Reference::new("id") + .greater_than(Datum::int(1)) + .bind(iceberg_schema.clone(), false) + .unwrap(); + + let reader_sub2 = ArrowReaderBuilder::new(file_io.clone(), Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .build(); + + let task_sub2 = FileScanTask::builder() + .with_file_size_in_bytes(file_size) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(iceberg_schema.clone()) + .with_project_field_ids(vec![1, 2]) + .with_predicate(Some(predicate_sub2)) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_path(pos_del_path.clone()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_file_size_in_bytes(std::fs::metadata(&pos_del_path).unwrap().len()) + .build() + .unwrap(), + ]) + .with_case_sensitive(false) + .build() + .unwrap(); + + let stream_sub2 = + Box::pin(futures::stream::iter(vec![Ok(task_sub2)])) as FileScanTaskStream; + let batches_sub2: Vec = reader_sub2 + .read(stream_sub2) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + let ids_sub2: Vec = batches_sub2 + .iter() + .flat_map(|b| { + b.column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied() + }) + .collect(); + + assert_eq!( + ids_sub2, + vec![2, 4, 5], + "positional deletes must be applied correctly even when page indexes are absent" + ); + } + + // Bloom filter pushdown: on-vs-off equivalence + // Pushdown must never change results. An encoding bug in the probe shows up as + // rows the bloom filter drops and the row filter keeps, so every case below + // reads the same file twice and compares. Fixtures spread each row group's + // values across the whole domain, so min/max statistics cannot prune and any + // reduction in bytes read is attributable to the bloom filter. + + const ROWS_PER_GROUP: i32 = 20_000; + const GROUPS: i32 = 3; + /// Gap between consecutive values in a row group, so a value can be absent from + /// the file yet still sit inside every row group's min/max range. + const STRIDE: i32 = 30; + + /// The `i`th value of row group `group`, spread across the whole domain. + fn interleaved(group: i32, i: i32) -> i32 { + i * STRIDE + group + } + + fn field_with_id(name: &str, ty: DataType, id: i32) -> Field { + Field::new(name, ty, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + id.to_string(), + )])) + } + + /// Writes one row group per batch, optionally with bloom filters. + fn write_row_groups( + path: &str, + arrow_schema: Arc, + row_groups: Vec, + with_bloom_filters: bool, + ) { + let mut props = WriterProperties::builder().set_compression(Compression::UNCOMPRESSED); + if with_bloom_filters { + props = props + .set_bloom_filter_enabled(true) + .set_bloom_filter_max_ndv(ROWS_PER_GROUP as u64); + } + + let file = File::create(path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props.build())).unwrap(); + for batch in row_groups { + writer.write(&batch).unwrap(); + // Force a row group boundary so each batch is independently prunable. + writer.flush().unwrap(); + } + writer.close().unwrap(); + } + + async fn read_once( + file_path: &str, + schema: Arc, + project_field_ids: Vec, + predicate: Predicate, + bloom_enabled: bool, + ) -> (Vec, u64) { + let file_io = FileIO::new_with_fs(); + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()) + .with_options( + ReaderOptions::builder() + .with_bloom_filter_enabled(bloom_enabled) + .build(), + ) + // Keep the fixed footer prefetch small so the byte measurement reflects + // row group I/O rather than a constant metadata read. + .with_parquet_options( + ParquetReadOptions::builder() + .with_metadata_size_hint(8 * 1024) + .build(), + ) + .build(); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(std::fs::metadata(file_path).unwrap().len()) + .with_start(0) + .with_length(0) + .with_data_file_path(file_path.to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids) + .with_predicate(Some(predicate.bind(schema, true).unwrap())) + .with_case_sensitive(false) + .build() + .unwrap(); + + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let result = reader.read(tasks).unwrap(); + let metrics = result.metrics().clone(); + let batches: Vec = result.stream().try_collect().await.unwrap(); + + (batches, metrics.bytes_read()) + } + + /// Batch boundaries differ when fewer row groups are read, so collapse to a + /// single batch before comparing. `None` means no rows at all. + fn collapse(batches: &[RecordBatch]) -> Option { + let first = batches.first()?; + Some(arrow_select::concat::concat_batches(&first.schema(), batches).unwrap()) + } + + /// Reads the file with pushdown on and off, asserts the rows are identical, and + /// returns the rows plus (bytes_on, bytes_off). + async fn assert_pushdown_agrees( + case: &str, + file_path: &str, + schema: Arc, + project_field_ids: Vec, + predicate: Predicate, + ) -> (Option, u64, u64) { + let (off, bytes_off) = read_once( + file_path, + schema.clone(), + project_field_ids.clone(), + predicate.clone(), + false, + ) + .await; + let (on, bytes_on) = read_once(file_path, schema, project_field_ids, predicate, true).await; + + let off = collapse(&off); + let on = collapse(&on); + assert_eq!( + on, off, + "{case}: bloom filter pushdown changed the rows returned" + ); + + (on, bytes_on, bytes_off) + } + + fn rows(batch: &Option) -> usize { + batch.as_ref().map_or(0, |b| b.num_rows()) + } + + fn value_schema(iceberg_type: Type) -> Arc { + Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![NestedField::required(1, "v", iceberg_type).into()]) + .build() + .unwrap(), + ) + } + + /// Writes a single-column file, one row group per group index. + fn write_value_fixture( + path: &str, + data_type: DataType, + make_group: impl Fn(i32) -> ArrayRef, + with_bloom_filters: bool, + ) { + let arrow_schema = Arc::new(ArrowSchema::new(vec![field_with_id("v", data_type, 1)])); + let row_groups = (0..GROUPS) + .map(|g| RecordBatch::try_new(arrow_schema.clone(), vec![make_group(g)]).unwrap()) + .collect(); + write_row_groups(path, arrow_schema, row_groups, with_bloom_filters); + } + + /// The core check for one physical encoding: a value that is present must + /// survive pushdown, and one that is absent must prune every row group. The + /// first catches a probe that encodes wrongly and drops real rows; the second + /// catches a probe that silently never prunes. + async fn assert_prunes_and_agrees( + case: &str, + file_path: &str, + schema: Arc, + present: Datum, + absent: Datum, + ) { + let (on, bytes_on, bytes_off) = assert_pushdown_agrees( + &format!("{case}/present"), + file_path, + schema.clone(), + vec![1], + Reference::new("v").equal_to(present), + ) + .await; + assert_eq!(rows(&on), 1, "{case}/present: expected one matching row"); + assert!( + bytes_on < bytes_off, + "{case}/present: pushdown must skip row groups: {bytes_on} vs {bytes_off}" + ); + + let (on, bytes_on, bytes_off) = assert_pushdown_agrees( + &format!("{case}/absent"), + file_path, + schema, + vec![1], + Reference::new("v").equal_to(absent), + ) + .await; + assert_eq!(rows(&on), 0, "{case}/absent: expected no matching rows"); + assert!( + bytes_on * 2 < bytes_off, + "{case}/absent: pruning every row group should cut reads sharply: \ + {bytes_on} vs {bytes_off}" + ); + } + + /// Value present in exactly one row group: `interleaved(1, 51)`. + const PRESENT_INT: i32 = 51 * STRIDE + 1; + /// Not congruent to any group index mod STRIDE, so absent from the file while + /// still inside every row group's min/max range. + const ABSENT_INT: i32 = 7; + + fn int_group(g: i32) -> ArrayRef { + Arc::new(Int32Array::from( + (0..ROWS_PER_GROUP) + .map(|i| interleaved(g, i)) + .collect::>(), + )) + } + + #[tokio::test] + async fn test_bloom_pushdown_int32_eq() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/int32.parquet", tmp.path().to_str().unwrap()); + write_value_fixture(&path, DataType::Int32, int_group, true); + + assert_prunes_and_agrees( + "int32", + &path, + value_schema(Type::Primitive(PrimitiveType::Int)), + Datum::int(PRESENT_INT), + Datum::int(ABSENT_INT), + ) + .await; + } + + /// `IN` takes a different evaluator path from `eq`: it must keep the row group + /// when any literal might be present, and prune only when all are absent. + #[tokio::test] + async fn test_bloom_pushdown_int32_in() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/int32_in.parquet", tmp.path().to_str().unwrap()); + write_value_fixture(&path, DataType::Int32, int_group, true); + let schema = value_schema(Type::Primitive(PrimitiveType::Int)); + + // One present literal keeps its row group; the absent one must not suppress it. + let (on, bytes_on, bytes_off) = assert_pushdown_agrees( + "int32_in/mixed", + &path, + schema.clone(), + vec![1], + Reference::new("v").is_in([Datum::int(PRESENT_INT), Datum::int(ABSENT_INT)]), + ) + .await; + assert_eq!(rows(&on), 1); + assert!(bytes_on < bytes_off, "{bytes_on} vs {bytes_off}"); + + // All literals absent: every row group prunes. + let (on, bytes_on, bytes_off) = assert_pushdown_agrees( + "int32_in/all_absent", + &path, + schema, + vec![1], + Reference::new("v").is_in([Datum::int(ABSENT_INT), Datum::int(ABSENT_INT + 1)]), + ) + .await; + assert_eq!(rows(&on), 0); + assert!(bytes_on * 2 < bytes_off, "{bytes_on} vs {bytes_off}"); + } + + #[tokio::test] + async fn test_bloom_pushdown_string_eq() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/string.parquet", tmp.path().to_str().unwrap()); + write_value_fixture( + &path, + DataType::Utf8, + |g| { + Arc::new(StringArray::from( + (0..ROWS_PER_GROUP) + .map(|i| format!("{:016}", interleaved(g, i))) + .collect::>(), + )) + }, + true, + ); + + assert_prunes_and_agrees( + "string", + &path, + value_schema(Type::Primitive(PrimitiveType::String)), + Datum::string(format!("{PRESENT_INT:016}")), + Datum::string(format!("{ABSENT_INT:016}")), + ) + .await; + } + + /// Decimal precision decides the Parquet physical type, and the probe has to be + /// encoded to match: precision 9 -> INT32, 15 -> INT64, 25 -> FIXED_LEN_BYTE_ARRAY. + /// Each width is a separate arm of `check_in_bloom_filter`. + async fn check_decimal_width(name: &str, precision: u32) { + const SCALE: u32 = 2; + let tmp = TempDir::new().unwrap(); + let path = format!("{}/{name}.parquet", tmp.path().to_str().unwrap()); + + write_value_fixture( + &path, + DataType::Decimal128(precision as u8, SCALE as i8), + |g| { + let values: Vec = (0..ROWS_PER_GROUP) + .map(|i| i128::from(interleaved(g, i))) + .collect(); + Arc::new( + Decimal128Array::from(values) + .with_precision_and_scale(precision as u8, SCALE as i8) + .unwrap(), + ) + }, + true, + ); + + let datum = |mantissa: i32| { + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale( + i128::from(mantissa), + SCALE, + ), + precision, + ) + .unwrap() + }; + + assert_prunes_and_agrees( + name, + &path, + value_schema(Type::Primitive(PrimitiveType::Decimal { + precision, + scale: SCALE, + })), + datum(PRESENT_INT), + datum(ABSENT_INT), + ) + .await; + } + + #[tokio::test] + async fn test_bloom_pushdown_decimal_int32() { + check_decimal_width("decimal_int32", 9).await; + } + + #[tokio::test] + async fn test_bloom_pushdown_decimal_int64() { + check_decimal_width("decimal_int64", 15).await; + } + + #[tokio::test] + async fn test_bloom_pushdown_decimal_fixed_len_byte_array() { + check_decimal_width("decimal_flba", 25).await; + } + + /// Widening a decimal's precision does not rewrite existing files, so the file + /// keeps its original physical width while the bound literal arrives at the new + /// precision. Deriving the probe encoding from the schema instead of the file + /// would miss every entry the writer inserted and prune matching row groups. + #[tokio::test] + async fn test_bloom_pushdown_decimal_widened_precision() { + const SCALE: u32 = 2; + let tmp = TempDir::new().unwrap(); + let path = format!("{}/decimal_widened.parquet", tmp.path().to_str().unwrap()); + + // File written at precision 9 -> INT32. + write_value_fixture( + &path, + DataType::Decimal128(9, SCALE as i8), + |g| { + let values: Vec = (0..ROWS_PER_GROUP) + .map(|i| i128::from(interleaved(g, i))) + .collect(); + Arc::new( + Decimal128Array::from(values) + .with_precision_and_scale(9, SCALE as i8) + .unwrap(), + ) + }, + true, + ); + + // Table schema has since widened to precision 30. + let schema = value_schema(Type::Primitive(PrimitiveType::Decimal { + precision: 30, + scale: SCALE, + })); + let datum = |mantissa: i32| { + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale( + i128::from(mantissa), + SCALE, + ), + 30, + ) + .unwrap() + }; + + assert_prunes_and_agrees( + "decimal_widened", + &path, + schema, + datum(PRESENT_INT), + datum(ABSENT_INT), + ) + .await; + } + + /// `Sbbf` hashes raw IEEE bytes, so `-0.0` and `0.0` are distinct entries. Arrow's + /// float equality is bitwise too, for an independent reason: `ArrowNativeTypeOp::is_eq` + /// implements total-order (`total_cmp`) semantics as `to_bits() == to_bits()`, so + /// `-0.0 != 0.0` and `NaN == NaN` there. That agreement is what makes it safe to prune + /// a row group holding only the opposite zero: if `is_eq` ever became IEEE-lenient, + /// those pruned rows would be rows the row filter should have returned. A probe that + /// canonicalized the sign instead only over-keeps, costing pruning rather than + /// correctness — but it would also hide such a change from this test. + /// + /// Each row group pairs its zero with a larger filler value so its min/max range spans + /// the other zero. Neither statistics path can prune it, so the bloom-off read really + /// does evaluate `is_eq` against `ROWS_PER_GROUP - 1` copies of the opposite zero: that + /// arm is the control pinning arrow's semantics, and IEEE-lenient equality would double + /// the row count it returns. Pairing with a filler also avoids relying on the writer + /// widening a single-value group's bounds to `[-0.0, +0.0]`. + #[tokio::test] + async fn test_bloom_pushdown_float_signed_zero() { + const FILLER: f32 = 5.0; + let tmp = TempDir::new().unwrap(); + let path = format!("{}/float_zero.parquet", tmp.path().to_str().unwrap()); + + // Row group 0 holds -0.0 and never +0.0; row group 1 the reverse; row group 2 + // neither zero. + write_value_fixture( + &path, + DataType::Float32, + |g| { + let zero = match g { + 0 => -0.0f32, + 1 => 0.0f32, + _ => 7.5f32, + }; + let mut values = vec![zero; ROWS_PER_GROUP as usize - 1]; + values.push(FILLER); + Arc::new(Float32Array::from(values)) + }, + true, + ); + + let schema = value_schema(Type::Primitive(PrimitiveType::Float)); + let zero_rows = ROWS_PER_GROUP as usize - 1; + + for (label, probe) in [("positive_zero", 0.0f32), ("negative_zero", -0.0f32)] { + let (on, _, _) = assert_pushdown_agrees( + label, + &path, + schema.clone(), + vec![1], + Reference::new("v").equal_to(Datum::float(probe)), + ) + .await; + // Only the group holding this exact zero matches, on both paths. + assert_eq!( + rows(&on), + zero_rows, + "{label}: expected only the row group holding this zero to match" + ); + } + } + + /// A file with no bloom filters at all must read identically with the option on, + /// exercising the conservative might-match path end to end. + #[tokio::test] + async fn test_bloom_pushdown_file_without_bloom_filters() { + let tmp = TempDir::new().unwrap(); + let path = format!("{}/no_filters.parquet", tmp.path().to_str().unwrap()); + write_value_fixture(&path, DataType::Int32, int_group, false); + + let (on, _, _) = assert_pushdown_agrees( + "no_filters", + &path, + value_schema(Type::Primitive(PrimitiveType::Int)), + vec![1], + Reference::new("v").equal_to(Datum::int(PRESENT_INT)), + ) + .await; + assert_eq!(rows(&on), 1); + } +} diff --git a/crates/iceberg/src/arrow/reader/row_lineage.rs b/crates/iceberg/src/arrow/reader/row_lineage.rs new file mode 100644 index 0000000000..739a7544a8 --- /dev/null +++ b/crates/iceberg/src/arrow/reader/row_lineage.rs @@ -0,0 +1,284 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Reader-level synthesis of the v3 `_row_id` metadata column. +//! +//! `_row_id` is `first_row_id + pos`, overridden by a physically-stored `_row_id` where the +//! file carries one. The position comes from the reader-produced `_pos` (`RowNumber`) +//! column, which is the true global file position under filter pushdown, row-group pruning, +//! and page-index pruning -- so synthesis is done here, over the record-batch stream, using +//! that position rather than recomputing it. + +use std::sync::Arc; + +use arrow_arith::boolean::is_not_null; +use arrow_arith::numeric::add; +use arrow_array::{Array, ArrayRef, Int64Array, RecordBatch}; +use arrow_schema::{DataType, Field, Schema}; +use arrow_select::zip::zip; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + +use crate::error::invalid_data; +use crate::metadata_columns::{ + RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, +}; +use crate::{Error, ErrorKind, Result}; + +/// Appends the synthesized `_row_id` column to `batch`. +/// +/// - `first_row_id == None`: the file carries no row lineage, so `_row_id` is all-null +/// (matching Java `ValueReaders.rowIds`). +/// - `first_row_id == Some(base)`: `_row_id` is the physically-stored value where the file +/// carries one and it is non-null, else `base + pos` (from the `_pos`/`RowNumber` column). +/// +/// A physically-stored `_row_id` leaf already in `batch` is replaced by the synthesized +/// column (its values are folded in via the coalesce), so the result carries exactly one +/// column tagged with the reserved `_row_id` field id. +pub(crate) fn synthesize_row_id_column( + batch: RecordBatch, + first_row_id: Option, +) -> Result { + let row_id: ArrayRef = match first_row_id { + None => Arc::new(Int64Array::new_null(batch.num_rows())), + Some(base) => { + let pos = column_by_field_id(&batch, RESERVED_FIELD_ID_POS).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "_row_id synthesis requires the _pos position column in the record batch", + ) + })?; + if pos.data_type() != &DataType::Int64 { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "_pos position column must be Int64, got {}", + pos.data_type() + ), + )); + } + + // base + pos, the fallback for every row. + let fallback = add(pos, &Int64Array::new_scalar(base))?; + match column_by_field_id(&batch, RESERVED_FIELD_ID_ROW_ID) { + Some(id) => { + if id.data_type() != &DataType::Int64 { + return Err(invalid_data!( + "_row_id source must be Int64, got {}", + id.data_type() + )); + } + + zip(&is_not_null(id)?, id, &fallback)? + } + None => fallback, + } + } + }; + + append_row_id(batch, row_id) +} + +/// Returns the batch column tagged with `field_id` (via `PARQUET_FIELD_ID_META_KEY`), if any. +fn column_by_field_id(batch: &RecordBatch, field_id: i32) -> Option<&ArrayRef> { + batch + .schema() + .fields() + .iter() + .position(|f| field_id_of(f) == Some(field_id)) + .map(|idx| batch.column(idx)) +} + +fn field_id_of(field: &Field) -> Option { + field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .and_then(|id| id.parse::().ok()) +} + +/// Rebuilds `batch` with `row_id` as its `_row_id` column: any existing `_row_id`-tagged +/// column is dropped (already folded into `row_id`) and the synthesized column is appended. +fn append_row_id(batch: RecordBatch, row_id: ArrayRef) -> Result { + let schema = batch.schema(); + let mut fields: Vec> = Vec::with_capacity(schema.fields().len() + 1); + let mut columns: Vec = Vec::with_capacity(schema.fields().len() + 1); + + for (idx, field) in schema.fields().iter().enumerate() { + if field_id_of(field) != Some(RESERVED_FIELD_ID_ROW_ID) { + fields.push(field.clone()); + columns.push(batch.column(idx).clone()); + } + } + + fields.push(Arc::new( + Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true).with_metadata([( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_ROW_ID.to_string(), + )]), + )); + columns.push(row_id); + + Ok(RecordBatch::try_new( + Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone())), + columns, + )?) +} + +#[cfg(test)] +mod tests { + use arrow_array::cast::AsArray; + use arrow_array::types::Int64Type; + use arrow_array::{Int32Array, Int64Array}; + + use super::*; + use crate::metadata_columns::RESERVED_COL_NAME_POS; + + fn field_with_id(name: &str, dt: DataType, id: i32) -> Arc { + Arc::new( + Field::new(name, dt, true) + .with_metadata([(PARQUET_FIELD_ID_META_KEY.to_string(), id.to_string())]), + ) + } + + /// A batch with `id`, a `_pos` column, and (when `physical` is `Some`) a physically + /// stored `_row_id` column — mirroring what the Parquet reader produces. + fn batch(pos: Vec, physical: Option>>) -> RecordBatch { + let n = pos.len() as i32; + let mut fields = vec![ + field_with_id("id", DataType::Int32, 1), + field_with_id( + RESERVED_COL_NAME_POS, + DataType::Int64, + RESERVED_FIELD_ID_POS, + ), + ]; + let mut columns: Vec = vec![ + Arc::new(Int32Array::from((0..n).collect::>())), + Arc::new(Int64Array::from(pos)), + ]; + if let Some(vals) = physical { + fields.push(field_with_id( + RESERVED_COL_NAME_ROW_ID, + DataType::Int64, + RESERVED_FIELD_ID_ROW_ID, + )); + columns.push(Arc::new(Int64Array::from(vals))); + } + RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap() + } + + fn row_ids(rb: &RecordBatch) -> Vec> { + let v = rb + .column_by_name(RESERVED_COL_NAME_ROW_ID) + .unwrap() + .as_primitive::(); + (0..v.len()) + .map(|i| (!v.is_null(i)).then(|| v.value(i))) + .collect() + } + + #[test] + fn pure_synthesis() { + let out = synthesize_row_id_column(batch(vec![0, 1, 2], None), Some(100)).unwrap(); + assert_eq!(row_ids(&out), vec![Some(100), Some(101), Some(102)]); + } + + #[test] + fn non_zero_start_position() { + let out = synthesize_row_id_column(batch(vec![10, 11, 12], None), Some(100)).unwrap(); + assert_eq!(row_ids(&out), vec![Some(110), Some(111), Some(112)]); + } + + #[test] + fn coalesce_with_physical_column() { + let out = synthesize_row_id_column( + batch(vec![0, 1, 2], Some(vec![Some(5), None, Some(8)])), + Some(100), + ) + .unwrap(); + // Physical value where non-null, else first_row_id + pos. + assert_eq!(row_ids(&out), vec![Some(5), Some(101), Some(8)]); + // The physical leaf was folded in -- exactly one _row_id column (id, _pos, _row_id). + assert_eq!(out.schema().fields().len(), 3); + } + + #[test] + fn all_physical_present() { + let out = synthesize_row_id_column( + batch(vec![0, 1, 2], Some(vec![Some(5), Some(6), Some(7)])), + Some(100), + ) + .unwrap(); + assert_eq!(row_ids(&out), vec![Some(5), Some(6), Some(7)]); + } + + #[test] + fn all_fallback() { + let out = + synthesize_row_id_column(batch(vec![0, 1, 2], Some(vec![None, None, None])), Some(50)) + .unwrap(); + assert_eq!(row_ids(&out), vec![Some(50), Some(51), Some(52)]); + } + + #[test] + fn null_first_row_id_is_all_null() { + let out = synthesize_row_id_column(batch(vec![0, 1, 2], None), None).unwrap(); + assert_eq!(row_ids(&out), vec![None, None, None]); + } + + #[test] + fn empty_batch() { + let out = synthesize_row_id_column(batch(vec![], None), Some(100)).unwrap(); + assert_eq!(out.num_rows(), 0); + assert_eq!(row_ids(&out), Vec::>::new()); + } + + #[test] + fn non_int64_physical_column_is_rejected() { + let fields = vec![ + field_with_id( + RESERVED_COL_NAME_POS, + DataType::Int64, + RESERVED_FIELD_ID_POS, + ), + field_with_id( + RESERVED_COL_NAME_ROW_ID, + DataType::Int32, + RESERVED_FIELD_ID_ROW_ID, + ), + ]; + let columns: Vec = vec![ + Arc::new(Int64Array::from(vec![0i64, 1, 2])), + Arc::new(Int32Array::from(vec![Some(5), None, Some(8)])), + ]; + let rb = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(); + + let err = synthesize_row_id_column(rb, Some(100)).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(format!("{err}").contains("_row_id source must be Int64")); + } + + #[test] + fn missing_pos_is_rejected() { + let fields = vec![field_with_id("id", DataType::Int32, 1)]; + let columns: Vec = vec![Arc::new(Int32Array::from(vec![0, 1, 2]))]; + let rb = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(); + + let err = synthesize_row_id_column(rb, Some(100)).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::Unexpected); + assert!(format!("{err}").contains("_pos position column")); + } +} diff --git a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs index 704a4e9c15..af3c99a979 100644 --- a/crates/iceberg/src/arrow/record_batch_partition_splitter.rs +++ b/crates/iceberg/src/arrow/record_batch_partition_splitter.rs @@ -19,145 +19,162 @@ use std::collections::HashMap; use std::sync::Arc; use arrow_array::{ArrayRef, BooleanArray, RecordBatch, StructArray}; -use arrow_schema::{DataType, SchemaRef as ArrowSchemaRef}; +use arrow_buffer::BooleanBufferBuilder; use arrow_select::filter::filter_record_batch; -use itertools::Itertools; -use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use super::arrow_struct_to_literal; -use super::record_batch_projector::RecordBatchProjector; -use crate::arrow::type_to_arrow_type; -use crate::spec::{Literal, PartitionSpecRef, SchemaRef, Struct, StructType, Type}; -use crate::transform::{BoxedTransformFunction, create_transform_function}; -use crate::{Error, ErrorKind, Result}; +use super::partition_value_calculator::PartitionValueCalculator; +use crate::Result; +use crate::error::invalid_data; +use crate::spec::{Literal, PartitionKey, PartitionSpecRef, SchemaRef, StructType}; + +/// Column name for the projected partition values struct +pub const PROJECTED_PARTITION_VALUE_COLUMN: &str = "_partition"; /// The splitter used to split the record batch into multiple record batches by the partition spec. /// 1. It will project and transform the input record batch based on the partition spec, get the partitioned record batch. /// 2. Split the input record batch into multiple record batches based on the partitioned record batch. -// # TODO -// Remove this after partition writer supported. -#[allow(dead_code)] +/// +/// # Partition Value Modes +/// +/// The splitter supports two modes for obtaining partition values: +/// - **Computed mode** (`calculator` is `Some`): Computes partition values from source columns using transforms +/// - **Pre-computed mode** (`calculator` is `None`): Expects a `_partition` column in the input batch pub struct RecordBatchPartitionSplitter { schema: SchemaRef, partition_spec: PartitionSpecRef, - projector: RecordBatchProjector, - transform_functions: Vec, - + calculator: Option, partition_type: StructType, - partition_arrow_type: DataType, } -// # TODO -// Remove this after partition writer supported. -#[allow(dead_code)] impl RecordBatchPartitionSplitter { - pub fn new( - input_schema: ArrowSchemaRef, + /// Create a new RecordBatchPartitionSplitter. + /// + /// # Arguments + /// + /// * `iceberg_schema` - The Iceberg schema reference + /// * `partition_spec` - The partition specification reference + /// * `calculator` - Optional calculator for computing partition values from source columns. + /// - `Some(calculator)`: Compute partition values from source columns using transforms + /// - `None`: Expect a pre-computed `_partition` column in the input batch + /// + /// # Returns + /// + /// Returns a new `RecordBatchPartitionSplitter` instance or an error if initialization fails. + pub fn try_new( iceberg_schema: SchemaRef, partition_spec: PartitionSpecRef, + calculator: Option, ) -> Result { - let projector = RecordBatchProjector::new( - input_schema, - &partition_spec - .fields() - .iter() - .map(|field| field.source_id) - .collect::>(), - // The source columns, selected by ids, must be a primitive type and cannot be contained in a map or list, but may be nested in a struct. - // ref: https://iceberg.apache.org/spec/#partitioning - |field| { - if !field.data_type().is_primitive() { - return Ok(None); - } - field - .metadata() - .get(PARQUET_FIELD_ID_META_KEY) - .map(|s| { - s.parse::() - .map_err(|e| Error::new(ErrorKind::Unexpected, e.to_string())) - }) - .transpose() - }, - |_| true, - )?; - let transform_functions = partition_spec - .fields() - .iter() - .map(|field| create_transform_function(&field.transform)) - .collect::>>()?; - let partition_type = partition_spec.partition_type(&iceberg_schema)?; - let partition_arrow_type = type_to_arrow_type(&Type::Struct(partition_type.clone()))?; Ok(Self { schema: iceberg_schema, partition_spec, - projector, - transform_functions, + calculator, partition_type, - partition_arrow_type, }) } - fn partition_columns_to_struct(&self, partition_columns: Vec) -> Result> { - let arrow_struct_array = { - let partition_arrow_fields = { - let DataType::Struct(fields) = &self.partition_arrow_type else { - return Err(Error::new( - ErrorKind::DataInvalid, - "The partition arrow type is not a struct type", - )); - }; - fields.clone() - }; - Arc::new(StructArray::try_new( - partition_arrow_fields, - partition_columns, - None, - )?) as ArrayRef - }; - let struct_array = { - let struct_array = arrow_struct_to_literal(&arrow_struct_array, &self.partition_type)?; + /// Create a new RecordBatchPartitionSplitter with computed partition values. + /// + /// This is a convenience method that creates a calculator and initializes the splitter + /// to compute partition values from source columns. + /// + /// # Arguments + /// + /// * `iceberg_schema` - The Iceberg schema reference + /// * `partition_spec` - The partition specification reference + /// + /// # Returns + /// + /// Returns a new `RecordBatchPartitionSplitter` instance or an error if initialization fails. + pub fn try_new_with_computed_values( + iceberg_schema: SchemaRef, + partition_spec: PartitionSpecRef, + ) -> Result { + let calculator = PartitionValueCalculator::try_new(&partition_spec, &iceberg_schema)?; + Self::try_new(iceberg_schema, partition_spec, Some(calculator)) + } + + /// Create a new RecordBatchPartitionSplitter expecting pre-computed partition values. + /// + /// This is a convenience method that initializes the splitter to expect a `_partition` + /// column in the input batches. + /// + /// # Arguments + /// + /// * `iceberg_schema` - The Iceberg schema reference + /// * `partition_spec` - The partition specification reference + /// + /// # Returns + /// + /// Returns a new `RecordBatchPartitionSplitter` instance or an error if initialization fails. + pub fn try_new_with_precomputed_values( + iceberg_schema: SchemaRef, + partition_spec: PartitionSpecRef, + ) -> Result { + Self::try_new(iceberg_schema, partition_spec, None) + } + + /// Split the record batch into multiple record batches based on the partition spec. + pub fn split(&self, batch: &RecordBatch) -> Result> { + let partition_structs = if let Some(calculator) = &self.calculator { + // Compute partition values from source columns using calculator + let partition_array = calculator.calculate(batch)?; + let struct_array = arrow_struct_to_literal(&partition_array, &self.partition_type)?; + struct_array .into_iter() .map(|s| { - if let Some(s) = s { - if let Literal::Struct(s) = s { - Ok(s) - } else { - Err(Error::new( - ErrorKind::DataInvalid, - "The struct is not a struct literal", - )) - } + if let Some(Literal::Struct(s)) = s { + Ok(s) } else { - Err(Error::new(ErrorKind::DataInvalid, "The struct is null")) + Err(invalid_data!( + "Partition value is not a struct literal or is null" + )) } }) .collect::>>()? - }; + } else { + // Extract partition values from pre-computed partition column + let partition_column = batch + .column_by_name(PROJECTED_PARTITION_VALUE_COLUMN) + .ok_or_else(|| { + invalid_data!( + "Partition column '{PROJECTED_PARTITION_VALUE_COLUMN}' not found in batch" + ) + })?; - Ok(struct_array) - } + let partition_struct_array = partition_column + .as_any() + .downcast_ref::() + .ok_or_else(|| invalid_data!("Partition column is not a StructArray"))?; - /// Split the record batch into multiple record batches based on the partition spec. - pub fn split(&self, batch: &RecordBatch) -> Result> { - let source_columns = self.projector.project_column(batch.columns())?; - let partition_columns = source_columns - .into_iter() - .zip_eq(self.transform_functions.iter()) - .map(|(source_column, transform_function)| transform_function.transform(source_column)) - .collect::>>()?; + let arrow_struct_array = Arc::new(partition_struct_array.clone()) as ArrayRef; + let struct_array = arrow_struct_to_literal(&arrow_struct_array, &self.partition_type)?; - let partition_structs = self.partition_columns_to_struct(partition_columns)?; + struct_array + .into_iter() + .map(|s| { + if let Some(Literal::Struct(s)) = s { + Ok(s) + } else { + Err(invalid_data!( + "Partition value is not a struct literal or is null" + )) + } + }) + .collect::>>()? + }; // Group the batch by row value. let mut group_ids = HashMap::new(); partition_structs - .iter() + .into_iter() .enumerate() .for_each(|(row_id, row)| { - group_ids.entry(row.clone()).or_insert(vec![]).push(row_id); + group_ids.entry(row).or_insert(vec![]).push(row_id); }); // Partition the batch with same partition partition_values @@ -165,15 +182,23 @@ impl RecordBatchPartitionSplitter { for (row, row_ids) in group_ids.into_iter() { // generate the bool filter array from column_ids let filter_array: BooleanArray = { - let mut filter = vec![false; batch.num_rows()]; - row_ids.into_iter().for_each(|row_id| { - filter[row_id] = true; - }); - filter.into() + let mut builder = BooleanBufferBuilder::new(batch.num_rows()); + builder.append_n(batch.num_rows(), false); + for row_id in row_ids { + builder.set_bit(row_id, true); + } + BooleanArray::new(builder.finish(), None) }; + // Create PartitionKey from the partition struct + let partition_key = PartitionKey::new( + self.partition_spec.as_ref().clone(), + self.schema.clone(), + row, + ); + // filter the RecordBatch - partition_batches.push((row, filter_record_batch(batch, &filter_array)?)); + partition_batches.push((partition_key, filter_record_batch(batch, &filter_array)?)); } Ok(partition_batches) @@ -184,12 +209,14 @@ impl RecordBatchPartitionSplitter { mod tests { use std::sync::Arc; - use arrow_array::{Int32Array, RecordBatch, StringArray}; + use arrow_array::{Float64Array, Int32Array, RecordBatch, StringArray}; + use arrow_schema::DataType; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use super::*; use crate::arrow::schema_to_arrow_schema; use crate::spec::{ - NestedField, PartitionSpecBuilder, PrimitiveLiteral, Schema, Transform, + NestedField, PartitionSpecBuilder, PrimitiveLiteral, Schema, Struct, Transform, Type, UnboundPartitionField, }; @@ -217,24 +244,28 @@ mod tests { let partition_spec = Arc::new( PartitionSpecBuilder::new(schema.clone()) .with_spec_id(1) - .add_unbound_field(UnboundPartitionField { - source_id: 1, - field_id: None, - name: "id_bucket".to_string(), - transform: Transform::Identity, - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(), ); - let input_schema = Arc::new(schema_to_arrow_schema(&schema).unwrap()); - let partition_splitter = - RecordBatchPartitionSplitter::new(input_schema.clone(), schema.clone(), partition_spec) - .expect("Failed to create splitter"); + let partition_splitter = RecordBatchPartitionSplitter::try_new_with_computed_values( + schema.clone(), + partition_spec, + ) + .expect("Failed to create splitter"); + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema).unwrap()); let id_array = Int32Array::from(vec![1, 2, 1, 3, 2, 3, 1]); let data_array = StringArray::from(vec!["a", "b", "c", "d", "e", "f", "g"]); - let batch = RecordBatch::try_new(input_schema.clone(), vec![ + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![ Arc::new(id_array), Arc::new(data_array), ]) @@ -243,8 +274,8 @@ mod tests { let mut partitioned_batches = partition_splitter .split(&batch) .expect("Failed to split RecordBatch"); - partitioned_batches.sort_by_key(|(row, _)| { - if let PrimitiveLiteral::Int(i) = row.fields()[0] + partitioned_batches.sort_by_key(|(partition_key, _)| { + if let PrimitiveLiteral::Int(i) = partition_key.data().fields()[0] .as_ref() .unwrap() .as_primitive_literal() @@ -260,7 +291,7 @@ mod tests { // check the first partition let expected_id_array = Int32Array::from(vec![1, 1, 1]); let expected_data_array = StringArray::from(vec!["a", "c", "g"]); - let expected_batch = RecordBatch::try_new(input_schema.clone(), vec![ + let expected_batch = RecordBatch::try_new(arrow_schema.clone(), vec![ Arc::new(expected_id_array), Arc::new(expected_data_array), ]) @@ -271,7 +302,7 @@ mod tests { // check the second partition let expected_id_array = Int32Array::from(vec![2, 2]); let expected_data_array = StringArray::from(vec!["b", "e"]); - let expected_batch = RecordBatch::try_new(input_schema.clone(), vec![ + let expected_batch = RecordBatch::try_new(arrow_schema.clone(), vec![ Arc::new(expected_id_array), Arc::new(expected_data_array), ]) @@ -282,7 +313,7 @@ mod tests { // check the third partition let expected_id_array = Int32Array::from(vec![3, 3]); let expected_data_array = StringArray::from(vec!["d", "f"]); - let expected_batch = RecordBatch::try_new(input_schema.clone(), vec![ + let expected_batch = RecordBatch::try_new(arrow_schema.clone(), vec![ Arc::new(expected_id_array), Arc::new(expected_data_array), ]) @@ -292,7 +323,7 @@ mod tests { let partition_values = partitioned_batches .iter() - .map(|(row, _)| row.clone()) + .map(|(partition_key, _)| partition_key.data().clone()) .collect::>(); // check partition value is struct(1), struct(2), struct(3) assert_eq!(partition_values, vec![ @@ -301,4 +332,219 @@ mod tests { Struct::from_iter(vec![Some(Literal::int(3))]), ]); } + + #[test] + fn test_record_batch_partition_split_with_partition_column() { + use arrow_array::StructArray; + use arrow_schema::{Field, Schema as ArrowSchema}; + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "id", + Type::Primitive(crate::spec::PrimitiveType::Int), + ) + .into(), + NestedField::required( + 2, + "name", + Type::Primitive(crate::spec::PrimitiveType::String), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = Arc::new( + PartitionSpecBuilder::new(schema.clone()) + .with_spec_id(1) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build() + .unwrap(), + ); + + // Create input schema with _partition column + // Note: partition field IDs start from 1000 by default + let partition_field = Field::new("id_bucket", DataType::Int32, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1000".to_string())]), + ); + let partition_struct_field = Field::new( + PROJECTED_PARTITION_VALUE_COLUMN, + DataType::Struct(vec![partition_field.clone()].into()), + false, + ); + + let input_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + partition_struct_field, + ])); + + // Create splitter expecting pre-computed partition column + let partition_splitter = RecordBatchPartitionSplitter::try_new_with_precomputed_values( + schema.clone(), + partition_spec, + ) + .expect("Failed to create splitter"); + + // Create test data with pre-computed partition column + let id_array = Int32Array::from(vec![1, 2, 1, 3, 2, 3, 1]); + let data_array = StringArray::from(vec!["a", "b", "c", "d", "e", "f", "g"]); + + // Create partition column (same values as id for Identity transform) + let partition_values = Int32Array::from(vec![1, 2, 1, 3, 2, 3, 1]); + let partition_struct = StructArray::from(vec![( + Arc::new(partition_field), + Arc::new(partition_values) as ArrayRef, + )]); + + let batch = RecordBatch::try_new(input_schema.clone(), vec![ + Arc::new(id_array), + Arc::new(data_array), + Arc::new(partition_struct), + ]) + .expect("Failed to create RecordBatch"); + + // Split using the pre-computed partition column + let mut partitioned_batches = partition_splitter + .split(&batch) + .expect("Failed to split RecordBatch"); + + partitioned_batches.sort_by_key(|(partition_key, _)| { + if let PrimitiveLiteral::Int(i) = partition_key.data().fields()[0] + .as_ref() + .unwrap() + .as_primitive_literal() + .unwrap() + { + i + } else { + panic!("The partition value is not a int"); + } + }); + + assert_eq!(partitioned_batches.len(), 3); + + // Helper to extract id and name values from a batch + let extract_values = |batch: &RecordBatch| -> (Vec, Vec) { + let id_col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let name_col = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + ( + id_col.values().to_vec(), + name_col.iter().map(|s| s.unwrap().to_string()).collect(), + ) + }; + + // Verify partition 1: id=1, names=["a", "c", "g"] + let (key, batch) = &partitioned_batches[0]; + assert_eq!(key.data(), &Struct::from_iter(vec![Some(Literal::int(1))])); + let (ids, names) = extract_values(batch); + assert_eq!(ids, vec![1, 1, 1]); + assert_eq!(names, vec!["a", "c", "g"]); + + // Verify partition 2: id=2, names=["b", "e"] + let (key, batch) = &partitioned_batches[1]; + assert_eq!(key.data(), &Struct::from_iter(vec![Some(Literal::int(2))])); + let (ids, names) = extract_values(batch); + assert_eq!(ids, vec![2, 2]); + assert_eq!(names, vec!["b", "e"]); + + // Verify partition 3: id=3, names=["d", "f"] + let (key, batch) = &partitioned_batches[2]; + assert_eq!(key.data(), &Struct::from_iter(vec![Some(Literal::int(3))])); + let (ids, names) = extract_values(batch); + assert_eq!(ids, vec![3, 3]); + assert_eq!(names, vec!["d", "f"]); + } + + #[test] + fn test_record_batch_partition_split_signed_zero() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "id", + Type::Primitive(crate::spec::PrimitiveType::Int), + ) + .into(), + NestedField::required( + 2, + "d", + Type::Primitive(crate::spec::PrimitiveType::Double), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = Arc::new( + PartitionSpecBuilder::new(schema.clone()) + .with_spec_id(1) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("d".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build() + .unwrap(), + ); + let partition_splitter = RecordBatchPartitionSplitter::try_new_with_computed_values( + schema.clone(), + partition_spec, + ) + .expect("Failed to create splitter"); + + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema).unwrap()); + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Float64Array::from(vec![-0.0, 0.0, -0.0])), + ]) + .expect("Failed to create RecordBatch"); + + let mut partitions: Vec<(Struct, Vec)> = partition_splitter + .split(&batch) + .expect("Failed to split RecordBatch") + .into_iter() + .map(|(key, batch)| { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + (key.data().clone(), ids) + }) + .collect(); + partitions.sort_by_key(|(_, ids)| ids[0]); + + // -0.0 and 0.0 are different partition values, as in iceberg-java. + assert_eq!(partitions, vec![ + (Struct::from_iter([Some(Literal::double(-0.0))]), vec![1, 3]), + (Struct::from_iter([Some(Literal::double(0.0))]), vec![2]), + ]); + } } diff --git a/crates/iceberg/src/arrow/record_batch_projector.rs b/crates/iceberg/src/arrow/record_batch_projector.rs index 7ca28c25c7..ca559933f7 100644 --- a/crates/iceberg/src/arrow/record_batch_projector.rs +++ b/crates/iceberg/src/arrow/record_batch_projector.rs @@ -17,16 +17,19 @@ use std::sync::Arc; -use arrow_array::{ArrayRef, RecordBatch, StructArray, make_array}; +use arrow_array::{Array, ArrayRef, RecordBatch, StructArray, make_array}; use arrow_buffer::NullBuffer; +use arrow_cast::cast; use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; -use crate::error::Result; +use crate::arrow::schema::{schema_to_arrow_schema, try_get_field_id_from_metadata}; +use crate::error::{Result, invalid_data}; +use crate::spec::Schema as IcebergSchema; use crate::{Error, ErrorKind}; /// Help to project specific field from `RecordBatch`` according to the fields id. -#[derive(Clone, Debug)] -pub(crate) struct RecordBatchProjector { +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecordBatchProjector { // A vector of vectors, where each inner vector represents the index path to access a specific field in a nested structure. // E.g. [[0], [1, 2]] means the first field is accessed directly from the first column, // while the second field is accessed from the second column and then from its third subcolumn (second column must be a struct column). @@ -77,6 +80,34 @@ impl RecordBatchProjector { }) } + /// Create RecordBatchProjector using Iceberg schema. + /// + /// This constructor converts the Iceberg schema to Arrow schema with field ID metadata, + /// then uses the standard field ID lookup for projection. + /// + /// # Arguments + /// * `iceberg_schema` - The Iceberg schema for field ID mapping + /// * `target_field_ids` - The field IDs to project + pub fn from_iceberg_schema( + iceberg_schema: Arc, + target_field_ids: &[i32], + ) -> Result { + let arrow_schema_with_ids = Arc::new(schema_to_arrow_schema(&iceberg_schema)?); + + let field_id_fetch_func = |field: &Field| -> Result> { + Ok(try_get_field_id_from_metadata(field)?.map(|id| id as i64)) + }; + + let searchable_field_func = |_field: &Field| -> bool { true }; + + Self::new( + arrow_schema_with_ids, + target_field_ids, + field_id_fetch_func, + searchable_field_func, + ) + } + fn fetch_field_index( fields: &Fields, index_vec: &mut Vec, @@ -90,25 +121,24 @@ impl RecordBatchProjector { { for (pos, field) in fields.iter().enumerate() { let id = field_id_fetch_func(field)?; - if let Some(id) = id { - if target_field_id == id { - index_vec.push(pos); - return Ok(Some(field.clone())); - } + if let Some(id) = id + && target_field_id == id + { + index_vec.push(pos); + return Ok(Some(field.clone())); } - if let DataType::Struct(inner) = field.data_type() { - if searchable_field_func(field) { - if let Some(res) = Self::fetch_field_index( - inner, - index_vec, - target_field_id, - field_id_fetch_func, - searchable_field_func, - )? { - index_vec.push(pos); - return Ok(Some(res)); - } - } + if let DataType::Struct(inner) = field.data_type() + && searchable_field_func(field) + && let Some(res) = Self::fetch_field_index( + inner, + index_vec, + target_field_id, + field_id_fetch_func, + searchable_field_func, + )? + { + index_vec.push(pos); + return Ok(Some(res)); } } Ok(None) @@ -125,33 +155,51 @@ impl RecordBatchProjector { self.projected_schema.clone(), self.project_column(batch.columns())?, ) - .map_err(|err| Error::new(ErrorKind::DataInvalid, format!("{err}"))) + .map_err(|err| invalid_data!("{err}")) } /// Do projection with columns - pub(crate) fn project_column(&self, batch: &[ArrayRef]) -> Result> { - self.field_indices + pub fn project_column(&self, batch: &[ArrayRef]) -> Result> { + let mut columns = Vec::with_capacity(self.field_indices.len()); + for (index_vec, field) in self + .field_indices .iter() - .map(|index_vec| Self::get_column_by_field_index(batch, index_vec)) - .collect::>>() + .zip(self.projected_schema.fields()) + { + let column = Self::get_column_by_field_index(batch, index_vec)?; + let column = match (column.data_type(), field.data_type()) { + (DataType::LargeBinary, DataType::Binary) => cast(&column, &DataType::Binary) + .map_err(|err| { + invalid_data!("Failed to cast column {} to Binary", field.name()) + .with_source(err) + })?, + _ => column, + }; + columns.push(column); + } + Ok(columns) } fn get_column_by_field_index(batch: &[ArrayRef], field_index: &[usize]) -> Result { let mut rev_iterator = field_index.iter().rev(); - let mut array = batch[*rev_iterator.next().unwrap()].clone(); - let mut null_buffer = array.logical_nulls(); + let mut array = &batch[*rev_iterator.next().unwrap()]; + let mut ancestor_nulls = None; for idx in rev_iterator { - array = array + let struct_array = array .as_any() .downcast_ref::() - .ok_or(Error::new( - ErrorKind::Unexpected, - "Cannot convert Array to StructArray", - ))? - .column(*idx) - .clone(); - null_buffer = NullBuffer::union(null_buffer.as_ref(), array.logical_nulls().as_ref()); + .ok_or_else(|| { + Error::new(ErrorKind::Unexpected, "Cannot convert Array to StructArray") + })?; + ancestor_nulls = NullBuffer::union(ancestor_nulls.as_ref(), struct_array.nulls()); + array = struct_array.column(*idx); } + let Some(ancestor_nulls) = ancestor_nulls else { + // Iceberg leaf types handled here have logical_nulls() == nulls(), so no null + // materialization is needed when no ancestor contributes nulls. + return Ok(Arc::clone(array)); + }; + let null_buffer = NullBuffer::union(Some(&ancestor_nulls), array.logical_nulls().as_ref()); Ok(make_array( array.to_data().into_builder().nulls(null_buffer).build()?, )) @@ -162,12 +210,40 @@ impl RecordBatchProjector { mod test { use std::sync::Arc; - use arrow_array::{ArrayRef, Int32Array, RecordBatch, StringArray, StructArray}; + use arrow_array::{Array, ArrayRef, Int32Array, RecordBatch, StringArray, StructArray}; + use arrow_buffer::NullBuffer; use arrow_schema::{DataType, Field, Fields, Schema}; use crate::arrow::record_batch_projector::RecordBatchProjector; + use crate::spec::{NestedField, PrimitiveType, Schema as IcebergSchema, Type}; use crate::{Error, ErrorKind}; + fn nested_projector() -> (RecordBatchProjector, Field, Field) { + let leaf_field = Field::new("leaf", DataType::Int32, true); + let inner_field = Field::new( + "inner", + DataType::Struct(Fields::from(vec![leaf_field.clone()])), + true, + ); + let outer_field = Field::new( + "outer", + DataType::Struct(Fields::from(vec![inner_field.clone()])), + true, + ); + let schema = Arc::new(Schema::new(vec![outer_field])); + let field_id_fetch_func = |field: &Field| -> crate::Result> { + match field.name().as_str() { + "outer" => Ok(Some(1)), + "inner" => Ok(Some(2)), + "leaf" => Ok(Some(3)), + _ => Ok(None), + } + }; + let projector = + RecordBatchProjector::new(schema, &[3], field_id_fetch_func, |_| true).unwrap(); + (projector, inner_field, leaf_field) + } + #[test] fn test_record_batch_projector_nested_level() { let inner_fields = vec![ @@ -231,6 +307,102 @@ mod test { assert_eq!(projected_inner_int_array.values(), &[4, 5, 6]); } + #[test] + fn test_record_batch_projector_top_level_nullable_column() { + let iceberg_schema = IcebergSchema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let projector = + RecordBatchProjector::from_iceberg_schema(Arc::new(iceberg_schema), &[1]).unwrap(); + let input = Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef; + + let projected = projector + .project_column(std::slice::from_ref(&input)) + .unwrap(); + assert!(Arc::ptr_eq(&projected[0], &input)); + let projected_array = projected[0].as_any().downcast_ref::().unwrap(); + + assert_eq!(projected_array.value(0), 10); + assert_eq!(projected_array.null_count(), 1); + assert!(projected_array.is_null(1)); + assert_eq!(projected_array.value(2), 30); + } + + #[test] + fn test_record_batch_projector_nested_nullable_leaf_without_parent_nulls() { + let (projector, inner_field, leaf_field) = nested_projector(); + let leaf = Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef; + let inner = Arc::new(StructArray::new( + Fields::from(vec![leaf_field]), + vec![leaf.clone()], + None, + )) as ArrayRef; + let outer = Arc::new(StructArray::new( + Fields::from(vec![inner_field]), + vec![inner], + None, + )) as ArrayRef; + + let projected = projector.project_column(&[outer]).unwrap(); + assert!(Arc::ptr_eq(&projected[0], &leaf)); + } + + #[test] + fn test_record_batch_projector_nested_all_valid_slice_reuses_leaf() { + let (projector, inner_field, leaf_field) = nested_projector(); + let leaf = Arc::new(Int32Array::from(vec![Some(0), Some(10), None, Some(30)])) as ArrayRef; + let inner = Arc::new(StructArray::new( + Fields::from(vec![leaf_field]), + vec![leaf], + None, + )) as ArrayRef; + let outer = StructArray::new( + Fields::from(vec![inner_field]), + vec![inner], + Some(NullBuffer::from(vec![false, true, true, true])), + ) + .slice(1, 3); + assert_eq!(outer.null_count(), 0); + assert!(outer.nulls().is_some()); + + let sliced_inner = outer + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let sliced_leaf = sliced_inner.column(0).clone(); + let projected = projector + .project_column(&[Arc::new(outer) as ArrayRef]) + .unwrap(); + assert!(Arc::ptr_eq(&projected[0], &sliced_leaf)); + } + + #[test] + fn test_record_batch_projector_propagates_nested_parent_nulls() { + let (projector, inner_field, leaf_field) = nested_projector(); + let leaf = Arc::new(Int32Array::from(vec![Some(10), Some(20), Some(30), None])) as ArrayRef; + let inner = Arc::new(StructArray::new( + Fields::from(vec![leaf_field]), + vec![leaf], + Some(NullBuffer::from(vec![true, true, false, true])), + )) as ArrayRef; + let outer = Arc::new(StructArray::new( + Fields::from(vec![inner_field]), + vec![inner], + Some(NullBuffer::from(vec![true, false, true, true])), + )) as ArrayRef; + + let projected = projector.project_column(&[outer]).unwrap(); + assert_eq!( + projected[0].as_ref(), + &Int32Array::from(vec![Some(10), None, None, None]) + ); + } + #[test] fn test_field_not_found() { let inner_fields = vec![ @@ -293,4 +465,25 @@ mod test { RecordBatchProjector::new(schema.clone(), &[3], field_id_fetch_func, |_| true); assert!(projector.is_ok()); } + + #[test] + fn test_from_iceberg_schema() { + let iceberg_schema = IcebergSchema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "age", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let projector = + RecordBatchProjector::from_iceberg_schema(Arc::new(iceberg_schema), &[1, 3]).unwrap(); + + assert_eq!(projector.field_indices.len(), 2); + assert_eq!(projector.projected_schema_ref().fields().len(), 2); + assert_eq!(projector.projected_schema_ref().field(0).name(), "id"); + assert_eq!(projector.projected_schema_ref().field(1).name(), "age"); + } } diff --git a/crates/iceberg/src/arrow/record_batch_transformer.rs b/crates/iceberg/src/arrow/record_batch_transformer.rs index 779f1cc625..9cd626547e 100644 --- a/crates/iceberg/src/arrow/record_batch_transformer.rs +++ b/crates/iceberg/src/arrow/record_batch_transformer.rs @@ -15,23 +15,129 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use arrow_arith::boolean::is_not_null; +use arrow_array::cast::AsArray; use arrow_array::{ - Array as ArrowArray, ArrayRef, BinaryArray, BooleanArray, Float32Array, Float64Array, - Int32Array, Int64Array, NullArray, RecordBatch, RecordBatchOptions, StringArray, + Array as ArrowArray, ArrayRef, Int32Array, Int64Array, ListArray, MapArray, RecordBatch, + RecordBatchOptions, RunArray, StructArray, }; use arrow_cast::cast; use arrow_schema::{ - DataType, FieldRef, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, SchemaRef, + DataType, Field, FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, + SchemaRef, }; +use arrow_select::zip::zip; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; -use crate::arrow::schema_to_arrow_schema; -use crate::spec::{Literal, PrimitiveLiteral, Schema as IcebergSchema}; +use crate::arrow::schema::{ + get_field_id_from_metadata, is_variant_field, try_get_field_id_from_metadata, + validate_unshredded_variant, +}; +use crate::arrow::value::{create_primitive_array_repeated, create_primitive_array_single_element}; +use crate::arrow::{ + datum_to_arrow_type_with_ree, primitive_type_to_arrow_type_with_ree, schema_to_arrow_schema, + type_to_arrow_type, +}; +use crate::error::invalid_data; +use crate::metadata_columns::{ + RESERVED_COL_NAME_PARTITION, RESERVED_FIELD_ID_PARTITION, get_metadata_field, +}; +use crate::spec::{ + Datum, Literal, PartitionSpec, PrimitiveLiteral, Schema as IcebergSchema, Struct, StructType, + Transform, +}; use crate::{Error, ErrorKind, Result}; +/// Create an Arrow Field with PARQUET_FIELD_ID_META_KEY metadata attached. +fn field_with_id( + name: impl Into, + data_type: DataType, + nullable: bool, + field_id: i32, +) -> Field { + Field::new(name, data_type, nullable).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + field_id.to_string(), + )])) +} + +/// Build a map of field ID to constant value (as Datum) for identity-partitioned fields. +/// +/// Implements Iceberg spec "Column Projection" rule #1: use partition metadata constants +/// only for identity-transformed fields. Non-identity transforms (bucket, truncate, year, etc.) +/// store derived values in partition metadata, so source columns must be read from data files. +/// +/// Example: For `bucket(4, id)`, partition metadata has `id_bucket = 2` (bucket number), +/// but the actual `id` values (100, 200, 300) are only in the data file. +/// +/// Matches Java's `PartitionUtil.constantsMap()` which filters `if (field.transform().isIdentity())`. +/// +/// # References +/// - Spec: https://iceberg.apache.org/spec/#column-projection +/// - Java: core/src/main/java/org/apache/iceberg/util/PartitionUtil.java:constantsMap() +fn constants_map( + partition_spec: &PartitionSpec, + partition_data: &Struct, + schema: &IcebergSchema, +) -> Result> { + let mut constants = HashMap::new(); + + for (pos, field) in partition_spec.fields().iter().enumerate() { + // Only identity transforms should use constant values from partition metadata + if matches!(field.transform, Transform::Identity) { + // The source column may have been dropped from the schema after the spec was + // created. It cannot be projected in that case, so no constant is needed. + let Some(iceberg_field) = schema.field_by_id(field.source_id) else { + continue; + }; + + // Ensure the field type is primitive + let prim_type = match &*iceberg_field.field_type { + crate::spec::Type::Primitive(prim_type) => prim_type, + _ => { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Partition field {} has non-primitive type {:?}", + field.source_id, iceberg_field.field_type + ), + )); + } + }; + + // Get the partition value for this field + // Handle both None (null) and Some(Literal::Primitive) cases + match &partition_data[pos] { + None => { + // Skip null partition values - they will be resolved as null per Iceberg spec rule #4. + // When a partition value is null, we don't add it to the constants map, + // allowing downstream column resolution to handle it correctly. + continue; + } + Some(Literal::Primitive(value)) => { + // Create a Datum from the primitive type and value + let datum = Datum::new(prim_type.clone(), value.clone()); + constants.insert(field.source_id, datum); + } + Some(literal) => { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Partition field {} has non-primitive value: {:?}", + field.source_id, literal + ), + )); + } + } + } + } + + Ok(constants) +} + /// Indicates how a particular column in a processed RecordBatch should /// be sourced. #[derive(Debug)] @@ -44,25 +150,44 @@ pub(crate) enum ColumnSource { // signifies that a column from the file's RecordBatch has undergone // type promotion so the source column with the given index needs - // to be promoted to the specified type + // to be promoted according to the given plan Promote { - target_type: DataType, + plan: PromotePlan, source_index: usize, }, - // Signifies that a new column has been inserted before the column - // with index `index`. (we choose "before" rather than "after" so - // that we can use usize; if we insert after, then we need to - // be able to store -1 here to signify that a new - // column is to be added at the front of the column list). - // If multiple columns need to be inserted at a given - // location, they should all be given the same index, as the index - // here refers to the original RecordBatch, not the interim state after - // a preceding operation. + // Materializes a column by looking up its per-file constant in the + // transformer's `constant_fields` map by `field_id`, producing a column of + // `target_type`. Handles the three non-coalesce `ColumnConstant` variants: + // scalar values (`_file`, `_spec_id`, identity partitions), the `_partition` + // struct, and all-null metadata columns. The coalesce variant routes to + // `Coalesce` instead. Resolved at apply time rather than baking the value here. + ConstantLookup { + field_id: i32, + target_type: DataType, + }, + + // Inserts a schema-evolution default value (the field's `initial_default`, + // or null for an absent optional field). Distinct from `ConstantLookup`: + // this value comes from the table schema, not the per-file constant map. Add { target_type: DataType, value: Option, }, + + // A metadata column read from the file and coalesced with a per-file + // fallback looked up in `constant_fields` by `field_id`: the source column's + // value where non-null, else the fallback constant. The entry at `field_id` + // must be a `ColumnConstant::CoalesceLastUpdatedSeq` (the apply arm rejects + // any other variant). Used for `_last_updated_sequence_number` when the file + // physically carries the per-row column. The result is cast to `target_type` + // so it matches the (run-end-encoded) type the constant and null paths + // produce for the column. + Coalesce { + field_id: i32, + source_index: usize, + target_type: DataType, + }, // The iceberg spec refers to other permissible schema evolution actions // (see https://iceberg.apache.org/spec/#schema-evolution): // renaming fields, deleting fields and reordering fields. @@ -73,6 +198,339 @@ pub(crate) enum ColumnSource { // post-processing step by using the projection mask. } +/// Reconciles a source array to the target type by matching nested fields on +/// field id rather than position, mirroring iceberg-java's nested readers. +/// +/// The plan is resolved once per file, when the `BatchTransform` is built, so +/// applying it per batch is just index lookups and array assembly. +#[derive(Debug)] +pub(crate) enum PromotePlan { + PassThrough, + Cast(DataType), + Struct { + fields: Fields, + children: Vec, + }, + List { + field: FieldRef, + element: Box, + }, + Map { + field: FieldRef, + entry_fields: Fields, + entries: Vec, + sorted: bool, + }, +} + +#[derive(Debug)] +pub(crate) enum ChildPlan { + FromSource { + source_index: usize, + plan: PromotePlan, + }, + Add { + target_type: DataType, + value: Option, + }, +} + +impl PromotePlan { + /// Plans one field, dispatching on the target field so a variant is seen by its extension. + fn build_field( + source: &Field, + target: &Field, + snapshot_schema: &IcebergSchema, + constant_fields: &HashMap, + ) -> Result { + if is_variant_field(target) { + return Self::build_variant(source, target); + } + Self::build( + source.data_type(), + target.data_type(), + snapshot_schema, + constant_fields, + ) + } + + /// Checks the storage as projection does. A variant's `metadata` and `value` carry no field + /// ids, so they are matched by name. + fn build_variant(source: &Field, target: &Field) -> Result { + validate_unshredded_variant(source, get_field_id_from_metadata(target)?)?; + let (source_type, target_type) = (source.data_type(), target.data_type()); + if source_type == target_type { + return Ok(PromotePlan::PassThrough); + } + let mismatch = || { + invalid_data!("cannot read variant stored as {source_type:?}, expected {target_type:?}") + }; + let (DataType::Struct(source_fields), DataType::Struct(target_fields)) = + (source_type, target_type) + else { + return Err(mismatch()); + }; + let children = target_fields + .iter() + .map(|target_field| { + let (source_index, source_field) = source_fields + .find(target_field.name()) + .ok_or_else(mismatch)?; + let plan = if source_field.data_type() == target_field.data_type() { + PromotePlan::PassThrough + } else { + PromotePlan::Cast(target_field.data_type().clone()) + }; + Ok(ChildPlan::FromSource { source_index, plan }) + }) + .collect::>()?; + Ok(PromotePlan::Struct { + fields: target_fields.clone(), + children, + }) + } + + fn build( + source: &DataType, + target: &DataType, + snapshot_schema: &IcebergSchema, + constant_fields: &HashMap, + ) -> Result { + if source == target { + return Ok(PromotePlan::PassThrough); + } + match (source, target) { + (DataType::Struct(source_fields), DataType::Struct(target_fields)) => { + Ok(PromotePlan::Struct { + children: Self::build_struct_children( + source_fields, + target_fields, + snapshot_schema, + constant_fields, + )?, + fields: target_fields.clone(), + }) + } + // A file's Arrow schema hint can give either offset width; `apply` converts it. + // Table schemas are always `List` (`ToArrowSchemaConverter::list`). + ( + DataType::List(source_field) | DataType::LargeList(source_field), + DataType::List(target_field), + ) => Ok(PromotePlan::List { + element: Box::new(Self::build_field( + source_field, + target_field, + snapshot_schema, + constant_fields, + )?), + field: target_field.clone(), + }), + (DataType::Map(source_entries, _), DataType::Map(target_entries, sorted)) => { + match (source_entries.data_type(), target_entries.data_type()) { + // Key and value match by position, as in iceberg-java's map readers. + (DataType::Struct(source_fields), DataType::Struct(target_fields)) + if source_fields.len() == target_fields.len() => + { + Ok(PromotePlan::Map { + entries: source_fields + .iter() + .zip(target_fields.iter()) + .enumerate() + .map(|(source_index, (source_field, target_field))| { + Ok(ChildPlan::FromSource { + source_index, + plan: Self::build_field( + source_field, + target_field, + snapshot_schema, + constant_fields, + )?, + }) + }) + .collect::>()?, + entry_fields: target_fields.clone(), + field: target_entries.clone(), + sorted: *sorted, + }) + } + (DataType::Struct(source_fields), DataType::Struct(target_fields)) => { + Err(invalid_data!( + "map entries have {} source fields, expected {}", + source_fields.len(), + target_fields.len() + )) + } + _ => Err(invalid_data!( + "expected struct-typed map entries, got {source_entries:?} and {target_entries:?}" + )), + } + } + (_, DataType::Struct(_) | DataType::List(_) | DataType::LargeList(_)) + | (_, DataType::Map(_, _)) + | (DataType::Struct(_) | DataType::List(_) | DataType::LargeList(_), _) + | (DataType::Map(_, _), _) => { + Err(invalid_data!("cannot promote {source:?} to {target:?}")) + } + _ => Ok(PromotePlan::Cast(target.clone())), + } + } + + fn build_struct_children( + source_fields: &Fields, + target_fields: &Fields, + snapshot_schema: &IcebergSchema, + constant_fields: &HashMap, + ) -> Result> { + let mut source_by_id = HashMap::with_capacity(source_fields.len()); + for (idx, field) in source_fields.iter().enumerate() { + if let Some(id) = try_get_field_id_from_metadata(field)? + && source_by_id.insert(id, idx).is_some() + { + return Err(invalid_data!("duplicate field id {id} in struct")); + } + } + // Fully id-less children (no field ids and no name mapping) match no target + // field and read as absent, as in iceberg-java. A partial set of ids errors. + if !source_by_id.is_empty() && source_by_id.len() != source_fields.len() { + return Err(invalid_data!( + "cannot reconcile struct fields by id: source fields do not all have field ids" + )); + } + + target_fields + .iter() + .map(|target_field| { + let field_id = try_get_field_id_from_metadata(target_field)?; + match field_id.and_then(|id| source_by_id.get(&id).copied()) { + Some(source_index) => Ok(ChildPlan::FromSource { + plan: Self::build_field( + &source_fields[source_index], + target_field, + snapshot_schema, + constant_fields, + )?, + source_index, + }), + None => { + // Identity partition constants apply inside structs too. A missing + // child uses that value before initial-default (#3261) or null. + if let Some(ColumnConstant::Scalar(datum)) = + field_id.and_then(|id| constant_fields.get(&id)) + { + return Ok(ChildPlan::Add { + target_type: target_field.data_type().clone(), + value: Some(datum.literal().clone()), + }); + } + // Missing nested fields are null-filled. Applying an + // initial-default here is not implemented yet (#3261). + let iceberg_field = + field_id.and_then(|id| snapshot_schema.field_by_id(id)); + if iceberg_field.is_some_and(|f| f.initial_default.is_some()) { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "initial-default of nested field {} is not supported, see https://github.com/apache/iceberg-rust/issues/3261", + target_field.name() + ), + )); + } + if iceberg_field.map_or(!target_field.is_nullable(), |f| f.required) { + return Err(invalid_data!( + "required nested field {} is absent from the data file and has no initial-default", + target_field.name() + )); + } + Ok(ChildPlan::Add { + target_type: target_field.data_type().clone(), + value: None, + }) + } + } + }) + .collect() + } + + fn apply(&self, array: &ArrayRef) -> Result { + match self { + PromotePlan::PassThrough => Ok(Arc::clone(array)), + PromotePlan::Cast(target_type) => Ok(cast(array.as_ref(), target_type)?), + PromotePlan::Struct { fields, children } => { + let source = array + .as_struct_opt() + .ok_or_else(|| promote_err(array, "struct"))?; + Ok(Arc::new(Self::apply_struct(source, fields, children)?)) + } + PromotePlan::List { field, element } => { + let array = match array.data_type() { + DataType::LargeList(f) => cast(array.as_ref(), &DataType::List(Arc::clone(f)))?, + _ => Arc::clone(array), + }; + let source = array + .as_list_opt::() + .ok_or_else(|| promote_err(&array, "list"))?; + Ok(Arc::new(ListArray::try_new( + field.clone(), + source.offsets().clone(), + element.apply(source.values())?, + source.nulls().cloned(), + )?)) + } + PromotePlan::Map { + field, + entry_fields, + entries, + sorted, + } => { + let source = array + .as_map_opt() + .ok_or_else(|| promote_err(array, "map"))?; + let entries = Self::apply_struct(source.entries(), entry_fields, entries)?; + Ok(Arc::new(MapArray::try_new( + field.clone(), + source.offsets().clone(), + entries, + source.nulls().cloned(), + *sorted, + )?)) + } + } + } + + fn apply_struct( + source: &StructArray, + fields: &Fields, + children: &[ChildPlan], + ) -> Result { + let columns = children + .iter() + .map(|child| match child { + ChildPlan::FromSource { source_index, plan } => { + plan.apply(source.column(*source_index)) + } + ChildPlan::Add { target_type, value } => { + RecordBatchTransformer::create_column(target_type, value.as_ref(), source.len()) + } + }) + .collect::>>()?; + Ok(StructArray::try_new( + fields.clone(), + columns, + source.nulls().cloned(), + )?) + } +} + +fn promote_err(array: &ArrayRef, expected: &str) -> Error { + Error::new( + ErrorKind::Unexpected, + format!( + "expected a {expected} array to promote, got {:?}", + array.data_type() + ), + ) +} + #[derive(Debug)] enum BatchTransform { // Indicates that no changes need to be performed to the RecordBatches @@ -107,32 +565,236 @@ enum SchemaComparison { Different, } +/// Builder for RecordBatchTransformer to improve ergonomics when constructing with optional parameters. +/// +/// All per-file constants (scalar metadata like `_file`, identity partition values, +/// and the `_partition` struct) are stored in a single `constant_fields` map keyed by +/// field_id (via [`ColumnConstant`]). Each is resolved at apply time from that single +/// map (through a `ConstantLookup` or `Coalesce` action) rather than baked into the +/// action. #[derive(Debug)] -pub(crate) struct RecordBatchTransformer { +pub(crate) struct RecordBatchTransformerBuilder { snapshot_schema: Arc, projected_iceberg_field_ids: Vec, + constant_fields: HashMap, + virtual_fields: HashSet, +} - // BatchTransform gets lazily constructed based on the schema of - // the first RecordBatch we receive from the file - batch_transform: Option, +/// How a metadata (or identity-partition) column's values are supplied. +/// +/// Covers scalar constants (metadata columns like `_file` and `_spec_id`, and +/// identity partition source fields), the struct `_partition` column, an all-null +/// column, and a per-row column coalesced with a per-file fallback. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum ColumnConstant { + /// A scalar constant (e.g., `_file` path, `_spec_id`, identity partition values). + /// The Datum carries both the Iceberg type and the value. + Scalar(Datum), + /// A struct constant (the `_partition` column). Each child is a primitive constant + /// or null (for partition evolution gaps). + Struct(StructConstant), + /// An all-null metadata column, carrying its Arrow type. Used when a metadata + /// column resolves to null (e.g. `_last_updated_sequence_number` for a file + /// with a null `first_row_id`). A distinct variant (rather than `Scalar` with a + /// null value) because `Datum` always represents a non-null value; the type is + /// computed once so the schema and column-source paths cannot derive it apart. + Null(DataType), + /// A metadata column physically present in the file that must be coalesced with + /// a per-file fallback: read the per-row value, substitute this Datum where the + /// per-row value is null (e.g. `_last_updated_sequence_number` on a file that + /// carries the column, falling back to the data sequence number). + CoalesceLastUpdatedSeq(Datum), } -impl RecordBatchTransformer { - /// Build a RecordBatchTransformer for a given - /// Iceberg snapshot schema and list of projected field ids. - pub(crate) fn build( +/// Pre-computed data for a struct constant column. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct StructConstant { + fields: Fields, + child_values: Vec>, +} + +impl StructConstant { + /// Create a new StructConstant, validating that fields and child_values have + /// the same length. + pub(crate) fn new(fields: Fields, child_values: Vec>) -> Result { + if fields.len() != child_values.len() { + return Err(invalid_data!( + "StructConstant: fields length ({}) != child_values length ({})", + fields.len(), + child_values.len() + )); + } + Ok(Self { + fields, + child_values, + }) + } + + pub(crate) fn fields(&self) -> &Fields { + &self.fields + } + + pub(crate) fn child_values(&self) -> &[Option] { + &self.child_values + } +} + +impl RecordBatchTransformerBuilder { + pub(crate) fn new( snapshot_schema: Arc, projected_iceberg_field_ids: &[i32], ) -> Self { - let projected_iceberg_field_ids = projected_iceberg_field_ids.to_vec(); - Self { snapshot_schema, - projected_iceberg_field_ids, + projected_iceberg_field_ids: projected_iceberg_field_ids.to_vec(), + constant_fields: HashMap::new(), + virtual_fields: HashSet::new(), + } + } + + /// Add a scalar constant value for a specific field ID. + /// This is used for virtual/metadata fields like _file that have constant values per batch. + pub(crate) fn with_constant(mut self, field_id: i32, datum: Datum) -> Self { + self.constant_fields + .insert(field_id, ColumnConstant::Scalar(datum)); + self + } + + /// Add a coalesced `_last_updated_sequence_number` column for a specific field ID: + /// the file's per-row value where non-null, falling back to `datum` where null. + /// Used when the file physically carries the column. + pub(crate) fn with_coalesced_last_updated_seq_column( + mut self, + field_id: i32, + datum: Datum, + ) -> Self { + self.constant_fields + .insert(field_id, ColumnConstant::CoalesceLastUpdatedSeq(datum)); + self + } + + /// Add an all-null metadata column for a specific field ID. + /// Used for metadata columns that resolve to null (e.g. + /// `_last_updated_sequence_number` when the file has a null `first_row_id`). + pub(crate) fn with_null_metadata_column(mut self, field_id: i32) -> Result { + let iceberg_field = get_metadata_field(field_id)?; + let prim_type = iceberg_field + .field_type + .as_primitive_type() + .ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!( + "Metadata column {} has non-primitive type {:?}", + iceberg_field.name, iceberg_field.field_type + ), + ) + })?; + // Use the same run-end-encoded type as the non-null constant path, so a + // column keeps one Arrow type whether a given file resolves it to a value + // or to null. + let arrow_type = primitive_type_to_arrow_type_with_ree(prim_type); + self.constant_fields + .insert(field_id, ColumnConstant::Null(arrow_type)); + Ok(self) + } + + /// Set partition spec and data together for identifying identity-transformed partition columns. + /// + /// Both partition_spec and partition_data must be provided together since the spec defines + /// which fields are identity-partitioned, and the data provides their constant values. + /// This method computes the partition constants and merges them into constant_fields. + pub(crate) fn with_partition( + mut self, + partition_spec: Arc, + partition_data: Struct, + ) -> Result { + let partition_constants = + constants_map(&partition_spec, &partition_data, &self.snapshot_schema)?; + + for (field_id, datum) in partition_constants { + self.constant_fields + .insert(field_id, ColumnConstant::Scalar(datum)); + } + + Ok(self) + } + + /// Set virtual fields such as '_pos'. Caller must have registered a corresponding + /// ArrowReaderOptions::with_virtual_columns call on the reader + pub(crate) fn with_virtual_field(mut self, field_id: i32) -> Self { + self.virtual_fields.insert(field_id); + self + } + + /// Set a pre-computed _partition column constant directly. + pub(crate) fn with_partition_constant(mut self, partition_column: StructConstant) -> Self { + self.constant_fields.insert( + RESERVED_FIELD_ID_PARTITION, + ColumnConstant::Struct(partition_column), + ); + self + } + + pub(crate) fn build(self) -> RecordBatchTransformer { + RecordBatchTransformer { + snapshot_schema: self.snapshot_schema, + projected_iceberg_field_ids: self.projected_iceberg_field_ids, + constant_fields: self.constant_fields, + virtual_fields: self.virtual_fields, batch_transform: None, } } +} + +/// Transforms RecordBatches from Parquet files to match the Iceberg table schema. +/// +/// Handles schema evolution, column reordering, type promotion, and implements the Iceberg spec's +/// "Column Projection" rules for resolving field IDs "not present" in data files: +/// 1. Return the value from partition metadata if an Identity Transform exists +/// 2. Use schema.name-mapping.default metadata to map field id to columns without field id (applied in ArrowReader) +/// 3. Return the default value if it has a defined initial-default +/// 4. Return null in all other cases +/// +/// # Field ID Resolution +/// +/// Field ID resolution happens in ArrowReader before data is read (matching Java's ReadConf): +/// - If file has embedded field IDs: trust them (ParquetSchemaUtil.hasIds() = true) +/// - If file lacks IDs and name_mapping exists: apply name mapping (ParquetSchemaUtil.applyNameMapping()) +/// - If file lacks IDs and no name_mapping: use position-based fallback (ParquetSchemaUtil.addFallbackIds()) +/// +/// By the time RecordBatchTransformer processes data, all field IDs are trustworthy. +/// This transformer only handles remaining projection rules (#1, #3, #4) for fields still "not present". +/// +/// # Partition Spec and Data +/// +/// **Bucket partitioning**: Distinguish identity transforms (use partition metadata constants) +/// from non-identity transforms like bucket (read from data file) to enable runtime filtering on +/// bucket-partitioned columns. For example, `bucket(4, id)` stores only the bucket number in +/// partition metadata, so actual `id` values must be read from the data file. +/// +/// # References +/// - Spec: https://iceberg.apache.org/spec/#column-projection +/// - Java: parquet/src/main/java/org/apache/iceberg/parquet/ReadConf.java (field ID resolution) +/// - Java: core/src/main/java/org/apache/iceberg/util/PartitionUtil.java (partition constants) +#[derive(Debug)] +pub(crate) struct RecordBatchTransformer { + snapshot_schema: Arc, + projected_iceberg_field_ids: Vec, + // Unified map of all per-file constant columns (metadata, identity partition, _partition struct). + constant_fields: HashMap, + + // Field IDs whose data is delivered by the source batch as an arrow-rs virtual column + // (e.g. _pos via RowNumber). These fields bypass the snapshot-schema lookup and the + // Iceberg projection rules (name mapping / initial-default / null) + virtual_fields: HashSet, + + // BatchTransform gets lazily constructed based on the schema of + // the first RecordBatch we receive from the file + batch_transform: Option, +} +impl RecordBatchTransformer { pub(crate) fn process_record_batch( &mut self, record_batch: RecordBatch, @@ -147,7 +809,7 @@ impl RecordBatchTransformer { .with_match_field_names(false) .with_row_count(Some(record_batch.num_rows())); RecordBatch::try_new_with_options( - target_schema.clone(), + Arc::clone(target_schema), self.transform_columns(record_batch.columns(), operations)?, &options, )? @@ -157,7 +819,7 @@ impl RecordBatchTransformer { .with_match_field_names(false) .with_row_count(Some(record_batch.num_rows())); RecordBatch::try_new_with_options( - target_schema.clone(), + Arc::clone(target_schema), record_batch.columns().to_vec(), &options, )? @@ -167,6 +829,8 @@ impl RecordBatchTransformer { record_batch.schema_ref(), self.snapshot_schema.as_ref(), &self.projected_iceberg_field_ids, + &self.constant_fields, + &self.virtual_fields, )?); self.process_record_batch(record_batch)? @@ -185,6 +849,8 @@ impl RecordBatchTransformer { source_schema: &ArrowSchemaRef, snapshot_schema: &IcebergSchema, projected_iceberg_field_ids: &[i32], + constant_fields: &HashMap, + virtual_fields: &HashSet, ) -> Result { let mapped_unprojected_arrow_schema = Arc::new(schema_to_arrow_schema(snapshot_schema)?); let field_id_to_mapped_schema_map = @@ -195,6 +861,86 @@ impl RecordBatchTransformer { let fields: Result> = projected_iceberg_field_ids .iter() .map(|field_id| { + match constant_fields.get(field_id) { + Some(ColumnConstant::Struct(pc)) + if *field_id == RESERVED_FIELD_ID_PARTITION => + { + let struct_type = DataType::Struct(pc.fields().clone()); + let nullable = pc.fields().is_empty(); + let arrow_field = field_with_id( + RESERVED_COL_NAME_PARTITION, + struct_type, + nullable, + *field_id, + ); + return Ok(Arc::new(arrow_field)); + } + Some(ColumnConstant::Struct(_)) => { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Struct column constants are only supported for the `_partition` \ + metadata column (field id {RESERVED_FIELD_ID_PARTITION}), but \ + one was set for field id {field_id}" + ), + )); + } + Some(ColumnConstant::Scalar(datum)) => { + if let Ok(iceberg_field) = get_metadata_field(*field_id) { + let arrow_type = datum_to_arrow_type_with_ree(datum); + let arrow_field = field_with_id( + &iceberg_field.name, + arrow_type, + !iceberg_field.required, + iceberg_field.id, + ); + return Ok(Arc::new(arrow_field)); + } + // Identity partition constant -- fall through to use the + // mapped schema field (read from file if present). + } + Some(ColumnConstant::Null(arrow_type)) => { + let iceberg_field = get_metadata_field(*field_id)?; + // Always nullable: this arm produces an all-null column regardless + // of the field's declared optionality. + let arrow_field = field_with_id( + &iceberg_field.name, + arrow_type.clone(), + true, + iceberg_field.id, + ); + return Ok(Arc::new(arrow_field)); + } + Some(ColumnConstant::CoalesceLastUpdatedSeq(datum)) => { + // Same run-end-encoded type as the constant/null paths, so the + // column has one Arrow type whether a file coalesces, uses the + // constant, or is nulled. + let iceberg_field = get_metadata_field(*field_id)?; + let arrow_field = field_with_id( + &iceberg_field.name, + datum_to_arrow_type_with_ree(datum), + true, + iceberg_field.id, + ); + return Ok(Arc::new(arrow_field)); + } + None => {} + } + + if virtual_fields.contains(field_id) { + let virtual_field = get_metadata_field(*field_id)?; + let arrow_type = type_to_arrow_type(&virtual_field.field_type)?; + let arrow_field = field_with_id( + &virtual_field.name, + arrow_type, + !virtual_field.required, + virtual_field.id, + ); + return Ok(Arc::new(arrow_field)); + } + + // Regular fields and identity-partitioned constant fields both exist in the + // table schema, so use the mapped Arrow field as-is. Ok(field_id_to_mapped_schema_map .get(field_id) .ok_or(Error::new(ErrorKind::Unexpected, "field not found"))? @@ -206,7 +952,12 @@ impl RecordBatchTransformer { let target_schema = Arc::new(ArrowSchema::new(fields?)); match Self::compare_schemas(source_schema, &target_schema) { - SchemaComparison::Equivalent => Ok(BatchTransform::PassThrough), + SchemaComparison::Equivalent => Ok( + match Self::with_variant_extensions(source_schema, &target_schema) { + Some(target_schema) => BatchTransform::ModifySchema { target_schema }, + None => BatchTransform::PassThrough, + }, + ), SchemaComparison::NameChangesOnly => Ok(BatchTransform::ModifySchema { target_schema }), SchemaComparison::Different => Ok(BatchTransform::Modify { operations: Self::generate_transform_operations( @@ -214,24 +965,28 @@ impl RecordBatchTransformer { snapshot_schema, projected_iceberg_field_ids, field_id_to_mapped_schema_map, + constant_fields, + virtual_fields, )?, target_schema, }), } } - /// Compares the source and target schemas - /// Determines if they have changed in any meaningful way: - /// * If they have different numbers of fields, then we need to modify - /// the incoming RecordBatch schema AND columns - /// * If they have the same number of fields, but some of them differ in - /// either data type or nullability, then we need to modify the - /// incoming RecordBatch schema AND columns - /// * If the schemas differ only in the column names, then we need - /// to modify the RecordBatch schema BUT we can keep the - /// original column data unmodified - /// * If the schemas are identical (or differ only in inconsequential - /// ways) then we can pass through the original RecordBatch unmodified + /// Compares the source and target schemas to decide how much work the + /// transform must do. The checks are applied in this order: + /// 1. If the schemas have different numbers of fields, they are + /// `Different`: we must rebuild both the schema AND the columns. + /// 2. If every field matches positionally in name, data type, and + /// nullability, they are `Equivalent`: the original RecordBatch can be + /// passed through unmodified. + /// 3. If they contain the same set of fields but in a different order + /// (detected via set-equality, since field names are unique), they are + /// `Different`: the columns must be rebuilt in the target order. + /// 4. Otherwise, the fields align positionally: if any differs in data + /// type or nullability the schemas are `Different` (rebuild schema AND + /// columns); if they differ only in names, it is `NameChangesOnly` and + /// we can relabel the schema BUT keep the original column data. fn compare_schemas( source_schema: &ArrowSchemaRef, target_schema: &ArrowSchemaRef, @@ -240,87 +995,275 @@ impl RecordBatchTransformer { return SchemaComparison::Different; } - let mut names_changed = false; + let field_eq = |a: &FieldRef, b: &FieldRef| { + a.name() == b.name() + && a.data_type() == b.data_type() + && a.is_nullable() == b.is_nullable() + }; - for (source_field, target_field) in source_schema + let positional = || source_schema.fields().iter().zip(target_schema.fields()); + + // Field-by-field identical: nothing to do. + if positional().all(|(s, t)| field_eq(s, t)) { + return SchemaComparison::Equivalent; + } + + // Same set of fields but reordered: rebuild in target (selection) order. + // Lengths are equal and field names are unique, so this is set-equality. + let reordered = target_schema .fields() .iter() - .zip(target_schema.fields().iter()) - { - if source_field.data_type() != target_field.data_type() - || source_field.is_nullable() != target_field.is_nullable() - { - return SchemaComparison::Different; - } - - if source_field.name() != target_field.name() { - names_changed = true; - } + .all(|t| source_schema.fields().iter().any(|s| field_eq(s, t))); + if reordered { + return SchemaComparison::Different; } - if names_changed { - SchemaComparison::NameChangesOnly + // Not a reordering: any type/nullability difference needs a full + // rebuild; otherwise only names differ and we can just relabel. + if positional() + .any(|(s, t)| s.data_type() != t.data_type() || s.is_nullable() != t.is_nullable()) + { + SchemaComparison::Different } else { - SchemaComparison::Equivalent + SchemaComparison::NameChangesOnly } } + /// Copies the variant extension from `target_schema` onto source fields that lack it, so + /// an otherwise unchanged batch still reports its variant columns as variants. `None` + /// when no field needs it. + fn with_variant_extensions( + source_schema: &ArrowSchemaRef, + target_schema: &ArrowSchemaRef, + ) -> Option { + let needs_extension = + |(s, t): (&FieldRef, &FieldRef)| is_variant_field(t) && !is_variant_field(s); + let pairs = || source_schema.fields().iter().zip(target_schema.fields()); + if !pairs().any(needs_extension) { + return None; + } + let fields: Vec = pairs() + .map(|pair| { + if needs_extension(pair) { + Arc::clone(pair.1) + } else { + Arc::clone(pair.0) + } + }) + .collect(); + Some(Arc::new(ArrowSchema::new_with_metadata( + fields, + source_schema.metadata().clone(), + ))) + } + fn generate_transform_operations( source_schema: &ArrowSchemaRef, snapshot_schema: &IcebergSchema, projected_iceberg_field_ids: &[i32], field_id_to_mapped_schema_map: HashMap, + constant_fields: &HashMap, + virtual_fields: &HashSet, ) -> Result> { let field_id_to_source_schema_map = Self::build_field_id_to_arrow_schema_map(source_schema)?; - projected_iceberg_field_ids.iter().map(|field_id|{ - let (target_field, _) = field_id_to_mapped_schema_map.get(field_id).ok_or( - Error::new(ErrorKind::Unexpected, "could not find field in schema") - )?; - let target_type = target_field.data_type(); + projected_iceberg_field_ids + .iter() + .map(|field_id| { + match constant_fields.get(field_id) { + Some(ColumnConstant::Struct(pc)) + if *field_id == RESERVED_FIELD_ID_PARTITION => + { + // The struct apply arm rebuilds the column from the map entry's own + // fields/child_values, so `target_type` is a placeholder here and is + // not read for the struct case (only the scalar/null cases consult it). + return Ok(ColumnSource::ConstantLookup { + field_id: *field_id, + target_type: DataType::Struct(pc.fields().clone()), + }); + } + Some(ColumnConstant::Struct(_)) => { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Struct column constants are only supported for the `_partition` \ + metadata column (field id {RESERVED_FIELD_ID_PARTITION}), but \ + one was set for field id {field_id}" + ), + )); + } + Some(ColumnConstant::Scalar(datum)) => { + // Metadata/virtual fields (like _file, _spec_id) never exist in the + // data file, so they always use their pre-computed constant value. + // + // For identity-partitioned fields, the Iceberg spec's "Column + // Projection" rules only apply to "field ids which are not present in + // a data file". When the column IS present in the Parquet file, it + // must be read from the file; the partition metadata constant is only + // a fallback for when the column is absent (e.g. add_files). + let is_metadata = get_metadata_field(*field_id).is_ok(); + let present_in_file = + field_id_to_source_schema_map.contains_key(field_id); - Ok(if let Some((source_field, source_index)) = field_id_to_source_schema_map.get(field_id) { - // column present in source + if is_metadata || !present_in_file { + let arrow_type = if is_metadata { + datum_to_arrow_type_with_ree(datum) + } else { + field_id_to_mapped_schema_map + .get(field_id) + .ok_or(Error::new( + ErrorKind::Unexpected, + "could not find field in schema", + ))? + .0 + .data_type() + .clone() + }; - if source_field.data_type().equals_datatype(target_type) { - // no promotion required - ColumnSource::PassThrough { - source_index: *source_index + return Ok(ColumnSource::ConstantLookup { + field_id: *field_id, + target_type: arrow_type, + }); + } + // Identity partition field present in the file -- fall through + // to read from the file instead of using the constant. } - } else { - // promotion required - ColumnSource::Promote { - target_type: target_type.clone(), - source_index: *source_index, + Some(ColumnConstant::Null(arrow_type)) => { + return Ok(ColumnSource::ConstantLookup { + field_id: *field_id, + target_type: arrow_type.clone(), + }); } + Some(ColumnConstant::CoalesceLastUpdatedSeq(datum)) => { + // Registered only for `_last_updated_sequence_number`, and only after + // the pipeline detects the embedded field id, so the source column is + // always present in the file. + let (_, source_index) = + field_id_to_source_schema_map.get(field_id).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!( + "coalesce registered for field id {field_id} but the \ + column is absent from the file batch" + ), + ) + })?; + return Ok(ColumnSource::Coalesce { + field_id: *field_id, + source_index: *source_index, + target_type: datum_to_arrow_type_with_ree(datum), + }); + } + None => {} } - } else { - // column must be added - let iceberg_field = snapshot_schema.field_by_id(*field_id).ok_or( - Error::new(ErrorKind::Unexpected, "Field not found in snapshot schema") - )?; - let default_value = if let Some(iceberg_default_value) = - &iceberg_field.initial_default - { - let Literal::Primitive(primitive_literal) = iceberg_default_value else { - return Err(Error::new( + if virtual_fields.contains(field_id) { + let (_, source_index) = field_id_to_source_schema_map + .get(field_id) + .ok_or_else(|| Error::new( ErrorKind::Unexpected, - format!("Default value for column must be primitive type, but encountered {:?}", iceberg_default_value) - )); - }; - Some(primitive_literal.clone()) + format!("virtual column for field id {field_id} is not present in source batch — ensure ArrowReaderOptions::with_virtual_columns was applied"), + ))?; + return Ok(ColumnSource::PassThrough { source_index: *source_index }); + } + + let (target_field, _) = + field_id_to_mapped_schema_map + .get(field_id) + .ok_or(Error::new( + ErrorKind::Unexpected, + format!("could not find field {field_id} in schema"), + ))?; + let target_type = target_field.data_type(); + + let iceberg_field = snapshot_schema.field_by_id(*field_id).ok_or(Error::new( + ErrorKind::Unexpected, + "Field not found in snapshot schema", + ))?; + + // Iceberg spec's "Column Projection" rules (https://iceberg.apache.org/spec/#column-projection). + // For fields "not present" in data files: + // 1. Use partition metadata (identity transforms only) + // 2. Use name mapping + // 3. Use initial_default + // 4. Return null + // + // Why check partition constants before Parquet field IDs (Java: BaseParquetReaders.java:299): + // In add_files scenarios, partition columns may exist in BOTH Parquet AND partition metadata. + // Partition metadata is authoritative - it defines which partition this file belongs to. + + // Field ID resolution now happens in ArrowReader via: + // 1. Embedded field IDs (ParquetSchemaUtil.hasIds() = true) - trust them + // 2. Name mapping (ParquetSchemaUtil.applyNameMapping()) - applied upfront + // 3. Position-based fallback (ParquetSchemaUtil.addFallbackIds()) - applied upfront + // + // At this point, all field IDs in the source schema are trustworthy. + // No conflict detection needed - schema resolution happened in reader.rs. + // + // Apply spec's fallback steps for "not present" fields. + // Rule #1 (constants) is handled at the beginning of this function + let column_source = if let Some((source_field, source_index)) = + field_id_to_source_schema_map.get(field_id) + { + match PromotePlan::build_field( + source_field, + target_field, + snapshot_schema, + constant_fields, + )? { + PromotePlan::PassThrough => ColumnSource::PassThrough { + source_index: *source_index, + }, + plan => ColumnSource::Promote { + plan, + source_index: *source_index, + }, + } } else { - None + // Rules #2, #3 and #4: + // Rule #2 (name mapping) was already applied in reader.rs if needed. + // If field_id is still not found, the column doesn't exist in the Parquet file. + // Fall through to rule #3 (initial_default) or rule #4 (null). + // + // Per the spec's "Default values", null is only a valid default for an + // optional field. A required field that is absent with no initial-default + // therefore has no valid value and must error, rather than producing a null + // column that violates the field's required constraint. This matches + // Iceberg-Java's Parquet readers (BaseParquetReaders / SparkParquetReaders), + // which raise "Missing required field: ". + if iceberg_field.initial_default.is_none() && iceberg_field.required { + return Err(invalid_data!("Missing required field: {}", iceberg_field.name)); + } + + // TODO: Support the spec-defined non-null struct initial default `{}` by + // creating a non-null struct and applying each child's initial default. + // Tracked in https://github.com/apache/iceberg-rust/issues/3261. + let default_value = match iceberg_field.initial_default.as_ref() { + None => None, + Some(Literal::Primitive(prim)) => Some(prim.clone()), + Some(_) => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Cannot read field {} that is absent from the data file: \ + applying a non-primitive initial-default for {} is not yet \ + supported", + iceberg_field.name, iceberg_field.field_type + ), + )); + } + }; + + ColumnSource::Add { + value: default_value, + target_type: target_type.clone(), + } }; - ColumnSource::Add { - value: default_value, - target_type: target_type.clone(), - } + Ok(column_source) }) - }).collect() + .collect() } fn build_field_id_to_arrow_schema_map( @@ -328,25 +1271,11 @@ impl RecordBatchTransformer { ) -> Result> { let mut field_id_to_source_schema = HashMap::new(); for (source_field_idx, source_field) in source_schema.fields.iter().enumerate() { - let this_field_id = source_field - .metadata() - .get(PARQUET_FIELD_ID_META_KEY) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "field ID not present in parquet metadata", - ) - })? - .parse() - .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("field id not parseable as an i32: {}", e), - ) - })?; - - field_id_to_source_schema - .insert(this_field_id, (source_field.clone(), source_field_idx)); + if let Some(this_field_id) = try_get_field_id_from_metadata(source_field)? { + field_id_to_source_schema + .insert(this_field_id, (source_field.clone(), source_field_idx)); + } + // If field doesn't have a field ID, skip it - name mapping will handle it } Ok(field_id_to_source_schema) @@ -368,98 +1297,964 @@ impl RecordBatchTransformer { Ok(match op { ColumnSource::PassThrough { source_index } => columns[*source_index].clone(), - ColumnSource::Promote { - target_type, - source_index, - } => cast(&*columns[*source_index], target_type)?, + ColumnSource::Promote { plan, source_index } => { + plan.apply(&columns[*source_index])? + } ColumnSource::Add { target_type, value } => { - Self::create_column(target_type, value, num_rows)? + Self::create_column(target_type, value.as_ref(), num_rows)? + } + + // Resolve the per-file constant for this field id from the map and + // materialize it as the target type. + ColumnSource::ConstantLookup { + field_id, + target_type, + } => match self.constant_fields.get(field_id) { + Some(ColumnConstant::Scalar(datum)) => { + Self::create_column(target_type, Some(datum.literal()), num_rows)? + } + Some(ColumnConstant::Null(_)) => { + Self::create_column(target_type, None, num_rows)? + } + Some(ColumnConstant::Struct(sc)) => { + Self::create_struct_column(sc.fields(), sc.child_values(), num_rows)? + } + other => { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "ConstantLookup for field id {field_id} expected a scalar, \ + null, or struct constant, found {other:?}" + ), + )); + } + }, + + ColumnSource::Coalesce { + field_id, + source_index, + target_type, + } => { + let Some(ColumnConstant::CoalesceLastUpdatedSeq(datum)) = + self.constant_fields.get(field_id) + else { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Coalesce for field id {field_id} expected a coalesce \ + fallback constant in the constant map" + ), + )); + }; + Self::create_coalesce_column( + &columns[*source_index], + datum.literal(), + target_type, + )? } }) }) .collect() } - fn create_column( + /// Builds a coalesced column: the source column's per-row value where non-null, + /// else the scalar `fallback`, cast to `target_type` (the run-end-encoded type + /// the column's other paths produce). + fn create_coalesce_column( + source: &ArrayRef, + fallback: &PrimitiveLiteral, target_type: &DataType, - prim_lit: &Option, - num_rows: usize, ) -> Result { - Ok(match (target_type, prim_lit) { - (DataType::Boolean, Some(PrimitiveLiteral::Boolean(value))) => { - Arc::new(BooleanArray::from(vec![*value; num_rows])) - } - (DataType::Boolean, None) => { - let vals: Vec> = vec![None; num_rows]; - Arc::new(BooleanArray::from(vals)) - } - (DataType::Int32, Some(PrimitiveLiteral::Int(value))) => { - Arc::new(Int32Array::from(vec![*value; num_rows])) - } - (DataType::Int32, None) => { - let vals: Vec> = vec![None; num_rows]; - Arc::new(Int32Array::from(vals)) - } - (DataType::Int64, Some(PrimitiveLiteral::Long(value))) => { - Arc::new(Int64Array::from(vec![*value; num_rows])) - } - (DataType::Int64, None) => { - let vals: Vec> = vec![None; num_rows]; - Arc::new(Int64Array::from(vals)) - } - (DataType::Float32, Some(PrimitiveLiteral::Float(value))) => { - Arc::new(Float32Array::from(vec![value.0; num_rows])) - } - (DataType::Float32, None) => { - let vals: Vec> = vec![None; num_rows]; - Arc::new(Float32Array::from(vals)) - } - (DataType::Float64, Some(PrimitiveLiteral::Double(value))) => { - Arc::new(Float64Array::from(vec![value.0; num_rows])) - } - (DataType::Float64, None) => { - let vals: Vec> = vec![None; num_rows]; - Arc::new(Float64Array::from(vals)) - } - (DataType::Utf8, Some(PrimitiveLiteral::String(value))) => { - Arc::new(StringArray::from(vec![value.clone(); num_rows])) - } - (DataType::Utf8, None) => { - let vals: Vec> = vec![None; num_rows]; - Arc::new(StringArray::from(vals)) - } - (DataType::Binary, Some(PrimitiveLiteral::Binary(value))) => { - Arc::new(BinaryArray::from_vec(vec![value; num_rows])) - } - (DataType::Binary, None) => { - let vals: Vec> = vec![None; num_rows]; - Arc::new(BinaryArray::from_opt_vec(vals)) - } - (DataType::Null, _) => Arc::new(NullArray::new(num_rows)), - (dt, _) => { - return Err(Error::new( - ErrorKind::Unexpected, - format!("unexpected target column type {}", dt), - )); - } - }) + if source.data_type() != &DataType::Int64 { + return Err(invalid_data!( + "coalesce source must be Int64, got {}", + source.data_type() + )); + } + let PrimitiveLiteral::Long(seq) = fallback else { + return Err(Error::new( + ErrorKind::Unexpected, + format!("coalesce fallback must be a long, got {fallback:?}"), + )); + }; + let scalar = Int64Array::new_scalar(*seq); + let mask = is_not_null(source)?; + let coalesced = zip(&mask, source, &scalar)?; + // Cast to the run-end-encoded target type for column-type uniformity across the + // constant/null/coalesce paths (not for compression -- a per-row-varying column + // gains nothing from REE). + Ok(cast(&coalesced, target_type)?) } -} -#[cfg(test)] -mod test { - use std::collections::HashMap; - use std::sync::Arc; + fn create_column( + target_type: &DataType, + prim_lit: Option<&PrimitiveLiteral>, + num_rows: usize, + ) -> Result { + // Check if this is a RunEndEncoded type (for constant fields) + if let DataType::RunEndEncoded(_, values_field) = target_type { + // Helper to create a Run-End Encoded array + let create_ree_array = |values_array: ArrayRef| -> Result { + let run_ends = if num_rows == 0 { + Int32Array::from(Vec::::new()) + } else { + Int32Array::from(vec![num_rows as i32]) + }; + Ok(Arc::new( + RunArray::try_new(&run_ends, &values_array).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + "Failed to create RunArray for constant value", + ) + .with_source(e) + })?, + )) + }; + + // Create the values array using the helper function + let values_array = + create_primitive_array_single_element(values_field.data_type(), prim_lit)?; + + // Wrap in Run-End Encoding + create_ree_array(values_array) + } else { + // Non-REE type (simple arrays for non-constant fields) + create_primitive_array_repeated(target_type, prim_lit, num_rows) + } + } + + fn create_struct_column( + fields: &Fields, + child_values: &[Option], + num_rows: usize, + ) -> Result { + if fields.is_empty() { + let nulls = arrow_buffer::NullBuffer::new_null(num_rows); + return Ok(Arc::new(StructArray::new_empty_fields( + num_rows, + Some(nulls), + ))); + } + + let child_arrays: Vec = fields + .iter() + .zip(child_values.iter()) + .map(|(field, value)| { + create_primitive_array_repeated(field.data_type(), value.as_ref(), num_rows) + }) + .collect::>()?; + + Ok(Arc::new(StructArray::try_new( + fields.clone(), + child_arrays, + None, + )?)) + } +} + +/// Builds a [`StructConstant`] from the unified partition type and a file's +/// partition spec/data. +/// +/// If `unified_partition_type` has no fields (unpartitioned table), returns an empty constant +/// that renders as a null struct column. +/// +/// For each field in the unified partition type: +/// - If it corresponds to a field in this file's partition spec, use the value from partition_data +/// - Otherwise (partition evolution), use null +pub(crate) fn build_partition_constant( + unified_partition_type: &StructType, + partition_spec: &PartitionSpec, + partition_data: &Struct, +) -> Result { + // Unpartitioned table: empty struct rendered as null + if unified_partition_type.fields().is_empty() { + return StructConstant::new(Fields::empty(), vec![]); + } + + use crate::arrow::type_to_arrow_type; + + let spec_fields = partition_spec.fields(); + + let mut arrow_fields = Vec::with_capacity(unified_partition_type.fields().len()); + let mut child_values = Vec::with_capacity(unified_partition_type.fields().len()); + + for unified_field in unified_partition_type.fields() { + let arrow_type = type_to_arrow_type(&unified_field.field_type)?; + // Don't attach PARQUET_FIELD_ID_META_KEY metadata to child fields -- + // Spark's output schema for _partition doesn't include it, and Arrow's + // Field::eq checks metadata, causing the schema adapter to insert an + // unnecessary cast operation per batch. + // Always nullable: partition evolution means any field may be absent + // for files written under a different spec. + let arrow_field = Field::new(&unified_field.name, arrow_type, true); + arrow_fields.push(Arc::new(arrow_field)); + + // Find matching field in this file's partition spec by field_id + let value = spec_fields + .iter() + .position(|f| f.field_id == unified_field.id) + .and_then(|pos| { + // Get the value from partition_data at this position + match &partition_data[pos] { + Some(Literal::Primitive(prim)) => Some(prim.clone()), + _ => None, + } + }); + + child_values.push(value); + } + + StructConstant::new(Fields::from(arrow_fields), child_values) +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + use std::sync::Arc; + use arrow_array::cast::AsArray; + use arrow_array::types::Int32Type; use arrow_array::{ - Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray, + Array, ArrayRef, Date32Array, Float32Array, Float64Array, Int32Array, Int64Array, + LargeListArray, ListArray, MapArray, RecordBatch, StringArray, StructArray, new_null_array, }; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use arrow_buffer::{NullBuffer, OffsetBuffer}; + use arrow_cast::cast; + use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; + + use super::{field_with_id, schema_to_arrow_schema}; + use crate::ErrorKind; + use crate::arrow::record_batch_transformer::{ + PromotePlan, RecordBatchTransformer, RecordBatchTransformerBuilder, + }; + use crate::arrow::{DEFAULT_MAP_FIELD_NAME, VariantExtensionType, build_partition_constant}; + use crate::spec::{ + ListType, Literal, MapType, NestedField, PrimitiveType, Schema, Struct, StructType, Type, + }; + + fn promote(source: &ArrayRef, target: &DataType, schema: &Schema) -> crate::Result { + PromotePlan::build(source.data_type(), target, schema, &HashMap::new())?.apply(source) + } + + fn empty_schema() -> Schema { + Schema::builder().build().unwrap() + } + + fn unevolved_struct_type() -> DataType { + DataType::Struct(Fields::from(vec![field_with_id( + "x", + DataType::Int32, + true, + 5, + )])) + } + + fn evolved_struct_type() -> DataType { + DataType::Struct(Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ])) + } + + fn unevolved_struct_data(x_values: Vec) -> Arc { + Arc::new(StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(x_values)) as ArrayRef], + None, + )) + } + + fn transform_top_level(column: NestedField, file_column: ArrayRef) -> crate::Result { + let name = column.name.clone(); + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + column.into(), + ]) + .build() + .unwrap(), + ); + let mut transformer = RecordBatchTransformerBuilder::new(snapshot_schema, &[1, 2]).build(); + let file_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id(name, file_column.data_type().clone(), true, 2), + ])); + let batch = RecordBatch::try_new(file_schema, vec![ + Arc::new(Int32Array::from(vec![1; file_column.len()])) as ArrayRef, + file_column, + ]) + .unwrap(); + Ok(transformer.process_record_batch(batch)?.column(1).clone()) + } + + #[test] + fn promote_struct_fills_added_fields_by_id() { + let source = Arc::new(StructArray::new( + Fields::from(vec![ + field_with_id("a", DataType::Int32, true, 1), + field_with_id("c", DataType::Utf8, true, 3), + ]), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(StringArray::from(vec!["x", "y"])) as ArrayRef, + ], + None, + )) as ArrayRef; + let fields = Fields::from(vec![ + field_with_id("a", DataType::Int32, true, 1), + field_with_id("b", DataType::Int32, true, 2), + field_with_id("c", DataType::Utf8, true, 3), + field_with_id("d", DataType::Int32, true, 4), + ]); + let target = DataType::Struct(fields.clone()); + + let out = promote(&source, &target, &empty_schema()).unwrap(); + let expected = StructArray::new( + fields, + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None])) as ArrayRef, + Arc::new(StringArray::from(vec!["x", "y"])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None])) as ArrayRef, + ], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_list_element_struct_fills_added_field_by_id() { + let source = Arc::new(ListArray::new( + Arc::new(field_with_id("element", unevolved_struct_type(), true, 4)), + OffsetBuffer::new(vec![0, 1, 2].into()), + unevolved_struct_data(vec![10, 20]), + None, + )) as ArrayRef; + let target = DataType::List(Arc::new(field_with_id( + "element", + evolved_struct_type(), + true, + 4, + ))); + + let out = promote(&source, &target, &empty_schema()).unwrap(); + let values = StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None])) as ArrayRef, + ], + None, + ); + let expected = ListArray::new( + Arc::new(field_with_id("element", evolved_struct_type(), true, 4)), + OffsetBuffer::new(vec![0, 1, 2].into()), + Arc::new(values), + None, + ); + assert_eq!(out.as_list::(), &expected); + } + + #[test] + fn promote_sliced_list_with_null_slot() { + let source = ListArray::new( + Arc::new(field_with_id("element", unevolved_struct_type(), true, 4)), + OffsetBuffer::new(vec![0, 1, 1, 3, 4].into()), + unevolved_struct_data(vec![10, 30, 31, 40]), + Some(NullBuffer::from(vec![true, false, true, true])), + ); + let source = Arc::new(source.slice(1, 3)) as ArrayRef; + let target = DataType::List(Arc::new(field_with_id( + "element", + evolved_struct_type(), + true, + 4, + ))); + + let out = promote(&source, &target, &empty_schema()).unwrap(); + let values = StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int32Array::from(vec![30, 31, 40])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None, None])) as ArrayRef, + ], + None, + ); + let expected = ListArray::new( + Arc::new(field_with_id("element", evolved_struct_type(), true, 4)), + OffsetBuffer::new(vec![0, 0, 2, 3].into()), + Arc::new(values), + Some(NullBuffer::from(vec![false, true, true])), + ); + assert_eq!(out.as_list::(), &expected); + } + + #[test] + fn promote_map_value_struct_fills_added_field_by_id() { + let entries = StructArray::new( + Fields::from(vec![ + field_with_id("key", DataType::Utf8, false, 7), + field_with_id("value", unevolved_struct_type(), true, 8), + ]), + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])) as ArrayRef, + unevolved_struct_data(vec![100, 200]), + ], + None, + ); + let source = Arc::new(MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + OffsetBuffer::new(vec![0, 1, 2].into()), + entries, + None, + false, + )) as ArrayRef; + let target_entries = DataType::Struct(Fields::from(vec![ + field_with_id("key", DataType::Utf8, false, 7), + field_with_id("value", evolved_struct_type(), true, 8), + ])); + let target = DataType::Map( + Arc::new(Field::new("entries", target_entries, false)), + false, + ); + + let out = promote(&source, &target, &empty_schema()).unwrap(); + let values = StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int32Array::from(vec![100, 200])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None])) as ArrayRef, + ], + None, + ); + let expected_entries = StructArray::new( + Fields::from(vec![ + field_with_id("key", DataType::Utf8, false, 7), + field_with_id("value", evolved_struct_type(), true, 8), + ]), + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])) as ArrayRef, + Arc::new(values) as ArrayRef, + ], + None, + ); + let expected = MapArray::new( + Arc::new(Field::new( + "entries", + expected_entries.data_type().clone(), + false, + )), + OffsetBuffer::new(vec![0, 1, 2].into()), + expected_entries, + None, + false, + ); + assert_eq!(out.data_type(), expected.data_type()); + assert_eq!(out.as_map(), &expected); + } + + #[test] + fn promote_struct_dropped_and_readded_same_name_nulls_by_id() { + let file = StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef], + None, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(6, "x", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ), + Arc::new(file), + ) + .unwrap(); + let expected = StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 6)]), + vec![Arc::new(Int32Array::from(vec![None, None])) as ArrayRef], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_struct_without_source_field_ids_reads_null() { + let nulls = NullBuffer::from(vec![true, true, false]); + let source = Arc::new(StructArray::new( + Fields::from(vec![Field::new("x", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef], + Some(nulls.clone()), + )) as ArrayRef; + + // Fields read as null and the struct keeps the file's validity, as Spark's reader does. + let out = promote(&source, &evolved_struct_type(), &empty_schema()).unwrap(); + let expected = StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ]), + vec![ + new_null_array(&DataType::Int32, 3), + new_null_array(&DataType::Int32, 3), + ], + Some(nulls), + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_struct_partial_field_ids_errors() { + let source = Arc::new(StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + Field::new("y", DataType::Int32, true), + ]), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![3, 4])) as ArrayRef, + ], + None, + )) as ArrayRef; + + let err = promote(&source, &evolved_struct_type(), &empty_schema()).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("do not all have field ids")); + } + + #[test] + fn promote_rejects_invalid_source_at_build() { + let duplicate_ids = DataType::Struct(Fields::from(vec![ + field_with_id("x", DataType::Int32, true, 5), + field_with_id("y", DataType::Int32, true, 5), + ])); + let map_of = + |entries| DataType::Map(Arc::new(Field::new("entries", entries, false)), false); + for (source, target, message) in [ + (duplicate_ids, evolved_struct_type(), "duplicate field id 5"), + (unevolved_struct_type(), DataType::Int32, "cannot promote"), + ( + map_of(DataType::Int32), + map_of(DataType::Int64), + "expected struct-typed map entries", + ), + ( + map_of(unevolved_struct_type()), + map_of(evolved_struct_type()), + "map entries have 1 source fields, expected 2", + ), + ] { + let err = + PromotePlan::build(&source, &target, &empty_schema(), &HashMap::new()).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains(message), "{err}"); + } + } + + fn variant_target() -> Field { + field_with_id( + "v", + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, true), + ])), + true, + 2, + ) + .with_extension_type(VariantExtensionType) + } - use crate::arrow::record_batch_transformer::RecordBatchTransformer; - use crate::spec::{Literal, NestedField, PrimitiveType, Schema, Type}; + #[test] + fn promote_variant_matches_storage_fields_by_name() { + use arrow_array::{BinaryArray, LargeBinaryArray}; + let source = Arc::new(StructArray::new( + Fields::from(vec![ + Field::new("value", DataType::LargeBinary, false), + Field::new("metadata", DataType::LargeBinary, false), + ]), + vec![ + Arc::new(LargeBinaryArray::from(vec![b"\x0a".as_ref()])) as ArrayRef, + Arc::new(LargeBinaryArray::from(vec![b"\x01".as_ref()])) as ArrayRef, + ], + None, + )) as ArrayRef; + let target = variant_target(); + let DataType::Struct(target_fields) = target.data_type() else { + unreachable!() + }; + + let promoted = PromotePlan::build_field( + &Field::new("v", source.data_type().clone(), true), + &target, + &empty_schema(), + &HashMap::new(), + ) + .unwrap() + .apply(&source) + .unwrap(); + + let expected = StructArray::new( + target_fields.clone(), + vec![ + Arc::new(BinaryArray::from(vec![b"\x01".as_ref()])) as ArrayRef, + Arc::new(BinaryArray::from(vec![b"\x0a".as_ref()])) as ArrayRef, + ], + None, + ); + assert_eq!(promoted.as_struct(), &expected); + } + + /// Reports the same errors as projection, which validates data files before this runs. + #[test] + fn promote_variant_rejects_unsupported_storage() { + let binary = |name: &str, nullable: bool| Field::new(name, DataType::Binary, nullable); + let invalid = |problem: &str| { + ( + ErrorKind::DataInvalid, + format!("Variant column `v` (field id 2) {problem}"), + ) + }; + for (storage, expected) in [ + ( + DataType::Struct(Fields::from(vec![ + binary("metadata", true), + binary("value", true), + Field::new("typed_value", DataType::Int64, true), + ])), + ( + ErrorKind::FeatureUnsupported, + "Reading shredded variant column `v` (field id 2) is not supported yet, see https://github.com/apache/iceberg-rust/issues/3606".to_string(), + ), + ), + ( + DataType::Struct(Fields::from(vec![ + binary("metadata", true), + binary("value", true), + ])), + invalid("has an optional `metadata` field, which must be required"), + ), + ( + DataType::Struct(Fields::from(vec![ + binary("metadata", false), + binary("foo", true), + ])), + invalid("is missing the `value` field"), + ), + ( + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Utf8, false), + binary("value", true), + ])), + invalid("has a non-binary `metadata` field: Utf8"), + ), + ( + DataType::Binary, + invalid("must be stored as a group, found Binary"), + ), + ] { + let err = PromotePlan::build_field( + &Field::new("v", storage, true), + &variant_target(), + &empty_schema(), + &HashMap::new(), + ) + .unwrap_err(); + assert_eq!((err.kind(), err.message().to_string()), expected); + } + } + + #[test] + fn promote_required_nested_field_absent_without_default_errors() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(6, "y", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let source = unevolved_struct_data(vec![1, 2]) as ArrayRef; + + let err = promote(&source, &evolved_struct_type(), &schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("required nested field")); + } + + #[test] + fn promote_nested_field_with_initial_default_errors() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(6, "y", Type::Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::int(42)) + .into(), + ]) + .build() + .unwrap(); + let source = unevolved_struct_data(vec![1, 2]) as ArrayRef; + + let err = promote(&source, &evolved_struct_type(), &schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert!(err.to_string().contains("initial-default")); + } + + #[test] + fn promote_struct_reorders_children_by_id_via_process_record_batch() { + let file = StructArray::new( + Fields::from(vec![ + field_with_id("a", DataType::Int32, true, 5), + field_with_id("b", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![100, 200])) as ArrayRef, + ], + None, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(6, "b", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(5, "a", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ), + Arc::new(file), + ) + .unwrap(); + let expected = StructArray::new( + Fields::from(vec![ + field_with_id("b", DataType::Int32, true, 6), + field_with_id("a", DataType::Int32, true, 5), + ]), + vec![ + Arc::new(Int32Array::from(vec![100, 200])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + ], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_struct_renames_child_via_process_record_batch() { + let file = StructArray::new( + Fields::from(vec![field_with_id("x_old", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef], + None, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(5, "x", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ), + Arc::new(file), + ) + .unwrap(); + let expected = StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef], + None, + ); + assert_eq!(out.as_struct(), &expected); + } + + #[test] + fn promote_idless_map_struct_value_reads_null_fields() { + let value = StructArray::new( + Fields::from(vec![Field::new("v", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef], + None, + ); + let entries = StructArray::new( + Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", value.data_type().clone(), true), + ]), + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])) as ArrayRef, + Arc::new(value) as ArrayRef, + ], + None, + ); + let map = MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + OffsetBuffer::new(vec![0, 2].into()), + entries, + None, + false, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "m", + Type::Map(MapType::optional( + 3, + Type::Primitive(PrimitiveType::String), + 4, + Type::Struct(StructType::new(vec![ + NestedField::optional(5, "v", Type::Primitive(PrimitiveType::Int)).into(), + ])), + )), + ), + Arc::new(map), + ) + .unwrap(); + let value_fields = Fields::from(vec![field_with_id("v", DataType::Int32, true, 5)]); + let expected_entries = StructArray::new( + Fields::from(vec![ + field_with_id("key", DataType::Utf8, false, 3), + field_with_id("value", DataType::Struct(value_fields.clone()), true, 4), + ]), + vec![ + Arc::new(StringArray::from(vec!["k1", "k2"])) as ArrayRef, + Arc::new(StructArray::new( + value_fields, + vec![new_null_array(&DataType::Int32, 2)], + None, + )) as ArrayRef, + ], + None, + ); + let expected = MapArray::new( + Arc::new(Field::new( + DEFAULT_MAP_FIELD_NAME, + expected_entries.data_type().clone(), + false, + )), + OffsetBuffer::new(vec![0, 2].into()), + expected_entries, + None, + false, + ); + assert_eq!(out.data_type(), expected.data_type()); + assert_eq!(out.as_map(), &expected); + } + + #[test] + fn promote_large_list_file_column_to_list() { + let element = Arc::new(field_with_id("element", DataType::Int32, true, 3)); + let file = LargeListArray::new( + element.clone(), + OffsetBuffer::new(vec![0i64, 2, 3].into()), + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + None, + ); + let out = transform_top_level( + NestedField::optional( + 2, + "l", + Type::List(ListType::new( + NestedField::list_element(3, Type::Primitive(PrimitiveType::Int), false).into(), + )), + ), + Arc::new(file), + ) + .unwrap(); + let expected = ListArray::new( + element, + OffsetBuffer::new(vec![0, 2, 3].into()), + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + None, + ); + assert_eq!(out.as_list::(), &expected); + } + + #[test] + fn promote_evolved_nested_struct_via_process_record_batch() { + // Written before x was promoted to long and renamed from x_old, and before y was added. + let file = StructArray::new( + Fields::from(vec![field_with_id("x_old", DataType::Int32, true, 5)]), + vec![Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef], + Some(NullBuffer::from(vec![true, true, false])), + ); + let out = transform_top_level( + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(5, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(6, "y", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ), + Arc::new(file), + ) + .unwrap(); + let expected = StructArray::new( + Fields::from(vec![ + field_with_id("x", DataType::Int64, true, 5), + field_with_id("y", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(Int64Array::from(vec![10i64, 20, 30])) as ArrayRef, + Arc::new(Int32Array::from(vec![None, None, None])) as ArrayRef, + ], + Some(NullBuffer::from(vec![true, true, false])), + ); + assert_eq!(out.as_struct(), &expected); + } + + /// Helper to extract string values from either StringArray or RunEndEncoded + /// Returns empty string for null values + fn get_string_value(array: &dyn Array, index: usize) -> String { + if let Some(string_array) = array.as_any().downcast_ref::() { + if string_array.is_null(index) { + String::new() + } else { + string_array.value(index).to_string() + } + } else if let Some(run_array) = array + .as_any() + .downcast_ref::>() + { + let values = run_array.values(); + let string_values = values + .as_any() + .downcast_ref::() + .expect("REE values should be StringArray"); + // For REE, all rows have the same value (index 0 in the values array) + if string_values.is_null(0) { + String::new() + } else { + string_values.value(0).to_string() + } + } else { + panic!("Expected StringArray or RunEndEncoded"); + } + } + + /// Helper to extract int values from either Int32Array or RunEndEncoded + fn get_int_value(array: &dyn Array, index: usize) -> i32 { + if let Some(int_array) = array.as_any().downcast_ref::() { + int_array.value(index) + } else if let Some(run_array) = array + .as_any() + .downcast_ref::>() + { + let values = run_array.values(); + let int_values = values + .as_any() + .downcast_ref::() + .expect("REE values should be Int32Array"); + int_values.value(0) + } else { + panic!("Expected Int32Array or RunEndEncoded"); + } + } #[test] fn build_field_id_to_source_schema_map_works() { @@ -484,7 +2279,9 @@ mod test { let snapshot_schema = Arc::new(iceberg_table_schema()); let projected_iceberg_field_ids = [13, 14]; - let mut inst = RecordBatchTransformer::build(snapshot_schema, &projected_iceberg_field_ids); + let mut inst = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_iceberg_field_ids) + .build(); let result = inst .process_record_batch(source_record_batch_no_migration_required()) @@ -500,7 +2297,9 @@ mod test { let snapshot_schema = Arc::new(iceberg_table_schema()); let projected_iceberg_field_ids = [10, 11, 12, 14, 15]; // a, b, c, e, f - let mut inst = RecordBatchTransformer::build(snapshot_schema, &projected_iceberg_field_ids); + let mut inst = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_iceberg_field_ids) + .build(); let result = inst.process_record_batch(source_record_batch()).unwrap(); @@ -509,6 +2308,371 @@ mod test { assert_eq!(result, expected); } + #[test] + fn schema_evolution_adds_date_column_with_nulls() { + // Reproduces TestSelect.readAndWriteWithBranchAfterSchemaChange from iceberg-spark. + // When reading old snapshots after adding a DATE column, the transformer must + // populate the new column with NULL values since old files lack this field. + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "date_col", Type::Primitive(PrimitiveType::Date)) + .into(), + ]) + .build() + .unwrap(), + ); + let projected_iceberg_field_ids = [1, 2, 3]; + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_iceberg_field_ids) + .build(); + + let file_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("name", DataType::Utf8, true, 2), + ])); + + let file_batch = RecordBatch::try_new(file_schema, vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec![ + Some("Alice"), + Some("Bob"), + Some("Charlie"), + ])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(file_batch).unwrap(); + + assert_eq!(result.num_columns(), 3); + assert_eq!(result.num_rows(), 3); + + let id_column = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(id_column.values(), &[1, 2, 3]); + + let name_column = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name_column.value(0), "Alice"); + assert_eq!(name_column.value(1), "Bob"); + assert_eq!(name_column.value(2), "Charlie"); + + let date_column = result + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(date_column.is_null(0)); + assert!(date_column.is_null(1)); + assert!(date_column.is_null(2)); + } + + #[test] + fn schema_evolution_required_field_absent_without_default_errors() { + // Per the spec's "Default values", null is only a valid default for an optional field, so + // a required field that is absent from the data file with no initial-default has no valid + // value and must error. Matches Iceberg-Java's Parquet readers, which raise + // "Missing required field: ". + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "missing_str", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(), + ); + let projected_iceberg_field_ids = [1, 2]; + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_iceberg_field_ids) + .build(); + + let file_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "id", + DataType::Int32, + false, + 1, + )])); + let file_batch = + RecordBatch::try_new(file_schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]) + .unwrap(); + + let err = transformer.process_record_batch(file_batch).unwrap_err(); + assert!( + err.to_string() + .contains("Missing required field: missing_str"), + "unexpected error: {err}" + ); + } + + #[test] + fn schema_evolution_absent_struct_with_initial_default_errors() { + for required in [false, true] { + let added_field = NestedField::new( + 2, + "added_struct", + Type::Struct(StructType::new(vec![ + NestedField::optional(3, "child", Type::Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::int(42)) + .into(), + ])), + required, + ) + .with_initial_default(Literal::Struct(Struct::from_iter(vec![None]))); + let snapshot_schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + added_field.into(), + ]) + .build() + .unwrap(), + ); + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &[1, 2]).build(); + let file_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "id", + DataType::Int32, + false, + 1, + )])); + let file_batch = + RecordBatch::try_new(file_schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]) + .unwrap(); + + let err = transformer + .process_record_batch(file_batch) + .expect_err(&format!( + "required={required}: complex default should be rejected" + )); + assert_eq!( + err.kind(), + ErrorKind::FeatureUnsupported, + "required={required}" + ); + assert!( + err.to_string().contains("Cannot read field added_struct"), + "required={required}: {err}" + ); + assert!( + err.to_string() + .contains("applying a non-primitive initial-default"), + "required={required}: {err}" + ); + } + } + + #[test] + fn schema_evolution_adds_struct_column_with_nulls() { + // Test that when a struct column is added after data files are written, + // the transformer can materialize the missing struct column with null values. + // This reproduces the scenario from Iceberg 1.10.0 TestSparkReaderDeletes tests + // where binaryData and structData columns were added to the schema. + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional( + 3, + "struct_col", + Type::Struct(StructType::new(vec![ + NestedField::optional( + 100, + "inner_field", + Type::Primitive(PrimitiveType::String), + ) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let projected_iceberg_field_ids = [1, 2, 3]; + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_iceberg_field_ids) + .build(); + + let file_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("data", DataType::Utf8, false, 2), + ])); + + let file_batch = RecordBatch::try_new(file_schema, vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(file_batch).unwrap(); + + assert_eq!(result.num_columns(), 3); + assert_eq!(result.num_rows(), 3); + + let id_column = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(id_column.values(), &[1, 2, 3]); + + let data_column = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(data_column.value(0), "a"); + assert_eq!(data_column.value(1), "b"); + assert_eq!(data_column.value(2), "c"); + + let struct_column = result + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(struct_column.is_null(0)); + assert!(struct_column.is_null(1)); + assert!(struct_column.is_null(2)); + } + + /// Evolved table schema for the #2618 regression test: `id` plus three + /// later-added optional nested columns — a list, a map, and a struct that + /// itself contains a nested list (`ys`). The nested-in-struct list is the + /// case a per-type NULL-fill would miss. + fn schema_with_added_nested_columns() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional( + 2, + "xs", + Type::List(ListType { + element_field: NestedField::list_element( + 3, + Type::Primitive(PrimitiveType::Int), + false, + ) + .into(), + }), + ) + .into(), + NestedField::optional( + 4, + "props", + Type::Map(MapType { + key_field: NestedField::map_key_element( + 5, + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::map_value_element( + 6, + Type::Primitive(PrimitiveType::Int), + false, + ) + .into(), + }), + ) + .into(), + NestedField::optional( + 7, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(8, "a", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::optional( + 9, + "ys", + Type::List(ListType { + element_field: NestedField::list_element( + 10, + Type::Primitive(PrimitiveType::Long), + false, + ) + .into(), + }), + ) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap() + } + + #[test] + fn schema_evolution_adds_list_map_and_nested_struct_columns_with_nulls() { + // Regression test for https://github.com/apache/iceberg-rust/issues/2618. + // + // The story the test tells, in order: + // 1. An old data file was written with only the `id` column. + // 2. The table schema has since evolved, adding optional list / map / + // struct columns (see `schema_with_added_nested_columns`). + // 3. Reading the old file against the evolved schema must fill those + // absent columns with typed all-NULL arrays — previously this errored + // with "unexpected target column type" for the nested types. + + // (1) The old data file: just `id`. + let file_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "id", + DataType::Int32, + false, + 1, + )])); + let file_batch = + RecordBatch::try_new(file_schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]) + .unwrap(); + + // (2) Read it against the evolved schema, projecting id + the three added columns. + let snapshot_schema = Arc::new(schema_with_added_nested_columns()); + let projected_iceberg_field_ids = [1, 2, 4, 7]; + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_iceberg_field_ids) + .build(); + let result = transformer.process_record_batch(file_batch).unwrap(); + + // (3a) `id` survives unchanged. + assert_eq!(result.num_columns(), 4); + assert_eq!(result.num_rows(), 3); + let id_column = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(id_column.values(), &[1, 2, 3]); + + // (3b) The added columns carry the evolved schema's Arrow types, including nested + // field names, nullability, and field IDs, and are all-NULL. + let expected_schema = schema_to_arrow_schema(&schema_with_added_nested_columns()).unwrap(); + assert_eq!(result.schema().as_ref(), &expected_schema); + for (idx, name) in [(1, "xs"), (2, "props"), (3, "s")] { + assert_eq!( + result.column(idx).null_count(), + 3, + "added nested column `{name}` should be all-NULL" + ); + } + } + pub fn source_record_batch() -> RecordBatch { RecordBatch::try_new( arrow_schema_promotion_addition_and_renaming_required(), @@ -589,35 +2753,1351 @@ mod test { fn arrow_schema_already_same_as_target() -> Arc { Arc::new(ArrowSchema::new(vec![ - simple_field("a", DataType::Utf8, true, "10"), - simple_field("b", DataType::Int64, false, "11"), - simple_field("c", DataType::Float64, false, "12"), - simple_field("e", DataType::Utf8, true, "14"), - simple_field("f", DataType::Utf8, false, "15"), + field_with_id("a", DataType::Utf8, true, 10), + field_with_id("b", DataType::Int64, false, 11), + field_with_id("c", DataType::Float64, false, 12), + field_with_id("e", DataType::Utf8, true, 14), + field_with_id("f", DataType::Utf8, false, 15), ])) } fn arrow_schema_promotion_addition_and_renaming_required() -> Arc { Arc::new(ArrowSchema::new(vec![ - simple_field("b", DataType::Int32, false, "11"), - simple_field("c", DataType::Float32, false, "12"), - simple_field("d", DataType::Int32, false, "13"), - simple_field("e_old", DataType::Utf8, true, "14"), + field_with_id("b", DataType::Int32, false, 11), + field_with_id("c", DataType::Float32, false, 12), + field_with_id("d", DataType::Int32, false, 13), + field_with_id("e_old", DataType::Utf8, true, 14), ])) } fn arrow_schema_no_promotion_addition_or_renaming_required() -> Arc { Arc::new(ArrowSchema::new(vec![ - simple_field("d", DataType::Int32, false, "13"), - simple_field("e", DataType::Utf8, true, "14"), + field_with_id("d", DataType::Int32, false, 13), + field_with_id("e", DataType::Utf8, true, 14), ])) } - /// Create a simple arrow field with metadata. - fn simple_field(name: &str, ty: DataType, nullable: bool, value: &str) -> Field { - Field::new(name, ty, nullable).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - value.to_string(), - )])) + /// Test for add_files with Parquet files that have NO field IDs (Hive tables). + /// + /// This reproduces the scenario from Iceberg spec where: + /// - Hive-style partitioned Parquet files are imported via add_files procedure + /// - Parquet files originally DO NOT have field IDs (typical for Hive tables) + /// - ArrowReader applies name mapping to assign correct Iceberg field IDs + /// - Iceberg schema assigns field IDs: id (1), name (2), dept (3), subdept (4) + /// - Partition columns (id, dept) have initial_default values + /// + /// Per the Iceberg spec (https://iceberg.apache.org/spec/#column-projection), + /// this scenario requires `schema.name-mapping.default` from table metadata + /// to correctly map Parquet columns by name to Iceberg field IDs. + /// This mapping is now applied in ArrowReader before data is processed. + /// + /// Expected behavior: + /// 1. id=1 (from initial_default) - spec rule #3 + /// 2. name="John Doe" (from Parquet with field_id=2 assigned by reader) - found by field ID + /// 3. dept="hr" (from initial_default) - spec rule #3 + /// 4. subdept="communications" (from Parquet with field_id=4 assigned by reader) - found by field ID + #[test] + fn add_files_with_name_mapping_applied_in_reader() { + // Iceberg schema after add_files: id (partition), name, dept (partition), subdept + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::int(1)) + .into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "dept", Type::Primitive(PrimitiveType::String)) + .with_initial_default(Literal::string("hr")) + .into(), + NestedField::optional(4, "subdept", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(), + ); + + // Simulate ArrowReader having applied name mapping: + // Original Parquet: name, subdept (NO field IDs) + // After reader.rs applies name mapping: name (field_id=2), subdept (field_id=4) + // + // Note: Partition columns (id, dept) are NOT in the Parquet file - they're in directory paths + use std::collections::HashMap; + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, true).with_metadata(HashMap::from([( + "PARQUET:field_id".to_string(), + "2".to_string(), + )])), + Field::new("subdept", DataType::Utf8, true).with_metadata(HashMap::from([( + "PARQUET:field_id".to_string(), + "4".to_string(), + )])), + ])); + + let projected_field_ids = [1, 2, 3, 4]; // id, name, dept, subdept + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids).build(); + + // Create a Parquet RecordBatch with data for: name="John Doe", subdept="communications" + let parquet_batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(StringArray::from(vec!["John Doe"])), + Arc::new(StringArray::from(vec!["communications"])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + // Verify the transformed RecordBatch has: + // - id=1 (from initial_default, not from Parquet) + // - name="John Doe" (from Parquet with correct field_id=2) + // - dept="hr" (from initial_default, not from Parquet) + // - subdept="communications" (from Parquet with correct field_id=4) + assert_eq!(result.num_columns(), 4); + assert_eq!(result.num_rows(), 1); + + let id_column = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(id_column.value(0), 1); + + let name_column = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name_column.value(0), "John Doe"); + + let dept_column = result + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(dept_column.value(0), "hr"); + + let subdept_column = result + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(subdept_column.value(0), "communications"); + } + + /// Test for bucket partitioning where source columns must be read from data files. + /// + /// This test verifies correct implementation of the Iceberg spec's "Column Projection" rules: + /// > "Return the value from partition metadata if an **Identity Transform** exists for the field" + /// + /// # Why this test is critical + /// + /// The key insight is that partition metadata stores TRANSFORMED values, not source values: + /// - For `bucket(4, id)`, partition metadata has `id_bucket = 2` (the bucket number) + /// - The actual `id` column values (100, 200, 300) are ONLY in the data file + /// + /// If iceberg-rust incorrectly treated bucket-partitioned fields as constants, it would: + /// 1. Replace all `id` values with the constant `2` from partition metadata + /// 2. Break runtime filtering (e.g., `WHERE id = 100` would match no rows) + /// 3. Return incorrect query results + /// + /// # What this test verifies + /// + /// - Bucket-partitioned fields (e.g., `bucket(4, id)`) are read from the data file + /// - The source column `id` contains actual values (100, 200, 300), not constants + /// - Java's `PartitionUtil.constantsMap()` behavior is correctly replicated: + /// ```java + /// if (field.transform().isIdentity()) { // FALSE for bucket transforms + /// idToConstant.put(field.sourceId(), converted); + /// } + /// ``` + /// + /// # Real-world impact + /// + /// This reproduces the failure scenario from Iceberg Java's TestRuntimeFiltering: + /// - Tables partitioned by `bucket(N, col)` are common for load balancing + /// - Queries filter on the source column: `SELECT * FROM tbl WHERE col = value` + /// - Runtime filtering pushes predicates down to Iceberg file scans + /// - Without this fix, the filter would match against constant partition values instead of data + /// + /// # References + /// - Iceberg spec: format/spec.md "Column Projection" + "Partition Transforms" + /// - Java impl: core/src/main/java/org/apache/iceberg/util/PartitionUtil.java + /// - Java test: spark/src/test/java/.../TestRuntimeFiltering.java + #[test] + fn bucket_partitioning_reads_source_column_from_file() { + use crate::spec::{Struct, Transform}; + + // Table schema: id (data column), name (data column), id_bucket (partition column) + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + // Partition spec: bucket(4, id) - the id field is bucketed + let partition_spec = Arc::new( + crate::spec::PartitionSpec::builder(snapshot_schema.clone()) + .with_spec_id(0) + .add_partition_field("id", "id_bucket", Transform::Bucket(4)) + .unwrap() + .build() + .unwrap(), + ); + + // Partition data: bucket value is 2 + // In Iceberg, partition data is a Struct where each field corresponds to a partition field + let partition_data = Struct::from_iter(vec![Some(Literal::int(2))]); + + // Parquet file contains both id and name columns + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("name", DataType::Utf8, true, 2), + ])); + + let projected_field_ids = [1, 2]; // id, name + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_partition(partition_spec, partition_data) + .expect("Failed to add partition constants") + .build(); + + // Create a Parquet RecordBatch with actual data + // The id column MUST be read from here, not treated as a constant + let parquet_batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(Int32Array::from(vec![100, 200, 300])), + Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + // Verify the transformed RecordBatch correctly reads id from the file + // (NOT as a constant from partition metadata) + assert_eq!(result.num_columns(), 2); + assert_eq!(result.num_rows(), 3); + + let id_column = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + // These values MUST come from the Parquet file, not be replaced by constants + assert_eq!(id_column.value(0), 100); + assert_eq!(id_column.value(1), 200); + assert_eq!(id_column.value(2), 300); + + let name_column = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name_column.value(0), "Alice"); + assert_eq!(name_column.value(1), "Bob"); + assert_eq!(name_column.value(2), "Charlie"); + } + + /// Test that identity-transformed partition fields ARE treated as constants. + /// + /// This is the complement to `bucket_partitioning_reads_source_column_from_file`, + /// verifying that constants_map() correctly identifies identity-transformed + /// partition fields per the Iceberg spec. + /// + /// # Spec requirement (format/spec.md "Column Projection") + /// + /// > "Return the value from partition metadata if an Identity Transform exists for the field + /// > and the partition value is present in the `partition` struct on `data_file` object + /// > in the manifest. This allows for metadata only migrations of Hive tables." + /// + /// # Why identity transforms use constants + /// + /// Unlike bucket/truncate/year/etc., identity transforms don't modify the value: + /// - `identity(dept)` stores the actual `dept` value in partition metadata + /// - Partition metadata has `dept = "engineering"` (the real value, not a hash/bucket) + /// - This value can be used directly without reading the data file + /// + /// # Performance benefit + /// + /// For Hive migrations where partition columns aren't in data files: + /// - Partition metadata provides the column values + /// - No need to read from data files (metadata-only query optimization) + /// - Common pattern: `dept=engineering/subdept=backend/file.parquet` + /// - `dept` and `subdept` are in directory structure, not in `file.parquet` + /// - Iceberg populates these from partition metadata as constants + /// + /// # What this test verifies + /// + /// - Identity-partitioned fields use constants from partition metadata + /// - The `dept` column is populated with `"engineering"` (not read from file) + /// - Java's `PartitionUtil.constantsMap()` behavior is matched: + /// ```java + /// if (field.transform().isIdentity()) { // TRUE for identity + /// idToConstant.put(field.sourceId(), converted); + /// } + /// ``` + /// + /// # References + /// - Iceberg spec: format/spec.md "Column Projection" + /// - Java impl: core/src/main/java/org/apache/iceberg/util/PartitionUtil.java + #[test] + fn identity_partition_uses_constant_from_metadata() { + use crate::spec::{Struct, Transform}; + + // Table schema: id (data column), dept (partition column), name (data column) + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "dept", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + // Partition spec: identity(dept) - the dept field uses identity transform + let partition_spec = Arc::new( + crate::spec::PartitionSpec::builder(snapshot_schema.clone()) + .with_spec_id(0) + .add_partition_field("dept", "dept", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + + // Partition data: dept="engineering" + let partition_data = Struct::from_iter(vec![Some(Literal::string("engineering"))]); + + // Parquet file contains only id and name (dept is in partition path) + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("name", DataType::Utf8, true, 3), + ])); + + let projected_field_ids = [1, 2, 3]; // id, dept, name + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_partition(partition_spec, partition_data) + .expect("Failed to add partition constants") + .build(); + + let parquet_batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(Int32Array::from(vec![100, 200])), + Arc::new(StringArray::from(vec!["Alice", "Bob"])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + // Verify the dept column is populated with the constant from partition metadata + assert_eq!(result.num_columns(), 3); + assert_eq!(result.num_rows(), 2); + + // Use helpers to handle both simple and REE arrays + assert_eq!(get_int_value(result.column(0).as_ref(), 0), 100); + assert_eq!(get_int_value(result.column(0).as_ref(), 1), 200); + + // dept column comes from partition metadata (constant) - will be REE + assert_eq!( + get_string_value(result.column(1).as_ref(), 0), + "engineering" + ); + assert_eq!( + get_string_value(result.column(1).as_ref(), 1), + "engineering" + ); + + // name column comes from file + assert_eq!(get_string_value(result.column(2).as_ref(), 0), "Alice"); + assert_eq!(get_string_value(result.column(2).as_ref(), 1), "Bob"); + } + + /// `identity(s.region)` with `region` absent from the file struct. The constant + /// is stored on field id 5; the struct promote has to read it. + #[test] + fn identity_partition_fills_missing_nested_field() { + use crate::spec::{Struct, Transform}; + + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional( + 5, + "region", + Type::Primitive(PrimitiveType::String), + ) + .into(), + NestedField::optional(6, "x", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = Arc::new( + crate::spec::PartitionSpec::builder(snapshot_schema.clone()) + .with_spec_id(0) + .add_partition_field("s.region", "region", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let partition_data = Struct::from_iter(vec![Some(Literal::string("us"))]); + + let file = StructArray::new( + Fields::from(vec![field_with_id("x", DataType::Int32, true, 6)]), + vec![Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef], + None, + ); + let parquet_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "s", + file.data_type().clone(), + true, + 2, + )])); + let mut transformer = RecordBatchTransformerBuilder::new(snapshot_schema, &[2]) + .with_partition(partition_spec, partition_data) + .unwrap() + .build(); + let batch = RecordBatch::try_new(parquet_schema, vec![Arc::new(file) as ArrayRef]).unwrap(); + let result = transformer.process_record_batch(batch).unwrap(); + + let expected = StructArray::new( + Fields::from(vec![ + field_with_id("region", DataType::Utf8, true, 5), + field_with_id("x", DataType::Int32, true, 6), + ]), + vec![ + Arc::new(StringArray::from(vec!["us", "us"])) as ArrayRef, + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + ], + None, + ); + assert_eq!(result.column(0).as_struct(), &expected); + } + + /// Test that a dropped identity partition source column is skipped rather than erroring. + /// + /// A historical spec may use `identity(col)` where `col` was later dropped from the + /// schema. `constants_map()` must skip that field instead of failing, so the file can + /// still be read (the dropped column is not projected). + #[test] + fn dropped_identity_partition_source_column_is_skipped() { + use crate::spec::{Struct, Transform}; + + // Original schema used to build the spec: has id, dept, name. + let original_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "dept", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + // Spec: identity(dept). Built against the original schema where dept still exists. + let partition_spec = Arc::new( + crate::spec::PartitionSpec::builder(original_schema.clone()) + .with_spec_id(0) + .add_partition_field("dept", "dept", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + + // Evolved snapshot schema: dept (field id 2) has been dropped. + let evolved_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + let partition_data = Struct::from_iter(vec![Some(Literal::string("engineering"))]); + + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("name", DataType::Utf8, true, 3), + ])); + + let projected_field_ids = [1, 3]; // id, name -- dept is gone + + // The dropped source column must not cause an error here. + let mut transformer = + RecordBatchTransformerBuilder::new(evolved_schema, &projected_field_ids) + .with_partition(partition_spec, partition_data) + .expect("dropped partition source column should be skipped, not error") + .build(); + + let parquet_batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(Int32Array::from(vec![100, 200])), + Arc::new(StringArray::from(vec!["Alice", "Bob"])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + // Only the live columns are projected; no constant is added for the dropped dept. + assert_eq!(result.num_columns(), 2); + assert_eq!(result.num_rows(), 2); + assert_eq!(get_int_value(result.column(0).as_ref(), 0), 100); + assert_eq!(get_int_value(result.column(0).as_ref(), 1), 200); + assert_eq!(get_string_value(result.column(1).as_ref(), 0), "Alice"); + assert_eq!(get_string_value(result.column(1).as_ref(), 1), "Bob"); + } + + /// Test bucket partitioning with renamed source column. + /// + /// This verifies correct behavior for TestRuntimeFiltering.testRenamedSourceColumnTable() in Iceberg Java. + /// When a source column is renamed after partitioning is established, field-ID-based mapping + /// must still correctly identify the column in Parquet files. + /// + /// # Scenario + /// + /// 1. Table created with `bucket(4, id)` partitioning + /// 2. Data written to Parquet files (field_id=1, name="id") + /// 3. Column renamed: `ALTER TABLE ... RENAME COLUMN id TO row_id` + /// 4. Iceberg schema now has: field_id=1, name="row_id" + /// 5. Parquet files still have: field_id=1, name="id" + /// + /// # Expected Behavior Per Iceberg Spec + /// + /// Per the Iceberg spec "Column Projection" section and Java's PartitionUtil.constantsMap(): + /// - Bucket transforms are NON-identity, so partition metadata stores bucket numbers (0-3), not source values + /// - Source columns for non-identity transforms MUST be read from data files + /// - Field-ID-based mapping should find the column by field_id=1 (ignoring name mismatch) + /// - Runtime filtering on `row_id` should work correctly + /// + /// # What This Tests + /// + /// This test ensures that when FileScanTask provides partition_spec and partition_data: + /// - constants_map() correctly identifies that bucket(4, row_id) is NOT an identity transform + /// - The source column (field_id=1) is NOT added to constants_map + /// - Field-ID-based mapping reads actual values from the Parquet file + /// - Values [100, 200, 300] are read, not replaced with bucket constant 2 + /// + /// # References + /// - Java test: spark/src/test/java/.../TestRuntimeFiltering.java::testRenamedSourceColumnTable + /// - Java impl: core/src/main/java/org/apache/iceberg/util/PartitionUtil.java::constantsMap() + /// - Iceberg spec: format/spec.md "Column Projection" section + #[test] + fn test_bucket_partitioning_with_renamed_source_column() { + use crate::spec::{Struct, Transform}; + + // Iceberg schema after rename: row_id (was id), name + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "row_id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + // Partition spec: bucket(4, row_id) - but source_id still points to field_id=1 + let partition_spec = Arc::new( + crate::spec::PartitionSpec::builder(snapshot_schema.clone()) + .with_spec_id(0) + .add_partition_field("row_id", "row_id_bucket", Transform::Bucket(4)) + .unwrap() + .build() + .unwrap(), + ); + + // Partition data: bucket value is 2 + let partition_data = Struct::from_iter(vec![Some(Literal::int(2))]); + + // Parquet file has OLD column name "id" but SAME field_id=1 + // Field-ID-based mapping should find this despite name mismatch + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("name", DataType::Utf8, true, 2), + ])); + + let projected_field_ids = [1, 2]; // row_id (field_id=1), name (field_id=2) + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_partition(partition_spec, partition_data) + .expect("Failed to add partition constants") + .build(); + + // Create a Parquet RecordBatch with actual data + // Despite column rename, data should be read via field_id=1 + let parquet_batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(Int32Array::from(vec![100, 200, 300])), + Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + // Verify the transformed RecordBatch correctly reads data despite name mismatch + assert_eq!(result.num_columns(), 2); + assert_eq!(result.num_rows(), 3); + + let row_id_column = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + // These values MUST come from the Parquet file via field_id=1, + // not be replaced by the bucket constant (2) + assert_eq!(row_id_column.value(0), 100); + assert_eq!(row_id_column.value(1), 200); + assert_eq!(row_id_column.value(2), 300); + + let name_column = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name_column.value(0), "Alice"); + assert_eq!(name_column.value(1), "Bob"); + assert_eq!(name_column.value(2), "Charlie"); + } + + /// Comprehensive integration test that verifies all 4 Iceberg spec rules work correctly. + /// + /// Per the Iceberg spec (https://iceberg.apache.org/spec/#column-projection), + /// "Values for field ids which are not present in a data file must be resolved + /// according the following rules:" + /// + /// This test creates a scenario where each rule is exercised: + /// - Rule #1: dept (identity-partitioned) -> constant from partition metadata + /// - Rule #2: data (via name mapping) -> read from Parquet file by name + /// - Rule #3: category (initial_default) -> use default value + /// - Rule #4: notes (no default) -> return null + /// + /// # References + /// - Iceberg spec: format/spec.md "Column Projection" section + #[test] + fn test_all_four_spec_rules() { + use crate::spec::Transform; + + // Iceberg schema with columns designed to exercise each spec rule + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + // Field in Parquet by field ID (normal case) + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + // Rule #1: Identity-partitioned field - should use partition metadata + NestedField::required(2, "dept", Type::Primitive(PrimitiveType::String)).into(), + // Rule #2: Field resolved by name mapping (ArrowReader already applied) + NestedField::required(3, "data", Type::Primitive(PrimitiveType::String)).into(), + // Rule #3: Field with initial_default + NestedField::optional(4, "category", Type::Primitive(PrimitiveType::String)) + .with_initial_default(Literal::string("default_category")) + .into(), + // Rule #4: Field with no default - should be null + NestedField::optional(5, "notes", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(), + ); + + // Partition spec: identity transform on dept + let partition_spec = Arc::new( + crate::spec::PartitionSpec::builder(snapshot_schema.clone()) + .with_spec_id(0) + .add_partition_field("dept", "dept", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + + // Partition data: dept="engineering" + let partition_data = Struct::from_iter(vec![Some(Literal::string("engineering"))]); + + // Parquet schema: simulates post-ArrowReader state where name mapping already applied + // Has id (field_id=1) and data (field_id=3, assigned by ArrowReader via name mapping) + // Missing: dept (in partition), category (has default), notes (no default) + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("data", DataType::Utf8, false, 3), + ])); + + let projected_field_ids = [1, 2, 3, 4, 5]; // id, dept, data, category, notes + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_partition(partition_spec, partition_data) + .expect("Failed to add partition constants") + .build(); + + let parquet_batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(Int32Array::from(vec![100, 200])), + Arc::new(StringArray::from(vec!["value1", "value2"])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + assert_eq!(result.num_columns(), 5); + assert_eq!(result.num_rows(), 2); + + // Verify each column demonstrates the correct spec rule: + + // Normal case: id from Parquet by field ID + // Use helpers to handle both simple and REE arrays + assert_eq!(get_int_value(result.column(0).as_ref(), 0), 100); + assert_eq!(get_int_value(result.column(0).as_ref(), 1), 200); + + // Rule #1: dept from partition metadata (identity transform) - will be REE + assert_eq!( + get_string_value(result.column(1).as_ref(), 0), + "engineering" + ); + assert_eq!( + get_string_value(result.column(1).as_ref(), 1), + "engineering" + ); + + // Rule #2: data from Parquet via name mapping - will be regular array + assert_eq!(get_string_value(result.column(2).as_ref(), 0), "value1"); + assert_eq!(get_string_value(result.column(2).as_ref(), 1), "value2"); + + // Rule #3: category from initial_default - will be REE + assert_eq!( + get_string_value(result.column(3).as_ref(), 0), + "default_category" + ); + assert_eq!( + get_string_value(result.column(3).as_ref(), 1), + "default_category" + ); + + // Rule #4: notes is null (no default, not in Parquet, not in partition) - will be REE with null + // For null REE arrays, we still use the helper which handles extraction + assert_eq!(get_string_value(result.column(4).as_ref(), 0), ""); + assert_eq!(get_string_value(result.column(4).as_ref(), 1), ""); + } + + /// Test handling of null values in identity-partitioned columns. + /// + /// Reproduces TestPartitionValues.testNullPartitionValue() from iceberg-java, which + /// writes records where the partition column has null values. Before the fix in #1922, + /// this would error with "Partition field X has null value for identity transform". + #[test] + fn null_identity_partition_value() { + use crate::spec::{Struct, Transform}; + + let schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "data", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + let partition_spec = Arc::new( + crate::spec::PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("data", "data", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + + // Partition has null value for the data column + let partition_data = Struct::from_iter(vec![None]); + + let file_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "id", + DataType::Int32, + true, + 1, + )])); + + let projected_field_ids = [1, 2]; + + let mut transformer = RecordBatchTransformerBuilder::new(schema, &projected_field_ids) + .with_partition(partition_spec, partition_data) + .expect("Should handle null partition values") + .build(); + + let file_batch = + RecordBatch::try_new(file_schema, vec![Arc::new(Int32Array::from(vec![1, 2, 3]))]) + .unwrap(); + + let result = transformer.process_record_batch(file_batch).unwrap(); + + assert_eq!(result.num_columns(), 2); + assert_eq!(result.num_rows(), 3); + + let id_col = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(id_col.values(), &[1, 2, 3]); + + // Partition column with null value should produce nulls + let data_col = result.column(1); + assert!(data_col.is_null(0)); + assert!(data_col.is_null(1)); + assert!(data_col.is_null(2)); + } + + #[test] + fn pos_column() { + use crate::metadata_columns::RESERVED_FIELD_ID_POS; + + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + // Simulate what arrow-rs's virtual-column reader produces: file columns + // followed by the _pos Int64 column carrying absolute file row indices. + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("name", DataType::Utf8, true, 2), + field_with_id("_pos", DataType::Int64, false, RESERVED_FIELD_ID_POS), + ])); + + let projected_field_ids = [1, 2, RESERVED_FIELD_ID_POS]; + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_virtual_field(RESERVED_FIELD_ID_POS) + .build(); + + let parquet_batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(Int32Array::from(vec![100, 200, 300])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + Arc::new(Int64Array::from(vec![0, 1, 2])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + assert_eq!(result.num_columns(), 3); + assert_eq!(result.num_rows(), 3); + + // id column from file + let id_col = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(id_col.values(), &[100, 200, 300]); + + // name column from file + let name_col = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name_col.value(0), "a"); + + // _pos column + let pos_col = result + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(pos_col.len(), 3); + assert_eq!(pos_col.values(), &[0, 1, 2]); + } + + #[test] + fn last_updated_sequence_number_constant() { + use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER; + use crate::spec::Datum; + + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + // The column is not physically present in the file; it is materialized as + // a per-file constant equal to the data sequence number. + let parquet_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "id", + DataType::Int32, + false, + 1, + )])); + + let projected_field_ids = [1, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER]; + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_constant( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + Datum::long(7), + ) + .build(); + + let parquet_batch = + RecordBatch::try_new(parquet_schema, vec![Arc::new(Int32Array::from(vec![ + 10, 20, 30, + ]))]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + assert_eq!(result.num_columns(), 2); + assert_eq!(result.num_rows(), 3); + + // Every row's logical value is 7 (the data sequence number), regardless of + // the physical (run-end) encoding. + assert_last_updated_seq_column(&result, &[Some(7), Some(7), Some(7)]); + } + + #[test] + fn last_updated_sequence_number_null_column() { + use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER; + + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + + let parquet_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "id", + DataType::Int32, + false, + 1, + )])); + + let projected_field_ids = [1, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER]; + + // The null-gate path (a file with a null first_row_id): the whole column is null. + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_null_metadata_column(RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER) + .unwrap() + .build(); + + let parquet_batch = + RecordBatch::try_new(parquet_schema, vec![Arc::new(Int32Array::from(vec![ + 10, 20, 30, + ]))]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + // Every row's logical value is null. + assert_last_updated_seq_column(&result, &[None, None, None]); + } + + /// Builds a transformer + a file batch for the coalesce case: `id` plus a physical + /// `_last_updated_sequence_number` column carrying `seq_values`. + fn coalesce_transformer_and_batch( + seq_values: Vec>, + id_values: Vec, + fallback: i64, + ) -> (RecordBatchTransformer, RecordBatch) { + use crate::metadata_columns::{ + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + }; + use crate::spec::Datum; + + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id( + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + DataType::Int64, + true, + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + ), + ])); + let projected_field_ids = [1, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER]; + let transformer = RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_coalesced_last_updated_seq_column( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + Datum::long(fallback), + ) + .build(); + let batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(Int32Array::from(id_values)), + Arc::new(Int64Array::from(seq_values)), + ]) + .unwrap(); + (transformer, batch) + } + + /// Reads the `_last_updated_sequence_number` column by name (encoding-agnostic: casts + /// through the run-end-encoding to logical Int64) and asserts its per-row values. + fn assert_last_updated_seq_column(result: &RecordBatch, expected: &[Option]) { + use crate::metadata_columns::RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER; + + let col = result + .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER) + .expect("_last_updated_sequence_number column should be present"); + let logical = cast(col, &DataType::Int64).unwrap(); + let values = logical.as_any().downcast_ref::().unwrap(); + let actual: Vec> = (0..values.len()) + .map(|i| (!values.is_null(i)).then(|| values.value(i))) + .collect(); + assert_eq!(&actual, expected); + } + + #[test] + fn last_updated_sequence_number_coalesce_column() { + let (mut transformer, batch) = + coalesce_transformer_and_batch(vec![Some(5), None, Some(8)], vec![10, 20, 30], 9); + let result = transformer.process_record_batch(batch).unwrap(); + + // Per-row value where non-null; the fallback (9) where null. + assert_last_updated_seq_column(&result, &[Some(5), Some(9), Some(8)]); + } + + #[test] + fn last_updated_sequence_number_coalesce_all_null() { + // Every row falls back. This is the only input that distinguishes the `is_not_null` + // mask from `is_null` -- a mixed input passes under either polarity. + let (mut transformer, batch) = + coalesce_transformer_and_batch(vec![None, None, None], vec![10, 20, 30], 9); + let result = transformer.process_record_batch(batch).unwrap(); + + assert_last_updated_seq_column(&result, &[Some(9), Some(9), Some(9)]); + } + + #[test] + fn last_updated_sequence_number_coalesce_all_non_null() { + // Nothing falls back: every per-row value passes through, and the run-end-encoding + // cast still yields a valid array. + let (mut transformer, batch) = + coalesce_transformer_and_batch(vec![Some(5), Some(6), Some(7)], vec![10, 20, 30], 9); + let result = transformer.process_record_batch(batch).unwrap(); + + assert_last_updated_seq_column(&result, &[Some(5), Some(6), Some(7)]); + } + + #[test] + fn last_updated_sequence_number_coalesce_empty_batch() { + // A 0-row batch (reachable when a predicate or positional deletes clear a row + // group) must still produce a well-formed, empty column of the shared type. + let (mut transformer, batch) = coalesce_transformer_and_batch(vec![], vec![], 9); + let result = transformer.process_record_batch(batch).unwrap(); + + assert_eq!(result.num_rows(), 0); + assert_last_updated_seq_column(&result, &[]); + } + + /// field 1 and RESERVED_FIELD_ID_POS are of identical type + /// swapping their projection order should not lead to [`SchemaComparison::NameChangesOnly`] + #[test] + fn reorder_pos_column_with_columns_of_identical_type() { + use crate::metadata_columns::RESERVED_FIELD_ID_POS; + + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id_long", Type::Primitive(PrimitiveType::Long)) + .into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id_long", DataType::Int64, false, 1), + field_with_id("name", DataType::Utf8, true, 2), + field_with_id("_pos", DataType::Int64, false, RESERVED_FIELD_ID_POS), + ])); + + let projected_field_ids = [RESERVED_FIELD_ID_POS, 2, 1]; + + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_virtual_field(RESERVED_FIELD_ID_POS) + .build(); + + let parquet_batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(Int64Array::from(vec![100, 200, 300])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + Arc::new(Int64Array::from(vec![0, 1, 2])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + assert_eq!(result.num_columns(), 3); + assert_eq!(result.num_rows(), 3); + + let pos_col = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(pos_col.values(), &[0, 1, 2]); + + let name_col = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name_col.value(0), "a"); + assert_eq!(name_col.value(1), "b"); + assert_eq!(name_col.value(2), "c"); + + let id_col = result + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(id_col.len(), 3); + assert_eq!(id_col.values(), &[100, 200, 300]); + } + + #[test] + fn partition_column_struct_constant() { + use arrow_array::StructArray; + + use crate::metadata_columns::RESERVED_FIELD_ID_PARTITION; + use crate::spec::Transform; + + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + // Partition spec: identity(id) + let partition_spec = Arc::new( + crate::spec::PartitionSpec::builder(snapshot_schema.clone()) + .with_spec_id(0) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + + // Unified partition type: just one field (same as the spec's type) + let unified_partition_type = partition_spec.partition_type(&snapshot_schema).unwrap(); + + // Partition data: id=42 + let partition_data = Struct::from_iter(vec![Some(Literal::int(42))]); + + // Parquet file has both columns + let parquet_schema = Arc::new(ArrowSchema::new(vec![ + field_with_id("id", DataType::Int32, false, 1), + field_with_id("name", DataType::Utf8, true, 2), + ])); + + // Project id, name, and _partition + let projected_field_ids = [1, 2, RESERVED_FIELD_ID_PARTITION]; + + let partition_column = + build_partition_constant(&unified_partition_type, &partition_spec, &partition_data) + .unwrap(); + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_partition_constant(partition_column) + .build(); + + let parquet_batch = RecordBatch::try_new(parquet_schema, vec![ + Arc::new(Int32Array::from(vec![100, 200, 300])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + assert_eq!(result.num_columns(), 3); + assert_eq!(result.num_rows(), 3); + + // id column from file + let id_col = result + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(id_col.values(), &[100, 200, 300]); + + // name column from file + let name_col = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name_col.value(0), "a"); + + // _partition struct column + let partition_col = result + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(partition_col.num_columns(), 1); + assert_eq!(partition_col.len(), 3); + + let inner = partition_col + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(inner.value(0), 42); + assert_eq!(inner.value(1), 42); + assert_eq!(inner.value(2), 42); + } + + #[test] + fn partition_column_with_evolution() { + use arrow_array::StructArray; + + use crate::metadata_columns::RESERVED_FIELD_ID_PARTITION; + use crate::partitioning::compute_unified_partition_type; + use crate::spec::Transform; + + // Schema with two fields that could be partition sources + let snapshot_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "year", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "month", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "data", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + + // Old spec: partition by year only + let spec_v0 = crate::spec::PartitionSpec::builder(snapshot_schema.clone()) + .with_spec_id(0) + .add_partition_field("year", "year", Transform::Identity) + .unwrap() + .build() + .unwrap(); + + // New spec: partition by year and month + let spec_v1 = crate::spec::PartitionSpec::builder(snapshot_schema.clone()) + .with_spec_id(1) + .add_partition_field("year", "year", Transform::Identity) + .unwrap() + .add_partition_field("month", "month", Transform::Identity) + .unwrap() + .build() + .unwrap(); + + // Unified type includes both year and month + let unified_partition_type = + compute_unified_partition_type([&spec_v0, &spec_v1].into_iter(), &snapshot_schema) + .unwrap(); + + assert_eq!(unified_partition_type.fields().len(), 2); + + // File written with spec_v0 (only has year=2023) + let partition_data = Struct::from_iter(vec![Some(Literal::int(2023))]); + + let parquet_schema = Arc::new(ArrowSchema::new(vec![field_with_id( + "data", + DataType::Utf8, + true, + 3, + )])); + + let projected_field_ids = [3, RESERVED_FIELD_ID_PARTITION]; + + let partition_column = + build_partition_constant(&unified_partition_type, &spec_v0, &partition_data).unwrap(); + let mut transformer = + RecordBatchTransformerBuilder::new(snapshot_schema, &projected_field_ids) + .with_partition_constant(partition_column) + .build(); + + let parquet_batch = + RecordBatch::try_new(parquet_schema, vec![Arc::new(StringArray::from(vec![ + "hello", "world", + ]))]) + .unwrap(); + + let result = transformer.process_record_batch(parquet_batch).unwrap(); + + assert_eq!(result.num_columns(), 2); + assert_eq!(result.num_rows(), 2); + + // _partition struct has 2 fields: year (present) and month (null for this old spec file) + let partition_col = result + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(partition_col.num_columns(), 2); + + let year_col = partition_col + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(year_col.value(0), 2023); + assert_eq!(year_col.value(1), 2023); + + let month_col = partition_col + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert!(month_col.is_null(0)); + assert!(month_col.is_null(1)); } } diff --git a/crates/iceberg/src/arrow/scan_metrics.rs b/crates/iceberg/src/arrow/scan_metrics.rs new file mode 100644 index 0000000000..642190c57d --- /dev/null +++ b/crates/iceberg/src/arrow/scan_metrics.rs @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Scan metrics and I/O counting for Parquet data file reads. + +use std::ops::Range; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use bytes::Bytes; + +use crate::error::Result; +use crate::io::FileRead; +use crate::scan::ArrowRecordBatchStream; + +/// Wraps a [`FileRead`] to count bytes read via a shared atomic counter. +pub(crate) struct CountingFileRead { + inner: F, + bytes_read: Arc, +} + +impl CountingFileRead { + pub(crate) fn new(inner: F, bytes_read: Arc) -> Self { + Self { inner, bytes_read } + } +} + +#[async_trait::async_trait] +impl FileRead for CountingFileRead { + async fn read(&self, range: Range) -> Result { + debug_assert!(range.end >= range.start); + self.bytes_read + .fetch_add(range.end - range.start, Ordering::Relaxed); + self.inner.read(range).await + } +} + +/// Metrics collected during an Iceberg scan. +#[derive(Clone, Debug)] +pub struct ScanMetrics { + bytes_read: Arc, +} + +impl ScanMetrics { + pub(crate) fn new() -> Self { + Self { + bytes_read: Arc::new(AtomicU64::new(0)), + } + } + + pub(crate) fn bytes_read_counter(&self) -> &Arc { + &self.bytes_read + } + + /// Total bytes read from storage during this scan, including data files and delete files. + pub fn bytes_read(&self) -> u64 { + self.bytes_read.load(Ordering::Relaxed) + } +} + +/// Result of [`ArrowReader::read`](super::ArrowReader::read), containing the +/// record batch stream and metrics collected during the scan. +pub struct ScanResult { + stream: ArrowRecordBatchStream, + metrics: ScanMetrics, +} + +impl ScanResult { + pub(crate) fn new(stream: ArrowRecordBatchStream, metrics: ScanMetrics) -> Self { + Self { stream, metrics } + } + + /// Consumes the result, returning only the record batch stream. + pub fn stream(self) -> ArrowRecordBatchStream { + self.stream + } + + /// Returns a reference to the scan metrics. + pub fn metrics(&self) -> &ScanMetrics { + &self.metrics + } +} diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs index c214821dd0..35d7214a08 100644 --- a/crates/iceberg/src/arrow/schema.rs +++ b/crates/iceberg/src/arrow/schema.rs @@ -22,21 +22,26 @@ use std::sync::Arc; use arrow_array::types::{Decimal128Type, validate_decimal_precision_and_scale}; use arrow_array::{ - BooleanArray, Date32Array, Datum as ArrowDatum, Decimal128Array, FixedSizeBinaryArray, - Float32Array, Float64Array, Int32Array, Int64Array, Scalar, StringArray, - TimestampMicrosecondArray, + BinaryArray, BooleanArray, Date32Array, Datum as ArrowDatum, Decimal128Array, + FixedSizeBinaryArray, Float32Array, Float64Array, Int32Array, Int64Array, Scalar, StringArray, + TimestampMicrosecondArray, TimestampNanosecondArray, +}; +use arrow_schema::extension::{ExtensionType, Uuid as UuidExtensionType}; +use arrow_schema::{ + ArrowError, DataType, Field, FieldRef, Fields, Schema as ArrowSchema, TimeUnit, }; -use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema, TimeUnit}; -use num_bigint::BigInt; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::file::statistics::Statistics; -use rust_decimal::prelude::ToPrimitive; +use parquet_geospatial::{WkbEdges, WkbMetadata, WkbType, WkbTypeHint}; use uuid::Uuid; -use crate::error::Result; +use crate::error::{Result, invalid_data}; +use crate::spec::decimal_utils::i128_from_be_bytes; +use crate::spec::geospatial::EdgeInterpolationAlgorithm; use crate::spec::{ - Datum, ListType, MapType, NestedField, NestedFieldRef, PrimitiveLiteral, PrimitiveType, Schema, - SchemaVisitor, StructType, Type, + Datum, FIRST_FIELD_ID, GeographyType, GeometryType, LIST_FIELD_NAME, ListType, + MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedField, NestedFieldRef, + PrimitiveLiteral, PrimitiveType, Schema, SchemaVisitor, StructType, Type, VariantType, }; use crate::{Error, ErrorKind}; @@ -44,10 +49,186 @@ use crate::{Error, ErrorKind}; pub const DEFAULT_MAP_FIELD_NAME: &str = "key_value"; /// UTC time zone for Arrow timestamp type. pub const UTC_TIME_ZONE: &str = "+00:00"; +const UNSET_GEOSPATIAL_CRS: &str = "srid:0"; + +/// The canonical Arrow [`arrow.parquet.variant`] extension type. +/// +/// Iceberg stores a Variant as a `Struct { metadata: Binary, value: Binary }`. Attaching this +/// extension type to the enclosing field marks that struct as a single logical Variant value, +/// so Arrow consumers treat it as a Variant rather than an anonymous struct. It carries no +/// metadata. +/// +/// [`arrow.parquet.variant`]: https://arrow.apache.org/docs/format/CanonicalExtensions.html#parquet-variant +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct VariantExtensionType; + +impl ExtensionType for VariantExtensionType { + const NAME: &'static str = "arrow.parquet.variant"; + + type Metadata = (); + + fn metadata(&self) -> &Self::Metadata { + &() + } + + fn serialize_metadata(&self) -> Option { + None + } + + fn deserialize_metadata( + metadata: Option<&str>, + ) -> std::result::Result { + match metadata { + None | Some("") => Ok(()), + Some(other) => Err(ArrowError::InvalidArgumentError(format!( + "arrow.parquet.variant extension type takes no metadata, got {other:?}" + ))), + } + } + + fn supports_data_type(&self, data_type: &DataType) -> std::result::Result<(), ArrowError> { + match data_type { + DataType::Struct(_) => Ok(()), + other => Err(ArrowError::InvalidArgumentError(format!( + "arrow.parquet.variant extension type requires a Struct storage type, got {other}" + ))), + } + } + + fn try_new( + data_type: &DataType, + _metadata: Self::Metadata, + ) -> std::result::Result { + Self.supports_data_type(data_type)?; + Ok(Self) + } +} + +/// Whether `field` carries the [`VariantExtensionType`]. +pub(crate) fn is_variant_field(field: &Field) -> bool { + field.extension_type_name() == Some(VariantExtensionType::NAME) +} + +/// Whether `data_type` can hold a variant's `metadata` or `value` bytes. +fn is_variant_binary(data_type: &DataType) -> bool { + match data_type { + DataType::Binary | DataType::LargeBinary | DataType::BinaryView => true, + DataType::Dictionary(_, value_type) => is_variant_binary(value_type), + _ => false, + } +} + +/// Checks that `field` stores a variant unshredded: `metadata` and `value` bytes, matched by +/// name, with a required `metadata`. +pub(crate) fn validate_unshredded_variant(field: &Field, variant_field_id: i32) -> Result<()> { + let invalid = |problem: String| { + invalid_data!( + "Variant column `{}` (field id {variant_field_id}) {problem}", + field.name() + ) + }; + let DataType::Struct(subfields) = field.data_type() else { + return Err(invalid(format!( + "must be stored as a group, found {}", + field.data_type() + ))); + }; + if subfields.find("typed_value").is_some() { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Reading shredded variant column `{}` (field id {variant_field_id}) is not supported yet, see https://github.com/apache/iceberg-rust/issues/3606", + field.name() + ), + )); + } + for name in ["metadata", "value"] { + match subfields.find(name) { + Some((_, subfield)) if !is_variant_binary(subfield.data_type()) => { + return Err(invalid(format!( + "has a non-binary `{name}` field: {}", + subfield.data_type() + ))); + } + // The Parquet and Arrow variant specs require `metadata`, and the table type reads + // it as non-nullable. + Some((_, subfield)) if name == "metadata" && subfield.is_nullable() => { + return Err(invalid( + "has an optional `metadata` field, which must be required".to_string(), + )); + } + Some(_) => {} + None => return Err(invalid(format!("is missing the `{name}` field"))), + } + } + if subfields.len() != 2 { + return Err(invalid( + "has fields other than `metadata` and `value`".to_string(), + )); + } + Ok(()) +} + +fn to_arrow_wkb_edges(algorithm: EdgeInterpolationAlgorithm) -> WkbEdges { + match algorithm { + EdgeInterpolationAlgorithm::Spherical => WkbEdges::Spherical, + EdgeInterpolationAlgorithm::Vincenty => WkbEdges::Vincenty, + EdgeInterpolationAlgorithm::Thomas => WkbEdges::Thomas, + EdgeInterpolationAlgorithm::Andoyer => WkbEdges::Andoyer, + EdgeInterpolationAlgorithm::Karney => WkbEdges::Karney, + } +} + +impl From for EdgeInterpolationAlgorithm { + fn from(edges: WkbEdges) -> Self { + match edges { + WkbEdges::Spherical => Self::Spherical, + WkbEdges::Vincenty => Self::Vincenty, + WkbEdges::Thomas => Self::Thomas, + WkbEdges::Andoyer => Self::Andoyer, + WkbEdges::Karney => Self::Karney, + } + } +} + +fn iceberg_crs_from_wkb_metadata(metadata: &WkbMetadata) -> Result { + match metadata.crs.as_ref() { + None => Ok(UNSET_GEOSPATIAL_CRS.to_string()), + Some(serde_json::Value::String(crs)) => Ok(crs.clone()), + Some(serde_json::Value::Object(crs)) => { + let id = crs.get("id"); + let authority = id + .and_then(|id| id.get("authority")) + .and_then(serde_json::Value::as_str) + .filter(|authority| !authority.is_empty()); + let code = match id.and_then(|id| id.get("code")) { + Some(serde_json::Value::String(code)) => Some(code.clone()), + Some(serde_json::Value::Number(code)) => Some(code.to_string()), + _ => None, + }; + + match (authority, code) { + (Some(authority), Some(code)) => Ok(format!("{authority}:{code}")), + _ => Err(Error::new( + ErrorKind::DataInvalid, + "Cannot determine Iceberg geospatial type from PROJJSON CRS without an embedded authority/code", + )), + } + } + Some(_) => Err(Error::new( + ErrorKind::DataInvalid, + "Geospatial CRS metadata must be a string or PROJJSON object", + )), + } +} + +fn arrow_crs_from_iceberg(crs: &str) -> Option<&str> { + (!crs.eq_ignore_ascii_case(UNSET_GEOSPATIAL_CRS)).then_some(crs) +} /// A post order arrow schema visitor. /// -/// For order of methods called, please refer to [`visit_schema`]. +/// For order of methods called, please refer to the internal `visit_schema` function. pub trait ArrowSchemaVisitor { /// Return type of this visitor on arrow field. type T; @@ -55,43 +236,47 @@ pub trait ArrowSchemaVisitor { /// Return type of this visitor on arrow schema. type U; - /// Called before struct/list/map field. - fn before_field(&mut self, _field: &Field) -> Result<()> { + /// Called before each field of the schema or of a struct, whatever its type. + /// List elements and map keys and values use [`Self::before_list_element`], + /// [`Self::before_map_key`] and [`Self::before_map_value`] instead. + fn before_field(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } - /// Called after struct/list/map field. - fn after_field(&mut self, _field: &Field) -> Result<()> { + /// Called after each field of the schema or of a struct, whatever its type. + /// List elements and map keys and values use [`Self::after_list_element`], + /// [`Self::after_map_key`] and [`Self::after_map_value`] instead. + fn after_field(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } /// Called before list element. - fn before_list_element(&mut self, _field: &Field) -> Result<()> { + fn before_list_element(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } /// Called after list element. - fn after_list_element(&mut self, _field: &Field) -> Result<()> { + fn after_list_element(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } /// Called before map key. - fn before_map_key(&mut self, _field: &Field) -> Result<()> { + fn before_map_key(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } /// Called after map key. - fn after_map_key(&mut self, _field: &Field) -> Result<()> { + fn after_map_key(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } /// Called before map value. - fn before_map_value(&mut self, _field: &Field) -> Result<()> { + fn before_map_value(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } /// Called after map value. - fn after_map_value(&mut self, _field: &Field) -> Result<()> { + fn after_map_value(&mut self, _field: &FieldRef) -> Result<()> { Ok(()) } @@ -109,6 +294,35 @@ pub trait ArrowSchemaVisitor { /// Called when see a primitive type. fn primitive(&mut self, p: &DataType) -> Result; + + /// Called when a field carries the `arrow.parquet.variant` extension type. + /// + /// The default treats the variant as its underlying Arrow struct storage: it + /// re-enters normal traversal, preserving the behavior of visitors that don't + /// special-case variants. A visitor that produces Iceberg types (or otherwise + /// needs variant identity) should override this to fold the struct into a + /// single logical variant. + /// + /// Takes the `&FieldRef` rather than a `&DataType` because the variant signal + /// lives on the field's metadata, not on its data type. + fn variant(&mut self, field: &FieldRef) -> Result + where Self: Sized { + visit_type(field.data_type(), self) + } + + /// Called when a field carries the `arrow.uuid` extension type. + /// + /// The default treats the field as its underlying Arrow storage, preserving + /// the behavior of visitors that don't special-case UUIDs. A visitor that + /// produces Iceberg types should override this to fold the storage into a + /// single logical UUID. + /// + /// Takes the `&FieldRef` rather than a `&DataType` because the UUID signal + /// lives on the field's metadata, not on its data type. + fn uuid(&mut self, field: &FieldRef) -> Result + where Self: Sized { + visit_type(field.data_type(), self) + } } /// Visiting a type in post order. @@ -121,6 +335,7 @@ fn visit_type(r#type: &DataType, visitor: &mut V) -> Resu | DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + | DataType::Null | DataType::Binary | DataType::LargeBinary | DataType::BinaryView @@ -135,10 +350,7 @@ fn visit_type(r#type: &DataType, visitor: &mut V) -> Resu DataType::Map(field, _) => match field.data_type() { DataType::Struct(fields) => { if fields.len() != 2 { - return Err(Error::new( - ErrorKind::DataInvalid, - "Map field must have exactly 2 fields", - )); + return Err(invalid_data!("Map field must have exactly 2 fields")); } let key_field = &fields[0]; @@ -146,42 +358,50 @@ fn visit_type(r#type: &DataType, visitor: &mut V) -> Resu let key_result = { visitor.before_map_key(key_field)?; - let ret = visit_type(key_field.data_type(), visitor)?; + let ret = visit_field(key_field, visitor)?; visitor.after_map_key(key_field)?; ret }; let value_result = { visitor.before_map_value(value_field)?; - let ret = visit_type(value_field.data_type(), visitor)?; + let ret = visit_field(value_field, visitor)?; visitor.after_map_value(value_field)?; ret }; visitor.map(r#type, key_result, value_result) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "Map field must have struct type", - )), + _ => Err(invalid_data!("Map field must have struct type")), }, DataType::Struct(fields) => visit_struct(fields, visitor), DataType::Dictionary(_key_type, value_type) => visit_type(value_type, visitor), - other => Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot visit Arrow data type: {other}"), - )), + other => Err(invalid_data!("Cannot visit Arrow data type: {other}")), + } +} + +/// Dispatch a field, and fold it into following extension types when carried: +/// - `arrow.parquet.variant` -- fold into a variant, +/// - `arrow.uuid` -- fold it into a UUID type, +/// - and in all other cases we visit it's data. +fn visit_field(field: &FieldRef, visitor: &mut V) -> Result { + if field.extension_type_name() == Some(VariantExtensionType::NAME) { + visitor.variant(field) + } else if field.extension_type_name() == Some(UuidExtensionType::NAME) { + visitor.uuid(field) + } else { + visit_type(field.data_type(), visitor) } } /// Visit list types in post order. fn visit_list( data_type: &DataType, - element_field: &Field, + element_field: &FieldRef, visitor: &mut V, ) -> Result { visitor.before_list_element(element_field)?; - let value = visit_type(element_field.data_type(), visitor)?; + let value = visit_field(element_field, visitor)?; visitor.after_list_element(element_field)?; visitor.list(data_type, value) } @@ -191,7 +411,7 @@ fn visit_struct(fields: &Fields, visitor: &mut V) -> Resu let mut results = Vec::with_capacity(fields.len()); for field in fields { visitor.before_field(field)?; - let result = visit_type(field.data_type(), visitor)?; + let result = visit_field(field, visitor)?; visitor.after_field(field)?; results.push(result); } @@ -200,11 +420,14 @@ fn visit_struct(fields: &Fields, visitor: &mut V) -> Resu } /// Visit schema in post order. -fn visit_schema(schema: &ArrowSchema, visitor: &mut V) -> Result { +pub(crate) fn visit_schema( + schema: &ArrowSchema, + visitor: &mut V, +) -> Result { let mut results = Vec::with_capacity(schema.fields().len()); for field in schema.fields() { visitor.before_field(field)?; - let result = visit_type(field.data_type(), visitor)?; + let result = visit_field(field, visitor)?; visitor.after_field(field)?; results.push(result); } @@ -221,6 +444,19 @@ pub fn arrow_schema_to_schema(schema: &ArrowSchema) -> Result { visit_schema(schema, &mut visitor) } +/// Convert Arrow schema to Iceberg schema with automatically assigned field IDs. +/// +/// Unlike [`arrow_schema_to_schema`], this function does not require field IDs in the Arrow +/// schema metadata. Instead, it automatically assigns unique field IDs starting from 1, +/// following Iceberg's field ID assignment rules. +/// +/// This is useful when converting Arrow schemas that don't originate from Iceberg tables, +/// such as schemas from DataFusion or other Arrow-based systems. +pub fn arrow_schema_to_schema_auto_assign_ids(schema: &ArrowSchema) -> Result { + let mut visitor = ArrowSchemaConverter::new_with_field_ids_from(FIRST_FIELD_ID); + visit_schema(schema, &mut visitor) +} + /// Convert Arrow type to iceberg type. pub fn arrow_type_to_type(ty: &DataType) -> Result { let mut visitor = ArrowSchemaConverter::new(); @@ -229,50 +465,88 @@ pub fn arrow_type_to_type(ty: &DataType) -> Result { const ARROW_FIELD_DOC_KEY: &str = "doc"; -pub(super) fn get_field_id(field: &Field) -> Result { - if let Some(value) = field.metadata().get(PARQUET_FIELD_ID_META_KEY) { - return value.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to parse field id".to_string(), - ) - .with_context("value", value) - .with_source(e) - }); - } - Err(Error::new( - ErrorKind::DataInvalid, - "Field id not found in metadata", - )) +pub(super) fn try_get_field_id_from_metadata(field: &Field) -> Result> { + field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .map(|value| { + value.parse::().map_err(|e| { + invalid_data!("Failed to parse field id") + .with_context("value", value) + .with_source(e) + }) + }) + .transpose() } -fn get_field_doc(field: &Field) -> Option { +pub(super) fn get_field_id_from_metadata(field: &Field) -> Result { + try_get_field_id_from_metadata(field)? + .ok_or_else(|| invalid_data!("Field id not found in metadata")) +} + +fn get_field_doc(field: &FieldRef) -> Option { if let Some(value) = field.metadata().get(ARROW_FIELD_DOC_KEY) { return Some(value.clone()); } None } -struct ArrowSchemaConverter; +struct ArrowSchemaConverter { + /// When set, the schema builder will reassign field IDs starting from this value + /// using level-order traversal (breadth-first). + reassign_field_ids_from: Option, + /// Generates unique placeholder IDs for fields before reassignment. + /// Required because `ReassignFieldIds` builds an old-to-new ID mapping + /// that expects unique input IDs. + next_field_id: i32, +} impl ArrowSchemaConverter { fn new() -> Self { - Self {} + Self { + reassign_field_ids_from: None, + next_field_id: 0, + } + } + + fn new_with_field_ids_from(start_from: i32) -> Self { + Self { + reassign_field_ids_from: Some(start_from), + next_field_id: 0, + } + } + + fn get_field_id(&mut self, field: &FieldRef) -> Result { + if self.reassign_field_ids_from.is_some() { + // Field IDs will be reassigned by the schema builder. + // We need unique temporary IDs because ReassignFieldIds builds an + // old->new ID mapping that requires unique input IDs. + let temp_id = self.next_field_id; + self.next_field_id += 1; + Ok(temp_id) + } else { + // Get field ID from arrow field metadata + get_field_id_from_metadata(field) + } } - fn convert_fields(fields: &Fields, field_results: &[Type]) -> Result> { + fn convert_fields( + &mut self, + fields: &Fields, + field_results: &[Type], + ) -> Result> { let mut results = Vec::with_capacity(fields.len()); for i in 0..fields.len() { let field = &fields[i]; - let field_type = &field_results[i]; - let id = get_field_id(field)?; + let field_type = self.apply_arrow_extension(field, field_results[i].clone())?; + let id = self.get_field_id(field)?; let doc = get_field_doc(field); let nested_field = NestedField { id, doc, name: field.name().clone(), required: !field.is_nullable(), - field_type: Box::new(field_type.clone()), + field_type: Box::new(field_type), initial_default: None, write_default: None, }; @@ -280,6 +554,51 @@ impl ArrowSchemaConverter { } Ok(results) } + + fn apply_arrow_extension(&self, arrow_field: &FieldRef, iceberg_type: Type) -> Result { + if arrow_field.extension_type_name() != Some(WkbType::NAME) { + return Ok(iceberg_type); + } + + if !matches!(&iceberg_type, Type::Primitive(PrimitiveType::Binary)) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "WKB extension type on field {} requires binary storage, got {iceberg_type}", + arrow_field.name() + ), + )); + } + + let wkb_type = arrow_field.try_extension_type::().map_err(|err| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid geospatial Arrow extension metadata for field {}", + arrow_field.name() + ), + ) + .with_source(err) + })?; + + let crs = iceberg_crs_from_wkb_metadata(wkb_type.metadata())?; + + match wkb_type.metadata().type_hint() { + WkbTypeHint::Geometry => Ok(Type::Primitive(PrimitiveType::Geometry( + GeometryType::new(Some(crs))?, + ))), + WkbTypeHint::Geography => Ok(Type::Primitive(PrimitiveType::Geography( + GeographyType::new( + Some(crs), + wkb_type + .metadata() + .algorithm + .unwrap_or(WkbEdges::Spherical) + .into(), + )?, + ))), + } + } } impl ArrowSchemaVisitor for ArrowSchemaConverter { @@ -287,13 +606,16 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { type U = Schema; fn schema(&mut self, schema: &ArrowSchema, values: Vec) -> Result { - let fields = Self::convert_fields(schema.fields(), &values)?; - let builder = Schema::builder().with_fields(fields); + let fields = self.convert_fields(schema.fields(), &values)?; + let mut builder = Schema::builder().with_fields(fields); + if let Some(start_from) = self.reassign_field_ids_from { + builder = builder.with_reassigned_field_ids(start_from) + } builder.build() } fn r#struct(&mut self, fields: &Fields, results: Vec) -> Result { - let fields = Self::convert_fields(fields, &results)?; + let fields = self.convert_fields(fields, &results)?; Ok(Type::Struct(StructType::new(fields))) } @@ -303,14 +625,12 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::LargeList(element_field) => element_field, DataType::FixedSizeList(element_field, _) => element_field, _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - "List type must have list data type", - )); + return Err(invalid_data!("List type must have list data type")); } }; - let id = get_field_id(element_field)?; + let value = self.apply_arrow_extension(element_field, value)?; + let id = self.get_field_id(element_field)?; let doc = get_field_doc(element_field); let mut element_field = NestedField::list_element(id, value.clone(), !element_field.is_nullable()); @@ -326,16 +646,15 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::Map(field, _) => match field.data_type() { DataType::Struct(fields) => { if fields.len() != 2 { - return Err(Error::new( - ErrorKind::DataInvalid, - "Map field must have exactly 2 fields", - )); + return Err(invalid_data!("Map field must have exactly 2 fields")); } let key_field = &fields[0]; let value_field = &fields[1]; + let key_value = self.apply_arrow_extension(key_field, key_value)?; + let value = self.apply_arrow_extension(value_field, value)?; - let key_id = get_field_id(key_field)?; + let key_id = self.get_field_id(key_field)?; let key_doc = get_field_doc(key_field); let mut key_field = NestedField::map_key_element(key_id, key_value.clone()); if let Some(doc) = key_doc { @@ -343,7 +662,7 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { } let key_field = Arc::new(key_field); - let value_id = get_field_id(value_field)?; + let value_id = self.get_field_id(value_field)?; let value_doc = get_field_doc(value_field); let mut value_field = NestedField::map_value_element( value_id, @@ -360,20 +679,15 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { value_field, })) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "Map field must have struct type", - )), + _ => Err(invalid_data!("Map field must have struct type")), }, - _ => Err(Error::new( - ErrorKind::DataInvalid, - "Map type must have map data type", - )), + _ => Err(invalid_data!("Map type must have map data type")), } } fn primitive(&mut self, p: &DataType) -> Result { match p { + DataType::Null => Ok(Type::Primitive(PrimitiveType::Unknown)), DataType::Boolean => Ok(Type::Primitive(PrimitiveType::Boolean)), DataType::Int8 | DataType::Int16 | DataType::Int32 => { Ok(Type::Primitive(PrimitiveType::Int)) @@ -383,20 +697,14 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::Int64 => Ok(Type::Primitive(PrimitiveType::Long)), DataType::UInt64 => { // Block uint64 - no safe casting option - Err(Error::new( - ErrorKind::DataInvalid, - "UInt64 is not supported. Use Int64 for values ≤ 9,223,372,036,854,775,807 or Decimal(20,0) for full uint64 range.", + Err(invalid_data!( + "UInt64 is not supported. Use Int64 for values ≤ 9,223,372,036,854,775,807 or Decimal(20,0) for full uint64 range." )) } DataType::Float32 => Ok(Type::Primitive(PrimitiveType::Float)), DataType::Float64 => Ok(Type::Primitive(PrimitiveType::Double)), - DataType::Decimal128(p, s) => Type::decimal(*p as u32, *s as u32).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to create decimal type".to_string(), - ) - .with_source(e) - }), + DataType::Decimal128(p, s) => Type::decimal(*p as u32, *s as u32) + .map_err(|e| invalid_data!("Failed to create decimal type").with_source(e)), DataType::Date32 => Ok(Type::Primitive(PrimitiveType::Date)), DataType::Time64(unit) if unit == &TimeUnit::Microsecond => { Ok(Type::Primitive(PrimitiveType::Time)) @@ -428,11 +736,39 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter { DataType::Utf8View | DataType::Utf8 | DataType::LargeUtf8 => { Ok(Type::Primitive(PrimitiveType::String)) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Unsupported Arrow data type: {p}"), - )), + _ => Err(invalid_data!("Unsupported Arrow data type: {p}")), + } + } + + fn variant(&mut self, field: &FieldRef) -> Result { + // The extension may only sit on struct storage (mirrors + // `VariantExtensionType::supports_data_type`). + if !matches!(field.data_type(), DataType::Struct(_)) { + return Err(invalid_data!( + "arrow.parquet.variant extension requires Struct storage" + )); + } + // Fold the whole struct into a single logical variant without descending: + // the storage sub-fields carry no Iceberg field id, so visiting them would + // fail. The enclosing field's own id is read by the caller. + Ok(Type::Variant(VariantType)) + } + + fn uuid(&mut self, field: &FieldRef) -> Result { + // The extension expects a fixed size binary. + // See: Apache Arrow spec https://arrow.apache.org/docs/format/CanonicalExtensions.html#uuid + let field_data_type = field.data_type(); + let expected_data_type = DataType::FixedSizeBinary(16); + if *field_data_type != expected_data_type { + let error_message = format!( + "{} extension requires {expected_data_type} storage, found {field_data_type}", + UuidExtensionType::NAME, + ); + + return Err(Error::new(ErrorKind::DataInvalid, error_message)); } + + Ok(Type::Primitive(PrimitiveType::Uuid)) } } @@ -449,9 +785,9 @@ impl SchemaVisitor for ToArrowSchemaConverter { fn schema( &mut self, - _schema: &crate::spec::Schema, + _schema: &Schema, value: ArrowSchemaOrFieldOrType, - ) -> crate::Result { + ) -> Result { let struct_type = match value { ArrowSchemaOrFieldOrType::Type(DataType::Struct(fields)) => fields, _ => unreachable!(), @@ -463,9 +799,9 @@ impl SchemaVisitor for ToArrowSchemaConverter { fn field( &mut self, - field: &crate::spec::NestedFieldRef, + field: &NestedFieldRef, value: ArrowSchemaOrFieldOrType, - ) -> crate::Result { + ) -> Result { let ty = match value { ArrowSchemaOrFieldOrType::Type(ty) => ty, _ => unreachable!(), @@ -478,16 +814,41 @@ impl SchemaVisitor for ToArrowSchemaConverter { } else { HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), field.id.to_string())]) }; - Ok(ArrowSchemaOrFieldOrType::Field( - Field::new(field.name.clone(), ty, !field.required).with_metadata(metadata), - )) + let mut arrow_field = + Field::new(field.name.clone(), ty, !field.required).with_metadata(metadata); + + match field.field_type.as_ref() { + Type::Variant(_) => { + // A variant column's storage is a struct; tag the field with the canonical + // `arrow.parquet.variant` extension type so consumers read it as a Variant, not a struct. + arrow_field.try_with_extension_type(VariantExtensionType)?; + } + Type::Primitive(PrimitiveType::Uuid) => { + // Preserve the canonical UUID extension on its FixedSizeBinary(16) storage. + arrow_field.try_with_extension_type(UuidExtensionType)?; + } + Type::Primitive(PrimitiveType::Geometry(geometry)) => { + let metadata = WkbMetadata::new(arrow_crs_from_iceberg(geometry.crs()), None); + arrow_field.try_with_extension_type(WkbType::new(Some(metadata)))?; + } + Type::Primitive(PrimitiveType::Geography(geography)) => { + let metadata = WkbMetadata::new( + arrow_crs_from_iceberg(geography.crs()), + Some(to_arrow_wkb_edges(geography.algorithm())), + ); + arrow_field.try_with_extension_type(WkbType::new(Some(metadata)))?; + } + _ => {} + } + + Ok(ArrowSchemaOrFieldOrType::Field(arrow_field)) } fn r#struct( &mut self, - _: &crate::spec::StructType, + _: &StructType, results: Vec, - ) -> crate::Result { + ) -> Result { let fields = results .into_iter() .map(|result| match result { @@ -498,30 +859,14 @@ impl SchemaVisitor for ToArrowSchemaConverter { Ok(ArrowSchemaOrFieldOrType::Type(DataType::Struct(fields))) } - fn list( - &mut self, - list: &crate::spec::ListType, - value: ArrowSchemaOrFieldOrType, - ) -> crate::Result { + fn list(&mut self, list: &ListType, value: ArrowSchemaOrFieldOrType) -> Result { + // `field` already carries the element's field id, doc, and — for a variant element — + // the arrow.parquet.variant extension type. Don't overwrite its metadata here (doing so + // would drop the extension type for `list`). let field = match self.field(&list.element_field, value)? { ArrowSchemaOrFieldOrType::Field(field) => field, _ => unreachable!(), }; - let meta = if let Some(doc) = &list.element_field.doc { - HashMap::from([ - ( - PARQUET_FIELD_ID_META_KEY.to_string(), - list.element_field.id.to_string(), - ), - (ARROW_FIELD_DOC_KEY.to_string(), doc.clone()), - ]) - } else { - HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - list.element_field.id.to_string(), - )]) - }; - let field = field.with_metadata(meta); Ok(ArrowSchemaOrFieldOrType::Type(DataType::List(Arc::new( field, )))) @@ -529,10 +874,10 @@ impl SchemaVisitor for ToArrowSchemaConverter { fn map( &mut self, - map: &crate::spec::MapType, + map: &MapType, key_value: ArrowSchemaOrFieldOrType, value: ArrowSchemaOrFieldOrType, - ) -> crate::Result { + ) -> Result { let key_field = match self.field(&map.key_field, key_value)? { ArrowSchemaOrFieldOrType::Field(field) => field, _ => unreachable!(), @@ -554,93 +899,89 @@ impl SchemaVisitor for ToArrowSchemaConverter { ))) } - fn primitive( - &mut self, - p: &crate::spec::PrimitiveType, - ) -> crate::Result { + fn primitive(&mut self, p: &PrimitiveType) -> Result { match p { - crate::spec::PrimitiveType::Boolean => { - Ok(ArrowSchemaOrFieldOrType::Type(DataType::Boolean)) - } - crate::spec::PrimitiveType::Int => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int32)), - crate::spec::PrimitiveType::Long => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int64)), - crate::spec::PrimitiveType::Float => { - Ok(ArrowSchemaOrFieldOrType::Type(DataType::Float32)) - } - crate::spec::PrimitiveType::Double => { - Ok(ArrowSchemaOrFieldOrType::Type(DataType::Float64)) - } - crate::spec::PrimitiveType::Decimal { precision, scale } => { + PrimitiveType::Unknown => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Null)), + PrimitiveType::Boolean => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Boolean)), + PrimitiveType::Int => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int32)), + PrimitiveType::Long => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int64)), + PrimitiveType::Float => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Float32)), + PrimitiveType::Double => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Float64)), + PrimitiveType::Decimal { precision, scale } => { let (precision, scale) = { let precision: u8 = precision.to_owned().try_into().map_err(|err| { - Error::new( - crate::ErrorKind::DataInvalid, - "incompatible precision for decimal type convert", - ) - .with_source(err) + invalid_data!("incompatible precision for decimal type convert") + .with_source(err) })?; let scale = scale.to_owned().try_into().map_err(|err| { - Error::new( - crate::ErrorKind::DataInvalid, - "incompatible scale for decimal type convert", - ) - .with_source(err) + invalid_data!("incompatible scale for decimal type convert") + .with_source(err) })?; (precision, scale) }; validate_decimal_precision_and_scale::(precision, scale).map_err( |err| { - Error::new( - crate::ErrorKind::DataInvalid, - "incompatible precision and scale for decimal type convert", - ) - .with_source(err) + invalid_data!("incompatible precision and scale for decimal type convert") + .with_source(err) }, )?; Ok(ArrowSchemaOrFieldOrType::Type(DataType::Decimal128( precision, scale, ))) } - crate::spec::PrimitiveType::Date => { - Ok(ArrowSchemaOrFieldOrType::Type(DataType::Date32)) - } - crate::spec::PrimitiveType::Time => Ok(ArrowSchemaOrFieldOrType::Type( - DataType::Time64(TimeUnit::Microsecond), - )), - crate::spec::PrimitiveType::Timestamp => Ok(ArrowSchemaOrFieldOrType::Type( - DataType::Timestamp(TimeUnit::Microsecond, None), - )), - crate::spec::PrimitiveType::Timestamptz => Ok(ArrowSchemaOrFieldOrType::Type( + PrimitiveType::Date => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Date32)), + PrimitiveType::Time => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Time64( + TimeUnit::Microsecond, + ))), + PrimitiveType::Timestamp => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Timestamp( + TimeUnit::Microsecond, + None, + ))), + PrimitiveType::Timestamptz => Ok(ArrowSchemaOrFieldOrType::Type( // Timestampz always stored as UTC DataType::Timestamp(TimeUnit::Microsecond, Some(UTC_TIME_ZONE.into())), )), - crate::spec::PrimitiveType::TimestampNs => Ok(ArrowSchemaOrFieldOrType::Type( - DataType::Timestamp(TimeUnit::Nanosecond, None), - )), - crate::spec::PrimitiveType::TimestamptzNs => Ok(ArrowSchemaOrFieldOrType::Type( + PrimitiveType::TimestampNs => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Timestamp( + TimeUnit::Nanosecond, + None, + ))), + PrimitiveType::TimestamptzNs => Ok(ArrowSchemaOrFieldOrType::Type( // Store timestamptz_ns as UTC DataType::Timestamp(TimeUnit::Nanosecond, Some(UTC_TIME_ZONE.into())), )), - crate::spec::PrimitiveType::String => { - Ok(ArrowSchemaOrFieldOrType::Type(DataType::Utf8)) - } - crate::spec::PrimitiveType::Uuid => Ok(ArrowSchemaOrFieldOrType::Type( - DataType::FixedSizeBinary(16), - )), - crate::spec::PrimitiveType::Fixed(len) => Ok(ArrowSchemaOrFieldOrType::Type( - len.to_i32() + PrimitiveType::String => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Utf8)), + PrimitiveType::Uuid => Ok(ArrowSchemaOrFieldOrType::Type(DataType::FixedSizeBinary( + 16, + ))), + PrimitiveType::Fixed(len) => Ok(ArrowSchemaOrFieldOrType::Type( + i32::try_from(*len) + .ok() .map(DataType::FixedSizeBinary) .unwrap_or(DataType::LargeBinary), )), - crate::spec::PrimitiveType::Binary => { + PrimitiveType::Binary => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Binary)), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { Ok(ArrowSchemaOrFieldOrType::Type(DataType::LargeBinary)) } } } + + fn variant(&mut self, _v: &VariantType) -> Result { + // Variant is stored as a struct of two binary sub-fields (no field IDs on sub-fields). + // Uses Binary (not LargeBinary) matching the Parquet BINARY primitive directly. + // `metadata` is always present; `value` is nullable, since in a shredded variant the + // value may be absent. The enclosing field carries the `arrow.parquet.variant` extension type + // (attached in `field`). + let metadata_field = Field::new("metadata", DataType::Binary, false); + let value_field = Field::new("value", DataType::Binary, true); + Ok(ArrowSchemaOrFieldOrType::Type(DataType::Struct( + vec![metadata_field, value_field].into(), + ))) + } } /// Convert iceberg schema to an arrow schema. -pub fn schema_to_arrow_schema(schema: &crate::spec::Schema) -> crate::Result { +pub fn schema_to_arrow_schema(schema: &Schema) -> Result { let mut converter = ToArrowSchemaConverter; match crate::spec::visit_schema(schema, &mut converter)? { ArrowSchemaOrFieldOrType::Schema(schema) => Ok(schema), @@ -649,7 +990,7 @@ pub fn schema_to_arrow_schema(schema: &crate::spec::Schema) -> crate::Result crate::Result { +pub fn type_to_arrow_type(ty: &Type) -> Result { let mut converter = ToArrowSchemaConverter; match crate::spec::visit_type(ty, &mut converter)? { ArrowSchemaOrFieldOrType::Type(ty) => Ok(ty), @@ -670,14 +1011,17 @@ pub(crate) fn get_arrow_datum(datum: &Datum) -> Result { - Ok(Arc::new(Float32Array::new_scalar(value.to_f32().unwrap()))) + Ok(Arc::new(Float32Array::new_scalar(value.into_inner()))) } (PrimitiveType::Double, PrimitiveLiteral::Double(value)) => { - Ok(Arc::new(Float64Array::new_scalar(value.to_f64().unwrap()))) + Ok(Arc::new(Float64Array::new_scalar(value.into_inner()))) } (PrimitiveType::String, PrimitiveLiteral::String(value)) => { Ok(Arc::new(StringArray::new_scalar(value.as_str()))) } + (PrimitiveType::Binary, PrimitiveLiteral::Binary(value)) => { + Ok(Arc::new(BinaryArray::new_scalar(value.as_slice()))) + } (PrimitiveType::Date, PrimitiveLiteral::Int(value)) => { Ok(Arc::new(Date32Array::new_scalar(*value))) } @@ -687,6 +1031,12 @@ pub(crate) fn get_arrow_datum(datum: &Datum) -> Result Ok(Arc::new(Scalar::new( TimestampMicrosecondArray::new(vec![*value; 1].into(), None).with_timezone_utc(), ))), + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(value)) => { + Ok(Arc::new(TimestampNanosecondArray::new_scalar(*value))) + } + (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(value)) => Ok(Arc::new(Scalar::new( + TimestampNanosecondArray::new(vec![*value; 1].into(), None).with_timezone_utc(), + ))), (PrimitiveType::Decimal { precision, scale }, PrimitiveLiteral::Int128(value)) => { let array = Decimal128Array::from_value(*value, 1) .with_precision_and_scale(*precision as _, *scale as _) @@ -698,13 +1048,15 @@ pub(crate) fn get_arrow_datum(datum: &Datum) -> Result { + let array = FixedSizeBinaryArray::try_from_iter(std::iter::once(value.as_slice())) + .map_err(|e| invalid_data!("FixedSizeBinary conversion failed").with_source(e))?; + Ok(Arc::new(Scalar::new(array))) + } (primitive_type, _) => Err(Error::new( ErrorKind::FeatureUnsupported, - format!( - "Converting datum from type {:?} to arrow not supported yet.", - primitive_type - ), + format!("Converting datum from type {primitive_type:?} to arrow not supported yet."), )), } } @@ -770,7 +1122,13 @@ pub(crate) fn get_parquet_stat_min_as_datum( }; Some(Datum::new( primitive_type.clone(), - PrimitiveLiteral::Int128(i128::from_be_bytes(bytes.try_into()?)), + PrimitiveLiteral::Int128( + // An empty bound carries no value; reject it rather than + // decode it as zero, which could prune matching row groups. + i128_from_be_bytes(bytes) + .filter(|_| !bytes.is_empty()) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), )) } ( @@ -783,15 +1141,12 @@ pub(crate) fn get_parquet_stat_min_as_datum( let Some(bytes) = stats.min_bytes_opt() else { return Ok(None); }; - let unscaled_value = BigInt::from_signed_bytes_be(bytes); Some(Datum::new( primitive_type.clone(), - PrimitiveLiteral::Int128(unscaled_value.to_i128().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't convert bytes to i128: {:?}", bytes), - ) - })?), + PrimitiveLiteral::Int128( + i128_from_be_bytes(bytes) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), )) } ( @@ -917,7 +1272,13 @@ pub(crate) fn get_parquet_stat_max_as_datum( }; Some(Datum::new( primitive_type.clone(), - PrimitiveLiteral::Int128(i128::from_be_bytes(bytes.try_into()?)), + PrimitiveLiteral::Int128( + // An empty bound carries no value; reject it rather than + // decode it as zero, which could prune matching row groups. + i128_from_be_bytes(bytes) + .filter(|_| !bytes.is_empty()) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), )) } ( @@ -930,15 +1291,12 @@ pub(crate) fn get_parquet_stat_max_as_datum( let Some(bytes) = stats.max_bytes_opt() else { return Ok(None); }; - let unscaled_value = BigInt::from_signed_bytes_be(bytes); Some(Datum::new( primitive_type.clone(), - PrimitiveLiteral::Int128(unscaled_value.to_i128().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't convert bytes to i128: {:?}", bytes), - ) - })?), + PrimitiveLiteral::Int128( + i128_from_be_bytes(bytes) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), )) } ( @@ -1003,93 +1361,337 @@ pub(crate) fn get_parquet_stat_max_as_datum( }) } -impl TryFrom<&ArrowSchema> for crate::spec::Schema { +impl TryFrom<&ArrowSchema> for Schema { type Error = Error; - fn try_from(schema: &ArrowSchema) -> crate::Result { + fn try_from(schema: &ArrowSchema) -> Result { arrow_schema_to_schema(schema) } } -impl TryFrom<&crate::spec::Schema> for ArrowSchema { +impl TryFrom<&Schema> for ArrowSchema { type Error = Error; - fn try_from(schema: &crate::spec::Schema) -> crate::Result { + fn try_from(schema: &Schema) -> Result { schema_to_arrow_schema(schema) } } -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; +/// Converts a Datum (Iceberg type + primitive literal) to its corresponding Arrow DataType +/// with Run-End Encoding (REE). +/// +/// This function is used for constant fields in record batches, where all values are the same. +/// Run-End Encoding provides efficient storage for such constant columns. +/// +/// # Arguments +/// * `datum` - The Datum to convert, which contains both type and value information +/// +/// # Returns +/// Arrow DataType with Run-End Encoding applied +/// +/// # Example +/// ``` +/// use iceberg::arrow::datum_to_arrow_type_with_ree; +/// use iceberg::spec::Datum; +/// +/// let datum = Datum::string("test_file.parquet"); +/// let ree_type = datum_to_arrow_type_with_ree(&datum); +/// // Returns: RunEndEncoded(Int32, Utf8) +/// ``` +pub fn datum_to_arrow_type_with_ree(datum: &Datum) -> DataType { + primitive_type_to_arrow_type_with_ree(datum.data_type()) +} - use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; - use rust_decimal::Decimal; +/// Returns the run-end-encoded Arrow type used to materialize a per-file constant +/// column of the given Iceberg primitive type. Shared by both the value and the +/// all-null constant paths so a column keeps one Arrow type across files. +pub(crate) fn primitive_type_to_arrow_type_with_ree(primitive_type: &PrimitiveType) -> DataType { + // Helper to create REE type with the given values type. + // Note: values field is nullable as Arrow expects this when building the + // final Arrow schema with `RunArray::try_new`. + let make_ree = |values_type: DataType| -> DataType { + let run_ends_field = Arc::new(Field::new("run_ends", DataType::Int32, false)); + let values_field = Arc::new(Field::new("values", values_type, true)); + DataType::RunEndEncoded(run_ends_field, values_field) + }; + + match primitive_type { + PrimitiveType::Unknown => make_ree(DataType::Null), + PrimitiveType::Boolean => make_ree(DataType::Boolean), + PrimitiveType::Int => make_ree(DataType::Int32), + PrimitiveType::Long => make_ree(DataType::Int64), + PrimitiveType::Float => make_ree(DataType::Float32), + PrimitiveType::Double => make_ree(DataType::Float64), + PrimitiveType::Date => make_ree(DataType::Date32), + PrimitiveType::Time => make_ree(DataType::Int64), + PrimitiveType::Timestamp => make_ree(DataType::Int64), + PrimitiveType::Timestamptz => make_ree(DataType::Int64), + PrimitiveType::TimestampNs => make_ree(DataType::Int64), + PrimitiveType::TimestamptzNs => make_ree(DataType::Int64), + PrimitiveType::String => make_ree(DataType::Utf8), + PrimitiveType::Uuid => make_ree(DataType::Binary), + PrimitiveType::Fixed(_) => make_ree(DataType::Binary), + PrimitiveType::Binary => make_ree(DataType::Binary), + // Match the LargeBinary storage used by schema_to_arrow_schema in the read layer. + // Its 64-bit offsets cover the array's total WKB bytes, not only one geometry's size. + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => make_ree(DataType::LargeBinary), + PrimitiveType::Decimal { precision, scale } => { + make_ree(DataType::Decimal128(*precision as u8, *scale as i8)) + } + } +} - use super::*; - use crate::spec::{Literal, Schema}; +/// A visitor that strips metadata from an Arrow schema. +/// +/// This visitor recursively removes all metadata from fields at every level of the schema, +/// including nested struct, list, and map fields. This is useful for schema comparison +/// where metadata differences should be ignored. +struct MetadataStripVisitor { + /// Fields whose types are still being visited, innermost last + pending_fields: Vec, +} - /// Create a simple field with metadata. - fn simple_field(name: &str, ty: DataType, nullable: bool, value: &str) -> Field { - Field::new(name, ty, nullable).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - value.to_string(), - )])) +/// What `MetadataStripVisitor` keeps of a field until its type is visited +struct PendingField { + name: String, + nullable: bool, +} + +impl MetadataStripVisitor { + fn new() -> Self { + Self { + pending_fields: Vec::new(), + } } - fn arrow_schema_for_arrow_schema_to_schema_test() -> ArrowSchema { - let fields = Fields::from(vec![ - simple_field("key", DataType::Int32, false, "28"), - simple_field("value", DataType::Utf8, true, "29"), - ]); + /// Records a field whose type is about to be visited. + fn start_field(&mut self, name: &str, nullable: bool) -> Result<()> { + self.pending_fields.push(PendingField { + name: name.to_string(), + nullable, + }); + Ok(()) + } - let r#struct = DataType::Struct(fields); - let map = DataType::Map( - Arc::new(simple_field(DEFAULT_MAP_FIELD_NAME, r#struct, false, "17")), - false, - ); - let dictionary = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + /// Builds the innermost pending field with `data_type` and no metadata. + /// `kind` names the type in the error for an unbalanced stack. + fn finish_field(&mut self, data_type: DataType, kind: &str) -> Result { + let PendingField { name, nullable } = self.pending_fields.pop().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!("Field stack underflow in {kind}"), + ) + })?; + Ok(Field::new(name, data_type, nullable)) + } +} - let fields = Fields::from(vec![ - simple_field("aa", DataType::Int32, false, "18"), - simple_field("bb", DataType::Utf8, true, "19"), - simple_field( - "cc", - DataType::Timestamp(TimeUnit::Microsecond, None), - false, - "20", - ), - ]); +impl ArrowSchemaVisitor for MetadataStripVisitor { + type T = Field; + type U = ArrowSchema; - let r#struct = DataType::Struct(fields); + fn before_field(&mut self, field: &FieldRef) -> Result<()> { + self.start_field(field.name(), field.is_nullable()) + } - ArrowSchema::new(vec![ - simple_field("a", DataType::Int32, false, "2"), - simple_field("b", DataType::Int64, false, "1"), - simple_field("c", DataType::Utf8, false, "3"), - simple_field("n", DataType::Utf8, false, "21"), - simple_field( - "d", - DataType::Timestamp(TimeUnit::Microsecond, None), - true, - "4", - ), - simple_field("e", DataType::Boolean, true, "6"), - simple_field("f", DataType::Float32, false, "5"), - simple_field("g", DataType::Float64, false, "7"), - simple_field("p", DataType::Decimal128(10, 2), false, "27"), - simple_field("h", DataType::Date32, false, "8"), - simple_field("i", DataType::Time64(TimeUnit::Microsecond), false, "9"), - simple_field( - "j", - DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), - false, - "10", - ), - simple_field( - "k", - DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())), + fn before_list_element(&mut self, field: &FieldRef) -> Result<()> { + self.start_field(LIST_FIELD_NAME, field.is_nullable()) + } + + fn before_map_key(&mut self, _field: &FieldRef) -> Result<()> { + // Map keys are always required, as `schema_to_arrow_schema` builds them. + self.start_field(MAP_KEY_FIELD_NAME, false) + } + + fn before_map_value(&mut self, field: &FieldRef) -> Result<()> { + self.start_field(MAP_VALUE_FIELD_NAME, field.is_nullable()) + } + + fn schema(&mut self, _schema: &ArrowSchema, values: Vec) -> Result { + if !self.pending_fields.is_empty() { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "{} fields left on the field stack after traversal", + self.pending_fields.len() + ), + )); + } + Ok(ArrowSchema::new(values)) + } + + fn r#struct(&mut self, _fields: &Fields, results: Vec) -> Result { + self.finish_field(DataType::Struct(Fields::from(results)), "struct") + } + + fn list(&mut self, list: &DataType, value: Self::T) -> Result { + let list_type = match list { + DataType::List(_) => DataType::List(Arc::new(value)), + DataType::LargeList(_) => DataType::LargeList(Arc::new(value)), + DataType::FixedSizeList(_, size) => DataType::FixedSizeList(Arc::new(value), *size), + _ => { + return Err(Error::new( + ErrorKind::Unexpected, + format!("Expected list type, got {list}"), + )); + } + }; + + self.finish_field(list_type, "list") + } + + fn map(&mut self, map: &DataType, key_value: Self::T, value: Self::T) -> Result { + // Reconstruct the map's struct field (contains key and value) + let struct_field = Field::new( + DEFAULT_MAP_FIELD_NAME, + DataType::Struct(Fields::from(vec![key_value, value])), + false, + ); + + // Get the sorted flag from the original map type + let sorted = match map { + DataType::Map(_, sorted) => *sorted, + _ => { + return Err(Error::new( + ErrorKind::Unexpected, + format!("Expected map type, got {map}"), + )); + } + }; + + self.finish_field(DataType::Map(Arc::new(struct_field), sorted), "map") + } + + fn primitive(&mut self, p: &DataType) -> Result { + self.finish_field(p.clone(), "primitive") + } +} + +/// Strips all metadata from an Arrow schema and its nested fields. +/// +/// This function recursively removes metadata from all fields at every level of the schema, +/// including nested struct, list, and map fields. This is useful for schema comparison +/// where metadata differences should be ignored. +/// +/// Some other differences are normalized away too, so that schemas that differ only in +/// them compare equal once stripped. The fields inside lists and maps are renamed as +/// [`schema_to_arrow_schema`] names them, and dictionaries are unwrapped: +/// - A list's element field is renamed to [`LIST_FIELD_NAME`], so `item` (the arrow-rs +/// default) becomes `element`. +/// - A map's entries field is renamed to [`DEFAULT_MAP_FIELD_NAME`] and made non-nullable. +/// - A map's key and value fields are renamed to [`MAP_KEY_FIELD_NAME`] and +/// [`MAP_VALUE_FIELD_NAME`], so `keys` and `values` (the arrow-rs `MapBuilder` default) +/// become `key` and `value`, and the key is made non-nullable. +/// - A dictionary-encoded field becomes a field of the dictionary's value type. +/// +/// The names of the schema's fields and of struct fields are kept. Extension type +/// metadata (`ARROW:extension:*`) is dropped with the rest, so a variant field becomes a +/// plain struct of its `metadata` and `value` fields. +/// +/// # Arguments +/// * `schema` - The Arrow schema to strip metadata from +/// +/// # Returns +/// A new Arrow schema with all metadata removed, or an error if the schema structure +/// is invalid. +/// +/// # Example +/// ``` +/// use std::collections::HashMap; +/// +/// use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +/// use iceberg::arrow::strip_metadata_from_schema; +/// +/// let mut metadata = HashMap::new(); +/// metadata.insert("key".to_string(), "value".to_string()); +/// +/// let field = Field::new("col1", DataType::Int32, false).with_metadata(metadata); +/// let schema = ArrowSchema::new(vec![field]); +/// +/// let stripped = strip_metadata_from_schema(&schema).unwrap(); +/// assert!(stripped.field(0).metadata().is_empty()); +/// ``` +pub fn strip_metadata_from_schema(schema: &ArrowSchema) -> Result { + let mut visitor = MetadataStripVisitor::new(); + visit_schema(schema, &mut visitor) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_schema::extension::ExtensionType; + use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit}; + use parquet_geospatial::WkbEdges; + + use super::*; + use crate::spec::decimal_utils::decimal_new; + use crate::spec::geospatial::{ + DEFAULT_GEOSPATIAL_CRS, EdgeInterpolationAlgorithm as IcebergEdgeInterpolationAlgorithm, + }; + use crate::spec::{Literal, Schema}; + + /// Create a simple field with metadata. + fn simple_field(name: &str, ty: DataType, nullable: bool, value: &str) -> Field { + Field::new(name, ty, nullable).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + value.to_string(), + )])) + } + + fn arrow_schema_for_arrow_schema_to_schema_test() -> ArrowSchema { + let fields = Fields::from(vec![ + simple_field("key", DataType::Int32, false, "28"), + simple_field("value", DataType::Utf8, true, "29"), + ]); + + let r#struct = DataType::Struct(fields); + let map = DataType::Map( + Arc::new(simple_field(DEFAULT_MAP_FIELD_NAME, r#struct, false, "17")), + false, + ); + let dictionary = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + + let fields = Fields::from(vec![ + simple_field("aa", DataType::Int32, false, "18"), + simple_field("bb", DataType::Utf8, true, "19"), + simple_field( + "cc", + DataType::Timestamp(TimeUnit::Microsecond, None), + false, + "20", + ), + ]); + + let r#struct = DataType::Struct(fields); + + ArrowSchema::new(vec![ + simple_field("a", DataType::Int32, false, "2"), + simple_field("b", DataType::Int64, false, "1"), + simple_field("c", DataType::Utf8, false, "3"), + simple_field("n", DataType::Utf8, false, "21"), + simple_field( + "d", + DataType::Timestamp(TimeUnit::Microsecond, None), + true, + "4", + ), + simple_field("e", DataType::Boolean, true, "6"), + simple_field("f", DataType::Float32, false, "5"), + simple_field("g", DataType::Float64, false, "7"), + simple_field("p", DataType::Decimal128(10, 2), false, "27"), + simple_field("h", DataType::Date32, false, "8"), + simple_field("i", DataType::Time64(TimeUnit::Microsecond), false, "9"), + simple_field( + "j", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + "10", + ), + simple_field( + "k", + DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())), false, "12", ), @@ -1330,6 +1932,105 @@ mod tests { pretty_assertions::assert_eq!(converted_schema, schema); } + #[test] + fn test_converting_uuid_from_arrow_to_iceberg_to_arrow_should_give_uuid() { + let arrow_schema = ArrowSchema::new(vec![ + simple_field("uuid_field", DataType::FixedSizeBinary(16), false, "1") + .with_extension_type(UuidExtensionType), + ]); + let output = + schema_to_arrow_schema(&arrow_schema_to_schema(&arrow_schema).unwrap()).unwrap(); + + assert_eq!(arrow_schema, output); + } + + #[test] + fn test_converting_uuid_from_iceberg_to_arrow_to_iceberg_should_give_uuid() { + let iceberg_schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "uuid_field", Type::Primitive(PrimitiveType::Uuid)).into(), + ]) + .build() + .unwrap(); + let output = + arrow_schema_to_schema(&schema_to_arrow_schema(&iceberg_schema).unwrap()).unwrap(); + + assert_eq!(iceberg_schema, output); + } + + #[test] + fn test_arrow_schema_to_schema_should_convert_uuid_when_fixed_size_binary() { + let converted_schema = arrow_schema_to_schema(&ArrowSchema::new(vec![ + simple_field("uuid_field", DataType::FixedSizeBinary(16), false, "1") + .with_extension_type(UuidExtensionType), + ])) + .unwrap(); + + let expected = Schema::builder() + .with_fields([NestedField::required( + 1, + "uuid_field", + Type::Primitive(PrimitiveType::Uuid), + ) + .into()]) + .build() + .unwrap(); + + pretty_assertions::assert_eq!(expected, converted_schema); + } + + #[test] + fn test_arrow_schema_to_schema_should_reject_uuid_when_not_a_fixed_size_binary() { + // The field must be built correctly, and then changed to the wrong type, + // to avoid Arrow's own validation and panic. + let field = simple_field_with_uuid_ext(DataType::Utf8); + + let error = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err(); + + pretty_assertions::assert_eq!( + "arrow.uuid extension requires FixedSizeBinary(16) storage, found Utf8", + error.message() + ); + } + + #[test] + fn test_arrow_schema_to_schema_should_reject_uuid_when_fixed_size_binary_that_is_less_than_16() + { + let field = simple_field_with_uuid_ext(DataType::FixedSizeBinary(8)); + let error = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err(); + + pretty_assertions::assert_eq!( + "arrow.uuid extension requires FixedSizeBinary(16) storage, found FixedSizeBinary(8)", + error.message() + ); + } + + #[test] + fn test_arrow_schema_to_schema_should_reject_uuid_when_fixed_size_binary_that_is_more_than_16() + { + let field = simple_field_with_uuid_ext(DataType::FixedSizeBinary(17)); + let error = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err(); + + pretty_assertions::assert_eq!( + "arrow.uuid extension requires FixedSizeBinary(16) storage, found FixedSizeBinary(17)", + error.message() + ); + } + + fn simple_field_with_uuid_ext(field_data_type: DataType) -> Field { + let mut field = simple_field("uuid_field", DataType::FixedSizeBinary(16), false, "1") + .with_extension_type(UuidExtensionType); + + // We want to allow setting a UUID extension to the wrong data type. + // This is to test our logic against bad schemas. + // + // To avoid Arrow's own internal validation and panic, we need to build + // the field correctly above, and then manually override the type here. + field = field.with_data_type(field_data_type); + + field + } + fn arrow_schema_for_schema_to_arrow_schema_test() -> ArrowSchema { let fields = Fields::from(vec![ simple_field("key", DataType::Int32, false, "28"), @@ -1384,8 +2085,8 @@ mod tests { false, "12", ), - simple_field("l", DataType::LargeBinary, false, "13"), - simple_field("o", DataType::LargeBinary, false, "22"), + simple_field("l", DataType::Binary, false, "13"), + simple_field("o", DataType::Binary, false, "22"), simple_field("m", DataType::FixedSizeBinary(10), false, "11"), simple_field( "list", @@ -1413,7 +2114,7 @@ mod tests { "fixed_list", DataType::List(Arc::new(simple_field( "element", - DataType::LargeBinary, + DataType::Binary, false, "26", ))), @@ -1422,7 +2123,23 @@ mod tests { ), simple_field("map", map, false, "16"), simple_field("struct", r#struct, false, "17"), - simple_field("uuid", DataType::FixedSizeBinary(16), false, "30"), + simple_field("uuid", DataType::FixedSizeBinary(16), false, "30") + .with_extension_type(UuidExtensionType), + Field::new( + "v", + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, true), + ])), + true, + ) + .with_metadata(HashMap::from([ + (PARQUET_FIELD_ID_META_KEY.to_string(), "31".to_string()), + ( + arrow_schema::extension::EXTENSION_TYPE_NAME_KEY.to_string(), + "arrow.parquet.variant".to_string(), + ), + ])), ]) } @@ -1606,6 +2323,12 @@ mod tests { "name":"uuid", "required":true, "type":"uuid" + }, + { + "id":31, + "name":"v", + "required":false, + "type":"variant" } ], "identifier-field-ids":[] @@ -1617,10 +2340,450 @@ mod tests { #[test] fn test_schema_to_arrow_schema() { - let arrow_schema = arrow_schema_for_schema_to_arrow_schema_test(); + let expected_arrow_schema = arrow_schema_for_schema_to_arrow_schema_test(); let schema = iceberg_schema_for_schema_to_arrow_schema(); let converted_arrow_schema = schema_to_arrow_schema(&schema).unwrap(); - assert_eq!(converted_arrow_schema, arrow_schema); + assert_eq!(converted_arrow_schema, expected_arrow_schema); + } + + #[test] + fn test_variant_type_to_arrow_type() { + // Variant maps to a struct with a required `metadata` and a nullable `value` binary + // field, with no field ids on the sub-fields, matching the Parquet BINARY layout. + let arrow_type = type_to_arrow_type(&Type::Variant(VariantType)).unwrap(); + assert_eq!( + arrow_type, + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, true), + ])) + ); + } + + #[test] + fn test_variant_field_carries_arrow_extension_type() { + // Converting a schema with a variant column tags the column's field with the + // canonical `arrow.parquet.variant` extension type (the struct storage stays as-is). + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(); + + let arrow_schema = schema_to_arrow_schema(&schema).unwrap(); + let field = arrow_schema.field_with_name("v").unwrap(); + + assert_eq!(field.extension_type_name(), Some("arrow.parquet.variant")); + // Attaching the extension type must not clobber the Iceberg field id. + assert_eq!( + field.metadata().get(PARQUET_FIELD_ID_META_KEY), + Some(&"1".to_string()) + ); + assert_eq!( + field.data_type(), + &DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, true), + ])) + ); + } + + #[test] + fn test_schema_to_arrow_schema_should_convert_uuid_to_arrow_uuid() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "uuid_field", Type::Primitive(PrimitiveType::Uuid)).into(), + ]) + .build() + .unwrap(); + + let arrow_schema = schema_to_arrow_schema(&schema).unwrap(); + let field = arrow_schema.field_with_name("uuid_field").unwrap(); + + assert_eq!(field.extension_type_name(), Some(UuidExtensionType::NAME)); + // Attaching the extension type must not clobber the Iceberg field id. + assert_eq!( + field.metadata().get(PARQUET_FIELD_ID_META_KEY), + Some(&"1".to_string()) + ); + assert_eq!(*field.data_type(), DataType::FixedSizeBinary(16)); + } + + #[test] + fn test_variant_nested_in_list_and_map_carries_arrow_extension_type() { + // A variant nested in a list element or map value keeps the arrow.parquet.variant + // extension type. Regression guard: the list converter must not overwrite the + // element field's metadata (which would drop the extension type). + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional( + 1, + "l", + Type::List(ListType::new( + NestedField::optional(2, "element", Type::Variant(VariantType)).into(), + )), + ) + .into(), + NestedField::optional( + 3, + "m", + Type::Map(MapType::new( + NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::map_value_element(5, Type::Variant(VariantType), false).into(), + )), + ) + .into(), + ]) + .build() + .unwrap(); + + let arrow_schema = schema_to_arrow_schema(&schema).unwrap(); + + let DataType::List(element) = arrow_schema.field_with_name("l").unwrap().data_type() else { + panic!("expected a list"); + }; + assert_eq!(element.extension_type_name(), Some("arrow.parquet.variant")); + + let DataType::Map(entries, _) = arrow_schema.field_with_name("m").unwrap().data_type() + else { + panic!("expected a map"); + }; + let DataType::Struct(kv) = entries.data_type() else { + panic!("expected a key_value struct"); + }; + let value = kv.iter().find(|f| f.name() == "value").unwrap(); + assert_eq!(value.extension_type_name(), Some("arrow.parquet.variant")); + } + + /// The unshredded Arrow storage of a variant: `metadata` (required) + `value` + /// (nullable) binary, with no field ids on the sub-fields. + fn variant_storage() -> DataType { + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, true), + ])) + } + + #[test] + fn test_variant_arrow_field_folds_to_iceberg_variant() { + // A field tagged with the arrow.parquet.variant extension is folded into an + // atomic Type::Variant; its storage sub-fields (which carry no field id) are + // never descended into. + let field = simple_field("v", variant_storage(), true, "1") + .with_extension_type(VariantExtensionType); + let arrow_schema = ArrowSchema::new(vec![field]); + + let converted = arrow_schema_to_schema(&arrow_schema).unwrap(); + let expected = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(); + pretty_assertions::assert_eq!(converted, expected); + } + + #[test] + fn test_variant_schema_round_trips() { + // Iceberg -> Arrow -> Iceberg is the identity for variants at every position: + // top-level, nested in a struct, as a list element, and as a map value. + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "v", Type::Variant(VariantType)).into(), + NestedField::optional( + 2, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(3, "sv", Type::Variant(VariantType)).into(), + ])), + ) + .into(), + NestedField::optional( + 4, + "l", + Type::List(ListType::new( + NestedField::optional(5, "element", Type::Variant(VariantType)).into(), + )), + ) + .into(), + NestedField::optional( + 6, + "m", + Type::Map(MapType::new( + NestedField::map_key_element(7, Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::map_value_element(8, Type::Variant(VariantType), false).into(), + )), + ) + .into(), + ]) + .build() + .unwrap(); + + let arrow_schema = schema_to_arrow_schema(&schema).unwrap(); + let round_tripped = arrow_schema_to_schema(&arrow_schema).unwrap(); + pretty_assertions::assert_eq!(round_tripped, schema); + } + + #[test] + fn test_variant_recognized_with_auto_assigned_ids() { + // Recognition also works when the Arrow schema has no field ids: the variant + // field gets an auto-assigned id and its storage is still not descended into. + let field = + Field::new("v", variant_storage(), true).with_extension_type(VariantExtensionType); + let arrow_schema = ArrowSchema::new(vec![field]); + + let converted = arrow_schema_to_schema_auto_assign_ids(&arrow_schema).unwrap(); + let expected = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(); + pretty_assertions::assert_eq!(converted, expected); + } + + #[test] + fn test_variant_extension_on_non_struct_storage_is_rejected() { + // The extension may only sit on struct storage. A hand-injected tag on a + // non-struct field is rejected rather than silently reinterpreted as a variant. + let field = Field::new("v", DataType::Int32, true).with_metadata(HashMap::from([ + (PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string()), + ( + arrow_schema::extension::EXTENSION_TYPE_NAME_KEY.to_string(), + VariantExtensionType::NAME.to_string(), + ), + ])); + let arrow_schema = ArrowSchema::new(vec![field]); + + let err = arrow_schema_to_schema(&arrow_schema).unwrap_err(); + assert!( + err.to_string().contains("requires Struct storage"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_geospatial_arrow_schema_roundtrip() { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry( + GeometryType::new(Some("EPSG:3857".to_string())).unwrap(), + )), + ) + .into(), + NestedField::optional( + 2, + "geog", + Type::Primitive(PrimitiveType::Geography( + GeographyType::new( + Some("OGC:CRS27".to_string()), + IcebergEdgeInterpolationAlgorithm::Karney, + ) + .unwrap(), + )), + ) + .into(), + NestedField::optional( + 3, + "geom_list", + Type::List(ListType::new( + NestedField::list_element( + 4, + Type::Primitive(PrimitiveType::Geometry(GeometryType::default())), + true, + ) + .into(), + )), + ) + .into(), + ]) + .build() + .unwrap(); + + let arrow_schema = schema_to_arrow_schema(&schema).unwrap(); + let geom = arrow_schema.field(0); + assert_eq!(geom.data_type(), &DataType::LargeBinary); + + let geog = arrow_schema.field(1); + let geog_wkb = geog.try_extension_type::().unwrap(); + assert_eq!(geog_wkb.metadata().algorithm, Some(WkbEdges::Karney)); + let geog_metadata: serde_json::Value = + serde_json::from_str(&geog_wkb.serialize_metadata().unwrap()).unwrap(); + assert_eq!(geog_metadata.get("edges").unwrap(), "karney"); + assert!(geog_metadata.get("algorithm").is_none()); + + let list = arrow_schema.field(2); + let DataType::List(element) = list.data_type() else { + panic!("Expected list field"); + }; + assert!( + matches!( + element + .try_extension_type::() + .unwrap() + .metadata() + .type_hint(), + WkbTypeHint::Geometry + ), + "Expected list element to retain WKB extension metadata" + ); + assert_eq!( + element + .try_extension_type::() + .unwrap() + .metadata() + .crs + .as_ref() + .and_then(serde_json::Value::as_str), + Some(DEFAULT_GEOSPATIAL_CRS) + ); + + let converted = arrow_schema_to_schema(&arrow_schema).unwrap(); + assert_eq!(converted.as_struct().fields(), schema.as_struct().fields()); + } + + #[test] + fn test_geospatial_arrow_crs_import() { + let mut geom = simple_field("geom", DataType::LargeBinary, true, "1"); + geom.try_with_extension_type(WkbType::new(Some(WkbMetadata::new( + Some(r#"{"id":{"authority":"EPSG","code":3857}}"#), + None, + )))) + .unwrap(); + let mut unset_geom = simple_field("unset_geom", DataType::LargeBinary, true, "2"); + unset_geom + .try_with_extension_type(WkbType::new(Some(WkbMetadata::new(None, None)))) + .unwrap(); + + let arrow_schema = ArrowSchema::new(vec![geom, unset_geom]); + let schema = arrow_schema_to_schema(&arrow_schema).unwrap(); + + assert_eq!( + schema.field_by_id(1).unwrap().field_type.as_ref(), + &Type::Primitive(PrimitiveType::Geometry( + GeometryType::new(Some("EPSG:3857".to_string())).unwrap() + )) + ); + assert_eq!( + schema.field_by_id(2).unwrap().field_type.as_ref(), + &Type::Primitive(PrimitiveType::Geometry( + GeometryType::new(Some(UNSET_GEOSPATIAL_CRS.to_string())).unwrap() + )) + ); + } + + #[test] + fn test_geospatial_arrow_unset_crs_export() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry( + GeometryType::new(Some(UNSET_GEOSPATIAL_CRS.to_string())).unwrap(), + )), + ) + .into(), + NestedField::optional( + 2, + "geog", + Type::Primitive(PrimitiveType::Geography( + GeographyType::new( + Some(UNSET_GEOSPATIAL_CRS.to_string()), + IcebergEdgeInterpolationAlgorithm::Spherical, + ) + .unwrap(), + )), + ) + .into(), + ]) + .build() + .unwrap(); + let arrow_schema = schema_to_arrow_schema(&schema).unwrap(); + for field in arrow_schema.fields() { + assert!( + field + .try_extension_type::() + .unwrap() + .metadata() + .crs + .is_none() + ); + } + assert_eq!(arrow_schema_to_schema(&arrow_schema).unwrap(), schema); + } + + #[test] + fn test_geospatial_arrow_invalid_crs_is_rejected() { + for crs in [r#"{}"#, "42"] { + let mut field = simple_field("geom", DataType::LargeBinary, true, "1"); + field + .try_with_extension_type(WkbType::new(Some(WkbMetadata::new(Some(crs), None)))) + .unwrap(); + + let err = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}"); + } + } + + #[test] + fn test_geospatial_arrow_extension_requires_binary_storage() { + use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY}; + + let metadata = WkbType::new(Some(WkbMetadata::new(None, None))) + .serialize_metadata() + .unwrap(); + let field = + simple_field("geom", DataType::Int32, true, "1").with_metadata(HashMap::from([ + (PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string()), + ( + EXTENSION_TYPE_NAME_KEY.to_string(), + WkbType::NAME.to_string(), + ), + (EXTENSION_TYPE_METADATA_KEY.to_string(), metadata), + ])); + + let err = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}"); + assert!( + err.message().contains("requires binary storage"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_geospatial_parquet_statistics_are_ignored() { + use parquet::data_type::ByteArray; + use parquet::file::statistics::ValueStatistics; + + let stats = Statistics::ByteArray(ValueStatistics::new( + Some(ByteArray::from(vec![1, 2, 3])), + Some(ByteArray::from(vec![4, 5, 6])), + None, + None, + false, + )); + + for primitive_type in [ + PrimitiveType::Geometry(GeometryType::default()), + PrimitiveType::Geography(GeographyType::default()), + ] { + assert_eq!( + get_parquet_stat_min_as_datum(&primitive_type, &stats).unwrap(), + None + ); + assert_eq!( + get_parquet_stat_max_as_datum(&primitive_type, &stats).unwrap(), + None + ); + } } #[test] @@ -1633,6 +2796,13 @@ mod tests { assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap()); } + { + let arrow_type = DataType::Null; + let iceberg_type = Type::Primitive(PrimitiveType::Unknown); + assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap()); + assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap()); + } + // test struct type { // no metadata will cause error @@ -1745,9 +2915,7 @@ mod tests { assert!( matches!(iceberg_field.field_type.as_ref(), Type::Primitive(t) if *t == expected_iceberg_type), - "Expected {:?} to map to {:?}", - arrow_type, - expected_iceberg_type + "Expected {arrow_type:?} to map to {expected_iceberg_type:?}" ); } @@ -1796,7 +2964,7 @@ mod tests { assert_eq!(array.value(0), 42); } { - let datum = Datum::float(42.42); + let datum = Datum::float(42.42_f32); let arrow_datum = get_arrow_datum(&datum).unwrap(); let (array, is_scalar) = arrow_datum.get(); let array = array.as_any().downcast_ref::().unwrap(); @@ -1819,6 +2987,14 @@ mod tests { assert!(is_scalar); assert_eq!(array.value(0), "abc"); } + { + let datum = Datum::binary(vec![1, 2, 3, 4]); + let arrow_datum = get_arrow_datum(&datum).unwrap(); + let (array, is_scalar) = arrow_datum.get(); + let array = array.as_any().downcast_ref::().unwrap(); + assert!(is_scalar); + assert_eq!(array.value(0), &[1, 2, 3, 4]); + } { let datum = Datum::date(42); let arrow_datum = get_arrow_datum(&datum).unwrap(); @@ -1851,7 +3027,7 @@ mod tests { assert_eq!(array.value(0), 42); } { - let datum = Datum::decimal_with_precision(Decimal::new(123, 2), 30).unwrap(); + let datum = Datum::decimal_with_precision(decimal_new(123, 2), 30).unwrap(); let arrow_datum = get_arrow_datum(&datum).unwrap(); let (array, is_scalar) = arrow_datum.get(); let array = array.as_any().downcast_ref::().unwrap(); @@ -1871,5 +3047,500 @@ mod tests { assert!(is_scalar); assert_eq!(array.value(0), [66u8; 16]); } + { + let datum = Datum::fixed(vec![1u8, 2, 3, 4, 5, 6, 7, 8]); + let arrow_datum = get_arrow_datum(&datum).unwrap(); + let (array, is_scalar) = arrow_datum.get(); + let array = array + .as_any() + .downcast_ref::() + .unwrap(); + assert!(is_scalar); + assert_eq!(array.value_length(), 8); + assert_eq!(array.value(0), &[1u8, 2, 3, 4, 5, 6, 7, 8]); + } + } + + #[test] + fn test_arrow_schema_to_schema_with_field_id() { + // Create a complex Arrow schema without field ID metadata + // Including: primitives, list, nested struct, map, and nested list of structs + let arrow_schema = ArrowSchema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + Field::new("price", DataType::Decimal128(10, 2), false), + Field::new("uuid", DataType::FixedSizeBinary(16), false) + .with_extension_type(UuidExtensionType), + Field::new( + "created_at", + DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())), + true, + ), + Field::new( + "tags", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + true, + ), + Field::new( + "address", + DataType::Struct(Fields::from(vec![ + Field::new("street", DataType::Utf8, true), + Field::new("city", DataType::Utf8, false), + Field::new("zip", DataType::Int32, true), + ])), + true, + ), + Field::new( + "attributes", + DataType::Map( + Arc::new(Field::new( + DEFAULT_MAP_FIELD_NAME, + DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Utf8, true), + ])), + false, + )), + false, + ), + true, + ), + Field::new( + "orders", + DataType::List(Arc::new(Field::new( + "element", + DataType::Struct(Fields::from(vec![ + Field::new("order_id", DataType::Int64, false), + Field::new("amount", DataType::Float64, false), + ])), + true, + ))), + true, + ), + ]); + + let schema = arrow_schema_to_schema_auto_assign_ids(&arrow_schema).unwrap(); + + // Build expected schema with exact field IDs following level-order assignment: + // Level 0: id=1, name=2, price=3, uuid=4, created_at=5, tags=6, address=7, attributes=8, orders=9 + // Level 1: tags.element=10, address.{street=11,city=12,zip=13}, attributes.{key=14,value=15}, orders.element=16 + // Level 2: orders.element.{order_id=17,amount=18} + let expected = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required( + 3, + "price", + Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }), + ) + .into(), + NestedField::required(4, "uuid", Type::Primitive(PrimitiveType::Uuid)).into(), + NestedField::optional(5, "created_at", Type::Primitive(PrimitiveType::Timestamptz)) + .into(), + NestedField::optional( + 6, + "tags", + Type::List(ListType { + element_field: NestedField::list_element( + 10, + Type::Primitive(PrimitiveType::String), + false, + ) + .into(), + }), + ) + .into(), + NestedField::optional( + 7, + "address", + Type::Struct(StructType::new(vec![ + NestedField::optional(11, "street", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::required(12, "city", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::optional(13, "zip", Type::Primitive(PrimitiveType::Int)) + .into(), + ])), + ) + .into(), + NestedField::optional( + 8, + "attributes", + Type::Map(MapType { + key_field: NestedField::map_key_element( + 14, + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::map_value_element( + 15, + Type::Primitive(PrimitiveType::String), + false, + ) + .into(), + }), + ) + .into(), + NestedField::optional( + 9, + "orders", + Type::List(ListType { + element_field: NestedField::list_element( + 16, + Type::Struct(StructType::new(vec![ + NestedField::required( + 17, + "order_id", + Type::Primitive(PrimitiveType::Long), + ) + .into(), + NestedField::required( + 18, + "amount", + Type::Primitive(PrimitiveType::Double), + ) + .into(), + ])), + false, + ) + .into(), + }), + ) + .into(), + ]) + .build() + .unwrap(); + + pretty_assertions::assert_eq!(schema, expected); + assert_eq!(schema.highest_field_id(), 18); + } + + fn decimal_byte_array_stats(min: Vec, max: Vec) -> Statistics { + Statistics::byte_array( + Some(parquet::data_type::ByteArray::from(min)), + Some(parquet::data_type::ByteArray::from(max)), + None, + Some(0), + false, + ) + } + + fn decimal_bounds(stats: &Statistics) -> Result<(Option, Option)> { + let ty = PrimitiveType::Decimal { + precision: 38, + scale: 2, + }; + Ok(( + get_parquet_stat_min_as_datum(&ty, stats)?, + get_parquet_stat_max_as_datum(&ty, stats)?, + )) + } + + fn decimal_datum(value: i128) -> Datum { + Datum::new( + PrimitiveType::Decimal { + precision: 38, + scale: 2, + }, + PrimitiveLiteral::Int128(value), + ) + } + + #[test] + fn test_byte_array_decimal_stats_short_bounds() { + // Minimal-length big-endian two's-complement: -1234 and 1234. + let stats = decimal_byte_array_stats(vec![0xFB, 0x2E], vec![0x04, 0xD2]); + let (min, max) = decimal_bounds(&stats).unwrap(); + assert_eq!(min, Some(decimal_datum(-1234))); + assert_eq!(max, Some(decimal_datum(1234))); + + // Single-byte bounds: sign bit set must sign-extend, not zero-extend. + let stats = decimal_byte_array_stats(vec![0x80], vec![0x7F]); + let (min, max) = decimal_bounds(&stats).unwrap(); + assert_eq!(min, Some(decimal_datum(-128))); + assert_eq!(max, Some(decimal_datum(127))); + } + + #[test] + fn test_byte_array_decimal_stats_full_width_bounds() { + let stats = decimal_byte_array_stats( + i128::MIN.to_be_bytes().to_vec(), + i128::MAX.to_be_bytes().to_vec(), + ); + let (min, max) = decimal_bounds(&stats).unwrap(); + assert_eq!(min, Some(decimal_datum(i128::MIN))); + assert_eq!(max, Some(decimal_datum(i128::MAX))); + } + + #[test] + fn test_byte_array_decimal_stats_empty_bound_errors() { + let stats = decimal_byte_array_stats(vec![], vec![0x01]); + let err = decimal_bounds(&stats).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + + let stats = decimal_byte_array_stats(vec![0x01], vec![]); + let err = decimal_bounds(&stats).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_byte_array_decimal_stats_oversized_bound_errors() { + let stats = decimal_byte_array_stats(vec![0; 17], vec![0x01]); + let err = decimal_bounds(&stats).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + + let stats = decimal_byte_array_stats(vec![0x01], vec![0; 17]); + let err = decimal_bounds(&stats).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + /// Every field is rebuilt without metadata, at any depth and inside any + /// container. Fields are rebuilt in nesting order rather than looked up by + /// name, so a path that repeats names (`a.b.a.b`) keeps each level's name + /// and nullability. + #[test] + fn test_strip_metadata_from_nested_schema() { + // The same schema, built with and without metadata on the schema and every field. + let schema = |with_metadata: bool| { + let metadata = |value: &str| { + if with_metadata { + HashMap::from([("k".to_string(), value.to_string())]) + } else { + HashMap::new() + } + }; + let field = |name: &str, ty: DataType, nullable: bool| { + Field::new(name, ty, nullable).with_metadata(metadata(name)) + }; + let element = |ty: DataType| Arc::new(field(LIST_FIELD_NAME, ty, false)); + let ints = || DataType::List(element(DataType::Int32)); + let map = |value: DataType, sorted: bool| { + let entries = Fields::from(vec![ + field(MAP_KEY_FIELD_NAME, DataType::Utf8, false), + field(MAP_VALUE_FIELD_NAME, value, true), + ]); + let entries = field(DEFAULT_MAP_FIELD_NAME, DataType::Struct(entries), false); + DataType::Map(Arc::new(entries), sorted) + }; + let struct_of = |fields: Vec| DataType::Struct(Fields::from(fields)); + + let a_b_a_b = struct_of(vec![field( + "b", + struct_of(vec![field( + "a", + struct_of(vec![field("b", DataType::Int32, true)]), + false, + )]), + false, + )]); + ArrowSchema::new(vec![ + field("list", ints(), true), + field( + "large_list", + DataType::LargeList(element(DataType::Int32)), + false, + ), + field( + "fixed_list", + DataType::FixedSizeList(element(DataType::Int32), 2), + true, + ), + field("list_of_list", DataType::List(element(ints())), false), + field("sorted_map", map(ints(), true), false), + field( + "map_of_struct", + map( + struct_of(vec![ + field("a", DataType::Int32, true), + field("b", ints(), false), + ]), + false, + ), + true, + ), + field( + "list_of_struct", + DataType::List(element(struct_of(vec![field( + "m", + map(ints(), false), + true, + )]))), + true, + ), + field("a", a_b_a_b, true), + ]) + .with_metadata(metadata("schema")) + }; + + let (input, expected) = (schema(true), schema(false)); + // Otherwise a builder that stopped adding metadata would pass trivially. + assert_ne!(input, expected); + pretty_assertions::assert_eq!(strip_metadata_from_schema(&input).unwrap(), expected); + } + + /// arrow-rs names a list's element `item`, and an arrow-rs `MapBuilder` names + /// a map's fields `entries`, `keys` and `values`. Once stripped, a schema + /// using those names, with a nullable map key, compares equal to the same + /// schema from `schema_to_arrow_schema`. + #[test] + fn test_strip_metadata_normalizes_container_field_names() { + // A list of ints and a map from string to a list of ints, with the given + // names for the list elements and the map's entries, key and value. + let schema = |[element, entries, key, value]: [&str; 4], key_nullable: bool| { + let ints = + |nullable| DataType::List(Arc::new(Field::new(element, DataType::Int32, nullable))); + let key_value = Fields::from(vec![ + Field::new(key, DataType::Utf8, key_nullable), + Field::new(value, ints(false), true), + ]); + let entries = Field::new(entries, DataType::Struct(key_value), false); + ArrowSchema::new(vec![ + Field::new("l", ints(true), true), + Field::new("m", DataType::Map(Arc::new(entries), false), true), + ]) + }; + let arrow_rs = schema(["item", "entries", "keys", "values"], true); + let expected = schema( + [ + LIST_FIELD_NAME, + DEFAULT_MAP_FIELD_NAME, + MAP_KEY_FIELD_NAME, + MAP_VALUE_FIELD_NAME, + ], + false, + ); + + // The same schema as an Iceberg table's. + let int_list = |id, required| { + let int = Type::Primitive(PrimitiveType::Int); + Type::List(ListType::new( + NestedField::list_element(id, int, required).into(), + )) + }; + let string_to_int_list = MapType::new( + NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String)).into(), + NestedField::map_value_element(5, int_list(6, true), false).into(), + ); + let iceberg = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "l", int_list(2, false)).into(), + NestedField::optional(3, "m", Type::Map(string_to_int_list)).into(), + ]) + .build() + .unwrap(); + + pretty_assertions::assert_eq!(strip_metadata_from_schema(&arrow_rs).unwrap(), expected); + pretty_assertions::assert_eq!( + strip_metadata_from_schema(&schema_to_arrow_schema(&iceberg).unwrap()).unwrap(), + expected + ); + } + + /// Names and nullability that `schema_to_arrow_schema` doesn't fix, such as a + /// struct field's name or a list element's nullability, still tell schemas apart. + #[test] + fn test_strip_metadata_keeps_other_differences() { + let schema = |child: &str, element_nullable: bool| { + let child = Field::new(child, DataType::Int32, true); + let element = Field::new("item", DataType::Int32, element_nullable); + ArrowSchema::new(vec![ + Field::new("s", DataType::Struct(Fields::from(vec![child])), true), + Field::new("l", DataType::List(Arc::new(element)), true), + ]) + }; + let strip = |schema: ArrowSchema| strip_metadata_from_schema(&schema).unwrap(); + + assert_ne!(strip(schema("a", true)), strip(schema("b", true))); + assert_ne!(strip(schema("a", true)), strip(schema("a", false))); + } + + /// Stripping drops the `arrow.parquet.variant` extension type with the rest of + /// the metadata, so a variant becomes a plain struct, in a list or map too. + #[test] + fn test_strip_metadata_turns_variants_into_structs() { + let variant = || Type::Variant(VariantType); + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "v", variant()).into(), + NestedField::optional( + 2, + "l", + Type::List(ListType::new( + NestedField::list_element(3, variant(), false).into(), + )), + ) + .into(), + NestedField::optional( + 4, + "m", + Type::Map(MapType::new( + NestedField::map_key_element(5, Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::map_value_element(6, variant(), false).into(), + )), + ) + .into(), + ]) + .build() + .unwrap(); + + let storage = || { + DataType::Struct(Fields::from(vec![ + Field::new("metadata", DataType::Binary, false), + Field::new("value", DataType::Binary, true), + ])) + }; + let entries = Fields::from(vec![ + Field::new(MAP_KEY_FIELD_NAME, DataType::Utf8, false), + Field::new(MAP_VALUE_FIELD_NAME, storage(), true), + ]); + let entries = Field::new(DEFAULT_MAP_FIELD_NAME, DataType::Struct(entries), false); + let expected = ArrowSchema::new(vec![ + Field::new("v", storage(), true), + Field::new( + "l", + DataType::List(Arc::new(Field::new(LIST_FIELD_NAME, storage(), true))), + true, + ), + Field::new("m", DataType::Map(Arc::new(entries), false), true), + ]); + + pretty_assertions::assert_eq!( + strip_metadata_from_schema(&schema_to_arrow_schema(&schema).unwrap()).unwrap(), + expected + ); + } + + /// A dictionary-encoded field compares equal to a plain field of its value + /// type once stripped, at any depth. + #[test] + fn test_strip_metadata_unwraps_dictionaries() { + let schema = |dictionary_encoded: bool| { + let maybe_dictionary = |key: DataType, value: DataType| { + if dictionary_encoded { + DataType::Dictionary(Box::new(key), Box::new(value)) + } else { + value + } + }; + let element = Field::new( + LIST_FIELD_NAME, + maybe_dictionary(DataType::Int8, DataType::LargeUtf8), + false, + ); + ArrowSchema::new(vec![ + Field::new("d", maybe_dictionary(DataType::Int32, DataType::Utf8), true), + Field::new("l", DataType::List(Arc::new(element)), true), + ]) + }; + + pretty_assertions::assert_eq!( + strip_metadata_from_schema(&schema(true)).unwrap(), + schema(false) + ); } } diff --git a/crates/iceberg/src/arrow/test_utils.rs b/crates/iceberg/src/arrow/test_utils.rs new file mode 100644 index 0000000000..b4e1beed34 --- /dev/null +++ b/crates/iceberg/src/arrow/test_utils.rs @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fs::File; + +use arrow_array::RecordBatch; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::{ArrowReaderOptions, ParquetRecordBatchReaderBuilder}; +use parquet::basic::Compression; +use parquet::encryption::decrypt::FileDecryptionProperties; +use parquet::encryption::encrypt::FileEncryptionProperties; +use parquet::file::properties::WriterProperties; + +/// Writes `batch` to `path` as a Parquet file encrypted with `key` and `aad_prefix`. +pub(crate) fn write_encrypted_parquet( + path: &str, + batch: &RecordBatch, + key: &[u8], + aad_prefix: Option<&[u8]>, +) { + let mut builder = FileEncryptionProperties::builder(key.to_vec()); + if let Some(aad) = aad_prefix { + builder = builder.with_aad_prefix(aad.to_vec()); + } + let encryption_properties = builder.build().unwrap(); + + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .with_file_encryption_properties(encryption_properties) + .build(); + + let file = File::create(path).unwrap(); + let mut writer = ArrowWriter::try_new(file, batch.schema(), Some(props)).unwrap(); + writer.write(batch).expect("Writing batch"); + writer.close().unwrap(); +} + +/// Reads the Parquet file at `path` encrypted with `key` and `aad_prefix`, returning +/// all record batches. +pub(crate) fn read_encrypted_parquet( + path: &str, + key: &[u8], + aad_prefix: Option<&[u8]>, +) -> Vec { + let mut builder = FileDecryptionProperties::builder(key.to_vec()); + if let Some(aad) = aad_prefix { + builder = builder.with_aad_prefix(aad.to_vec()); + } + let options = + ArrowReaderOptions::new().with_file_decryption_properties(builder.build().unwrap()); + + let file = File::open(path).unwrap(); + ParquetRecordBatchReaderBuilder::try_new_with_options(file, options) + .unwrap() + .build() + .unwrap() + .map(|b| b.unwrap()) + .collect() +} diff --git a/crates/iceberg/src/arrow/value.rs b/crates/iceberg/src/arrow/value.rs index cc3a561d57..1d0a6e9a0f 100644 --- a/crates/iceberg/src/arrow/value.rs +++ b/crates/iceberg/src/arrow/value.rs @@ -15,19 +15,24 @@ // specific language governing permissions and limitations // under the License. +use std::sync::Arc; + use arrow_array::{ - Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Decimal128Array, FixedSizeBinaryArray, - FixedSizeListArray, Float32Array, Float64Array, Int32Array, Int64Array, LargeBinaryArray, - LargeListArray, LargeStringArray, ListArray, MapArray, StringArray, StructArray, - Time64MicrosecondArray, TimestampMicrosecondArray, TimestampNanosecondArray, + Array, ArrayRef, BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Decimal128Array, + FixedSizeBinaryArray, FixedSizeListArray, Float32Array, Float64Array, Int32Array, Int64Array, + LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, MapArray, StringArray, + StructArray, Time64MicrosecondArray, TimestampMicrosecondArray, TimestampNanosecondArray, + new_null_array, }; -use arrow_schema::{DataType, FieldRef}; +use arrow_buffer::BooleanBuffer; +use arrow_schema::{DataType, FieldRef, TimeUnit}; use uuid::Uuid; -use super::get_field_id; +use super::get_field_id_from_metadata; +use crate::error::invalid_data; use crate::spec::{ - ListType, Literal, Map, MapType, NestedField, PartnerAccessor, PrimitiveType, - SchemaWithPartnerVisitor, Struct, StructType, Type, visit_struct_with_partner, + ListType, Literal, Map, MapType, NestedField, PartnerAccessor, PrimitiveLiteral, PrimitiveType, + SchemaWithPartnerVisitor, Struct, StructType, Type, VariantType, visit_struct_with_partner, visit_type_with_partner, }; use crate::{Error, ErrorKind, Result}; @@ -54,12 +59,9 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { ) -> Result>> { // Make there is no null value if the field is required if field.required && value.iter().any(Option::is_none) { - return Err(Error::new( - ErrorKind::DataInvalid, - "The field is required but has null value", - ) - .with_context("field_id", field.id.to_string()) - .with_context("field_name", &field.name)); + return Err(invalid_data!("The field is required but has null value") + .with_context("field_id", field.id.to_string()) + .with_context("field_name", &field.name)); } Ok(value) } @@ -72,12 +74,11 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { ) -> Result>> { let row_len = results.first().map(|column| column.len()).unwrap_or(0); if let Some(col) = results.iter().find(|col| col.len() != row_len) { - return Err(Error::new( - ErrorKind::DataInvalid, - "The struct columns have different row length", - ) - .with_context("first col length", row_len.to_string()) - .with_context("actual col length", col.len().to_string())); + return Err( + invalid_data!("The struct columns have different row length") + .with_context("first col length", row_len.to_string()) + .with_context("actual col length", col.len().to_string()), + ); } let mut struct_literals = Vec::with_capacity(row_len); @@ -108,19 +109,14 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { elements: Vec>, ) -> Result>> { if list.element_field.required && elements.iter().any(Option::is_none) { - return Err(Error::new( - ErrorKind::DataInvalid, - "The list should not have null value", - )); + return Err(invalid_data!("The list should not have null value")); } match array.data_type() { DataType::List(_) => { let offset = array .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a list array") - })? + .ok_or_else(|| invalid_data!("The partner is not a list array"))? .offsets(); // combine the result according to the offset let mut result = Vec::with_capacity(offset.len() - 1); @@ -135,12 +131,7 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { let offset = array .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a large list array", - ) - })? + .ok_or_else(|| invalid_data!("The partner is not a large list array"))? .offsets(); // combine the result according to the offset let mut result = Vec::with_capacity(offset.len() - 1); @@ -160,10 +151,7 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { } Ok(result) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a list type", - )), + _ => Err(invalid_data!("The partner is not a list type")), } } @@ -176,16 +164,15 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { ) -> Result>> { // Make sure key_value and value have the same row length if key_values.len() != values.len() { - return Err(Error::new( - ErrorKind::DataInvalid, - "The key value and value of map should have the same row length", + return Err(invalid_data!( + "The key value and value of map should have the same row length" )); } let offsets = partner .as_any() .downcast_ref::() - .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "The partner is not a map array"))? + .ok_or_else(|| invalid_data!("The partner is not a map array"))? .offsets(); // combine the result according to the offset let mut result = Vec::with_capacity(offsets.len() - 1); @@ -203,71 +190,53 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { fn primitive(&mut self, p: &PrimitiveType, partner: &ArrayRef) -> Result>> { match p { + PrimitiveType::Unknown => Ok(vec![None; partner.len()]), PrimitiveType::Boolean => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a boolean array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a boolean array"))?; Ok(array.iter().map(|v| v.map(Literal::bool)).collect()) } PrimitiveType::Int => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a int32 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a int32 array"))?; Ok(array.iter().map(|v| v.map(Literal::int)).collect()) } PrimitiveType::Long => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a int64 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a int64 array"))?; Ok(array.iter().map(|v| v.map(Literal::long)).collect()) } PrimitiveType::Float => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a float32 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a float32 array"))?; Ok(array.iter().map(|v| v.map(Literal::float)).collect()) } PrimitiveType::Double => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a float64 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a float64 array"))?; Ok(array.iter().map(|v| v.map(Literal::double)).collect()) } PrimitiveType::Decimal { precision, scale } => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a decimal128 array", - ) - })?; - if let DataType::Decimal128(arrow_precision, arrow_scale) = array.data_type() { - if *arrow_precision as u32 != *precision || *arrow_scale as u32 != *scale { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "The precision or scale ({},{}) of arrow decimal128 array is not compatible with iceberg decimal type ({},{})", - arrow_precision, arrow_scale, precision, scale - ), - )); - } + .ok_or_else(|| invalid_data!("The partner is not a decimal128 array"))?; + if let DataType::Decimal128(arrow_precision, arrow_scale) = array.data_type() + && (*arrow_precision as u32 != *precision || *arrow_scale as u32 != *scale) + { + return Err(invalid_data!( + "The precision or scale ({arrow_precision},{arrow_scale}) of arrow decimal128 array is not compatible with iceberg decimal type ({precision},{scale})" + )); } Ok(array.iter().map(|v| v.map(Literal::decimal)).collect()) } @@ -275,54 +244,35 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a date32 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a date32 array"))?; Ok(array.iter().map(|v| v.map(Literal::date)).collect()) } PrimitiveType::Time => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a time64 array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a time64 array"))?; Ok(array.iter().map(|v| v.map(Literal::time)).collect()) } PrimitiveType::Timestamp => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a timestamp array", - ) - })?; + .ok_or_else(|| invalid_data!("The partner is not a timestamp array"))?; Ok(array.iter().map(|v| v.map(Literal::timestamp)).collect()) } PrimitiveType::Timestamptz => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a timestamptz array", - ) - })?; + .ok_or_else(|| invalid_data!("The partner is not a timestamptz array"))?; Ok(array.iter().map(|v| v.map(Literal::timestamptz)).collect()) } PrimitiveType::TimestampNs => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a timestamp_ns array", - ) - })?; + .ok_or_else(|| invalid_data!("The partner is not a timestamp_ns array"))?; Ok(array .iter() .map(|v| v.map(Literal::timestamp_nano)) @@ -332,12 +282,7 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The partner is not a timestamptz_ns array", - ) - })?; + .ok_or_else(|| invalid_data!("The partner is not a timestamptz_ns array"))?; Ok(array .iter() .map(|v| v.map(Literal::timestamptz_nano)) @@ -349,54 +294,37 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { } else if let Some(array) = partner.as_any().downcast_ref::() { Ok(array.iter().map(|v| v.map(Literal::string)).collect()) } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a string array", - )); + Err(invalid_data!("The partner is not a string array")) } } PrimitiveType::Uuid => { if let Some(array) = partner.as_any().downcast_ref::() { if array.value_length() != 16 { - return Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a uuid array", - )); + return Err(invalid_data!("The partner is not a uuid array")); } Ok(array .iter() .map(|v| { v.map(|v| { Ok(Literal::uuid(Uuid::from_bytes(v.try_into().map_err( - |_| { - Error::new( - ErrorKind::DataInvalid, - "Failed to convert binary to uuid", - ) - }, + |_| invalid_data!("Failed to convert binary to uuid"), )?))) }) .transpose() }) .collect::>>()?) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a uuid array", - )) + Err(invalid_data!("The partner is not a uuid array")) } } PrimitiveType::Fixed(len) => { let array = partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The partner is not a fixed array") - })?; + .ok_or_else(|| invalid_data!("The partner is not a fixed array"))?; if array.value_length() != *len as i32 { - return Err(Error::new( - ErrorKind::DataInvalid, - "The length of fixed size binary array is not compatible with iceberg fixed type", + return Err(invalid_data!( + "The length of fixed size binary array is not compatible with iceberg fixed type" )); } Ok(array @@ -404,7 +332,7 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { .map(|v| v.map(|v| Literal::fixed(v.iter().cloned()))) .collect()) } - PrimitiveType::Binary => { + PrimitiveType::Binary | PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { if let Some(array) = partner.as_any().downcast_ref::() { Ok(array .iter() @@ -415,15 +343,24 @@ impl SchemaWithPartnerVisitor for ArrowArrayToIcebergStructConverter { .iter() .map(|v| v.map(|v| Literal::binary(v.to_vec()))) .collect()) + } else if let Some(array) = partner.as_any().downcast_ref::() { + Ok(array + .iter() + .map(|v| v.map(|v| Literal::binary(v.to_vec()))) + .collect()) } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "The partner is not a binary array", - )); + Err(invalid_data!("The partner is not a binary array")) } } } } + + fn variant(&mut self, _v: &VariantType, _partner: &ArrayRef) -> Result>> { + Err(Error::new( + ErrorKind::FeatureUnsupported, + "Converting variant Arrow array to Iceberg literal is not supported yet", + )) + } } /// Defines how Arrow fields are matched with Iceberg fields when converting data. @@ -448,7 +385,7 @@ impl FieldMatchMode { /// Determines if an Arrow field matches an Iceberg field based on the matching mode. pub fn match_field(&self, arrow_field: &FieldRef, iceberg_field: &NestedField) -> bool { match self { - FieldMatchMode::Id => get_field_id(arrow_field) + FieldMatchMode::Id => get_field_id_from_metadata(arrow_field) .map(|id| id == iceberg_field.id) .unwrap_or(false), FieldMatchMode::Name => arrow_field.name() == &iceberg_field.name, @@ -484,10 +421,7 @@ impl Default for ArrowArrayAccessor { impl PartnerAccessor for ArrowArrayAccessor { fn struct_partner<'a>(&self, schema_partner: &'a ArrayRef) -> Result<&'a ArrayRef> { if !matches!(schema_partner.data_type(), DataType::Struct(_)) { - return Err(Error::new( - ErrorKind::DataInvalid, - "The schema partner is not a struct type", - )); + return Err(invalid_data!("The schema partner is not a struct type")); } Ok(schema_partner) @@ -502,12 +436,8 @@ impl PartnerAccessor for ArrowArrayAccessor { .as_any() .downcast_ref::() .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "The struct partner is not a struct array, partner: {:?}", - struct_partner - ), + invalid_data!( + "The struct partner is not a struct array, partner: {struct_partner:?}" ) })?; @@ -515,12 +445,7 @@ impl PartnerAccessor for ArrowArrayAccessor { .fields() .iter() .position(|arrow_field| self.match_mode.match_field(arrow_field, field)) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Field id {} not found in struct array", field.id), - ) - })?; + .ok_or_else(|| invalid_data!("Field id {} not found in struct array", field.id))?; Ok(struct_array.column(field_pos)) } @@ -531,24 +456,14 @@ impl PartnerAccessor for ArrowArrayAccessor { let list_array = list_partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The list partner is not a list array", - ) - })?; + .ok_or_else(|| invalid_data!("The list partner is not a list array"))?; Ok(list_array.values()) } DataType::LargeList(_) => { let list_array = list_partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The list partner is not a large list array", - ) - })?; + .ok_or_else(|| invalid_data!("The list partner is not a large list array"))?; Ok(list_array.values()) } DataType::FixedSizeList(_, _) => { @@ -556,17 +471,11 @@ impl PartnerAccessor for ArrowArrayAccessor { .as_any() .downcast_ref::() .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "The list partner is not a fixed size list array", - ) + invalid_data!("The list partner is not a fixed size list array") })?; Ok(list_array.values()) } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "The list partner is not a list type", - )), + _ => Err(invalid_data!("The list partner is not a list type")), } } @@ -574,9 +483,7 @@ impl PartnerAccessor for ArrowArrayAccessor { let map_array = map_partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The map partner is not a map array") - })?; + .ok_or_else(|| invalid_data!("The map partner is not a map array"))?; Ok(map_array.keys()) } @@ -584,9 +491,7 @@ impl PartnerAccessor for ArrowArrayAccessor { let map_array = map_partner .as_any() .downcast_ref::() - .ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "The map partner is not a map array") - })?; + .ok_or_else(|| invalid_data!("The map partner is not a map array"))?; Ok(map_array.values()) } } @@ -619,12 +524,222 @@ pub fn arrow_primitive_to_literal( ) } +/// Create a single-element array from a primitive literal. +/// +/// This is used for creating constant arrays (Run-End Encoded arrays) where we need +/// a single value that represents all rows. +pub(crate) fn create_primitive_array_single_element( + data_type: &DataType, + prim_lit: Option<&PrimitiveLiteral>, +) -> Result { + // No value: a single NULL of any (possibly nested) type. + if prim_lit.is_none() { + return Ok(new_null_array(data_type, 1)); + } + match (data_type, prim_lit) { + (DataType::Boolean, Some(PrimitiveLiteral::Boolean(v))) => { + Ok(Arc::new(BooleanArray::from(vec![*v]))) + } + (DataType::Int32, Some(PrimitiveLiteral::Int(v))) => { + Ok(Arc::new(Int32Array::from(vec![*v]))) + } + (DataType::Date32, Some(PrimitiveLiteral::Int(v))) => { + Ok(Arc::new(Date32Array::from(vec![*v]))) + } + (DataType::Int64, Some(PrimitiveLiteral::Long(v))) => { + Ok(Arc::new(Int64Array::from(vec![*v]))) + } + (DataType::Timestamp(TimeUnit::Microsecond, timezone), Some(PrimitiveLiteral::Long(v))) => { + let array = TimestampMicrosecondArray::from(vec![*v]); + if let Some(timezone) = timezone { + Ok(Arc::new(array.with_timezone(timezone.clone()))) + } else { + Ok(Arc::new(array)) + } + } + (DataType::Timestamp(TimeUnit::Nanosecond, timezone), Some(PrimitiveLiteral::Long(v))) => { + let array = TimestampNanosecondArray::from(vec![*v]); + if let Some(timezone) = timezone { + Ok(Arc::new(array.with_timezone(timezone.clone()))) + } else { + Ok(Arc::new(array)) + } + } + (DataType::Float32, Some(PrimitiveLiteral::Float(v))) => { + Ok(Arc::new(Float32Array::from(vec![v.0]))) + } + (DataType::Float64, Some(PrimitiveLiteral::Double(v))) => { + Ok(Arc::new(Float64Array::from(vec![v.0]))) + } + (DataType::Utf8, Some(PrimitiveLiteral::String(v))) => { + Ok(Arc::new(StringArray::from(vec![v.as_str()]))) + } + (DataType::Binary, Some(PrimitiveLiteral::Binary(v))) => { + Ok(Arc::new(BinaryArray::from_vec(vec![v.as_slice()]))) + } + (DataType::LargeBinary, Some(PrimitiveLiteral::Binary(v))) => { + Ok(Arc::new(LargeBinaryArray::from_vec(vec![v.as_slice()]))) + } + (DataType::Decimal128(precision, scale), Some(PrimitiveLiteral::Int128(v))) => { + let array = Decimal128Array::from(vec![{ *v }]) + .with_precision_and_scale(*precision, *scale) + .map_err(|e| { + invalid_data!( + "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" + ) + })?; + Ok(Arc::new(array)) + } + (DataType::Decimal128(precision, scale), Some(PrimitiveLiteral::UInt128(v))) => { + let array = Decimal128Array::from(vec![*v as i128]) + .with_precision_and_scale(*precision, *scale) + .map_err(|e| { + invalid_data!( + "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" + ) + })?; + Ok(Arc::new(array)) + } + _ => Err(Error::new( + ErrorKind::Unexpected, + format!("Unsupported constant type combination: {data_type:?} with {prim_lit:?}"), + )), + } +} + +/// Create a repeated array from a primitive literal for a given number of rows. +/// +/// This is used for creating non-constant arrays where we need the same value +/// repeated for each row. +pub(crate) fn create_primitive_array_repeated( + data_type: &DataType, + prim_lit: Option<&PrimitiveLiteral>, + num_rows: usize, +) -> Result { + // No value to repeat: an all-NULL column of any (possibly nested) type. + if prim_lit.is_none() { + return Ok(new_null_array(data_type, num_rows)); + } + Ok(match (data_type, prim_lit) { + // --- Primitive Some arms --- + (DataType::Boolean, Some(PrimitiveLiteral::Boolean(value))) => { + let buffer = if *value { + BooleanBuffer::new_set(num_rows) + } else { + BooleanBuffer::new_unset(num_rows) + }; + Arc::new(BooleanArray::new(buffer, None)) + } + (DataType::Int32, Some(PrimitiveLiteral::Int(value))) => { + Arc::new(Int32Array::from(vec![*value; num_rows])) + } + (DataType::Date32, Some(PrimitiveLiteral::Int(value))) => { + Arc::new(Date32Array::from(vec![*value; num_rows])) + } + (DataType::Int64, Some(PrimitiveLiteral::Int(value))) => { + Arc::new(Int64Array::from(vec![i64::from(*value); num_rows])) + } + (DataType::Int64, Some(PrimitiveLiteral::Long(value))) => { + Arc::new(Int64Array::from(vec![*value; num_rows])) + } + ( + DataType::Timestamp(TimeUnit::Microsecond, timezone), + Some(PrimitiveLiteral::Long(value)), + ) => { + let array = TimestampMicrosecondArray::from(vec![*value; num_rows]); + if let Some(timezone) = timezone { + Arc::new(array.with_timezone(timezone.clone())) + } else { + Arc::new(array) + } + } + ( + DataType::Timestamp(TimeUnit::Nanosecond, timezone), + Some(PrimitiveLiteral::Long(value)), + ) => { + let array = TimestampNanosecondArray::from(vec![*value; num_rows]); + if let Some(timezone) = timezone { + Arc::new(array.with_timezone(timezone.clone())) + } else { + Arc::new(array) + } + } + (DataType::Float32, Some(PrimitiveLiteral::Float(value))) => { + Arc::new(Float32Array::from(vec![value.0; num_rows])) + } + (DataType::Float64, Some(PrimitiveLiteral::Double(value))) => { + Arc::new(Float64Array::from(vec![value.0; num_rows])) + } + (DataType::Utf8, Some(PrimitiveLiteral::String(value))) => Arc::new( + StringArray::from_iter_values(std::iter::repeat_n(value.as_str(), num_rows)), + ), + (DataType::Binary, Some(PrimitiveLiteral::Binary(value))) => Arc::new( + BinaryArray::from_iter_values(std::iter::repeat_n(value.as_slice(), num_rows)), + ), + (DataType::LargeBinary, Some(PrimitiveLiteral::Binary(value))) => Arc::new( + LargeBinaryArray::from_iter_values(std::iter::repeat_n(value.as_slice(), num_rows)), + ), + (DataType::FixedSizeBinary(len), Some(PrimitiveLiteral::Binary(value))) => { + // try_from_iter infers the width from the data, never from `len`, so a + // wrong-width literal would otherwise produce a FixedSizeBinary of the + // wrong width instead of erroring. + if value.len() != *len as usize { + return Err(invalid_data!( + "FixedSizeBinary literal length {} does not match declared width {len}", + value.len() + )); + } + + Arc::new( + FixedSizeBinaryArray::try_from_iter(std::iter::repeat_n(value.as_slice(), num_rows)) + .map_err(|e| { + invalid_data!("Failed to create FixedSizeBinary({len}) array: {e}") + })?, + ) + } + (DataType::Time64(TimeUnit::Microsecond), Some(PrimitiveLiteral::Long(value))) => { + Arc::new(Time64MicrosecondArray::from(vec![*value; num_rows])) + } + (DataType::Decimal128(precision, scale), Some(PrimitiveLiteral::Int128(value))) => { + Arc::new( + Decimal128Array::from(vec![*value; num_rows]) + .with_precision_and_scale(*precision, *scale) + .map_err(|e| { + invalid_data!( + "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" + ) + })?, + ) + } + (DataType::Decimal128(precision, scale), Some(PrimitiveLiteral::UInt128(value))) => { + Arc::new( + Decimal128Array::from(vec![*value as i128; num_rows]) + .with_precision_and_scale(*precision, *scale) + .map_err(|e| { + invalid_data!( + "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}" + ) + })?, + ) + } + + (dt, _) => { + return Err(Error::new( + ErrorKind::Unexpected, + format!("unexpected target column type {dt}, prim_lit {prim_lit:?}"), + )); + } + }) +} + #[cfg(test)] mod test { use std::collections::HashMap; use std::sync::Arc; - use arrow_array::builder::{Int32Builder, ListBuilder, MapBuilder, StructBuilder}; + use arrow_array::builder::{ + Int32Builder, ListBuilder, MapBuilder, MapFieldNames, StructBuilder, + }; use arrow_array::{ ArrayRef, BinaryArray, BooleanArray, Date32Array, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array, StringArray, StructArray, Time64MicrosecondArray, @@ -848,7 +963,7 @@ mod test { Some(Literal::bool(true)), Some(Literal::int(3)), Some(Literal::long(5)), - Some(Literal::float(1.1)), + Some(Literal::float(1.1_f32)), Some(Literal::double(3.3)), Some(Literal::decimal(1000)), Some(Literal::date(18628)), @@ -862,7 +977,7 @@ mod test { Some(Literal::bool(false)), Some(Literal::int(4)), Some(Literal::long(6)), - Some(Literal::float(2.2)), + Some(Literal::float(2.2_f32)), Some(Literal::double(4.4)), Some(Literal::decimal(2000)), Some(Literal::date(18629)), @@ -965,6 +1080,74 @@ mod test { assert_eq!(result, vec![None; 0]); } + #[test] + fn test_arrow_variant_to_literal_is_unsupported() { + // Converting a variant Arrow array back to an Iceberg literal is not implemented; + // the visitor must reject it rather than silently decode it incorrectly. + let variant_child = Arc::new(StructArray::from(vec![ + ( + Arc::new(Field::new("metadata", DataType::Binary, false)), + Arc::new(BinaryArray::from(vec![Some(b"m".as_ref())])) as ArrayRef, + ), + ( + Arc::new(Field::new("value", DataType::Binary, false)), + Arc::new(BinaryArray::from(vec![Some(b"v".as_ref())])) as ArrayRef, + ), + ])) as ArrayRef; + + let struct_array = Arc::new(StructArray::from(vec![( + Arc::new( + Field::new("v", variant_child.data_type().clone(), false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]), + ), + ), + variant_child, + )])) as ArrayRef; + + let ty = StructType::new(vec![ + NestedField::required(1, "v", Type::Variant(VariantType)).into(), + ]); + + let err = arrow_struct_to_literal(&struct_array, &ty).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert!( + err.to_string() + .contains("Converting variant Arrow array to Iceberg literal is not supported yet"), + "{err}" + ); + } + + #[test] + fn test_arrow_geospatial_to_literal() { + let values = Arc::new(LargeBinaryArray::from(vec![ + Some(b"wkb-1".as_slice()), + None, + Some(b"wkb-2".as_slice()), + ])) as ArrayRef; + let mut converter = ArrowArrayToIcebergStructConverter; + + let result = converter + .primitive(&PrimitiveType::Geometry(Default::default()), &values) + .unwrap(); + + assert_eq!(result, vec![ + Some(Literal::binary(b"wkb-1".to_vec())), + None, + Some(Literal::binary(b"wkb-2".to_vec())), + ]); + } + + #[test] + fn test_arrow_geospatial_binary_view_to_literal() { + let wkb = parquet_geospatial::testing::wkb_point_xy(1.0, 2.0); + let values = Arc::new(BinaryViewArray::from(vec![Some(wkb.as_slice()), None])) as ArrayRef; + let ty = Type::Primitive(PrimitiveType::Geography(Default::default())); + + let result = arrow_primitive_to_literal(&values, &ty).unwrap(); + + assert_eq!(result, vec![Some(Literal::binary(wkb)), None]); + } + #[test] fn test_find_field_by_id() { // Create Arrow arrays for the nested structure @@ -1225,7 +1408,17 @@ mod test { let map_key_builder = Int32Builder::new(); let map_value_builder = Int32Builder::new(); - let map_builder = MapBuilder::new(None, map_key_builder, map_value_builder); + // Arrow 60 `MapBuilder` defaults to `key` and `value`, but this fixture's + // Arrow type names the map fields `keys` and `values`. + let map_builder = MapBuilder::new( + Some(MapFieldNames { + entry: "entries".to_string(), + key: "keys".to_string(), + value: "values".to_string(), + }), + map_key_builder, + map_value_builder, + ); let b_builder = ListBuilder::new(map_builder); let inner_list_item_builder = Int32Builder::new(); @@ -1469,4 +1662,271 @@ mod test { ]))), ]); } + + #[test] + fn test_create_decimal_array_respects_precision() { + // Decimal128Array::from() uses Arrow's default precision (38) instead of the + // target precision, causing RecordBatch construction to fail when schemas don't match. + let target_precision = 18u8; + let target_scale = 10i8; + let target_type = DataType::Decimal128(target_precision, target_scale); + let value = PrimitiveLiteral::Int128(10000000000); + + let array = create_primitive_array_single_element(&target_type, Some(&value)) + .expect("Failed to create decimal array"); + + match array.data_type() { + DataType::Decimal128(precision, scale) => { + assert_eq!(*precision, target_precision); + assert_eq!(*scale, target_scale); + } + other => panic!("Expected Decimal128, got {other:?}"), + } + } + + #[test] + fn test_create_null_arrays_preserve_type_and_length() { + let data_types = [ + DataType::Decimal128(10, 2), + DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())), + DataType::Struct( + vec![ + Field::new("a", DataType::Utf8, true), + Field::new( + "ys", + DataType::List(Arc::new(Field::new("element", DataType::Int64, true))), + true, + ), + ] + .into(), + ), + DataType::Null, + ]; + + // NullArray has no validity bitmap; logical_null_count includes its implicit NULLs. + for data_type in data_types { + let single = create_primitive_array_single_element(&data_type, None) + .unwrap_or_else(|err| panic!("single, type={data_type:?}: {err}")); + assert_eq!(single.data_type(), &data_type, "single, type={data_type:?}"); + assert_eq!(single.len(), 1, "single, type={data_type:?}"); + if data_type != DataType::Null { + assert_eq!(single.null_count(), 1, "single, type={data_type:?}"); + } + assert_eq!(single.logical_null_count(), 1, "single, type={data_type:?}"); + + for num_rows in [0, 1, 3] { + let repeated = create_primitive_array_repeated(&data_type, None, num_rows) + .unwrap_or_else(|err| { + panic!("repeated, type={data_type:?}, rows={num_rows}: {err}") + }); + assert_eq!( + repeated.data_type(), + &data_type, + "repeated, type={data_type:?}, rows={num_rows}" + ); + assert_eq!( + repeated.len(), + num_rows, + "repeated, type={data_type:?}, rows={num_rows}" + ); + if data_type != DataType::Null { + assert_eq!( + repeated.null_count(), + num_rows, + "repeated, type={data_type:?}, rows={num_rows}" + ); + } + assert_eq!( + repeated.logical_null_count(), + num_rows, + "repeated, type={data_type:?}, rows={num_rows}" + ); + } + } + } + + #[test] + fn test_create_null_array_rejects_non_null_literal() { + let literal = Some(PrimitiveLiteral::Int(1)); + + assert!(create_primitive_array_single_element(&DataType::Null, literal.as_ref()).is_err()); + assert!(create_primitive_array_repeated(&DataType::Null, literal.as_ref(), 2).is_err()); + assert_eq!( + create_primitive_array_single_element(&DataType::Null, None) + .unwrap() + .len(), + 1 + ); + assert_eq!( + create_primitive_array_repeated(&DataType::Null, None, 2) + .unwrap() + .len(), + 2 + ); + } + + #[test] + fn test_create_decimal_array_repeated_respects_precision() { + // Ensure repeated arrays also respect target precision, not Arrow's default. + let target_precision = 18u8; + let target_scale = 10i8; + let target_type = DataType::Decimal128(target_precision, target_scale); + let value = PrimitiveLiteral::Int128(10000000000); + let num_rows = 5; + + let array = create_primitive_array_repeated(&target_type, Some(&value), num_rows) + .expect("Failed to create repeated decimal array"); + + match array.data_type() { + DataType::Decimal128(precision, scale) => { + assert_eq!(*precision, target_precision); + assert_eq!(*scale, target_scale); + } + other => panic!("Expected Decimal128, got {other:?}"), + } + + assert_eq!(array.len(), num_rows); + } + + #[test] + fn test_create_timestamp_microsecond_array_repeated() { + let target_type = DataType::Timestamp(TimeUnit::Microsecond, None); + let value = PrimitiveLiteral::Long(1_740_600_000_000_000); + let num_rows = 3; + + let array = create_primitive_array_repeated(&target_type, Some(&value), num_rows) + .expect("Failed to create repeated timestamp microsecond array"); + + assert_eq!(array.data_type(), &target_type); + assert_eq!(array.len(), num_rows); + } + + #[test] + fn test_create_timestamp_microsecond_with_timezone_array_repeated() { + let target_type = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())); + let value = PrimitiveLiteral::Long(1_740_600_000_000_000); + let num_rows = 2; + + let array = create_primitive_array_repeated(&target_type, Some(&value), num_rows) + .expect("Failed to create repeated timestamp microsecond array with timezone"); + + assert_eq!(array.data_type(), &target_type); + assert_eq!(array.len(), num_rows); + } + + #[test] + fn test_create_string_and_binary_arrays_repeated() { + let text = "partition-value-2026"; + let bytes: Vec = vec![0xDE, 0xAD, 0xBE, 0xEF]; + let num_rows = 4; + + let utf8 = create_primitive_array_repeated( + &DataType::Utf8, + Some(&PrimitiveLiteral::String(text.to_string())), + num_rows, + ) + .unwrap(); + let utf8 = utf8.as_any().downcast_ref::().unwrap(); + assert_eq!(utf8.len(), num_rows); + assert!((0..num_rows).all(|i| utf8.value(i) == text)); + + let binary = create_primitive_array_repeated( + &DataType::Binary, + Some(&PrimitiveLiteral::Binary(bytes.clone())), + num_rows, + ) + .unwrap(); + let binary = binary.as_any().downcast_ref::().unwrap(); + assert_eq!(binary.len(), num_rows); + assert!((0..num_rows).all(|i| binary.value(i) == bytes.as_slice())); + + let large = create_primitive_array_repeated( + &DataType::LargeBinary, + Some(&PrimitiveLiteral::Binary(bytes.clone())), + num_rows, + ) + .unwrap(); + let large = large.as_any().downcast_ref::().unwrap(); + assert_eq!(large.len(), num_rows); + assert!((0..num_rows).all(|i| large.value(i) == bytes.as_slice())); + + let fixed = create_primitive_array_repeated( + &DataType::FixedSizeBinary(bytes.len() as i32), + Some(&PrimitiveLiteral::Binary(bytes.clone())), + num_rows, + ) + .unwrap(); + let fixed = fixed + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(fixed.len(), num_rows); + assert!((0..num_rows).all(|i| fixed.value(i) == bytes.as_slice())); + } + + #[test] + fn test_create_boolean_array_repeated() { + let num_rows = 4; + + for value in [true, false] { + let array = create_primitive_array_repeated( + &DataType::Boolean, + Some(&PrimitiveLiteral::Boolean(value)), + num_rows, + ) + .unwrap(); + let array = array.as_any().downcast_ref::().unwrap(); + assert_eq!(array.len(), num_rows); + assert_eq!(array.null_count(), 0); + assert!((0..num_rows).all(|i| array.value(i) == value)); + } + + // num_rows == 0 must produce an empty (not one-element) array. + let empty = create_primitive_array_repeated( + &DataType::Boolean, + Some(&PrimitiveLiteral::Boolean(true)), + 0, + ) + .unwrap(); + assert_eq!(empty.len(), 0); + assert_eq!(empty.null_count(), 0); + } + + #[test] + fn test_create_string_array_repeated_empty() { + // num_rows == 0 must produce an empty (not one-element) array. + let array = create_primitive_array_repeated( + &DataType::Utf8, + Some(&PrimitiveLiteral::String("x".to_string())), + 0, + ) + .unwrap(); + assert_eq!(array.len(), 0); + } + + #[test] + fn test_create_fixed_size_binary_repeated_wrong_width_errors() { + // A literal whose length differs from the declared width must error rather + // than silently producing a FixedSizeBinary array of the literal's width. + let err = create_primitive_array_repeated( + &DataType::FixedSizeBinary(4), + Some(&PrimitiveLiteral::Binary(vec![0x01, 0x02, 0x03])), + 2, + ) + .unwrap_err(); + assert!(err.to_string().contains("does not match declared width")); + } + + #[test] + fn test_create_fixed_size_binary_repeated_empty_errors() { + // arrow's try_from_iter cannot infer the fixed width from an empty iterator, + // so num_rows == 0 over a fixed[n] column surfaces as an error rather than an + // empty fixed[n] array. + let result = create_primitive_array_repeated( + &DataType::FixedSizeBinary(4), + Some(&PrimitiveLiteral::Binary(vec![0xDE, 0xAD, 0xBE, 0xEF])), + 0, + ); + assert!(result.is_err()); + } } diff --git a/crates/iceberg/src/avro/deserializer.rs b/crates/iceberg/src/avro/deserializer.rs new file mode 100644 index 0000000000..e130925443 --- /dev/null +++ b/crates/iceberg/src/avro/deserializer.rs @@ -0,0 +1,457 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Deserialization straight from an Avro writer schema, with the parts of Avro +//! schema resolution that apache-avro's schema-aware deserializer leaves out. +//! +//! `Reader::into_deser_iter` decodes into serde types without building +//! `apache_avro::types::Value`s, but it requires the reader schema to equal the +//! writer schema and applies no resolution rules. [`ResolvingDeserializer`] +//! wraps it and routes every value through `deserialize_any`, which follows the +//! writer schema. As a result: +//! +//! - Avro record names don't have to match serde type names. +//! - A writer value that isn't a union reads into an `Option`. +//! - A writer union reads into a type that isn't an `Option`. A null value +//! returns an error. +//! - serde's numeric visitors convert between numeric types, so an `int` reads +//! into an `i64` and a `float` into an `f64`. An integer that doesn't fit the +//! target, such as a `long` above `i32::MAX` read into an `i32`, returns an +//! error. Conversions into `f32` or `f64` use `as` and can lose precision. +//! +//! apache-avro plans a `SchemaAwareResolvingDeserializer` that resolves against +//! a reader schema (). Once a +//! release includes it, readers can pass their reader schema to +//! `Reader::builder` and drop this module, provided it doesn't reject writer +//! record names that differ from the reader's. The Avro spec requires record +//! names to match, and the writers this crate reads from don't all agree on +//! them. + +use std::fmt; + +use serde::de::value::{ + BorrowedBytesDeserializer, BorrowedStrDeserializer, BytesDeserializer, EnumAccessDeserializer, + MapAccessDeserializer, SeqAccessDeserializer, +}; +use serde::de::{ + DeserializeSeed, Deserializer, EnumAccess, Error, IntoDeserializer, MapAccess, SeqAccess, + Visitor, +}; +use serde::{Deserialize, forward_to_deserialize_any}; + +/// Deserializes the wrapped type through [`ResolvingDeserializer`], for use +/// with `Reader::into_deser_iter`. +pub(crate) struct Resolved(pub T); + +impl<'de, T: Deserialize<'de>> Deserialize<'de> for Resolved { + fn deserialize>(deserializer: D) -> Result { + T::deserialize(ResolvingDeserializer(deserializer)).map(Resolved) + } +} + +struct ResolvingDeserializer(D); + +impl<'de, D: Deserializer<'de>> Deserializer<'de> for ResolvingDeserializer { + type Error = D::Error; + + fn deserialize_any>(self, visitor: V) -> Result { + self.0.deserialize_any(ResolvingVisitor(visitor)) + } + + fn deserialize_option>(self, visitor: V) -> Result { + self.0.deserialize_any(OptionVisitor(visitor)) + } + + fn deserialize_ignored_any>(self, visitor: V) -> Result { + self.0.deserialize_ignored_any(visitor) + } + + fn is_human_readable(&self) -> bool { + self.0.is_human_readable() + } + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string bytes byte_buf + unit unit_struct newtype_struct seq tuple tuple_struct map struct enum identifier + } +} + +/// Wraps the sequences and maps a visitor receives, so that their elements are +/// deserialized through [`ResolvingDeserializer`] too. +struct ResolvingVisitor(V); + +/// Forwards `visit_*` methods for values that contain no nested values. +macro_rules! forward_visit { + ($($method:ident($ty:ty)),* $(,)?) => { + $( + fn $method(self, v: $ty) -> Result { + self.0.$method(v) + } + )* + }; +} + +impl<'de, V: Visitor<'de>> Visitor<'de> for ResolvingVisitor { + type Value = V::Value; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.0.expecting(formatter) + } + + forward_visit! { + visit_bool(bool), visit_i8(i8), visit_i16(i16), visit_i32(i32), visit_i64(i64), + visit_i128(i128), visit_u8(u8), visit_u16(u16), visit_u32(u32), visit_u64(u64), + visit_u128(u128), visit_f32(f32), visit_f64(f64), visit_char(char), visit_str(&str), + visit_borrowed_str(&'de str), visit_string(String), visit_bytes(&[u8]), + visit_borrowed_bytes(&'de [u8]), visit_byte_buf(Vec), + } + + fn visit_none(self) -> Result { + self.0.visit_none() + } + + fn visit_unit(self) -> Result { + self.0.visit_unit() + } + + fn visit_some>(self, deserializer: D) -> Result { + self.0.visit_some(ResolvingDeserializer(deserializer)) + } + + fn visit_newtype_struct>( + self, + deserializer: D, + ) -> Result { + self.0 + .visit_newtype_struct(ResolvingDeserializer(deserializer)) + } + + fn visit_seq>(self, seq: A) -> Result { + self.0.visit_seq(ResolvingAccess(seq)) + } + + fn visit_map>(self, map: A) -> Result { + self.0.visit_map(ResolvingAccess(map)) + } + + fn visit_enum>(self, data: A) -> Result { + self.0.visit_enum(data) + } +} + +/// Reads an `Option` from any writer value. A null becomes `None`, and anything +/// else becomes `Some`, whether or not the writer schema is a union. +struct OptionVisitor(V); + +/// Implements `visit_*` methods that hand a value to `visit_some` through the +/// deserializer that `deserializer` builds from it. +macro_rules! visit_some { + ($($method:ident($ty:ty) => $deserializer:expr),* $(,)?) => { + $( + fn $method(self, v: $ty) -> Result { + self.0.visit_some($deserializer(v)) + } + )* + }; +} + +impl<'de, V: Visitor<'de>> Visitor<'de> for OptionVisitor { + type Value = V::Value; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + self.0.expecting(formatter) + } + + visit_some! { + visit_bool(bool) => IntoDeserializer::into_deserializer, + visit_i8(i8) => IntoDeserializer::into_deserializer, + visit_i16(i16) => IntoDeserializer::into_deserializer, + visit_i32(i32) => IntoDeserializer::into_deserializer, + visit_i64(i64) => IntoDeserializer::into_deserializer, + visit_i128(i128) => IntoDeserializer::into_deserializer, + visit_u8(u8) => IntoDeserializer::into_deserializer, + visit_u16(u16) => IntoDeserializer::into_deserializer, + visit_u32(u32) => IntoDeserializer::into_deserializer, + visit_u64(u64) => IntoDeserializer::into_deserializer, + visit_u128(u128) => IntoDeserializer::into_deserializer, + visit_f32(f32) => IntoDeserializer::into_deserializer, + visit_f64(f64) => IntoDeserializer::into_deserializer, + visit_char(char) => IntoDeserializer::into_deserializer, + visit_str(&str) => IntoDeserializer::into_deserializer, + visit_borrowed_str(&'de str) => BorrowedStrDeserializer::new, + visit_string(String) => IntoDeserializer::into_deserializer, + visit_bytes(&[u8]) => BytesDeserializer::new, + visit_borrowed_bytes(&'de [u8]) => BorrowedBytesDeserializer::new, + } + + fn visit_byte_buf(self, v: Vec) -> Result { + self.0.visit_some(BytesDeserializer::new(&v)) + } + + fn visit_none(self) -> Result { + self.0.visit_none() + } + + fn visit_unit(self) -> Result { + self.0.visit_none() + } + + fn visit_some>(self, deserializer: D) -> Result { + self.0.visit_some(ResolvingDeserializer(deserializer)) + } + + fn visit_seq>(self, seq: A) -> Result { + self.0 + .visit_some(SeqAccessDeserializer::new(ResolvingAccess(seq))) + } + + fn visit_map>(self, map: A) -> Result { + self.0 + .visit_some(MapAccessDeserializer::new(ResolvingAccess(map))) + } + + fn visit_enum>(self, data: A) -> Result { + self.0.visit_some(EnumAccessDeserializer::new(data)) + } +} + +/// Deserializes the elements of a sequence, or the keys and values of a map, +/// through [`ResolvingDeserializer`]. Keys need it because apache-avro's record +/// field names only support `deserialize_identifier` and `deserialize_any`, +/// and a key read as a `String` calls `deserialize_string`. +struct ResolvingAccess(A); + +impl<'de, A: SeqAccess<'de>> SeqAccess<'de> for ResolvingAccess { + type Error = A::Error; + + fn next_element_seed>( + &mut self, + seed: T, + ) -> Result, A::Error> { + self.0.next_element_seed(ResolvingSeed(seed)) + } + + fn size_hint(&self) -> Option { + self.0.size_hint() + } +} + +impl<'de, A: MapAccess<'de>> MapAccess<'de> for ResolvingAccess { + type Error = A::Error; + + fn next_key_seed>( + &mut self, + seed: K, + ) -> Result, A::Error> { + self.0.next_key_seed(ResolvingSeed(seed)) + } + + fn next_value_seed>(&mut self, seed: T) -> Result { + self.0.next_value_seed(ResolvingSeed(seed)) + } + + fn size_hint(&self) -> Option { + self.0.size_hint() + } +} + +struct ResolvingSeed(S); + +impl<'de, S: DeserializeSeed<'de>> DeserializeSeed<'de> for ResolvingSeed { + type Value = S::Value; + + fn deserialize>(self, deserializer: D) -> Result { + self.0.deserialize(ResolvingDeserializer(deserializer)) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::fmt::Debug; + + use apache_avro::types::Value as AvroValue; + use apache_avro::{Reader, Schema as AvroSchema, Writer}; + use serde::de::DeserializeOwned; + use serde_bytes::ByteBuf; + use serde_derive::Deserialize; + + use super::*; + + #[derive(Deserialize, Debug, PartialEq)] + struct Pair { + a: i64, + b: Option, + } + + #[derive(Deserialize, Debug, PartialEq)] + enum Suit { + Hearts, + Spades, + } + + /// Writes `value` with `writer_schema` and reads it back as a `T`. + fn read( + writer_schema: &str, + value: AvroValue, + ) -> apache_avro::AvroResult { + let schema = AvroSchema::parse_str(writer_schema).unwrap(); + let mut writer = Writer::new(&schema, Vec::new()).unwrap(); + writer.append_value(value).unwrap(); + let bs = writer.into_inner().unwrap(); + let mut values = Reader::new(bs.as_slice()) + .unwrap() + .into_deser_iter::>(); + values.next().unwrap().map(|value| value.0) + } + + fn check( + writer_schema: &str, + value: AvroValue, + expected: T, + ) { + assert_eq!( + read::(writer_schema, value).unwrap(), + expected, + "{writer_schema}" + ); + } + + const PAIR: &str = r#"{"type": "record", "name": "writer_name", "fields": [ + {"name": "b", "type": "string"}, + {"name": "extra", "type": "int"}, + {"name": "a", "type": "int"} + ]}"#; + + fn pair_value() -> AvroValue { + AvroValue::Record(vec![ + ("b".to_string(), AvroValue::String("x".to_string())), + ("extra".to_string(), AvroValue::Int(9)), + ("a".to_string(), AvroValue::Int(1)), + ]) + } + + fn pair() -> Pair { + Pair { + a: 1, + b: Some("x".to_string()), + } + } + + const SUIT: &str = r#"{"type": "enum", "name": "suit", "symbols": ["Hearts", "Spades"]}"#; + + #[test] + fn test_resolved_reads_writer_values() { + check(r#""boolean""#, AvroValue::Boolean(true), true); + check(r#""int""#, AvroValue::Int(-3), -3i64); + check(r#""long""#, AvroValue::Long(7), 7i32); + check(r#""float""#, AvroValue::Float(1.5), 1.5f64); + check(r#""double""#, AvroValue::Double(2.5), 2.5f64); + check( + r#""string""#, + AvroValue::String("s".to_string()), + "s".to_string(), + ); + check( + r#""bytes""#, + AvroValue::Bytes(vec![1, 2]), + ByteBuf::from(vec![1, 2]), + ); + check( + r#"{"type": "fixed", "name": "f", "size": 2}"#, + AvroValue::Fixed(2, vec![3, 4]), + ByteBuf::from(vec![3, 4]), + ); + check( + r#"{"type": "array", "items": "int"}"#, + AvroValue::Array(vec![AvroValue::Int(1), AvroValue::Int(2)]), + vec![1i64, 2], + ); + check( + r#"{"type": "map", "values": "long"}"#, + AvroValue::Map(HashMap::from([("k".to_string(), AvroValue::Long(5))])), + HashMap::from([("k".to_string(), 5i32)]), + ); + check(PAIR, pair_value(), pair()); + check(SUIT, AvroValue::Enum(1, "Spades".to_string()), Suit::Spades); + check( + r#"["null", "long"]"#, + AvroValue::Union(1, Box::new(AvroValue::Long(5))), + 5i64, + ); + } + + #[test] + fn test_resolved_reads_any_writer_value_as_option() { + check(r#""null""#, AvroValue::Null, None::); + check( + r#"["null", "long"]"#, + AvroValue::Union(0, Box::new(AvroValue::Null)), + None::, + ); + check( + r#"["null", "int"]"#, + AvroValue::Union(1, Box::new(AvroValue::Int(5))), + Some(5i64), + ); + check(r#""boolean""#, AvroValue::Boolean(true), Some(true)); + check(r#""int""#, AvroValue::Int(-3), Some(-3i64)); + check(r#""long""#, AvroValue::Long(7), Some(7i32)); + check(r#""float""#, AvroValue::Float(1.5), Some(1.5f64)); + check(r#""double""#, AvroValue::Double(2.5), Some(2.5f64)); + check( + r#""string""#, + AvroValue::String("s".to_string()), + Some("s".to_string()), + ); + check( + r#""bytes""#, + AvroValue::Bytes(vec![1, 2]), + Some(ByteBuf::from(vec![1, 2])), + ); + check( + r#"{"type": "array", "items": "int"}"#, + AvroValue::Array(vec![AvroValue::Int(1)]), + Some(vec![1i64]), + ); + check( + r#"{"type": "map", "values": "long"}"#, + AvroValue::Map(HashMap::from([("k".to_string(), AvroValue::Long(5))])), + Some(HashMap::from([("k".to_string(), 5i32)])), + ); + check(PAIR, pair_value(), Some(pair())); + check( + SUIT, + AvroValue::Enum(0, "Hearts".to_string()), + Some(Suit::Hearts), + ); + } + + #[test] + fn test_resolved_rejects_values_that_do_not_fit() { + assert!(read::(r#""long""#, AvroValue::Long(i64::from(i32::MAX) + 1)).is_err()); + assert!(read::(r#""int""#, AvroValue::Int(-1)).is_err()); + assert!( + read::( + r#"["null", "long"]"#, + AvroValue::Union(0, Box::new(AvroValue::Null)) + ) + .is_err() + ); + assert!(read::(r#""long""#, AvroValue::Long(1)).is_err()); + } +} diff --git a/crates/iceberg/src/avro/mod.rs b/crates/iceberg/src/avro/mod.rs index f2a9310e7c..6a723e613e 100644 --- a/crates/iceberg/src/avro/mod.rs +++ b/crates/iceberg/src/avro/mod.rs @@ -16,5 +16,49 @@ // under the License. //! Avro related codes. +mod deserializer; +mod named_types; mod schema; +pub(crate) use deserializer::Resolved; +pub(crate) use named_types::define_named_types_once; +#[cfg(test)] +pub(crate) use named_types::define_named_types_repeatedly; pub(crate) use schema::*; + +/// Creates an Avro writer with deflate compression. +pub(crate) fn new_writer( + schema: &apache_avro::Schema, + writer: W, +) -> apache_avro::AvroResult> { + apache_avro::Writer::with_codec( + schema, + writer, + apache_avro::Codec::Deflate(Default::default()), + ) +} + +#[cfg(test)] +pub(crate) fn assert_deflate_codec(bytes: &[u8]) { + use apache_avro::Schema; + use apache_avro::reader::datum::GenericDatumReader; + use apache_avro::schema::MapSchema; + use apache_avro::types::Value; + + let schema = Schema::Map(MapSchema { + types: Box::new(Schema::Bytes), + attributes: Default::default(), + }); + let mut body = bytes.strip_prefix(b"Obj\x01").unwrap(); + let Value::Map(metadata) = GenericDatumReader::builder(&schema) + .build() + .unwrap() + .read_value(&mut body) + .unwrap() + else { + panic!("Avro header metadata must be a map"); + }; + assert_eq!( + metadata.get("avro.codec"), + Some(&Value::Bytes(b"deflate".to_vec())) + ); +} diff --git a/crates/iceberg/src/avro/named_types.rs b/crates/iceberg/src/avro/named_types.rs new file mode 100644 index 0000000000..a006a84156 --- /dev/null +++ b/crates/iceberg/src/avro/named_types.rs @@ -0,0 +1,309 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Reading Avro files whose schema defines a named type more than once. +//! +//! The Avro specification allows one definition per full name. iceberg-rust +//! wrote manifests that define a `fixed` or `decimal` type once for every field +//! that uses it, while `apache-avro` 0.21 accepted that. `apache-avro` 0.22 +//! rejects such a file's header. Rewriting each repeated definition as a +//! reference to the first one keeps those files readable, because a reference +//! decodes the same bytes as the definition it refers to. + +use std::collections::HashMap; +use std::io::Cursor; + +use apache_avro::Schema as AvroSchema; +use apache_avro::reader::datum::GenericDatumReader; +use apache_avro::schema::MapSchema; +use apache_avro::types::Value as AvroValue; +use apache_avro::writer::datum::GenericDatumWriter; +use serde_json::{Map, Value as JsonValue}; + +use crate::Result; +use crate::error::invalid_data; + +const MAGIC: &[u8] = b"Obj\x01"; +const SCHEMA_KEY: &str = "avro.schema"; + +/// Rewrites the header of an Avro object container file so that each named type +/// is defined once and referenced by name afterwards. Returns the rewritten +/// file and the full names of the repeated types, or `None` if a repeated +/// definition differs from the first one, which leaves the file ambiguous. +pub(crate) fn define_named_types_once(bs: &[u8]) -> Result, Vec)>> { + rewrite_schema(bs, reference_repeated_definitions) +} + +/// Rewrites the schema in the header of an Avro object container file with +/// `rewrite`, leaving the data blocks as they are. Returns `None` if `rewrite` +/// does. +fn rewrite_schema( + bs: &[u8], + rewrite: impl FnOnce(&mut JsonValue) -> Option, +) -> Result, T)>> { + let body = bs + .strip_prefix(MAGIC) + .ok_or_else(|| invalid_data!("Not an Avro object container file"))?; + // The header's metadata is an Avro `map`, followed by the sync marker. + let metadata_schema = AvroSchema::Map(MapSchema { + types: Box::new(AvroSchema::Bytes), + attributes: Default::default(), + }); + let mut cursor = Cursor::new(body); + let AvroValue::Map(mut metadata) = GenericDatumReader::builder(&metadata_schema) + .build()? + .read_value(&mut cursor)? + else { + return Err(invalid_data!("Avro file metadata is not a map")); + }; + let rest = usize::try_from(cursor.position()) + .ok() + .and_then(|position| body.get(position..)) + .ok_or_else(|| invalid_data!("Avro file metadata extends past the end of the file"))?; + + let Some(AvroValue::Bytes(schema)) = metadata.get(SCHEMA_KEY) else { + return Err(invalid_data!("Avro file metadata has no {SCHEMA_KEY}")); + }; + let mut schema: JsonValue = serde_json::from_slice(schema)?; + let Some(rewritten) = rewrite(&mut schema) else { + return Ok(None); + }; + metadata.insert( + SCHEMA_KEY.to_string(), + AvroValue::Bytes(serde_json::to_vec(&schema)?), + ); + let metadata = GenericDatumWriter::builder(&metadata_schema) + .build()? + .write_value_to_vec(AvroValue::Map(metadata))?; + + Ok(Some(([MAGIC, &metadata, rest].concat(), rewritten))) +} + +/// Rewrites the header of an Avro object container file so that each reference +/// to a named type repeats its definition, as iceberg-rust wrote manifests with +/// `apache-avro` 0.21. +#[cfg(test)] +pub(crate) fn define_named_types_repeatedly(bs: &[u8]) -> Vec { + fn inline_references(schema: &mut JsonValue, defined: &mut HashMap) { + match schema { + JsonValue::String(name) => { + if let Some(definition) = defined.get(name.as_str()) { + *schema = definition.clone(); + } + } + JsonValue::Array(schemas) => schemas + .iter_mut() + .for_each(|schema| inline_references(schema, defined)), + JsonValue::Object(object) => { + if let (Some(JsonValue::String(name)), Some("record" | "enum" | "fixed")) = ( + object.get("name"), + object.get("type").and_then(JsonValue::as_str), + ) { + defined.insert(name.clone(), JsonValue::Object(object.clone())); + } + for key in ["type", "items", "values", "fields"] { + if let Some(child) = object.get_mut(key) { + inline_references(child, defined); + } + } + } + _ => {} + } + } + + rewrite_schema(bs, |schema| { + inline_references(schema, &mut HashMap::new()); + Some(()) + }) + .unwrap() + .unwrap() + .0 +} + +/// Replaces each repeated definition of a named type in `schema` with its full +/// name. Returns the replaced names, or `None` if a repeated definition differs +/// from the first one. +fn reference_repeated_definitions(schema: &mut JsonValue) -> Option> { + let mut defined = HashMap::new(); + let mut repeated = Vec::new(); + reference_repeated(schema, None, &mut defined, &mut repeated).then_some(repeated) +} + +/// Walks `schema` in the order Avro resolves names, recording each named type's +/// definition without its name. Returns `false` on a repeated name whose +/// definition differs from the first one. +fn reference_repeated( + schema: &mut JsonValue, + namespace: Option<&str>, + defined: &mut HashMap>, + repeated: &mut Vec, +) -> bool { + let object = match schema { + JsonValue::Array(variants) => { + return variants + .iter_mut() + .all(|variant| reference_repeated(variant, namespace, defined, repeated)); + } + JsonValue::Object(object) => object, + _ => return true, + }; + let type_name = match object.get_mut("type") { + Some(JsonValue::String(type_name)) => type_name.as_str(), + Some(nested) => return reference_repeated(nested, namespace, defined, repeated), + None => return true, + }; + match type_name { + "record" | "error" | "enum" | "fixed" => {} + "array" => { + return object + .get_mut("items") + .is_none_or(|items| reference_repeated(items, namespace, defined, repeated)); + } + "map" => { + return object + .get_mut("values") + .is_none_or(|values| reference_repeated(values, namespace, defined, repeated)); + } + _ => return true, + } + + let Some(full_name) = full_name(object, namespace) else { + // Leave a malformed schema for apache-avro to report. + return true; + }; + let mut definition = object.clone(); + definition.remove("name"); + definition.remove("namespace"); + if let Some(first) = defined.get(&full_name) { + if *first != definition { + return false; + } + *schema = JsonValue::String(full_name.clone()); + repeated.push(full_name); + return true; + } + defined.insert(full_name.clone(), definition); + + let record_namespace = full_name.rsplit_once('.').map(|(namespace, _)| namespace); + match object.get_mut("fields") { + Some(JsonValue::Array(fields)) => fields.iter_mut().all(|field| { + field.get_mut("type").is_none_or(|field_type| { + reference_repeated(field_type, record_namespace, defined, repeated) + }) + }), + _ => true, + } +} + +/// The full name of a named type, following the Avro specification: a name +/// containing a dot is already full, and otherwise the type's own `namespace` +/// or the enclosing namespace qualifies it. +fn full_name(object: &Map, enclosing_namespace: Option<&str>) -> Option { + let name = object.get("name")?.as_str()?; + if name.contains('.') { + return Some(name.to_string()); + } + let namespace = match object.get("namespace") { + Some(namespace) => namespace.as_str(), + None => enclosing_namespace, + }; + Some(match namespace { + Some(namespace) if !namespace.is_empty() => format!("{namespace}.{name}"), + _ => name.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn decimal_10_2() -> JsonValue { + json!({"type": "fixed", "name": "decimal_10_2", "size": 5, + "logicalType": "decimal", "precision": 10, "scale": 2}) + } + + #[test] + fn test_reference_repeated_definitions() { + let mut schema = json!({"type": "record", "name": "r102", "fields": [ + {"name": "a", "type": ["null", decimal_10_2()]}, + {"name": "b", "type": ["null", decimal_10_2()]}, + {"name": "c", "type": {"type": "array", "items": decimal_10_2()}}, + ]}); + + let repeated = reference_repeated_definitions(&mut schema); + + assert_eq!( + repeated, + Some(vec!["decimal_10_2".to_string(), "decimal_10_2".to_string()]) + ); + assert_eq!( + schema, + json!({"type": "record", "name": "r102", "fields": [ + {"name": "a", "type": ["null", decimal_10_2()]}, + {"name": "b", "type": ["null", "decimal_10_2"]}, + {"name": "c", "type": {"type": "array", "items": "decimal_10_2"}}, + ]}) + ); + } + + #[test] + fn test_reference_repeated_definitions_leaves_unique_names() { + let original = json!({"type": "record", "name": "r", "fields": [ + {"name": "a", "type": decimal_10_2()}, + {"name": "b", "type": "decimal_10_2"}, + {"name": "c", "type": {"type": "fixed", "name": "fixed_4", "size": 4}}, + ]}); + let mut schema = original.clone(); + + assert_eq!(reference_repeated_definitions(&mut schema), Some(vec![])); + assert_eq!(schema, original); + } + + #[test] + fn test_reference_repeated_definitions_uses_full_names() { + // The same short name in two namespaces names two different types. + let mut schema = json!({"type": "record", "name": "r", "fields": [ + {"name": "a", "type": {"type": "fixed", "name": "f", "namespace": "x", "size": 4}}, + {"name": "b", "type": {"type": "fixed", "name": "f", "namespace": "y", "size": 8}}, + {"name": "c", "type": {"type": "fixed", "name": "x.f", "size": 4}}, + {"name": "d", "type": {"type": "record", "name": "n", "namespace": "x", "fields": [ + {"name": "e", "type": {"type": "fixed", "name": "f", "size": 4}}, + ]}}, + ]}); + + let repeated = reference_repeated_definitions(&mut schema); + + assert_eq!(repeated, Some(vec!["x.f".to_string(), "x.f".to_string()])); + assert_eq!(schema["fields"][2]["type"], json!("x.f")); + assert_eq!( + schema["fields"][3]["type"]["fields"][0]["type"], + json!("x.f") + ); + } + + #[test] + fn test_reference_repeated_definitions_rejects_different_definitions() { + let mut schema = json!({"type": "record", "name": "r", "fields": [ + {"name": "a", "type": {"type": "fixed", "name": "f", "size": 4}}, + {"name": "b", "type": {"type": "fixed", "name": "f", "size": 8}}, + ]}); + + assert_eq!(reference_repeated_definitions(&mut schema), None); + } +} diff --git a/crates/iceberg/src/avro/schema.rs b/crates/iceberg/src/avro/schema.rs index fdbc680977..3caf99bfee 100644 --- a/crates/iceberg/src/avro/schema.rs +++ b/crates/iceberg/src/avro/schema.rs @@ -16,19 +16,20 @@ // under the License. //! Conversion between iceberg and avro schema. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use apache_avro::Schema as AvroSchema; use apache_avro::schema::{ - ArraySchema, DecimalSchema, FixedSchema, MapSchema, Name, RecordField as AvroRecordField, - RecordFieldOrder, RecordSchema, UnionSchema, + ArraySchema, DecimalSchema, FixedSchema, InnerDecimalSchema, MapSchema, Name, + RecordField as AvroRecordField, RecordSchema, UnionSchema, UuidSchema, }; use itertools::{Either, Itertools}; use serde_json::{Number, Value}; +use crate::error::invalid_data; use crate::spec::{ ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, Schema, SchemaVisitor, - StructType, Type, visit_schema, + StructType, Type, VariantType, visit_schema, }; use crate::{Error, ErrorKind, Result, ensure_data_valid}; @@ -42,6 +43,29 @@ const LOGICAL_TYPE: &str = "logicalType"; struct SchemaToAvroSchema { schema: String, + /// Names of the `fixed` types defined so far. Avro allows one definition per + /// name, and the visitor reaches fields in the order they're serialized. + defined_names: HashSet, +} + +impl SchemaToAvroSchema { + /// Returns a reference to `schema` by name if a type with its name was + /// already defined, and `schema` itself otherwise. + fn define_once(&mut self, schema: AvroSchema) -> AvroSchema { + let name = match &schema { + AvroSchema::Fixed(FixedSchema { name, .. }) + | AvroSchema::Decimal(DecimalSchema { + inner: InnerDecimalSchema::Fixed(FixedSchema { name, .. }), + .. + }) => name.clone(), + _ => return schema, + }; + if self.defined_names.insert(name.clone()) { + schema + } else { + AvroSchema::Ref { name } + } + } } type AvroSchemaOrField = Either; @@ -53,7 +77,7 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut avro_schema = value.unwrap_left(); if let AvroSchema::Record(record) = &mut avro_schema { - record.name = Name::from(self.schema.as_str()); + record.name = Name::new(self.schema.as_str())?; } else { return Err(Error::new( ErrorKind::Unexpected, @@ -71,11 +95,17 @@ impl SchemaVisitor for SchemaToAvroSchema { ) -> Result { let mut field_schema = avro_schema.unwrap_left(); if let AvroSchema::Record(record) = &mut field_schema { - record.name = Name::from(format!("r{}", field.id).as_str()); + record.name = Name::new(format!("r{}", field.id))?; } if !field.required { - field_schema = avro_optional(field_schema)?; + if is_avro_null(&field_schema) { + // Unknown maps directly to Avro null. Wrapping it as optional would create + // an invalid union with duplicate null variants. + field_schema = AvroSchema::Null; + } else { + field_schema = avro_optional(field_schema)?; + } } let default = if let Some(literal) = &field.initial_default { @@ -89,10 +119,8 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut avro_record_field = AvroRecordField { name: field.name.clone(), schema: field_schema, - order: RecordFieldOrder::Ignore, - position: 0, doc: field.doc.clone(), - aliases: None, + aliases: Vec::new(), default, custom_attributes: Default::default(), }; @@ -123,11 +151,17 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut field_schema = value.unwrap_left(); if let AvroSchema::Record(record) = &mut field_schema { - record.name = Name::from(format!("r{}", list.element_field.id).as_str()); + record.name = Name::new(format!("r{}", list.element_field.id))?; } if !list.element_field.required { - field_schema = avro_optional(field_schema)?; + if is_avro_null(&field_schema) { + // Unknown maps directly to Avro null. Wrapping it as optional would create + // an invalid union with duplicate null variants. + field_schema = AvroSchema::Null; + } else { + field_schema = avro_optional(field_schema)?; + } } Ok(Either::Left(AvroSchema::Array(ArraySchema { @@ -148,7 +182,13 @@ impl SchemaVisitor for SchemaToAvroSchema { let key_field_schema = key_value.unwrap_left(); let mut value_field_schema = value.unwrap_left(); if !map.value_field.required { - value_field_schema = avro_optional(value_field_schema)?; + if is_avro_null(&value_field_schema) { + // Unknown maps directly to Avro null. Wrapping it as optional would create + // an invalid union with duplicate null variants. + value_field_schema = AvroSchema::Null; + } else { + value_field_schema = avro_optional(value_field_schema)?; + } } if matches!(key_field_schema, AvroSchema::String) { @@ -172,11 +212,9 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut field = AvroRecordField { name: map.key_field.name.clone(), doc: None, - aliases: None, + aliases: Vec::new(), default: None, schema: key_field_schema, - order: RecordFieldOrder::Ascending, - position: 0, custom_attributes: Default::default(), }; field.custom_attributes.insert( @@ -190,11 +228,9 @@ impl SchemaVisitor for SchemaToAvroSchema { let mut field = AvroRecordField { name: map.value_field.name.clone(), doc: None, - aliases: None, + aliases: Vec::new(), default: None, schema: value_field_schema, - order: RecordFieldOrder::Ignore, - position: 0, custom_attributes: Default::default(), }; field.custom_attributes.insert( @@ -222,6 +258,7 @@ impl SchemaVisitor for SchemaToAvroSchema { fn primitive(&mut self, p: &PrimitiveType) -> Result { let avro_schema = match p { + PrimitiveType::Unknown => AvroSchema::Null, PrimitiveType::Boolean => AvroSchema::Boolean, PrimitiveType::Int => AvroSchema::Int, PrimitiveType::Long => AvroSchema::Long, @@ -234,21 +271,33 @@ impl SchemaVisitor for SchemaToAvroSchema { PrimitiveType::TimestampNs => AvroSchema::TimestampNanos, PrimitiveType::TimestamptzNs => AvroSchema::TimestampNanos, PrimitiveType::String => AvroSchema::String, - PrimitiveType::Uuid => AvroSchema::Uuid, - PrimitiveType::Fixed(len) => avro_fixed_schema((*len) as usize)?, - PrimitiveType::Binary => AvroSchema::Bytes, + // TODO(#2913): write the spec's `fixed[16]` uuid. The manifest reader + // already accepts both forms. + PrimitiveType::Uuid => AvroSchema::Uuid(UuidSchema::String), + PrimitiveType::Fixed(len) => self.define_once(avro_fixed_schema((*len) as usize)?), + PrimitiveType::Binary | PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { + AvroSchema::Bytes + } PrimitiveType::Decimal { precision, scale } => { - avro_decimal_schema(*precision as usize, *scale as usize)? + self.define_once(avro_decimal_schema(*precision as usize, *scale as usize)?) } }; Ok(Either::Left(avro_schema)) } + + fn variant(&mut self, _v: &VariantType) -> Result { + Err(Error::new( + ErrorKind::FeatureUnsupported, + "Converting a variant type to an Avro schema is not supported yet", + )) + } } /// Converting iceberg schema to avro schema. pub(crate) fn schema_to_avro_schema(name: impl ToString, schema: &Schema) -> Result { let mut converter = SchemaToAvroSchema { schema: name.to_string(), + defined_names: HashSet::new(), }; visit_schema(schema, &mut converter).map(Either::unwrap_left) @@ -278,7 +327,6 @@ pub(crate) fn avro_fixed_schema(len: usize) -> Result { doc: None, size: len, attributes: Default::default(), - default: None, })) } @@ -290,16 +338,15 @@ pub(crate) fn avro_decimal_schema(precision: usize, scale: usize) -> Result bool { } } +fn is_avro_null(avro_schema: &AvroSchema) -> bool { + matches!(avro_schema, AvroSchema::Null) +} + +fn unwrap_or_unknown(field_type: Option) -> Type { + field_type.unwrap_or(Type::Primitive(PrimitiveType::Unknown)) +} + /// Post order avro schema visitor. pub(crate) trait AvroSchemaVisitor { type T; @@ -367,9 +422,8 @@ pub(crate) fn visit(schema: &AvroSchema, visitor: &mut V) let value = visit(&record_schema.fields[1].schema, visitor)?; return visitor.map_array(record_schema, key, value); } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Can't convert avro map schema, item is not a record.", + return Err(invalid_data!( + "Can't convert avro map schema, item is not a record." )); } } else { @@ -403,25 +457,16 @@ impl AvroSchemaToSchema { ) -> Result { attributes .get(name) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro array schema, missing element id.", - ) - })? + .ok_or_else(|| invalid_data!("Can't convert avro array schema, missing element id."))? .as_i64() .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro array schema, element id is not a valid i64 number.", + invalid_data!( + "Can't convert avro array schema, element id is not a valid i64 number." ) })? .try_into() .map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro array schema, element id is not a valid i32.", - ) + invalid_data!("Can't convert avro array schema, element id is not a valid i32.") }) } } @@ -440,10 +485,10 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { let field_id = Self::get_element_id_from_attributes(&avro_field.custom_attributes, FIELD_ID_PROP)?; - let optional = is_avro_optional(&avro_field.schema); + let optional = is_avro_optional(&avro_field.schema) || is_avro_null(&avro_field.schema); - let mut field = - NestedField::new(field_id, &avro_field.name, field_type.unwrap(), !optional); + let field_type = unwrap_or_unknown(field_type); + let mut field = NestedField::new(field_id, &avro_field.name, field_type, !optional); if let Some(doc) = &avro_field.doc { field = field.with_doc(doc); @@ -475,7 +520,7 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { } if options.len() == 1 { - Ok(Some(options.remove(0).unwrap())) + Ok(Some(unwrap_or_unknown(options.remove(0)))) } else { Ok(Some(options.remove(1).unwrap())) } @@ -483,10 +528,11 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { fn array(&mut self, array: &ArraySchema, item: Option) -> Result { let element_field_id = Self::get_element_id_from_attributes(&array.attributes, ELEMENT_ID)?; + let item = unwrap_or_unknown(item); let element_field = NestedField::list_element( element_field_id, - item.unwrap(), - !is_avro_optional(&array.items), + item, + !is_avro_optional(&array.items) && !is_avro_null(&array.items), ) .into(); Ok(Some(Type::List(ListType { element_field }))) @@ -497,10 +543,11 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { let key_field = NestedField::map_key_element(key_field_id, Type::Primitive(PrimitiveType::String)); let value_field_id = Self::get_element_id_from_attributes(&map.attributes, VALUE_ID)?; + let value = unwrap_or_unknown(value); let value_field = NestedField::map_value_element( value_field_id, - value.unwrap(), - !is_avro_optional(&map.types), + value, + !is_avro_optional(&map.types) && !is_avro_null(&map.types), ); Ok(Some(Type::Map(MapType { key_field: key_field.into(), @@ -522,7 +569,7 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { AvroSchema::Long => Type::Primitive(PrimitiveType::Long), AvroSchema::Float => Type::Primitive(PrimitiveType::Float), AvroSchema::Double => Type::Primitive(PrimitiveType::Double), - AvroSchema::Uuid => Type::Primitive(PrimitiveType::Uuid), + AvroSchema::Uuid(_) => Type::Primitive(PrimitiveType::Uuid), AvroSchema::String | AvroSchema::Enum(_) => Type::Primitive(PrimitiveType::String), AvroSchema::Fixed(fixed) => Type::Primitive(PrimitiveType::Fixed(fixed.size as u64)), AvroSchema::Bytes => Type::Primitive(PrimitiveType::Binary), @@ -544,18 +591,9 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { key: Option, value: Option, ) -> Result { - let key = key.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro map schema, missing key schema.", - ) - })?; - let value = value.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't convert avro map schema, missing value schema.", - ) - })?; + let key = + key.ok_or_else(|| invalid_data!("Can't convert avro map schema, missing key schema."))?; + let value = unwrap_or_unknown(value); let key_id = Self::get_element_id_from_attributes( &array.fields[0].custom_attributes, FIELD_ID_PROP, @@ -568,7 +606,7 @@ impl AvroSchemaVisitor for AvroSchemaToSchema { let value_field = NestedField::map_value_element( value_id, value, - !is_avro_optional(&array.fields[1].schema), + !is_avro_optional(&array.fields[1].schema) && !is_avro_null(&array.fields[1].schema), ); Ok(Some(Type::Map(MapType { key_field: key_field.into(), @@ -597,9 +635,8 @@ pub(crate) fn avro_schema_to_schema(avro_schema: &AvroSchema) -> Result )) } } else { - Err(Error::new( - ErrorKind::DataInvalid, - "Can't convert non record avro schema to iceberg schema: {avro_schema}", + Err(invalid_data!( + "Can't convert non record avro schema to iceberg schema: {avro_schema}" )) } } @@ -610,11 +647,13 @@ mod tests { use std::sync::Arc; use apache_avro::Schema as AvroSchema; - use apache_avro::schema::{Namespace, UnionSchema}; + use apache_avro::schema::UnionSchema; use super::*; use crate::avro::schema::AvroSchemaToSchema; - use crate::spec::{ListType, MapType, NestedField, PrimitiveType, Schema, StructType, Type}; + use crate::spec::{ + ListType, MapType, NestedField, PrimitiveType, Schema, StructType, Type, VariantType, + }; fn read_test_data_file_to_avro_schema(filename: &str) -> AvroSchema { let input = read_to_string(format!( @@ -637,11 +676,9 @@ mod tests { assert_eq!(iceberg_schema, converted_iceberg_schema); // 2. iceberg to avro - let converted_avro_schema = schema_to_avro_schema( - avro_schema.name().unwrap().fullname(Namespace::None), - &iceberg_schema, - ) - .unwrap(); + let converted_avro_schema = + schema_to_avro_schema(avro_schema.name().unwrap().fullname(None), &iceberg_schema) + .unwrap(); assert_eq!(avro_schema, converted_avro_schema); // 3.iceberg to avro to iceberg back @@ -650,74 +687,106 @@ mod tests { assert_eq!(iceberg_schema, converted_avro_converted_iceberg_schema); } + #[test] + fn test_unknown_type_schema_conversion() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "empty", PrimitiveType::Unknown).into(), + NestedField::required( + 2, + "unknowns", + Type::List(ListType::new( + NestedField::list_element(3, PrimitiveType::Unknown, false).into(), + )), + ) + .into(), + NestedField::required( + 4, + "unknown_values", + Type::Map(MapType::optional( + 5, + PrimitiveType::String, + 6, + PrimitiveType::Unknown, + )), + ) + .into(), + ]) + .build() + .unwrap(); + + let avro_schema = schema_to_avro_schema("table", &schema).unwrap(); + let AvroSchema::Record(record) = &avro_schema else { + panic!("expected avro record schema"); + }; + assert!(is_avro_null(&record.fields[0].schema)); + assert_eq!(record.fields[0].default, Some(Value::Null)); + let AvroSchema::Array(array) = &record.fields[1].schema else { + panic!("expected Avro array schema"); + }; + assert!(is_avro_null(&array.items)); + let AvroSchema::Map(map) = &record.fields[2].schema else { + panic!("expected Avro map schema"); + }; + assert!(is_avro_null(&map.types)); + + assert_eq!(schema, avro_schema_to_schema(&avro_schema).unwrap()); + } + #[test] fn test_manifest_file_v1_schema() { let fields = vec![ - NestedField::required(500, "manifest_path", PrimitiveType::String.into()) + NestedField::required(500, "manifest_path", PrimitiveType::String) .with_doc("Location URI with FS scheme") .into(), - NestedField::required(501, "manifest_length", PrimitiveType::Long.into()) + NestedField::required(501, "manifest_length", PrimitiveType::Long) .with_doc("Total file size in bytes") .into(), - NestedField::required(502, "partition_spec_id", PrimitiveType::Int.into()) + NestedField::required(502, "partition_spec_id", PrimitiveType::Int) .with_doc("Spec ID used to write") .into(), - NestedField::optional(503, "added_snapshot_id", PrimitiveType::Long.into()) + NestedField::optional(503, "added_snapshot_id", PrimitiveType::Long) .with_doc("Snapshot ID that added the manifest") .into(), - NestedField::optional(504, "added_data_files_count", PrimitiveType::Int.into()) + NestedField::optional(504, "added_data_files_count", PrimitiveType::Int) .with_doc("Added entry count") .into(), - NestedField::optional(505, "existing_data_files_count", PrimitiveType::Int.into()) + NestedField::optional(505, "existing_data_files_count", PrimitiveType::Int) .with_doc("Existing entry count") .into(), - NestedField::optional(506, "deleted_data_files_count", PrimitiveType::Int.into()) + NestedField::optional(506, "deleted_data_files_count", PrimitiveType::Int) .with_doc("Deleted entry count") .into(), - NestedField::optional( - 507, - "partitions", - ListType { - element_field: NestedField::list_element( - 508, - StructType::new(vec![ - NestedField::required( - 509, - "contains_null", - PrimitiveType::Boolean.into(), - ) + NestedField::optional(507, "partitions", ListType { + element_field: NestedField::list_element( + 508, + StructType::new(vec![ + NestedField::required(509, "contains_null", PrimitiveType::Boolean) .with_doc("True if any file has a null partition value") .into(), - NestedField::optional( - 518, - "contains_nan", - PrimitiveType::Boolean.into(), - ) + NestedField::optional(518, "contains_nan", PrimitiveType::Boolean) .with_doc("True if any file has a nan partition value") .into(), - NestedField::optional(510, "lower_bound", PrimitiveType::Binary.into()) - .with_doc("Partition lower bound for all files") - .into(), - NestedField::optional(511, "upper_bound", PrimitiveType::Binary.into()) - .with_doc("Partition upper bound for all files") - .into(), - ]) - .into(), - true, - ) - .into(), - } + NestedField::optional(510, "lower_bound", PrimitiveType::Binary) + .with_doc("Partition lower bound for all files") + .into(), + NestedField::optional(511, "upper_bound", PrimitiveType::Binary) + .with_doc("Partition upper bound for all files") + .into(), + ]), + true, + ) .into(), - ) + }) .with_doc("Summary for each partition") .into(), - NestedField::optional(512, "added_rows_count", PrimitiveType::Long.into()) + NestedField::optional(512, "added_rows_count", PrimitiveType::Long) .with_doc("Added rows count") .into(), - NestedField::optional(513, "existing_rows_count", PrimitiveType::Long.into()) + NestedField::optional(513, "existing_rows_count", PrimitiveType::Long) .with_doc("Existing rows count") .into(), - NestedField::optional(514, "deleted_rows_count", PrimitiveType::Long.into()) + NestedField::optional(514, "deleted_rows_count", PrimitiveType::Long) .with_doc("Deleted rows count") .into(), ]; @@ -757,19 +826,10 @@ mod tests { let iceberg_schema = { Schema::builder() .with_fields(vec![ - NestedField::required( - 100, - "array_with_string", - ListType { - element_field: NestedField::list_element( - 101, - PrimitiveType::String.into(), - true, - ) + NestedField::required(100, "array_with_string", ListType { + element_field: NestedField::list_element(101, PrimitiveType::String, true) .into(), - } - .into(), - ) + }) .into(), ]) .build() @@ -808,19 +868,10 @@ mod tests { let iceberg_schema = { Schema::builder() .with_fields(vec![ - NestedField::required( - 100, - "array_with_string", - ListType { - element_field: NestedField::list_element( - 101, - PrimitiveType::String.into(), - true, - ) + NestedField::required(100, "array_with_string", ListType { + element_field: NestedField::list_element(101, PrimitiveType::String, true) .into(), - } - .into(), - ) + }) .into(), ]) .build() @@ -873,33 +924,19 @@ mod tests { let iceberg_schema = { Schema::builder() .with_fields(vec![ - NestedField::required( - 100, - "array_with_record", - ListType { - element_field: NestedField::list_element( - 101, - StructType::new(vec![ - NestedField::required( - 102, - "contains_null", - PrimitiveType::Boolean.into(), - ) + NestedField::required(100, "array_with_record", ListType { + element_field: NestedField::list_element( + 101, + StructType::new(vec![ + NestedField::required(102, "contains_null", PrimitiveType::Boolean) .into(), - NestedField::optional( - 103, - "contains_nan", - PrimitiveType::Boolean.into(), - ) + NestedField::optional(103, "contains_nan", PrimitiveType::Boolean) .into(), - ]) - .into(), - true, - ) - .into(), - } + ]), + true, + ) .into(), - ) + }) .into(), ]) .build() @@ -910,6 +947,77 @@ mod tests { } #[test] + fn test_schema_to_avro_schema_defines_named_types_before_references() { + let decimal = Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }); + let iceberg_schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "a", decimal.clone()).into(), + NestedField::required( + 2, + "list", + Type::List(ListType::new( + NestedField::list_element(3, decimal.clone(), false).into(), + )), + ) + .into(), + NestedField::required( + 4, + "map", + Type::Map(MapType { + key_field: NestedField::map_key_element(5, PrimitiveType::Int).into(), + value_field: NestedField::map_value_element(6, decimal, false).into(), + }), + ) + .into(), + ]) + .build() + .unwrap(); + + let json = schema_to_avro_schema("r", &iceberg_schema) + .unwrap() + .canonical_form(); + + // Parsing fails on a reference that precedes its definition. + AvroSchema::parse_str(&json).unwrap(); + assert_eq!(json.matches(r#""name":"decimal_10_2""#).count(), 1); + } + + #[test] + fn test_schema_with_array_map_without_parsing() { + // Unlike `test_schema_with_array_map`, this converts an Avro schema built + // in code, which keeps `logicalType: map` in apache-avro 0.22. + let map = |key_id, value_id, value_required| { + Type::Map(MapType { + key_field: NestedField::map_key_element(key_id, PrimitiveType::Boolean).into(), + value_field: NestedField::map_value_element( + value_id, + PrimitiveType::Boolean, + value_required, + ) + .into(), + }) + }; + let iceberg_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(100, "optional_values", map(102, 103, false)).into(), + NestedField::optional(104, "required_values", map(105, 106, true)).into(), + ]) + .build() + .unwrap(); + + let avro_schema = schema_to_avro_schema("avro_schema", &iceberg_schema).unwrap(); + + assert_eq!(avro_schema_to_schema(&avro_schema).unwrap(), iceberg_schema); + } + + #[test] + #[ignore = "apache-avro 0.22 drops `logicalType: map` when parsing schema JSON \ + (https://github.com/apache/avro-rs/issues/654), so avro_schema_to_schema \ + reads a parsed map array as a list. Fixed by \ + https://github.com/apache/avro-rs/pull/656, which is not yet released."] fn test_schema_with_array_map() { let avro_schema = { AvroSchema::parse_str( @@ -991,14 +1099,11 @@ mod tests { 100, "optional", Type::Map(MapType { - key_field: NestedField::map_key_element( - 102, - PrimitiveType::Boolean.into(), - ) - .into(), + key_field: NestedField::map_key_element(102, PrimitiveType::Boolean) + .into(), value_field: NestedField::map_value_element( 103, - PrimitiveType::Boolean.into(), + PrimitiveType::Boolean, false, ) .into(), @@ -1008,14 +1113,11 @@ mod tests { 104, "required", Type::Map(MapType { - key_field: NestedField::map_key_element( - 105, - PrimitiveType::Boolean.into(), - ) - .into(), + key_field: NestedField::map_key_element(105, PrimitiveType::Boolean) + .into(), value_field: NestedField::map_value_element( 106, - PrimitiveType::Boolean.into(), + PrimitiveType::Boolean, true, ) .into(), @@ -1025,14 +1127,11 @@ mod tests { 107, "string_map", Type::Map(MapType { - key_field: NestedField::map_key_element( - 108, - PrimitiveType::String.into(), - ) - .into(), + key_field: NestedField::map_key_element(108, PrimitiveType::String) + .into(), value_field: NestedField::map_value_element( 109, - PrimitiveType::Long.into(), + PrimitiveType::Long, false, ) .into(), @@ -1098,9 +1197,8 @@ mod tests { let mut converter = AvroSchemaToSchema; let iceberg_type = Type::Map(MapType { - key_field: NestedField::map_key_element(101, PrimitiveType::String.into()).into(), - value_field: NestedField::map_value_element(102, PrimitiveType::Long.into(), false) - .into(), + key_field: NestedField::map_key_element(101, PrimitiveType::String).into(), + value_field: NestedField::map_value_element(102, PrimitiveType::Long, false).into(), }); assert_eq!( @@ -1137,7 +1235,14 @@ mod tests { fn test_unknown_primitive() { let mut converter = AvroSchemaToSchema; - assert!(converter.primitive(&AvroSchema::Duration).is_err()); + let duration = AvroSchema::Duration(FixedSchema { + name: Name::new("duration").unwrap(), + aliases: None, + doc: None, + size: 12, + attributes: Default::default(), + }); + assert!(converter.primitive(&duration).is_err()); } #[test] @@ -1163,6 +1268,23 @@ mod tests { assert!(avro_schema_to_schema(&avro_schema).is_err()); } + #[test] + fn test_non_record_schema_error_interpolates_schema() { + let err = avro_schema_to_schema(&AvroSchema::String).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + !err.message().contains("{avro_schema}"), + "message should interpolate the schema, got: {}", + err.message() + ); + assert!( + err.message().ends_with(&AvroSchema::String.to_string()), + "message should end with the rendered schema, got: {}", + err.message() + ); + } + #[test] fn test_decimal_type() { let avro_schema = { @@ -1212,4 +1334,38 @@ mod tests { converter.primitive(&avro_schema).unwrap().unwrap() ); } + + #[test] + fn test_variant_to_avro_schema_is_unsupported() { + // Converting a variant to an Avro schema is not supported yet; it must error rather + // than emit an incorrect schema. + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(); + let err = schema_to_avro_schema("t", &schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + } + + #[test] + fn test_geospatial_to_avro_bytes_schema() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(Default::default())), + ) + .into(), + ]) + .build() + .unwrap(); + + let AvroSchema::Record(record) = schema_to_avro_schema("t", &schema).unwrap() else { + panic!("expected Avro record schema"); + }; + assert_eq!(record.fields[0].schema, AvroSchema::Bytes); + } } diff --git a/crates/iceberg/src/catalog/memory/catalog.rs b/crates/iceberg/src/catalog/memory/catalog.rs index 6ce3111419..b73a1d468e 100644 --- a/crates/iceberg/src/catalog/memory/catalog.rs +++ b/crates/iceberg/src/catalog/memory/catalog.rs @@ -18,18 +18,24 @@ //! This module contains memory catalog implementation. use std::collections::HashMap; +use std::str::FromStr; +use std::sync::Arc; use async_trait::async_trait; use futures::lock::{Mutex, MutexGuard}; +use iceberg_property_macro::Properties; use itertools::Itertools; use super::namespace_state::NamespaceState; -use crate::io::FileIO; +use crate::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use crate::error::invalid_data; +use crate::io::{FileIO, FileIOBuilder, MemoryStorageFactory, StorageFactory}; +use crate::runtime::Runtime; use crate::spec::{TableMetadata, TableMetadataBuilder}; use crate::table::Table; use crate::{ - Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, - TableCommit, TableCreation, TableIdent, + Catalog, CatalogBuilder, MetadataLocation, Namespace, NamespaceIdent, Result, TableCommit, + TableCreation, TableIdent, delete_removed_metadata_files, }; /// Memory catalog warehouse location @@ -39,86 +45,107 @@ pub const MEMORY_CATALOG_WAREHOUSE: &str = "warehouse"; const LOCATION: &str = "location"; /// Builder for [`MemoryCatalog`]. -#[derive(Debug)] -pub struct MemoryCatalogBuilder(MemoryCatalogConfig); - -impl Default for MemoryCatalogBuilder { - fn default() -> Self { - Self(MemoryCatalogConfig { - name: None, - warehouse: "".to_string(), - props: HashMap::new(), - }) - } +#[derive(Debug, Default)] +pub struct MemoryCatalogBuilder { + storage_factory: Option>, + kms_client_factory: Option>, + runtime: Option, } impl CatalogBuilder for MemoryCatalogBuilder { type C = MemoryCatalog; + fn with_storage_factory(mut self, storage_factory: Arc) -> Self { + self.storage_factory = Some(storage_factory); + self + } + + fn with_kms_client_factory(mut self, kms_client_factory: Arc) -> Self { + self.kms_client_factory = Some(kms_client_factory); + self + } + + fn with_runtime(mut self, runtime: Runtime) -> Self { + self.runtime = Some(runtime); + self + } + fn load( - mut self, + self, name: impl Into, props: HashMap, ) -> impl Future> + Send { - self.0.name = Some(name.into()); - - if props.contains_key(MEMORY_CATALOG_WAREHOUSE) { - self.0.warehouse = props - .get(MEMORY_CATALOG_WAREHOUSE) - .cloned() - .unwrap_or_default() - } + let name = name.into(); - // Collect other remaining properties - self.0.props = props - .into_iter() - .filter(|(k, _)| k != MEMORY_CATALOG_WAREHOUSE) - .collect(); - - let result = { - if self.0.name.is_none() { - Err(Error::new( - ErrorKind::DataInvalid, - "Catalog name is required", - )) - } else if self.0.warehouse.is_empty() { - Err(Error::new( - ErrorKind::DataInvalid, - "Catalog warehouse is required", - )) - } else { - MemoryCatalog::new(self.0) + async move { + let catalog_properties = MemoryCatalogProperties::from_properties(&props)?; + if catalog_properties.warehouse.is_empty() { + return Err(invalid_data!("Catalog warehouse is required")); } - }; - std::future::ready(result) + let runtime = self.runtime.unwrap_or_else(Runtime::current); + let kms_client = match self.kms_client_factory { + Some(factory) => Some(factory.create_kms_client(&props).await?), + None => None, + }; + MemoryCatalog::new( + name, + catalog_properties, + props, + self.storage_factory, + runtime, + kms_client, + ) + } } } -#[derive(Clone, Debug)] -pub(crate) struct MemoryCatalogConfig { - name: Option, +/// Memory catalog properties parsed from a catalog property map. +#[derive(Debug, Properties)] +pub(crate) struct MemoryCatalogProperties { + #[property(key = MEMORY_CATALOG_WAREHOUSE, default = "")] warehouse: String, - props: HashMap, } /// Memory catalog implementation. -#[derive(Debug)] pub struct MemoryCatalog { + name: String, + properties: MemoryCatalogProperties, root_namespace_state: Mutex, file_io: FileIO, - warehouse_location: String, + runtime: Runtime, + kms_client: Option>, +} + +impl std::fmt::Debug for MemoryCatalog { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryCatalog") + .field("name", &self.name) + .field("properties", &self.properties) + .finish_non_exhaustive() + } } impl MemoryCatalog { /// Creates a memory catalog. - fn new(config: MemoryCatalogConfig) -> Result { + fn new( + name: String, + properties: MemoryCatalogProperties, + props: HashMap, + storage_factory: Option>, + runtime: Runtime, + kms_client: Option>, + ) -> Result { + // Use provided factory or default to MemoryStorageFactory + let factory = storage_factory.unwrap_or_else(|| Arc::new(MemoryStorageFactory)); + Ok(Self { + name, + properties, + file_io: FileIOBuilder::new(factory).with_props(props).build(), root_namespace_state: Mutex::new(NamespaceState::default()), - file_io: FileIO::from_path(&config.warehouse)? - .with_props(config.props) - .build()?, - warehouse_location: config.warehouse, + runtime, + kms_client, }) } @@ -131,12 +158,16 @@ impl MemoryCatalog { let metadata_location = root_namespace_state.get_existing_table_location(table_ident)?; let metadata = TableMetadata::read_from(&self.file_io, metadata_location).await?; - Table::builder() + let mut builder = Table::builder() .identifier(table_ident.clone()) .metadata(metadata) .metadata_location(metadata_location.to_string()) .file_io(self.file_io.clone()) - .build() + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + builder.build() } } @@ -163,8 +194,12 @@ impl Catalog for MemoryCatalog { let namespaces = root_namespace_state .list_namespaces_under(parent_namespace_ident)? .into_iter() - .map(|name| NamespaceIdent::new(name.to_string())) - .collect_vec(); + .map(|name| { + let mut names = parent_namespace_ident.iter().cloned().collect::>(); + names.push(name.to_string()); + NamespaceIdent::from_vec(names) + }) + .collect::>>()?; Ok(namespaces) } @@ -252,41 +287,46 @@ impl Catalog for MemoryCatalog { let table_name = table_creation.name.clone(); let table_ident = TableIdent::new(namespace_ident.clone(), table_name); - let (table_creation, location) = match table_creation.location.clone() { - Some(location) => (table_creation, location), - None => { - let namespace_properties = root_namespace_state.get_properties(namespace_ident)?; - let location_prefix = match namespace_properties.get(LOCATION) { - Some(namespace_location) => namespace_location.clone(), - None => format!("{}/{}", self.warehouse_location, namespace_ident.join("/")), - }; - - let location = format!("{}/{}", location_prefix, table_ident.name()); - - let new_table_creation = TableCreation { - location: Some(location.clone()), - ..table_creation - }; - - (new_table_creation, location) + let table_creation = if table_creation.location.is_some() { + table_creation + } else { + let namespace_properties = root_namespace_state.get_properties(namespace_ident)?; + let location_prefix = match namespace_properties.get(LOCATION) { + Some(namespace_location) => namespace_location.clone(), + None => format!( + "{}/{}", + self.properties.warehouse, + namespace_ident.join("/") + ), + }; + + let location = format!("{}/{}", location_prefix, table_ident.name()); + + TableCreation { + location: Some(location), + ..table_creation } }; let metadata = TableMetadataBuilder::from_table_creation(table_creation)? .build()? .metadata; - let metadata_location = MetadataLocation::new_with_table_location(location).to_string(); + let metadata_location = MetadataLocation::try_new_with_metadata(&metadata)?; metadata.write_to(&self.file_io, &metadata_location).await?; - root_namespace_state.insert_new_table(&table_ident, metadata_location.clone())?; + root_namespace_state.insert_new_table(&table_ident, metadata_location.to_string())?; - Table::builder() + let mut builder = Table::builder() .file_io(self.file_io.clone()) - .metadata_location(metadata_location) + .metadata_location(metadata_location.to_string()) .metadata(metadata) .identifier(table_ident) - .build() + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + builder.build() } /// Load table from the catalog. @@ -301,8 +341,14 @@ impl Catalog for MemoryCatalog { async fn drop_table(&self, table_ident: &TableIdent) -> Result<()> { let mut root_namespace_state = self.root_namespace_state.lock().await; - let metadata_location = root_namespace_state.remove_existing_table(table_ident)?; - self.file_io.delete(&metadata_location).await + root_namespace_state.remove_existing_table(table_ident)?; + Ok(()) + } + + async fn purge_table(&self, table_ident: &TableIdent) -> Result<()> { + let table_info = self.load_table(table_ident).await?; + self.drop_table(table_ident).await?; + crate::catalog::utils::drop_table_data(&table_info).await } /// Check if a table exists in the catalog. @@ -341,12 +387,16 @@ impl Catalog for MemoryCatalog { let metadata = TableMetadata::read_from(&self.file_io, &metadata_location).await?; - Table::builder() + let mut builder = Table::builder() .file_io(self.file_io.clone()) .metadata_location(metadata_location) .metadata(metadata) .identifier(table_ident.clone()) - .build() + .runtime(self.runtime.clone()); + if let Some(kms_client) = self.kms_client.clone() { + builder = builder.kms_client(kms_client); + } + builder.build() } /// Update a table in the catalog. @@ -356,28 +406,36 @@ impl Catalog for MemoryCatalog { let current_table = self .load_table_from_locked_state(commit.identifier(), &root_namespace_state) .await?; + let base_metadata = current_table.metadata_ref(); // Apply TableCommit to get staged table let staged_table = commit.apply(current_table)?; // Write table metadata to the new location + let metadata_location = + MetadataLocation::from_str(staged_table.metadata_location_result()?)?; staged_table .metadata() - .write_to( - staged_table.file_io(), - staged_table.metadata_location_result()?, - ) + .write_to(staged_table.file_io(), &metadata_location) .await?; // Flip the pointer to reference the new metadata file. let updated_table = root_namespace_state.commit_table_update(staged_table)?; + drop(root_namespace_state); + + delete_removed_metadata_files( + updated_table.file_io(), + &base_metadata, + updated_table.metadata(), + ) + .await; Ok(updated_table) } } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::collections::HashSet; use std::hash::Hash; use std::iter::FromIterator; @@ -387,8 +445,13 @@ mod tests { use tempfile::TempDir; use super::*; - use crate::io::FileIOBuilder; - use crate::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, Type}; + use crate::ErrorKind; + use crate::encryption::kms::MemoryKmsClientFactory; + use crate::io::{FileIO, LocalFsStorageFactory}; + use crate::spec::{ + NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, TableProperties, Type, + }; + use crate::test_utils::test_runtime; use crate::transaction::{ApplyTransactionAction, Transaction}; fn temp_path() -> String { @@ -396,7 +459,58 @@ mod tests { temp_dir.path().to_str().unwrap().to_string() } - async fn new_memory_catalog() -> impl Catalog { + #[test] + fn test_catalog_properties() { + let properties = MemoryCatalogProperties::from_properties(&HashMap::from([( + MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )])) + .unwrap(); + + assert_eq!(properties.warehouse, "memory:///warehouse"); + } + + #[test] + fn test_catalog_properties_warehouse_defaults_to_empty() { + let missing = MemoryCatalogProperties::from_properties(&HashMap::new()).unwrap(); + let explicitly_empty = MemoryCatalogProperties::from_properties(&HashMap::from([( + MEMORY_CATALOG_WAREHOUSE.to_string(), + String::new(), + )])) + .unwrap(); + + assert_eq!(missing.warehouse, ""); + assert_eq!(explicitly_empty.warehouse, ""); + } + + #[tokio::test] + async fn test_catalog_forwards_properties_to_file_io() { + let catalog = MemoryCatalogBuilder::default() + .load( + "memory", + HashMap::from([ + ( + MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + ), + ("custom.property".to_string(), "value".to_string()), + ]), + ) + .await + .unwrap(); + let file_io_props = catalog.file_io.config().props(); + + assert_eq!( + file_io_props.get("custom.property"), + Some(&"value".to_string()) + ); + assert_eq!( + file_io_props.get(MEMORY_CATALOG_WAREHOUSE), + Some(&"memory:///warehouse".to_string()) + ); + } + + pub(crate) async fn new_memory_catalog() -> impl Catalog { let warehouse_location = temp_path(); MemoryCatalogBuilder::default() .load( @@ -506,9 +620,7 @@ mod tests { let regex = Regex::new(regex_str).unwrap(); assert!( regex.is_match(&actual), - "Expected metadata location to match regex, but got location: {} and regex: {}", - actual, - regex + "Expected metadata location to match regex, but got location: {actual} and regex: {regex}" ) } @@ -601,7 +713,7 @@ mod tests { .list_namespaces(Some(&namespace_ident_1)) .await .unwrap(), - vec![NamespaceIdent::new("b".into())] + vec![namespace_ident_2] ); } @@ -630,9 +742,9 @@ mod tests { .unwrap() ), to_set(vec![ - NamespaceIdent::new("a".into()), - NamespaceIdent::new("b".into()), - NamespaceIdent::new("c".into()), + namespace_ident_2, + namespace_ident_3, + namespace_ident_4, ]) ); } @@ -888,10 +1000,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "NamespaceNotFound => No such namespace: {:?}", - non_existent_namespace_ident - ) + format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}") ) } @@ -975,10 +1084,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "NamespaceNotFound => No such namespace: {:?}", - non_existent_namespace_ident - ) + format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}") ) } @@ -1058,10 +1164,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "NamespaceNotFound => No such namespace: {:?}", - non_existent_namespace_ident - ) + format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}") ) } @@ -1078,10 +1181,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "NamespaceNotFound => No such namespace: {:?}", - non_existent_namespace_ident - ) + format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}") ) } @@ -1168,10 +1268,8 @@ mod tests { let table_name = "tbl1"; let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - let expected_table_metadata_location_regex = format!( - "^{}/tbl1/metadata/00000-{}.metadata.json$", - namespace_location, UUID_REGEX_STR, - ); + let expected_table_metadata_location_regex = + format!("^{namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",); let table = catalog .create_table( @@ -1230,8 +1328,7 @@ mod tests { let expected_table_ident = TableIdent::new(nested_namespace_ident.clone(), table_name.into()); let expected_table_metadata_location_regex = format!( - "^{}/tbl1/metadata/00000-{}.metadata.json$", - nested_namespace_location, UUID_REGEX_STR, + "^{nested_namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$", ); let table = catalog @@ -1278,10 +1375,8 @@ mod tests { let table_name = "tbl1"; let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - let expected_table_metadata_location_regex = format!( - "^{}/a/tbl1/metadata/00000-{}.metadata.json$", - warehouse_location, UUID_REGEX_STR - ); + let expected_table_metadata_location_regex = + format!("^{warehouse_location}/a/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$"); let table = catalog .create_table( @@ -1335,8 +1430,7 @@ mod tests { let expected_table_ident = TableIdent::new(nested_namespace_ident.clone(), table_name.into()); let expected_table_metadata_location_regex = format!( - "^{}/a/b/tbl1/metadata/00000-{}.metadata.json$", - warehouse_location, UUID_REGEX_STR + "^{warehouse_location}/a/b/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$" ); let table = catalog @@ -1359,17 +1453,28 @@ mod tests { } #[tokio::test] - async fn test_create_table_throws_error_if_table_location_and_namespace_location_and_warehouse_location_are_missing() - { - let catalog = MemoryCatalogBuilder::default() + async fn test_load_throws_error_if_warehouse_is_missing() { + let error = MemoryCatalogBuilder::default() .load("memory", HashMap::from([])) - .await; + .await + .unwrap_err(); - assert!(catalog.is_err()); - assert_eq!( - catalog.unwrap_err().to_string(), - "DataInvalid => Catalog warehouse is required" - ); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert_eq!(error.message(), "Catalog warehouse is required"); + } + + #[tokio::test] + async fn test_load_throws_error_if_warehouse_is_empty() { + let error = MemoryCatalogBuilder::default() + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), String::new())]), + ) + .await + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert_eq!(error.message(), "Catalog warehouse is required"); } #[tokio::test] @@ -1499,10 +1604,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "NamespaceNotFound => No such namespace: {:?}", - non_existent_namespace_ident - ), + format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}"), ); } @@ -1549,10 +1651,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "NamespaceNotFound => No such namespace: {:?}", - non_existent_namespace_ident - ), + format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}"), ); } @@ -1570,10 +1669,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "TableNotFound => No such table: {:?}", - non_existent_table_ident - ), + format!("TableNotFound => No such table: {non_existent_table_ident:?}"), ); } @@ -1638,10 +1734,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "NamespaceNotFound => No such namespace: {:?}", - non_existent_namespace_ident - ), + format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}"), ); } @@ -1753,10 +1846,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "NamespaceNotFound => No such namespace: {:?}", - non_existent_src_namespace_ident - ), + format!("NamespaceNotFound => No such namespace: {non_existent_src_namespace_ident:?}"), ); } @@ -1777,10 +1867,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!( - "NamespaceNotFound => No such namespace: {:?}", - non_existent_dst_namespace_ident - ), + format!("NamespaceNotFound => No such namespace: {non_existent_dst_namespace_ident:?}"), ); } @@ -1798,7 +1885,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!("TableNotFound => No such table: {:?}", src_table_ident), + format!("TableNotFound => No such table: {src_table_ident:?}"), ); } @@ -1877,6 +1964,11 @@ mod tests { // Assert the table doesn't contain the update yet assert!(!table.metadata().properties().contains_key("key")); + // `last_updated_ms` is millisecond-resolution `chrono::Utc::now()`. + // Without this sleep, create and commit can land in the same + // millisecond and the `<` assertion below flakes. + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + // Update table metadata let tx = Transaction::new(&table); let updated_table = tx @@ -1895,7 +1987,6 @@ mod tests { assert_eq!(table.identifier(), updated_table.identifier()); assert_eq!(table.metadata().uuid(), updated_table.metadata().uuid()); - assert!(table.metadata().last_updated_ms() < updated_table.metadata().last_updated_ms()); assert_ne!(table.metadata_location(), updated_table.metadata_location()); assert!( @@ -1903,6 +1994,200 @@ mod tests { ); } + async fn create_table_with_properties( + catalog: &C, + properties: HashMap, + ) -> Table { + let namespace_ident = NamespaceIdent::new("abc".into()); + create_namespace(catalog, &namespace_ident).await; + catalog + .create_table( + &namespace_ident, + TableCreation::builder() + .name("test".into()) + .schema(simple_table_schema()) + .properties(properties) + .build(), + ) + .await + .unwrap() + } + + async fn commit_property_update(catalog: &C, table: &Table, value: &str) -> Table { + let tx = Transaction::new(table); + tx.update_table_properties() + .set("key".to_string(), value.to_string()) + .apply(tx) + .unwrap() + .commit(catalog) + .await + .unwrap() + } + + /// Commits `num_commits` property updates and returns the metadata location of every + /// table version, oldest first. + async fn commit_metadata_versions( + catalog: &C, + mut table: Table, + num_commits: usize, + ) -> Vec { + let mut locations = vec![table.metadata_location().unwrap().to_string()]; + for i in 0..num_commits { + table = commit_property_update(catalog, &table, &i.to_string()).await; + locations.push(table.metadata_location().unwrap().to_string()); + } + locations + } + + async fn existing_files(file_io: &FileIO, locations: &[String]) -> Vec { + let mut existing = Vec::new(); + for location in locations { + if file_io.exists(location).await.unwrap() { + existing.push(location.clone()); + } + } + existing + } + + #[tokio::test] + async fn test_update_table_deletes_metadata_files_removed_from_log_when_enabled() { + let catalog = new_memory_catalog().await; + let table = create_table_with_properties( + &catalog, + HashMap::from([ + ( + TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(), + "true".to_string(), + ), + ( + TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), + "1".to_string(), + ), + ]), + ) + .await; + let file_io = table.file_io().clone(); + + let locations = commit_metadata_versions(&catalog, table, 3).await; + + // The current metadata file and the one entry left in the metadata log remain. + assert_eq!( + existing_files(&file_io, &locations).await, + locations[2..].to_vec() + ); + } + + #[tokio::test] + async fn test_update_table_deletes_removed_metadata_files_on_commit_that_enables_cleanup() { + let catalog = new_memory_catalog().await; + let table = create_table_with_properties( + &catalog, + HashMap::from([( + TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), + "1".to_string(), + )]), + ) + .await; + let file_io = table.file_io().clone(); + let v0 = table.metadata_location().unwrap().to_string(); + let v1_table = commit_property_update(&catalog, &table, "1").await; + let v1 = v1_table.metadata_location().unwrap().to_string(); + let v2_table = commit_property_update(&catalog, &v1_table, "2").await; + let v2 = v2_table.metadata_location().unwrap().to_string(); + + let tx = Transaction::new(&v2_table); + let v3_table = tx + .update_table_properties() + .set( + TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(), + "true".to_string(), + ) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let v3 = v3_table.metadata_location().unwrap().to_string(); + + // v0 fell out of the log while cleanup was disabled, so only v1 is deleted. + assert_eq!( + existing_files(&file_io, &[v0.clone(), v1, v2.clone(), v3.clone()]).await, + vec![v0, v2, v3] + ); + } + + #[tokio::test] + async fn test_update_table_keeps_metadata_files_removed_from_log_when_not_enabled() { + for delete_after_commit in [None, Some("false")] { + let catalog = new_memory_catalog().await; + let mut properties = HashMap::from([( + TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), + "1".to_string(), + )]); + if let Some(value) = delete_after_commit { + properties.insert( + TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(), + value.to_string(), + ); + } + let table = create_table_with_properties(&catalog, properties).await; + let file_io = table.file_io().clone(); + + let locations = commit_metadata_versions(&catalog, table, 3).await; + + assert_eq!(existing_files(&file_io, &locations).await, locations); + } + } + + #[tokio::test] + async fn test_update_table_succeeds_when_deleting_removed_metadata_file_fails() { + let warehouse = TempDir::new().unwrap(); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([( + MEMORY_CATALOG_WAREHOUSE.to_string(), + warehouse.path().to_str().unwrap().to_string(), + )]), + ) + .await + .unwrap(); + let table = create_table_with_properties( + &catalog, + HashMap::from([ + ( + TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(), + "true".to_string(), + ), + ( + TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), + "1".to_string(), + ), + ]), + ) + .await; + let file_io = table.file_io().clone(); + let v0 = table.metadata_location().unwrap().to_string(); + let v1_table = commit_property_update(&catalog, &table, "1").await; + let v1 = v1_table.metadata_location().unwrap().to_string(); + + // A non-empty directory in place of v0 makes its deletion fail. + std::fs::remove_file(&v0).unwrap(); + std::fs::create_dir(&v0).unwrap(); + std::fs::write(format!("{v0}/blocker"), b"").unwrap(); + + let v2_table = commit_property_update(&catalog, &v1_table, "2").await; + let v2 = v2_table.metadata_location().unwrap().to_string(); + let v3_table = commit_property_update(&catalog, &v2_table, "3").await; + let v3 = v3_table.metadata_location().unwrap().to_string(); + + assert_eq!( + existing_files(&file_io, &[v0.clone(), v1, v2.clone(), v3.clone()]).await, + vec![v0, v2, v3] + ); + } + #[tokio::test] async fn test_update_table_fails_if_table_doesnt_exist() { let catalog = new_memory_catalog().await; @@ -1926,8 +2211,93 @@ mod tests { assert_eq!(err.kind(), ErrorKind::TableNotFound); } + /// Master key bytes used to generate the encrypted testdata fixtures. + /// See `testdata/manifests_lists/README.md`. + const FIXTURE_MASTER_KEY_ID: &str = "master-1"; + const FIXTURE_MASTER_KEY_BYTES: [u8; 16] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, + ]; + + /// Builds a `MemoryKmsClientFactory` seeded with the fixture master key. + fn fixture_kms_factory() -> MemoryKmsClientFactory { + use crate::encryption::SensitiveBytes; + + let factory = MemoryKmsClientFactory::new(); + factory + .add_master_key_bytes( + FIXTURE_MASTER_KEY_ID, + SensitiveBytes::new(FIXTURE_MASTER_KEY_BYTES), + ) + .unwrap(); + factory + } + + /// Loads the encrypted V3 metadata fixture and patches its snapshot's + /// manifest-list to point at the on-disk encrypted testdata file. + fn load_encrypted_fixture_metadata() -> TableMetadata { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + let metadata_json = std::fs::read_to_string(format!( + "{manifest_dir}/testdata/table_metadata/TableMetadataV3ValidEncryption.json" + )) + .unwrap(); + let mut metadata: TableMetadata = serde_json::from_str(&metadata_json).unwrap(); + + let manifest_list_path = + format!("{manifest_dir}/testdata/manifests_lists/manifest-list-v3-encrypted.avro"); + let snapshot = metadata.snapshots.get_mut(&1).unwrap(); + let mut patched = snapshot.as_ref().clone(); + patched.manifest_list = manifest_list_path; + *snapshot = Arc::new(patched); + + metadata + } + + #[tokio::test] + async fn catalog_kms_factory_client_reaches_table_encryption_manager() { + let warehouse = temp_path(); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .with_kms_client_factory(Arc::new(fixture_kms_factory())) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await + .unwrap(); + + let namespace_ident = NamespaceIdent::new("enc_ns".into()); + create_namespace(&catalog, &namespace_ident).await; + + let metadata = load_encrypted_fixture_metadata(); + let metadata_dir = TempDir::new().unwrap(); + let metadata_location = + format!("{}/v1.metadata.json", metadata_dir.path().to_str().unwrap()); + std::fs::write(&metadata_location, serde_json::to_vec(&metadata).unwrap()).unwrap(); + + let table_ident = TableIdent::new(namespace_ident, "enc".to_string()); + catalog + .register_table(&table_ident, metadata_location) + .await + .unwrap(); + + let table = catalog.load_table(&table_ident).await.unwrap(); + assert!( + table.encryption_manager().is_some(), + "factory-built KMS client should have reached the table's EncryptionManager" + ); + + let snapshot_ref = table.metadata().current_snapshot().unwrap(); + let manifest_list = table + .object_cache() + .get_manifest_list(snapshot_ref, &table.metadata_ref()) + .await + .unwrap(); + assert_eq!(manifest_list.entries().len(), 0); + } + fn build_table(ident: TableIdent) -> Table { - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let temp_dir = TempDir::new().unwrap(); let location = temp_dir.path().to_str().unwrap().to_string(); @@ -1947,6 +2317,7 @@ mod tests { .identifier(ident) .metadata(metadata) .file_io(file_io) + .runtime(test_runtime()) .build() .unwrap() } diff --git a/crates/iceberg/src/catalog/memory/namespace_state.rs b/crates/iceberg/src/catalog/memory/namespace_state.rs index 2fc481b767..d8361dd246 100644 --- a/crates/iceberg/src/catalog/memory/namespace_state.rs +++ b/crates/iceberg/src/catalog/memory/namespace_state.rs @@ -19,6 +19,7 @@ use std::collections::{HashMap, hash_map}; use itertools::Itertools; +use crate::error::invalid_data; use crate::table::Table; use crate::{Error, ErrorKind, NamespaceIdent, Result, TableIdent}; @@ -36,34 +37,28 @@ pub(crate) struct NamespaceState { fn no_such_namespace_err(namespace_ident: &NamespaceIdent) -> Result { Err(Error::new( ErrorKind::NamespaceNotFound, - format!("No such namespace: {:?}", namespace_ident), + format!("No such namespace: {namespace_ident:?}"), )) } fn no_such_table_err(table_ident: &TableIdent) -> Result { Err(Error::new( ErrorKind::TableNotFound, - format!("No such table: {:?}", table_ident), + format!("No such table: {table_ident:?}"), )) } fn namespace_already_exists_err(namespace_ident: &NamespaceIdent) -> Result { Err(Error::new( ErrorKind::NamespaceAlreadyExists, - format!( - "Cannot create namespace {:?}. Namespace already exists.", - namespace_ident - ), + format!("Cannot create namespace {namespace_ident:?}. Namespace already exists."), )) } fn table_already_exists_err(table_ident: &TableIdent) -> Result { Err(Error::new( ErrorKind::TableAlreadyExists, - format!( - "Cannot create table {:?}. Table already exists.", - table_ident - ), + format!("Cannot create table {table_ident:?}. Table already exists."), )) } @@ -117,10 +112,7 @@ impl NamespaceState { namespace_ident: &NamespaceIdent, ) -> Result<(&mut NamespaceState, String)> { match namespace_ident.split_last() { - None => Err(Error::new( - ErrorKind::DataInvalid, - "Namespace identifier can't be empty!", - )), + None => Err(invalid_data!("Namespace identifier can't be empty!")), Some((child_namespace_name, parent_name_parts)) => { let parent_namespace_state = if parent_name_parts.is_empty() { Ok(self) diff --git a/crates/iceberg/src/catalog/metadata_location.rs b/crates/iceberg/src/catalog/metadata_location.rs index 8cb5cb11d2..8240a326f3 100644 --- a/crates/iceberg/src/catalog/metadata_location.rs +++ b/crates/iceberg/src/catalog/metadata_location.rs @@ -20,72 +20,105 @@ use std::str::FromStr; use uuid::Uuid; +use crate::compression::CompressionCodec; +use crate::spec::TableMetadata; use crate::{Error, ErrorKind, Result}; -/// Helper for parsing a location of the format: `/metadata/-.metadata.json` +/// Default folder name for metadata files under the table location, used when the +/// `write.metadata.path` table property is not set. +pub(crate) const METADATA_FOLDER_NAME: &str = "metadata"; + +/// Helper for parsing a location of the format: `/-.metadata.json` +/// or with compression: `/-.gz.metadata.json` +/// +/// `` is set to the `write.metadata.path` table property and +/// it defaults to the `/metadata` when the property is not set. #[derive(Clone, Debug, PartialEq)] pub struct MetadataLocation { - table_location: String, + location: String, version: i32, id: Uuid, + compression_codec: CompressionCodec, } impl MetadataLocation { - /// Creates a completely new metadata location starting at version 0. - /// Only used for creating a new table. For updates, see `with_next_version`. - pub fn new_with_table_location(table_location: impl ToString) -> Self { - Self { - table_location: table_location.to_string(), + /// Creates a completely new metadata location starting at version 0, deriving the + /// metadata directory and compression settings from the table metadata. + /// Only used for creating a new table. For updates, see `with_next_version` and + /// `try_with_new_metadata`. + pub fn try_new_with_metadata(metadata: &TableMetadata) -> Result { + Ok(Self { + location: metadata.metadata_location()?, version: 0, id: Uuid::new_v4(), - } + compression_codec: metadata.metadata_compression_codec()?, + }) } /// Creates a new metadata location for an updated metadata file. + /// Increments the version number and generates a new UUID. pub fn with_next_version(&self) -> Self { Self { - table_location: self.table_location.clone(), + location: self.location.clone(), version: self.version + 1, id: Uuid::new_v4(), + compression_codec: self.compression_codec, } } - fn parse_metadata_path_prefix(path: &str) -> Result { - let prefix = path.strip_suffix("/metadata").ok_or(Error::new( + /// Updates the metadata location with the metadata directory + /// and compression settings from the new metadata. + pub fn try_with_new_metadata(&self, new_metadata: &TableMetadata) -> Result { + Ok(Self { + location: new_metadata.metadata_location()?, + version: self.version, + id: self.id, + compression_codec: new_metadata.metadata_compression_codec()?, + }) + } + + /// Returns the compression codec used for this metadata location. + pub fn compression_codec(&self) -> CompressionCodec { + self.compression_codec + } + + /// Parses a file name of the format `-.metadata.json` + /// or with compression: `-.gz.metadata.json`. + /// Parse errors for compression codec result in CompressionCodec::None. + fn parse_file_name(file_name: &str) -> Result<(i32, Uuid, CompressionCodec)> { + let stripped = file_name.strip_suffix(".metadata.json").ok_or(Error::new( ErrorKind::Unexpected, - format!( - "Metadata location not under \"/metadata\" subdirectory: {}", - path - ), + format!("Invalid metadata file ending: {file_name}"), ))?; - Ok(prefix.to_string()) - } + // Check for compression suffix (e.g., .gz) + let gzip_suffix = CompressionCodec::gzip_default().suffix()?; + let (stripped, compression_codec) = if let Some(s) = stripped.strip_suffix(gzip_suffix) { + (s, CompressionCodec::gzip_default()) + } else { + (stripped, CompressionCodec::None) + }; + + let (version, id) = stripped.split_once('-').ok_or(Error::new( + ErrorKind::Unexpected, + format!("Invalid metadata file name format: {file_name}"), + ))?; - /// Parses a file name of the format `-.metadata.json`. - fn parse_file_name(file_name: &str) -> Result<(i32, Uuid)> { - let (version, id) = file_name - .strip_suffix(".metadata.json") - .ok_or(Error::new( - ErrorKind::Unexpected, - format!("Invalid metadata file ending: {}", file_name), - ))? - .split_once('-') - .ok_or(Error::new( - ErrorKind::Unexpected, - format!("Invalid metadata file name format: {}", file_name), - ))?; - - Ok((version.parse::()?, Uuid::parse_str(id)?)) + Ok(( + version.parse::()?, + Uuid::parse_str(id)?, + compression_codec, + )) } } impl Display for MetadataLocation { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let suffix = self.compression_codec.suffix().unwrap_or(""); write!( f, - "{}/metadata/{:0>5}-{}.metadata.json", - self.table_location, self.version, self.id + "{}/{:0>5}-{}{}.metadata.json", + self.location, self.version, self.id, suffix ) } } @@ -94,29 +127,46 @@ impl FromStr for MetadataLocation { type Err = Error; fn from_str(s: &str) -> Result { - let (path, file_name) = s.rsplit_once('/').ok_or(Error::new( + let (location, file_name) = s.rsplit_once('/').ok_or(Error::new( ErrorKind::Unexpected, - format!("Invalid metadata location: {}", s), + format!("Invalid metadata location: {s}"), ))?; - let prefix = Self::parse_metadata_path_prefix(path)?; - let (version, id) = Self::parse_file_name(file_name)?; + let (version, id, compression_codec) = Self::parse_file_name(file_name)?; Ok(MetadataLocation { - table_location: prefix, + location: location.to_string(), version, id, + compression_codec, }) } } #[cfg(test)] mod test { + use std::collections::HashMap; use std::str::FromStr; use uuid::Uuid; - use crate::MetadataLocation; + use crate::compression::CompressionCodec; + use crate::spec::{Schema, TableMetadata, TableMetadataBuilder, TableProperties}; + use crate::{MetadataLocation, TableCreation}; + + fn create_test_metadata(properties: HashMap) -> TableMetadata { + let table_creation = TableCreation::builder() + .name("test_table".to_string()) + .location("/test/table".to_string()) + .schema(Schema::builder().build().unwrap()) + .properties(properties) + .build(); + TableMetadataBuilder::from_table_creation(table_creation) + .unwrap() + .build() + .unwrap() + .metadata + } #[test] fn test_metadata_location_from_string() { @@ -125,54 +175,70 @@ mod test { ( "/metadata/1234567-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", Ok(MetadataLocation { - table_location: "".to_string(), + location: "/metadata".to_string(), version: 1234567, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Some prefix ( "/abc/metadata/1234567-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", Ok(MetadataLocation { - table_location: "/abc".to_string(), + location: "/abc/metadata".to_string(), version: 1234567, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Longer prefix ( "/abc/def/metadata/1234567-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", Ok(MetadataLocation { - table_location: "/abc/def".to_string(), + location: "/abc/def/metadata".to_string(), version: 1234567, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Prefix with special characters ( "https://127.0.0.1/metadata/1234567-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", Ok(MetadataLocation { - table_location: "https://127.0.0.1".to_string(), + location: "https://127.0.0.1/metadata".to_string(), version: 1234567, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Another id ( "/abc/metadata/1234567-81056704-ce5b-41c4-bb83-eb6408081af6.metadata.json", Ok(MetadataLocation { - table_location: "/abc".to_string(), + location: "/abc/metadata".to_string(), version: 1234567, id: Uuid::from_str("81056704-ce5b-41c4-bb83-eb6408081af6").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Version 0 ( "/abc/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", Ok(MetadataLocation { - table_location: "/abc".to_string(), + location: "/abc/metadata".to_string(), version: 0, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, + }), + ), + // With gzip compression + ( + "/abc/metadata/1234567-2cd22b57-5127-4198-92ba-e4e67c79821b.gz.metadata.json", + Ok(MetadataLocation { + location: "/abc/metadata".to_string(), + version: 1234567, + id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::gzip_default(), }), ), // Negative version @@ -190,10 +256,15 @@ mod test { "/metadata/noversion-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", Err("".to_string()), ), - // No /metadata subdirectory + // Metadata dir does not need to be named "metadata" (e.g. a `write.metadata.path` location) ( "/wrongsubdir/1234567-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", - Err("".to_string()), + Ok(MetadataLocation { + location: "/wrongsubdir".to_string(), + version: 1234567, + id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, + }), ), // No .metadata.json suffix ( @@ -219,8 +290,9 @@ mod test { #[test] fn test_metadata_location_with_next_version() { + let metadata = create_test_metadata(HashMap::new()); let test_cases = vec![ - MetadataLocation::new_with_table_location("/abc"), + MetadataLocation::try_new_with_metadata(&metadata).unwrap(), MetadataLocation::from_str( "/abc/def/metadata/1234567-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", ) @@ -231,9 +303,155 @@ mod test { let next = MetadataLocation::from_str(&input.to_string()) .unwrap() .with_next_version(); - assert_eq!(next.table_location, input.table_location); + assert_eq!(next.location, input.location); assert_eq!(next.version, input.version + 1); assert_ne!(next.id, input.id); } } + + #[test] + fn test_with_next_version_preserves_compression() { + // Start from a parsed location with no compression + let location_none = MetadataLocation::from_str( + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", + ) + .unwrap(); + assert_eq!(location_none.compression_codec, CompressionCodec::None); + + let next_none = location_none.with_next_version(); + assert_eq!(next_none.compression_codec, CompressionCodec::None); + assert_eq!(next_none.version, 1); + + // Start from a parsed location with gzip compression + let location_gzip = MetadataLocation::from_str( + "/test/table/metadata/00005-81056704-ce5b-41c4-bb83-eb6408081af6.gz.metadata.json", + ) + .unwrap(); + assert_eq!( + location_gzip.compression_codec, + CompressionCodec::gzip_default() + ); + + let next_gzip = location_gzip.with_next_version(); + assert_eq!( + next_gzip.compression_codec, + CompressionCodec::gzip_default() + ); + assert_eq!(next_gzip.version, 6); + } + + #[test] + fn test_with_new_metadata_updates_compression() { + // Start from a parsed location with no compression + let location = MetadataLocation::from_str( + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", + ) + .unwrap(); + assert_eq!(location.compression_codec, CompressionCodec::None); + + // Update to gzip compression + let mut props_gzip = HashMap::new(); + props_gzip.insert( + "write.metadata.compression-codec".to_string(), + "gzip".to_string(), + ); + let metadata_gzip = create_test_metadata(props_gzip); + let updated_gzip = location.try_with_new_metadata(&metadata_gzip).unwrap(); + assert_eq!( + updated_gzip.compression_codec, + CompressionCodec::gzip_default() + ); + assert_eq!(updated_gzip.version, 0); + assert_eq!( + updated_gzip.to_string(), + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.gz.metadata.json" + ); + + // Update back to no compression + let props_none = HashMap::new(); + let metadata_none = create_test_metadata(props_none); + let updated_none = updated_gzip.try_with_new_metadata(&metadata_none).unwrap(); + assert_eq!(updated_none.compression_codec, CompressionCodec::None); + assert_eq!(updated_none.version, 0); + assert_eq!( + updated_none.to_string(), + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json" + ); + + // Test explicit "none" codec + let mut props_explicit_none = HashMap::new(); + props_explicit_none.insert( + "write.metadata.compression-codec".to_string(), + "none".to_string(), + ); + let metadata_explicit_none = create_test_metadata(props_explicit_none); + let updated_explicit = updated_gzip + .try_with_new_metadata(&metadata_explicit_none) + .unwrap(); + assert_eq!(updated_explicit.compression_codec, CompressionCodec::None); + assert_eq!( + updated_explicit.to_string(), + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json" + ); + } + + #[test] + fn test_with_new_metadata_re_derives_location() { + // Start from a parsed location under some existing metadata directory + let location = MetadataLocation::from_str( + "/old/table/metadata/00003-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", + ) + .unwrap(); + + // A new metadata (no `write.metadata.path`) re-derives to `/metadata`, + // preserving the version and id of the existing location + let relocated = create_test_metadata(HashMap::new()); + let updated = location.try_with_new_metadata(&relocated).unwrap(); + assert!( + updated.to_string().starts_with("/test/table/metadata/"), + "unexpected location: {updated}" + ); + assert_eq!(updated.version, location.version); + assert_eq!(updated.id, location.id); + + // A configured `write.metadata.path` is honored on updates too + let props = HashMap::from([( + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "s3://bucket/custom-meta".to_string(), + )]); + let with_meta_path = create_test_metadata(props); + let updated = location.try_with_new_metadata(&with_meta_path).unwrap(); + assert!( + updated.to_string().starts_with("s3://bucket/custom-meta/"), + "unexpected location: {updated}" + ); + } + + #[test] + fn test_new_with_metadata_honors_write_metadata_path() { + // Test metadata lives under `/metadata` by default + let default_meta = create_test_metadata(HashMap::new()); + let default_loc = MetadataLocation::try_new_with_metadata(&default_meta).unwrap(); + assert!( + default_loc + .to_string() + .starts_with("/test/table/metadata/00000-"), + "unexpected location: {default_loc}" + ); + + // Test a configured `write.metadata.path` is honored + let props = HashMap::from([( + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "s3://bucket/custom-meta".to_string(), + )]); + let custom_meta = create_test_metadata(props); + let custom_loc = MetadataLocation::try_new_with_metadata(&custom_meta).unwrap(); + assert!( + custom_loc + .to_string() + .starts_with("s3://bucket/custom-meta/00000-"), + "unexpected location: {custom_loc}" + ); + assert!(custom_loc.to_string().ends_with(".metadata.json")); + } } diff --git a/crates/iceberg/src/catalog/mod.rs b/crates/iceberg/src/catalog/mod.rs index 77b171ee30..88681040cd 100644 --- a/crates/iceberg/src/catalog/mod.rs +++ b/crates/iceberg/src/catalog/mod.rs @@ -19,6 +19,8 @@ pub mod memory; mod metadata_location; +mod session; +pub(crate) mod utils; use std::collections::HashMap; use std::fmt::{Debug, Display}; @@ -28,20 +30,25 @@ use std::ops::Deref; use std::str::FromStr; use std::sync::Arc; -use _serde::deserialize_snapshot; +use _serde::{deserialize_snapshot, serialize_snapshot}; use async_trait::async_trait; pub use memory::MemoryCatalog; pub use metadata_location::*; #[cfg(test)] use mockall::automock; use serde_derive::{Deserialize, Serialize}; +pub use session::*; use typed_builder::TypedBuilder; use uuid::Uuid; +use crate::encryption::kms::KmsClientFactory; +use crate::error::invalid_data; +use crate::io::StorageFactory; +use crate::runtime::Runtime; use crate::spec::{ - FormatVersion, PartitionStatisticsFile, Schema, SchemaId, Snapshot, SnapshotReference, - SortOrder, StatisticsFile, TableMetadata, TableMetadataBuilder, UnboundPartitionSpec, - ViewFormatVersion, ViewRepresentations, ViewVersion, + EncryptedKey, FormatVersion, PartitionStatisticsFile, Schema, SchemaId, Snapshot, + SnapshotReference, SortOrder, StatisticsFile, TableMetadata, TableMetadataBuilder, + UnboundPartitionSpec, ViewFormatVersion, ViewRepresentations, ViewVersion, }; use crate::table::Table; use crate::{Error, ErrorKind, Result}; @@ -97,6 +104,14 @@ pub trait Catalog: Debug + Sync + Send { /// Drop a table from the catalog, or returns error if it doesn't exist. async fn drop_table(&self, table: &TableIdent) -> Result<()>; + /// Drop a table from the catalog and delete the underlying table data. + /// + /// Implementations should load the table metadata, drop the table + /// from the catalog, then delete all associated data and metadata files. + /// The [`drop_table_data`](utils::drop_table_data) utility function can + /// be used for the file cleanup step. + async fn purge_table(&self, table: &TableIdent) -> Result<()>; + /// Check if a table exists in the catalog. async fn table_exists(&self, table: &TableIdent) -> Result; @@ -114,6 +129,62 @@ pub trait Catalog: Debug + Sync + Send { pub trait CatalogBuilder: Default + Debug + Send + Sync { /// The catalog type that this builder creates. type C: Catalog; + + /// Set a custom StorageFactory to use for storage operations. + /// + /// When a StorageFactory is provided, the catalog will use it to build FileIO + /// instances for all storage operations instead of using the default factory. + /// + /// # Arguments + /// + /// * `storage_factory` - The StorageFactory to use for creating storage instances + /// + /// # Example + /// + /// ```rust,ignore + /// use iceberg::CatalogBuilder; + /// use iceberg::io::StorageFactory; + /// use iceberg_storage_opendal::OpenDalStorageFactory; + /// use std::sync::Arc; + /// + /// let catalog = MyCatalogBuilder::default() + /// .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + /// customized_credential_load: None, + /// })) + /// .load("my_catalog", props) + /// .await?; + /// ``` + fn with_storage_factory(self, storage_factory: Arc) -> Self; + + /// Set a [`KmsClientFactory`] to enable table encryption. + /// + /// When provided, the catalog calls the factory once during + /// [`load`](Self::load) with the catalog properties to create a shared + /// [`KeyManagementClient`](crate::encryption::KeyManagementClient). + /// That client is then passed to each table's `TableBuilder` so tables + /// with `encryption.key-id` set can construct an `EncryptionManager`. + /// + /// # Example + /// + /// ```rust,ignore + /// use iceberg::CatalogBuilder; + /// use iceberg::encryption::kms::KmsClientFactory; + /// use std::sync::Arc; + /// + /// let catalog = MyCatalogBuilder::default() + /// .with_kms_client_factory(Arc::new(MyKmsClientFactory)) + /// .load("my_catalog", props) + /// .await?; + /// ``` + fn with_kms_client_factory(self, kms_client_factory: Arc) -> Self; + + /// Set a custom tokio Runtime to use for spawning async tasks. + /// + /// When a Runtime is provided, the catalog will propagate it to all tables + /// it creates. Tasks such as scan planning and delete file processing + /// will be spawned on this runtime. + fn with_runtime(self, runtime: Runtime) -> Self; + /// Create a new catalog instance. fn load( self, @@ -139,10 +210,7 @@ impl NamespaceIdent { /// Create a multi-level namespace identifier from vector. pub fn from_vec(names: Vec) -> Result { if names.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Namespace identifier can't be empty!", - )); + return Err(invalid_data!("Namespace identifier can't be empty!")); } Ok(Self(names)) } @@ -252,9 +320,9 @@ impl TableIdent { /// Try to create table identifier from an iterator of string. pub fn from_strs(iter: impl IntoIterator) -> Result { let mut vec: Vec = iter.into_iter().map(|s| s.to_string()).collect(); - let table_name = vec.pop().ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "Table identifier can't be empty!") - })?; + let table_name = vec + .pop() + .ok_or_else(|| invalid_data!("Table identifier can't be empty!"))?; let namespace_ident = NamespaceIdent::from_vec(vec)?; Ok(Self { @@ -291,6 +359,9 @@ pub struct TableCreation { props.into_iter().collect() }))] pub properties: HashMap, + /// Format version of the table. Defaults to V2. + #[builder(default = FormatVersion::V2)] + pub format_version: FormatVersion, } /// TableCommit represents the commit of a table in the catalog. @@ -350,13 +421,16 @@ impl TableCommit { metadata_builder = update.apply(metadata_builder)?; } - // Bump the version of metadata + // Build the new metadata + let new_metadata = metadata_builder.build()?.metadata; + let new_metadata_location = MetadataLocation::from_str(current_metadata_location)? .with_next_version() + .try_with_new_metadata(&new_metadata)? .to_string(); Ok(table - .with_metadata(Arc::new(metadata_builder.build()?.metadata)) + .with_metadata(Arc::new(new_metadata)) .with_metadata_location(new_metadata_location)) } } @@ -479,7 +553,10 @@ pub enum TableUpdate { #[serde(rename_all = "kebab-case")] AddSnapshot { /// Snapshot to add. - #[serde(deserialize_with = "deserialize_snapshot")] + #[serde( + deserialize_with = "deserialize_snapshot", + serialize_with = "serialize_snapshot" + )] snapshot: Snapshot, }, /// Set table's snapshot ref. @@ -554,6 +631,18 @@ pub enum TableUpdate { /// Schema IDs to remove. schema_ids: Vec, }, + /// Add an encryption key + #[serde(rename_all = "kebab-case")] + AddEncryptionKey { + /// The encryption key to add. + encryption_key: EncryptedKey, + }, + /// Remove an encryption key + #[serde(rename_all = "kebab-case")] + RemoveEncryptionKey { + /// The id of the encryption key to remove. + key_id: String, + }, } impl TableUpdate { @@ -598,6 +687,12 @@ impl TableUpdate { Ok(builder.remove_partition_statistics(snapshot_id)) } TableUpdate::RemoveSchemas { schema_ids } => builder.remove_schemas(&schema_ids), + TableUpdate::AddEncryptionKey { encryption_key } => { + Ok(builder.add_encryption_key(encryption_key)) + } + TableUpdate::RemoveEncryptionKey { key_id } => { + Ok(builder.remove_encryption_key(&key_id)) + } } } } @@ -662,7 +757,7 @@ impl TableRequirement { let snapshot_ref = snapshot_ref.ok_or( Error::new( ErrorKind::CatalogCommitConflicts, - format!("Requirement failed: Branch or tag `{}` not found", r#ref), + format!("Requirement failed: Branch or tag `{ref}` not found"), ) .with_retryable(true), )?; @@ -670,8 +765,7 @@ impl TableRequirement { return Err(Error::new( ErrorKind::CatalogCommitConflicts, format!( - "Requirement failed: Branch or tag `{}`'s snapshot has changed", - r#ref + "Requirement failed: Branch or tag `{ref}`'s snapshot has changed" ), ) .with_context("expected", snapshot_id.to_string()) @@ -682,10 +776,7 @@ impl TableRequirement { // a null snapshot ID means the ref should not exist already return Err(Error::new( ErrorKind::CatalogCommitConflicts, - format!( - "Requirement failed: Branch or tag `{}` already exists", - r#ref - ), + format!("Requirement failed: Branch or tag `{ref}` already exists"), ) .with_retryable(true)); } @@ -746,7 +837,7 @@ impl TableRequirement { } pub(super) mod _serde { - use serde::{Deserialize as _, Deserializer}; + use serde::{Deserialize as _, Deserializer, Serialize as _}; use super::*; use crate::spec::{SchemaId, Summary}; @@ -759,7 +850,18 @@ pub(super) mod _serde { Ok(buf.into()) } - #[derive(Debug, Deserialize, PartialEq, Eq)] + pub(super) fn serialize_snapshot( + snapshot: &Snapshot, + serializer: S, + ) -> std::result::Result + where + S: serde::Serializer, + { + let buf: CatalogSnapshot = snapshot.clone().into(); + buf.serialize(serializer) + } + + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] /// Defines the structure of a v2 snapshot for the catalog. /// Main difference to SnapshotV2 is that sequence-number is optional @@ -775,6 +877,12 @@ pub(super) mod _serde { summary: Summary, #[serde(skip_serializing_if = "Option::is_none")] schema_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + first_row_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + added_rows: Option, + #[serde(skip_serializing_if = "Option::is_none")] + key_id: Option, } impl From for Snapshot { @@ -787,6 +895,9 @@ pub(super) mod _serde { manifest_list, schema_id, summary, + first_row_id, + added_rows, + key_id, } = snapshot; let builder = Snapshot::builder() .with_snapshot_id(snapshot_id) @@ -794,11 +905,49 @@ pub(super) mod _serde { .with_sequence_number(sequence_number) .with_timestamp_ms(timestamp_ms) .with_manifest_list(manifest_list) - .with_summary(summary); - if let Some(schema_id) = schema_id { - builder.with_schema_id(schema_id).build() - } else { - builder.build() + .with_summary(summary) + .with_encryption_key_id(key_id); + let row_range = first_row_id.zip(added_rows); + match (schema_id, row_range) { + (None, None) => builder.build(), + (Some(schema_id), None) => builder.with_schema_id(schema_id).build(), + (None, Some((first_row_id, last_row_id))) => { + builder.with_row_range(first_row_id, last_row_id).build() + } + (Some(schema_id), Some((first_row_id, last_row_id))) => builder + .with_schema_id(schema_id) + .with_row_range(first_row_id, last_row_id) + .build(), + } + } + } + + impl From for CatalogSnapshot { + fn from(snapshot: Snapshot) -> Self { + let first_row_id = snapshot.first_row_id(); + let added_rows = snapshot.added_rows_count(); + let Snapshot { + snapshot_id, + parent_snapshot_id, + sequence_number, + timestamp_ms, + manifest_list, + summary, + schema_id, + row_range: _, + encryption_key_id: key_id, + } = snapshot; + CatalogSnapshot { + snapshot_id, + parent_snapshot_id, + sequence_number, + timestamp_ms, + manifest_list, + summary, + schema_id, + first_row_id, + added_rows, + key_id, } } } @@ -922,13 +1071,13 @@ mod _serde_set_statistics { snapshot_id, statistics, } = SetStatistics::deserialize(deserializer)?; - if let Some(snapshot_id) = snapshot_id { - if snapshot_id != statistics.snapshot_id { - return Err(serde::de::Error::custom(format!( - "Snapshot id to set {snapshot_id} does not match the statistics file snapshot id {}", - statistics.snapshot_id - ))); - } + if let Some(snapshot_id) = snapshot_id + && snapshot_id != statistics.snapshot_id + { + return Err(serde::de::Error::custom(format!( + "Snapshot id to set {snapshot_id} does not match the statistics file snapshot id {}", + statistics.snapshot_id + ))); } Ok(statistics) @@ -942,21 +1091,23 @@ mod tests { use std::fs::File; use std::io::BufReader; + use base64::Engine as _; use serde::Serialize; use serde::de::DeserializeOwned; use uuid::uuid; use super::ViewUpdate; - use crate::io::FileIOBuilder; + use crate::io::FileIO; use crate::spec::{ - BlobMetadata, FormatVersion, MAIN_BRANCH, NestedField, NullOrder, Operation, + BlobMetadata, EncryptedKey, FormatVersion, MAIN_BRANCH, NestedField, NullOrder, Operation, PartitionStatisticsFile, PrimitiveType, Schema, Snapshot, SnapshotReference, SnapshotRetention, SortDirection, SortField, SortOrder, SqlViewRepresentation, StatisticsFile, Summary, TableMetadata, TableMetadataBuilder, Transform, Type, - UnboundPartitionSpec, ViewFormatVersion, ViewRepresentation, ViewRepresentations, - ViewVersion, + UnboundPartitionField, UnboundPartitionSpec, ViewFormatVersion, ViewRepresentation, + ViewRepresentations, ViewVersion, }; use crate::table::Table; + use crate::test_utils::test_runtime; use crate::{ NamespaceIdent, TableCommit, TableCreation, TableIdent, TableRequirement, TableUpdate, }; @@ -1079,20 +1230,18 @@ mod tests { assert!(requirement.check(Some(&metadata)).is_ok()); // Add snapshot - let record = r#" - { - "snapshot-id": 3051729675574597004, - "sequence-number": 10, - "timestamp-ms": 9992191116217, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://b/wh/.../s1.avro", - "schema-id": 0 - } - "#; + let snapshot = Snapshot::builder() + .with_snapshot_id(3051729675574597004) + .with_sequence_number(10) + .with_timestamp_ms(9992191116217) + .with_manifest_list("s3://b/wh/.../s1.avro".to_string()) + .with_schema_id(0) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .build(); - let snapshot = serde_json::from_str::(record).unwrap(); let builder = metadata.into_builder(None); let builder = TableUpdate::AddSnapshot { snapshot: snapshot.clone(), @@ -1510,11 +1659,32 @@ mod tests { "#, TableUpdate::AddSpec { spec: UnboundPartitionSpec::builder() - .add_partition_field(4, "ts_day".to_string(), Transform::Day) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .name("ts_day") + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap() - .add_partition_field(1, "id_bucket".to_string(), Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() - .add_partition_field(2, "id_truncate".to_string(), Transform::Truncate(4)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("id_truncate") + .transform(Transform::Truncate(4)) + .build() + .unwrap(), + ) .unwrap() .build(), }, @@ -1670,6 +1840,50 @@ mod tests { assert_eq!(actual, update, "Parsed value is not equal to expected"); } + #[test] + fn test_add_snapshot_v3() { + let json = serde_json::json!( + { + "action": "add-snapshot", + "snapshot": { + "snapshot-id": 3055729675574597000i64, + "parent-snapshot-id": 3051729675574597000i64, + "timestamp-ms": 1555100955770i64, + "first-row-id":0, + "added-rows":2, + "key-id":"key123", + "summary": { + "operation": "append" + }, + "manifest-list": "s3://a/b/2.avro" + } + }); + + let update = TableUpdate::AddSnapshot { + snapshot: Snapshot::builder() + .with_snapshot_id(3055729675574597000) + .with_parent_snapshot_id(Some(3051729675574597000)) + .with_timestamp_ms(1555100955770) + .with_sequence_number(0) + .with_manifest_list("s3://a/b/2.avro") + .with_row_range(0, 2) + .with_encryption_key_id(Some("key123".to_string())) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::default(), + }) + .build(), + }; + + let actual: TableUpdate = serde_json::from_value(json).expect("Failed to parse from json"); + assert_eq!(actual, update, "Parsed value is not equal to expected"); + let restored: TableUpdate = serde_json::from_str( + &serde_json::to_string(&actual).expect("Failed to serialize to json"), + ) + .expect("Failed to parse from serialized json"); + assert_eq!(restored, update); + } + #[test] fn test_remove_snapshots() { let json = r#" @@ -2173,6 +2387,48 @@ mod tests { ); } + #[test] + fn test_add_encryption_key() { + let key_bytes = "key".as_bytes(); + let encoded_key = base64::engine::general_purpose::STANDARD.encode(key_bytes); + test_serde_json( + format!( + r#" + {{ + "action": "add-encryption-key", + "encryption-key": {{ + "key-id": "a", + "encrypted-key-metadata": "{encoded_key}", + "encrypted-by-id": "b" + }} + }} + "# + ), + TableUpdate::AddEncryptionKey { + encryption_key: EncryptedKey::builder() + .key_id("a") + .encrypted_key_metadata(key_bytes.to_vec()) + .encrypted_by_id("b") + .build(), + }, + ); + } + + #[test] + fn test_remove_encryption_key() { + test_serde_json( + r#" + { + "action": "remove-encryption-key", + "key-id": "a" + } + "#, + TableUpdate::RemoveEncryptionKey { + key_id: "a".to_string(), + }, + ); + } + #[test] fn test_table_commit() { let table = { @@ -2187,9 +2443,10 @@ mod tests { Table::builder() .metadata(resp) - .metadata_location("s3://bucket/test/location/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json".to_string()) + .metadata_location("s3://bucket/test/location/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json") .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .file_io(FileIOBuilder::new("memory").build().unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) .build() .unwrap() }; @@ -2229,12 +2486,13 @@ mod tests { "v2" ); - // metadata version should be bumped + // metadata version should be bumped, and the metadata directory should be + // re-derived from the updated table location assert!( updated_table .metadata_location() .unwrap() - .starts_with("s3://bucket/test/location/metadata/00001-") + .starts_with("s3://bucket/test/new_location/data/metadata/00001-") ); assert_eq!( diff --git a/crates/iceberg/src/catalog/session.rs b/crates/iceberg/src/catalog/session.rs new file mode 100644 index 0000000000..e506820983 --- /dev/null +++ b/crates/iceberg/src/catalog/session.rs @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Session catalog API for Apache Iceberg. + +use std::collections::HashMap; +use std::fmt::Debug; + +use async_trait::async_trait; +#[cfg(test)] +use mockall::automock; +use typed_builder::TypedBuilder; +use uuid::Uuid; + +use crate::sensitive::SensitiveString; +use crate::table::Table; +use crate::{Namespace, NamespaceIdent, Result, TableCommit, TableCreation, TableIdent}; + +/// Context for a session. +/// +/// # Example +/// ```rust +/// use iceberg::SessionContext; +/// +/// let session = SessionContext::builder() +/// .identity("user123".to_string()) +/// .build(); +/// +/// assert_eq!(session.identity(), Some("user123")); +/// assert!(!session.session_id().is_empty()); +/// ``` +#[derive(Debug, Clone, TypedBuilder)] +pub struct SessionContext { + /// The unique identifier for this session. + /// + /// Note that the session_id may be used for caching session-scoped state + /// and re-use of a session_id with different session context may result in + /// unexpected behavior. + #[builder(default=Uuid::new_v4().to_string())] + session_id: String, + + /// An optional user or principal associated with the session. + #[builder(default, setter(strip_option))] + identity: Option, + + #[builder(default)] + properties: HashMap, + + #[builder(default)] + credentials: HashMap, +} + +impl SessionContext { + /// Creates a new unique but empty session. + pub fn empty() -> Self { + Self::builder().build() + } + + /// Returns the identifier for this session. + /// + /// The identifier may be used for caching state within a session. + pub fn session_id(&self) -> &str { + &self.session_id + } + + /// Returns a string that identifies the current user or principal. + pub fn identity(&self) -> Option<&str> { + self.identity.as_deref() + } + + /// Returns a map of properties currently set for the session. + pub fn properties(&self) -> &HashMap { + &self.properties + } + + /// Returns the session's credential map. + pub fn credentials(&self) -> &HashMap { + &self.credentials + } +} + +/// The catalog API for Iceberg Rust that includes session handling. +#[async_trait] +#[cfg_attr(test, automock)] +pub trait SessionCatalog: Debug + Send + Sync { + /// List namespaces inside the catalog. + async fn list_namespaces( + &self, + context: &SessionContext, + parent: Option<&NamespaceIdent>, + ) -> Result>; + + /// Create a new namespace inside the catalog. + async fn create_namespace( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + properties: HashMap, + ) -> Result; + + /// Get a namespace information from the catalog. + async fn get_namespace( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + ) -> Result; + + /// Check if namespace exists in catalog. + async fn namespace_exists( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + ) -> Result; + + /// Update a namespace inside the catalog. + /// + /// # Behavior + /// + /// The properties must be the full set of namespace. + async fn update_namespace( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + properties: HashMap, + ) -> Result<()>; + + /// Drop a namespace from the catalog, or returns error if it doesn't exist. + async fn drop_namespace( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + ) -> Result<()>; + + /// List tables from namespace. + async fn list_tables( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + ) -> Result>; + + /// Create a new table inside the namespace. + async fn create_table( + &self, + context: &SessionContext, + namespace: &NamespaceIdent, + creation: TableCreation, + ) -> Result

; + + /// Load table from the catalog. + async fn load_table(&self, context: &SessionContext, table: &TableIdent) -> Result
; + + /// Drop a table from the catalog, or returns error if it doesn't exist. + async fn drop_table(&self, context: &SessionContext, table: &TableIdent) -> Result<()>; + + /// Drop a table from the catalog and delete the underlying table data. + /// + /// Implementations should load the table metadata, drop the table + /// from the catalog, then delete all associated data and metadata files. + /// The [`drop_table_data`](super::utils::drop_table_data) utility function can + /// be used for the file cleanup step. + async fn purge_table(&self, context: &SessionContext, table: &TableIdent) -> Result<()>; + + /// Check if a table exists in the catalog. + async fn table_exists(&self, context: &SessionContext, table: &TableIdent) -> Result; + + /// Rename a table in the catalog. + async fn rename_table( + &self, + context: &SessionContext, + src: &TableIdent, + dest: &TableIdent, + ) -> Result<()>; + + /// Register an existing table to the catalog. + async fn register_table( + &self, + context: &SessionContext, + table: &TableIdent, + metadata_location: String, + ) -> Result
; + + /// Update a table to the catalog. + async fn update_table(&self, context: &SessionContext, commit: TableCommit) -> Result
; +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use uuid::Uuid; + + use crate::sensitive::SensitiveString; + use crate::{SessionCatalog, SessionContext}; + + #[test] + fn test_empty_session_context_has_uuid_session_id() { + let session = SessionContext::empty(); + let session_id = session.session_id(); + + assert!(Uuid::parse_str(session_id).is_ok()); + } + + #[test] + fn test_empty_sessions_get_unique_id() { + let session1 = SessionContext::empty(); + let session2 = SessionContext::empty(); + + assert_ne!(session1.session_id(), session2.session_id()) + } + + #[test] + fn test_empty_sessions_get_unique_id_via_builder() { + let session1 = SessionContext::builder().build(); + let session2 = SessionContext::builder().build(); + + assert_ne!(session1.session_id(), session2.session_id()); + } + + #[test] + fn test_session_with_credentials_does_not_display_them() { + let sensitive_value = "my-pw-123456"; + let session = SessionContext::builder() + .credentials(HashMap::from([( + "key".to_string(), + SensitiveString::from(sensitive_value.to_string()), + )])) + .build(); + + let logged = format!("{:?}", session); + assert!(!logged.contains(sensitive_value)) + } + + #[test] + fn test_types_are_send_sync() { + assert_send_sync::(); + assert_send_sync::(); + + fn _dyn_compatible(_: &dyn SessionCatalog) {} + } + + fn assert_send_sync() {} +} diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs new file mode 100644 index 0000000000..3fb9b007d6 --- /dev/null +++ b/crates/iceberg/src/catalog/utils.rs @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Utility functions for catalog operations. + +use std::collections::{HashMap, HashSet}; + +use futures::{StreamExt, TryStreamExt, stream}; + +use crate::Result; +use crate::io::FileIO; +use crate::spec::{ManifestFile, ManifestReader, TableMetadata}; +use crate::table::Table; + +const DELETE_CONCURRENCY: usize = 10; + +/// Deletes all data and metadata files referenced by the given table metadata. +/// +/// This mirrors the Java implementation's `CatalogUtil.dropTableData`. +/// It collects all manifest files, manifest lists, previous metadata files, +/// statistics files, and partition statistics files, then deletes them. +/// +/// Data files within manifests are only deleted if the `gc.enabled` table +/// property is `true` (the default), to avoid corrupting other tables that +/// may share the same data files. +pub async fn drop_table_data(table_info: &Table) -> Result<()> { + let mut manifest_lists_to_delete: HashSet = HashSet::new(); + let mut manifests_to_delete: HashMap = HashMap::new(); + + let metadata = table_info.metadata_ref(); + let io = table_info.file_io(); + // Load all manifest lists concurrently + let results: Vec<_> = + futures::future::try_join_all(metadata.snapshots().map(|snapshot| async { + let manifest_list = table_info.manifest_list_reader(snapshot).load().await?; + Ok::<_, crate::Error>((snapshot.manifest_list().to_string(), manifest_list)) + })) + .await?; + + for (manifest_list_location, manifest_list) in results { + if !manifest_list_location.is_empty() { + manifest_lists_to_delete.insert(manifest_list_location); + } + for manifest_file in manifest_list.entries() { + manifests_to_delete.insert(manifest_file.manifest_path.clone(), manifest_file.clone()); + } + } + + // Delete data files only if gc.enabled is true, to avoid corrupting shared tables + if metadata.table_properties().gc_enabled()? { + delete_data_files(io, &manifests_to_delete).await?; + } + + // Delete manifest files + let manifest_paths: Vec = manifests_to_delete.into_keys().collect(); + io.delete_stream(stream::iter(manifest_paths)).await?; + + // Delete manifest lists + io.delete_stream(stream::iter(manifest_lists_to_delete)) + .await?; + + // Delete previous metadata files + let prev_metadata_paths: Vec = metadata + .metadata_log() + .iter() + .map(|m| m.metadata_file.clone()) + .collect(); + io.delete_stream(stream::iter(prev_metadata_paths)).await?; + + // Delete statistics files + let stats_paths: Vec = metadata + .statistics_iter() + .map(|s| s.statistics_path.clone()) + .collect(); + io.delete_stream(stream::iter(stats_paths)).await?; + + // Delete partition statistics files + let partition_stats_paths: Vec = metadata + .partition_statistics_iter() + .map(|s| s.statistics_path.clone()) + .collect(); + io.delete_stream(stream::iter(partition_stats_paths)) + .await?; + + // Delete the current metadata file + if let Some(location) = table_info.metadata_location() { + io.delete(location).await?; + } + + Ok(()) +} + +/// Deletes the previous metadata files tracked by `base` that `new` no longer tracks, if +/// `write.metadata.delete-after-commit.enabled` is `true` on `new`. +/// +/// This mirrors the Java implementation's `CatalogUtil.deleteRemovedMetadataFiles`. Call it +/// only after `new` has been committed over `base`. Deletion is best-effort: failures are +/// logged rather than returned, since the commit has already succeeded. +pub async fn delete_removed_metadata_files(io: &FileIO, base: &TableMetadata, new: &TableMetadata) { + match new + .table_properties() + .metadata_delete_after_commit_enabled() + { + Ok(true) => {} + Ok(false) => return, + Err(e) => { + tracing::warn!("Skipping deletion of removed metadata files: {e}"); + return; + } + } + + let retained: HashSet<&str> = new + .metadata_log() + .iter() + .map(|log| log.metadata_file.as_str()) + .collect(); + let removed = base + .metadata_log() + .iter() + .map(|log| log.metadata_file.as_str()) + .filter(|path| !retained.contains(path)); + + stream::iter(removed) + .for_each_concurrent(DELETE_CONCURRENCY, |path| async move { + if let Err(e) = io.delete(path).await { + tracing::warn!("Failed to delete removed metadata file {path}: {e}"); + } + }) + .await; +} + +/// Reads manifests concurrently and deletes the data files referenced within. +async fn delete_data_files( + io: &FileIO, + manifest_files: &HashMap, +) -> Result<()> { + stream::iter(manifest_files.values().map(Ok)) + .try_for_each_concurrent(DELETE_CONCURRENCY, |manifest_file| async move { + let manifest = ManifestReader::new(io.clone()).read(manifest_file).await?; + let data_file_paths = manifest + .entries() + .iter() + .map(|entry| entry.data_file.file_path().to_string()) + .collect::>(); + + io.delete_stream(stream::iter(data_file_paths)).await + }) + .await +} diff --git a/crates/iceberg/src/compression.rs b/crates/iceberg/src/compression.rs new file mode 100644 index 0000000000..dbd2c97881 --- /dev/null +++ b/crates/iceberg/src/compression.rs @@ -0,0 +1,334 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Compression codec support for data compression and decompression. + +use std::fmt; +use std::io::{Read, Write}; + +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use crate::{Error, ErrorKind, Result}; + +/// Default compression level for Zstandard (zstd). +const ZSTD_DEFAULT_LEVEL: u8 = 3; +/// Default compression level for Gzip. +const GZIP_DEFAULT_LEVEL: u8 = 6; +/// Maximum compression level for Gzip. +const GZIP_MAX_LEVEL: u8 = 9; +/// Default compression level for Brotli. +const BROTLI_DEFAULT_LEVEL: u8 = 1; + +/// Data compression formats +#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)] +pub enum CompressionCodec { + #[default] + /// No compression + None, + /// LZ4 single compression frame with content size present + Lz4, + /// LZ4 raw block compression (no frame). Used by Parquet as `lz4_raw`. + Lz4Raw, + /// Zstandard single compression frame with content size present. + /// Level range is 0–22, where 0 means default compression level (not no compression). + /// Use [`CompressionCodec::zstd_default`] to construct with the default level. + Zstd(u8), + /// Gzip compression. Level range is 0–9, where 0 means no compression. + /// Use [`CompressionCodec::gzip_default`] to construct with the default level. + Gzip(u8), + /// Brotli compression. Level range is 0–11. + /// Use [`CompressionCodec::brotli_default`] to construct with the default level. + Brotli(u8), + /// LZO compression + Lzo, + /// Snappy compression + Snappy, +} + +impl CompressionCodec { + /// Returns a Zstd codec with the default compression level. + pub const fn zstd_default() -> Self { + CompressionCodec::Zstd(ZSTD_DEFAULT_LEVEL) + } + + /// Returns a Gzip codec with the default compression level. + pub const fn gzip_default() -> Self { + CompressionCodec::Gzip(GZIP_DEFAULT_LEVEL) + } + + /// Returns a Brotli codec with the default compression level. + pub const fn brotli_default() -> Self { + CompressionCodec::Brotli(BROTLI_DEFAULT_LEVEL) + } + + /// Returns the codec name as used in serialization and error messages. + pub fn name(&self) -> &'static str { + match self { + CompressionCodec::None => "none", + CompressionCodec::Lz4 => "lz4", + CompressionCodec::Lz4Raw => "lz4_raw", + CompressionCodec::Zstd(_) => "zstd", + CompressionCodec::Gzip(_) => "gzip", + CompressionCodec::Brotli(_) => "brotli", + CompressionCodec::Lzo => "lzo", + CompressionCodec::Snappy => "snappy", + } + } +} + +// Note: serialize/deserialize do not round-trip the compression level. Iceberg configuration +// only the codec name (e.g. "zstd"), not the level, so deserialization always produces the +// default level. A `Zstd(5)` written to metadata will be read back as `Zstd(3)`. Some +// compression configuration (e.g. Avro metadata) has a separate level field alongside the codec name. +impl Serialize for CompressionCodec { + fn serialize(&self, serializer: S) -> std::result::Result { + serializer.serialize_str(self.name()) + } +} + +impl<'de> Deserialize<'de> for CompressionCodec { + fn deserialize>(deserializer: D) -> std::result::Result { + let s = String::deserialize(deserializer)?; + match s.to_lowercase().as_str() { + "none" | "uncompressed" => Ok(CompressionCodec::None), + "lz4" => Ok(CompressionCodec::Lz4), + "lz4_raw" => Ok(CompressionCodec::Lz4Raw), + "zstd" => Ok(CompressionCodec::zstd_default()), + "gzip" => Ok(CompressionCodec::gzip_default()), + "brotli" => Ok(CompressionCodec::brotli_default()), + "lzo" => Ok(CompressionCodec::Lzo), + "snappy" => Ok(CompressionCodec::Snappy), + other => Err(serde::de::Error::unknown_variant(other, &[ + "none", + "uncompressed", + "lz4", + "lz4_raw", + "zstd", + "gzip", + "brotli", + "lzo", + "snappy", + ])), + } + } +} + +impl fmt::Display for CompressionCodec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + CompressionCodec::None => write!(f, "None"), + CompressionCodec::Lz4 => write!(f, "Lz4"), + CompressionCodec::Lz4Raw => write!(f, "Lz4Raw"), + CompressionCodec::Zstd(level) => write!(f, "Zstd(level={level})"), + CompressionCodec::Gzip(level) => write!(f, "Gzip(level={level})"), + CompressionCodec::Brotli(level) => write!(f, "Brotli(level={level})"), + CompressionCodec::Lzo => write!(f, "Lzo"), + CompressionCodec::Snappy => write!(f, "Snappy"), + } + } +} + +impl CompressionCodec { + pub(crate) fn decompress(&self, bytes: Vec) -> Result> { + match self { + CompressionCodec::None => Ok(bytes), + CompressionCodec::Lz4 => Err(Error::new( + ErrorKind::FeatureUnsupported, + "LZ4 decompression is not supported currently", + )), + CompressionCodec::Zstd(_) => Ok(zstd::stream::decode_all(&bytes[..])?), + CompressionCodec::Gzip(_) => { + let mut decoder = GzDecoder::new(&bytes[..]); + let mut decompressed = Vec::new(); + decoder.read_to_end(&mut decompressed)?; + Ok(decompressed) + } + CompressionCodec::Lz4Raw => Err(Error::new( + ErrorKind::FeatureUnsupported, + "LZ4_RAW decompression is not supported currently", + )), + CompressionCodec::Brotli(_) => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Brotli decompression is not supported currently", + )), + CompressionCodec::Lzo => Err(Error::new( + ErrorKind::FeatureUnsupported, + "LZO decompression is not supported currently", + )), + CompressionCodec::Snappy => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snappy decompression is not supported currently", + )), + } + } + + pub(crate) fn compress(&self, bytes: Vec) -> Result> { + match self { + CompressionCodec::None => Ok(bytes), + CompressionCodec::Lz4 => Err(Error::new( + ErrorKind::FeatureUnsupported, + "LZ4 compression is not supported currently", + )), + CompressionCodec::Zstd(level) => { + let writer = Vec::::new(); + let mut encoder = zstd::stream::Encoder::new(writer, *level as i32)?; + encoder.include_checksum(true)?; + encoder.set_pledged_src_size(Some(bytes.len().try_into()?))?; + std::io::copy(&mut &bytes[..], &mut encoder)?; + Ok(encoder.finish()?) + } + CompressionCodec::Gzip(level) => { + let compression = Compression::new((*level).min(GZIP_MAX_LEVEL) as u32); + let mut encoder = GzEncoder::new(Vec::new(), compression); + encoder.write_all(&bytes)?; + Ok(encoder.finish()?) + } + CompressionCodec::Lz4Raw => Err(Error::new( + ErrorKind::FeatureUnsupported, + "LZ4_RAW compression is not supported currently", + )), + CompressionCodec::Brotli(_) => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Brotli compression is not supported currently", + )), + CompressionCodec::Lzo => Err(Error::new( + ErrorKind::FeatureUnsupported, + "LZO compression is not supported currently", + )), + CompressionCodec::Snappy => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snappy compression is not supported currently", + )), + } + } + + pub(crate) fn is_none(&self) -> bool { + matches!(self, CompressionCodec::None) + } + + /// Returns the file extension suffix for this compression codec. + /// Returns empty string for None, ".gz" for Gzip. + /// + /// # Errors + /// + /// Returns an error for Lz4 and Zstd as they are not fully supported. + pub fn suffix(&self) -> Result<&'static str> { + match self { + CompressionCodec::None => Ok(""), + CompressionCodec::Gzip(_) => Ok(".gz"), + codec @ (CompressionCodec::Lz4 + | CompressionCodec::Lz4Raw + | CompressionCodec::Zstd(_) + | CompressionCodec::Brotli(_) + | CompressionCodec::Lzo + | CompressionCodec::Snappy) => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("suffix not defined for {codec:?}"), + )), + } + } +} + +#[cfg(test)] +mod tests { + use super::CompressionCodec; + + #[tokio::test] + async fn test_compression_codec_none() { + let bytes_vec = [0_u8; 100].to_vec(); + + let codec = CompressionCodec::None; + let compressed = codec.compress(bytes_vec.clone()).unwrap(); + assert_eq!(bytes_vec, compressed); + let decompressed = codec.decompress(compressed).unwrap(); + assert_eq!(bytes_vec, decompressed); + } + + #[tokio::test] + async fn test_compression_codec_compress() { + let bytes_vec = [0_u8; 100].to_vec(); + + let compression_codecs = [ + CompressionCodec::zstd_default(), + CompressionCodec::gzip_default(), + ]; + + for codec in compression_codecs { + let compressed = codec.compress(bytes_vec.clone()).unwrap(); + assert!(compressed.len() < bytes_vec.len()); + let decompressed = codec.decompress(compressed).unwrap(); + assert_eq!(decompressed, bytes_vec); + } + } + + #[tokio::test] + async fn test_compression_codec_unsupported() { + let unsupported_codecs = [ + (CompressionCodec::Lz4, "LZ4"), + (CompressionCodec::Snappy, "Snappy"), + ]; + let bytes_vec = [0_u8; 100].to_vec(); + + for (codec, name) in unsupported_codecs { + assert_eq!( + codec.compress(bytes_vec.clone()).unwrap_err().to_string(), + format!("FeatureUnsupported => {name} compression is not supported currently"), + ); + + assert_eq!( + codec.decompress(bytes_vec.clone()).unwrap_err().to_string(), + format!("FeatureUnsupported => {name} decompression is not supported currently"), + ); + } + } + + #[test] + fn test_suffix() { + assert_eq!(CompressionCodec::None.suffix().unwrap(), ""); + assert_eq!(CompressionCodec::gzip_default().suffix().unwrap(), ".gz"); + + assert!(CompressionCodec::Lz4.suffix().is_err()); + assert!(CompressionCodec::zstd_default().suffix().is_err()); + assert!(CompressionCodec::Snappy.suffix().is_err()); + + let lz4_err = CompressionCodec::Lz4.suffix().unwrap_err(); + assert!(lz4_err.to_string().contains("suffix not defined for Lz4")); + + let zstd_err = CompressionCodec::zstd_default().suffix().unwrap_err(); + assert!(zstd_err.to_string().contains("suffix not defined for Zstd")); + } + + #[test] + fn test_display() { + assert_eq!(CompressionCodec::None.to_string(), "None"); + assert_eq!(CompressionCodec::Lz4.to_string(), "Lz4"); + assert_eq!( + CompressionCodec::zstd_default().to_string(), + "Zstd(level=3)" + ); + assert_eq!(CompressionCodec::Zstd(5).to_string(), "Zstd(level=5)"); + assert_eq!( + CompressionCodec::gzip_default().to_string(), + "Gzip(level=6)" + ); + assert_eq!(CompressionCodec::Gzip(9).to_string(), "Gzip(level=9)"); + assert_eq!(CompressionCodec::Snappy.to_string(), "Snappy"); + } +} diff --git a/crates/iceberg/src/cow_rewrite/mod.rs b/crates/iceberg/src/cow_rewrite/mod.rs new file mode 100644 index 0000000000..fa567834bd --- /dev/null +++ b/crates/iceberg/src/cow_rewrite/mod.rs @@ -0,0 +1,1672 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Copy-on-write rewrite primitives. +//! +//! This module plans candidate data files, reads their visible rows, applies a +//! caller-provided batch rewriter, and writes replacement data files. It returns +//! old and new file sets that can be committed by an overwrite-style transaction +//! action. +//! +//! The primitive does not parse SQL and does not commit metadata by itself. +//! Rewriters must emit batches compatible with the schema rows were read in +//! (the planned snapshot's schema) and must preserve each source file's +//! partition values; this primitive does not repartition rewritten rows. +//! +//! Replacement files deliberately mirror their source file's layout instead of +//! re-optimizing it — a deliberate divergence from Java's RewriteDataFiles and +//! Spark DML: they are written in the planned snapshot's schema rather than +//! promoted to the table's current schema, they keep the source file's +//! partition spec rather than being repartitioned into the table's default +//! spec, and they stay unsorted since no sort order is applied. An opt-in +//! promote-to-current-schema rewrite is left for follow-up work. +//! +//! The result carries data files only. A commit adapter consuming these file +//! lists may remove position delete files and deletion vectors only when they +//! exclusively reference removed data files. Equality delete files must be +//! retained while they can still apply to other live data files by sequence +//! number. +//! +//! ```rust,no_run +//! # use std::sync::Arc; +//! # use arrow_array::RecordBatch; +//! # use iceberg::cow_rewrite::{CowBatchRewrite, CowBatchRewriter, CowRewriteBuilder}; +//! # use iceberg::table::Table; +//! # use iceberg::Result; +//! struct KeepAll; +//! +//! impl CowBatchRewriter for KeepAll { +//! fn rewrite_batch(&self, batch: RecordBatch) -> Result { +//! Ok(CowBatchRewrite { +//! output: Some(batch), +//! changed: false, +//! }) +//! } +//! } +//! +//! # async fn example(table: &Table) -> Result<()> { +//! let result = CowRewriteBuilder::new(table) +//! .with_rewriter(Arc::new(KeepAll)) +//! .rewrite() +//! .await?; +//! +//! assert!(!result.has_changes()); +//! # Ok(()) +//! # } +//! ``` + +mod plan; +mod rewriter; +pub(crate) mod writer; + +use std::sync::Arc; + +use arrow_array::RecordBatch; +use futures::TryStreamExt; +pub use plan::CowRewriteFile; +pub use rewriter::{CowBatchRewrite, CowBatchRewriter}; + +use crate::arrow::ArrowReaderOptions; +use crate::expr::Predicate; +use crate::scan::FileScanTaskStream; +use crate::spec::{DataFile, PartitionKey}; +use crate::table::Table; +use crate::{Error, ErrorKind, Result}; + +/// Counters produced by a copy-on-write rewrite. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct CowRewriteStats { + /// Number of candidate files selected by planning. + pub candidate_files: usize, + /// Number of old files that have replacement output or are fully removed. + pub rewritten_files: usize, + /// Number of candidate files that did not change after row rewriting. + pub unchanged_files: usize, + /// Visible input row count read from candidate files. + pub input_rows: u64, + /// Output row count written to replacement files. + /// + /// Rows the rewriter emitted for files that turned out unchanged are not + /// counted, so this always matches the row counts of `added_data_files`. + pub output_rows: u64, + /// Number of input batches that changed, including batches the rewriter + /// dropped entirely (`output: None`) even if it did not flag them. + pub changed_batches: u64, +} + +/// Result of a copy-on-write rewrite operation. +#[derive(Debug, Default)] +pub struct CowRewriteResult { + /// Old data files that should be removed by the commit action. + pub removed_data_files: Vec, + /// New data files that should be added by the commit action. + pub added_data_files: Vec, + /// Candidate files that were read and left unchanged. + /// + /// Files whose visible rows were all removed by delete files are NOT + /// included here: they read as zero rows and are reported in + /// `removed_data_files` with no replacement. A commit adapter may remove + /// position deletes and deletion vectors that exclusively reference these + /// removed files, but must retain equality deletes that can still apply to + /// other live files. + pub unchanged_data_files: Vec, + /// Rewrite counters. + pub stats: CowRewriteStats, +} + +impl CowRewriteResult { + /// Returns true if the rewrite produced any table changes. + pub fn has_changes(&self) -> bool { + !self.removed_data_files.is_empty() || !self.added_data_files.is_empty() + } +} + +/// Builder for orchestrating copy-on-write data file rewrites. +pub struct CowRewriteBuilder<'a> { + table: &'a Table, + predicate: Predicate, + snapshot_id: Option, + batch_size: Option, + case_sensitive: bool, + rewriter: Option>, +} + +impl<'a> CowRewriteBuilder<'a> { + /// Creates a copy-on-write rewrite builder for `table`. + pub fn new(table: &'a Table) -> Self { + Self { + table, + predicate: Predicate::AlwaysTrue, + snapshot_id: None, + batch_size: None, + case_sensitive: true, + rewriter: None, + } + } + + /// Sets the row predicate used to plan candidate files. + pub fn with_predicate(mut self, predicate: Predicate) -> Self { + self.predicate = predicate; + self + } + + /// Sets the snapshot id used to plan candidate files. + pub fn with_snapshot_id(mut self, snapshot_id: i64) -> Self { + self.snapshot_id = Some(snapshot_id); + self + } + + /// Sets the Arrow reader batch size. + pub fn with_batch_size(mut self, batch_size: usize) -> Self { + self.batch_size = Some(batch_size); + self + } + + /// Sets the case sensitivity used to bind the planning predicate. + pub fn with_case_sensitive(mut self, case_sensitive: bool) -> Self { + self.case_sensitive = case_sensitive; + self + } + + /// Sets the record batch rewriter. + pub fn with_rewriter(mut self, rewriter: Arc) -> Self { + self.rewriter = Some(rewriter); + self + } + + /// Plans, reads, rewrites, and writes replacement data files. + /// + /// # Memory usage + /// + /// Batches read before a file's first changed batch are buffered so that + /// no replacement file is emitted for a source file that turns out + /// unchanged. A file that never changes — or whose first change sits at + /// its very end — therefore buffers its entire decoded contents in + /// memory. Files are processed sequentially, so peak usage is one decoded + /// source file at a time, but that can still be several GB for a + /// compaction-sized file. A size-capped fallback that starts writing the + /// replacement once the buffer crosses a threshold is left for follow-up + /// work. + pub async fn rewrite(self) -> Result { + let rewriter = self.rewriter.ok_or_else(|| { + Error::new( + ErrorKind::PreconditionFailed, + "COW rewrite requires a batch rewriter", + ) + })?; + let files = plan::plan_cow_rewrite_files( + self.table, + Some(self.predicate), + self.snapshot_id, + self.case_sensitive, + ) + .await?; + + let mut result = CowRewriteResult { + stats: CowRewriteStats { + candidate_files: files.len(), + ..CowRewriteStats::default() + }, + ..CowRewriteResult::default() + }; + + for file in files { + let CowRewriteFile { + old_data_file, + scan_task, + } = file; + // Schema the rows are read in (the planned snapshot's schema). The + // replacement files must be written with this schema so that batches + // remain compatible when the table's current schema has evolved past + // the snapshot the source files belong to. This preserves the older + // schema in replacement files rather than promoting them to the + // table's current schema during this rewrite. + let write_schema = scan_task.schema_ref(); + let has_delete_files = !scan_task.deletes().is_empty(); + + // Batches produced before the first changed batch. They are buffered + // rather than written immediately because the primitive must not + // emit a replacement file for a source file that turns out to be + // unchanged. Once a changed batch is observed the buffered prefix is + // flushed to the writer and all subsequent batches stream straight + // through. + // + // Worst-case footprint: for a file that never changes (or whose + // first change sits at its very end) the prefix holds the entire + // decoded source file in memory. Files are processed sequentially, + // so peak usage is one file at a time, but that can still be several + // GB for a compaction-sized file. A size-capped fallback that starts + // writing the replacement once the buffer crosses a threshold is + // left for follow-up work. + let mut prefix: Vec = Vec::new(); + let mut file_changed = false; + let mut file_input_rows = 0_u64; + let mut file_output_rows = 0_u64; + let mut writer: Option> = None; + // Debug-only tripwire for the "preserve partition values" + // contract; see `DebugPartitionGuard`. + #[cfg(debug_assertions)] + let partition_guard = + DebugPartitionGuard::new(self.table, &old_data_file, &write_schema)?; + + // Planning already cleared the row predicate (see + // `ManifestEntryContext::into_cow_rewrite_file`), so this task + // reads every row of the source file. + let tasks = Box::pin(futures::stream::iter(vec![Ok(scan_task)])) as FileScanTaskStream; + + // Each candidate file gets its own reader so the per-file prefix + // and lazy-writer semantics stay intact; the delete-file cache is + // therefore also per file, and equality deletes shared by several + // candidates are fetched once per file. + let mut reader_builder = self.table.reader_builder(); + if let Some(batch_size) = self.batch_size { + reader_builder = reader_builder.with_options( + ArrowReaderOptions::builder() + .with_batch_size(batch_size) + .build(), + ); + } + + let mut batches = reader_builder.build().read(tasks)?.stream(); + while let Some(batch) = batches.try_next().await? { + result.stats.input_rows += batch.num_rows() as u64; + file_input_rows += batch.num_rows() as u64; + + let rewrite = rewriter.rewrite_batch(batch)?; + // `output: None` means the batch is fully removed, which is + // itself a change. Derive the effective flag instead of + // trusting every rewriter to keep `changed` consistent with + // `output` — otherwise a `{changed: false, output: None}` + // batch would silently drop its rows while leaving the file + // marked unchanged. + let changed = rewrite.changed || rewrite.output.is_none(); + if changed { + file_changed = true; + result.stats.changed_batches += 1; + } + + // The guard must see every emitted batch, including ones that + // are only buffered into the prefix before the file flips to + // changed. + #[cfg(debug_assertions)] + if let (Some(guard), Some(output)) = (&partition_guard, &rewrite.output) { + guard.check(output)?; + } + + // A dropped batch can be the first change. Flush any kept + // prefix even when this batch has no output; defer opening the + // writer only if there are no rows to preserve yet. + if file_changed + && writer.is_none() + && (!prefix.is_empty() || rewrite.output.is_some()) + { + let partition_key = + source_partition_key(self.table, &old_data_file, &write_schema)?; + writer = Some( + writer::build_replacement_writer( + self.table, + write_schema.clone(), + Some(partition_key), + ) + .await?, + ); + } + + if let Some(writer) = writer.as_mut() { + for prefix_batch in prefix.drain(..) { + writer.write(prefix_batch).await?; + } + if let Some(output) = rewrite.output { + file_output_rows += output.num_rows() as u64; + writer.write(output).await?; + } + } else if let Some(output) = rewrite.output { + file_output_rows += output.num_rows() as u64; + prefix.push(output); + } + } + + // A candidate whose visible rows were all removed by its delete + // files reads as zero rows and the loop above never runs. Treat it + // the same as a rewriter that dropped every batch — changed with + // no replacement — so the file and its delete files can be + // compacted away instead of being pinned in the table forever. + let fully_removed_by_deletes = + !file_changed && file_input_rows == 0 && has_delete_files; + + if file_changed || fully_removed_by_deletes { + result.stats.rewritten_files += 1; + result.stats.output_rows += file_output_rows; + result.removed_data_files.push(old_data_file); + + if let Some(mut writer) = writer { + let added_data_files = writer.close().await?; + result.added_data_files.extend(added_data_files); + } + // If `writer` is `None`, no visible rows remain after delete + // files and batch rewriting, so no replacement is written. + } else { + result.stats.unchanged_files += 1; + result.unchanged_data_files.push(old_data_file); + // `prefix` is dropped here; no replacement file was written. + } + } + + Ok(result) + } +} + +fn source_partition_key( + table: &Table, + data_file: &DataFile, + schema: &crate::spec::SchemaRef, +) -> Result { + let spec = table + .metadata() + .partition_spec_by_id(data_file.partition_spec_id) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Missing partition spec {} for COW rewrite source file", + data_file.partition_spec_id + ), + ) + })? + .as_ref() + .clone(); + // `PartitionKey::new` does not bind the spec to the schema, so validate the + // binding here: a spec that is incompatible with the planned snapshot + // schema should fail with a clear error instead of producing a bad + // partition path when the writer later calls `PartitionKey::to_path`. + spec.partition_type(schema).map_err(|err| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Cannot bind partition spec {} to the planned snapshot schema for COW rewrite", + data_file.partition_spec_id + ), + ) + .with_source(err) + })?; + + Ok(PartitionKey::new( + spec, + schema.clone(), + data_file.partition().clone(), + )) +} + +/// Debug-only tripwire for the rewriter contract that output batches preserve +/// the source file's partition values: replacement files record the source +/// partition verbatim, so a rewriter that mutates a partition-source column +/// would silently write rows under a partition they do not belong to. Compiled +/// out in release builds, where per-row partition calculation would be too +/// expensive for the rewrite hot path. +#[cfg(debug_assertions)] +struct DebugPartitionGuard { + calculator: crate::arrow::PartitionValueCalculator, + partition_type: crate::spec::StructType, + expected: crate::spec::Struct, +} + +#[cfg(debug_assertions)] +impl DebugPartitionGuard { + /// Builds a guard for one candidate file, or `None` when the source + /// file's partition spec is unpartitioned. Mirrors the spec lookup and + /// schema binding of [`source_partition_key`]. + fn new( + table: &Table, + data_file: &DataFile, + schema: &crate::spec::SchemaRef, + ) -> Result> { + let spec = table + .metadata() + .partition_spec_by_id(data_file.partition_spec_id) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Missing partition spec {} for COW rewrite source file", + data_file.partition_spec_id + ), + ) + })?; + if spec.is_unpartitioned() { + return Ok(None); + } + + Ok(Some(Self { + calculator: crate::arrow::PartitionValueCalculator::try_new(spec, schema)?, + partition_type: spec.partition_type(schema)?, + expected: data_file.partition().clone(), + })) + } + + /// Asserts that every row in `output` computes to the source file's + /// partition under the source spec. + fn check(&self, output: &RecordBatch) -> Result<()> { + let partition_array = self.calculator.calculate(output)?; + let values = crate::arrow::arrow_struct_to_literal(&partition_array, &self.partition_type)?; + for value in values { + debug_assert_eq!( + value, + Some(crate::spec::Literal::Struct(self.expected.clone())), + "COW rewriter must preserve the source file's partition values: \ + replacement files record the source partition verbatim, so \ + mutating a partition-source column writes rows under a \ + partition they do not belong to" + ); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::sync::Arc; + + use arrow_array::{Array, ArrayRef, BooleanArray, Int32Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use futures::TryStreamExt; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use tempfile::TempDir; + + use crate::cow_rewrite::{CowBatchRewrite, CowBatchRewriter, CowRewriteBuilder}; + use crate::io::LocalFsStorageFactory; + use crate::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; + use crate::scan::{FileScanTask, FileScanTaskStream}; + use crate::spec::{ + DataFile, Literal, NestedField, PrimitiveType, Schema, Struct, TableProperties, Transform, + Type, + }; + use crate::table::Table; + use crate::test_utils::scan::TableTestFixture; + use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction}; + use crate::{Catalog, CatalogBuilder, Error, ErrorKind, NamespaceIdent, Result, TableCreation}; + + struct KeepAll; + + impl CowBatchRewriter for KeepAll { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + Ok(CowBatchRewrite { + output: Some(batch), + changed: false, + }) + } + } + + struct DeleteEvenIds; + + impl CowBatchRewriter for DeleteEvenIds { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let ids = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + + let keep = + BooleanArray::from_iter((0..ids.len()).map(|row| Some(ids.value(row) % 2 != 0))); + let filtered = arrow_select::filter::filter_record_batch(&batch, &keep) + .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?; + + Ok(CowBatchRewrite { + changed: filtered.num_rows() != batch.num_rows(), + output: (filtered.num_rows() > 0).then_some(filtered), + }) + } + } + + struct IncrementValueForEvenIds; + + impl CowBatchRewriter for IncrementValueForEvenIds { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let ids = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + let values = batch + .column_by_name("value") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?; + + let mut changed = false; + let updated_values = Int32Array::from_iter((0..values.len()).map(|row| { + let value = values.value(row); + if ids.value(row) % 2 == 0 { + changed = true; + Some(value + 10) + } else { + Some(value) + } + })); + let output = RecordBatch::try_new(batch.schema(), vec![ + batch.column(0).clone(), + Arc::new(updated_values), + ]) + .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?; + + Ok(CowBatchRewrite { + output: Some(output), + changed, + }) + } + } + + struct CowRewriteFixture { + _temp_dir: TempDir, + table: Table, + } + + async fn test_table_with_ids(ids: Vec) -> Result { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_rewrite_fixture".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef + ])?; + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&catalog).await?; + + Ok(CowRewriteFixture { + _temp_dir: temp_dir, + table, + }) + } + + async fn test_table_with_id_batches(batches: Vec>) -> Result { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_rewrite_fixture".to_string()) + .schema(schema) + .properties(HashMap::from([( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "1".to_string(), + )])) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let input = batches.into_iter().map(|ids| { + Ok(RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef, + ])?) + }); + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(input), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&catalog).await?; + + Ok(CowRewriteFixture { + _temp_dir: temp_dir, + table, + }) + } + + async fn test_table_with_id_value_rows(rows: Vec<(i32, i32)>) -> Result { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "value", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_rewrite_fixture".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("value", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + let ids = rows.iter().map(|(id, _)| *id).collect::>(); + let values = rows.iter().map(|(_, value)| *value).collect::>(); + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef, + Arc::new(Int32Array::from(values)) as ArrayRef, + ])?; + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&catalog).await?; + + Ok(CowRewriteFixture { + _temp_dir: temp_dir, + table, + }) + } + + async fn read_ids(table: &Table, files: &[DataFile]) -> Result> { + let schema = table.metadata().current_schema().clone(); + let project_field_ids = schema + .as_struct() + .fields() + .iter() + .map(|field| field.id) + .collect::>(); + let tasks = files + .iter() + .map(|data_file| { + FileScanTask::builder() + .with_file_size_in_bytes(data_file.file_size_in_bytes()) + .with_start(0) + .with_length(data_file.file_size_in_bytes()) + .with_record_count(Some(data_file.record_count())) + .with_data_file_path(data_file.file_path().to_string()) + .with_data_file_format(data_file.file_format()) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids.clone()) + .with_case_sensitive(true) + .build() + }) + .collect::>(); + let task_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream; + let batches = table + .reader_builder() + .build() + .read(task_stream)? + .stream() + .try_collect::>() + .await?; + + let mut ids = Vec::new(); + for batch in batches { + let column = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + ids.extend((0..column.len()).map(|row| column.value(row))); + } + ids.sort_unstable(); + Ok(ids) + } + + async fn read_ids_and_values(table: &Table, files: &[DataFile]) -> Result> { + let schema = table.metadata().current_schema().clone(); + let project_field_ids = schema + .as_struct() + .fields() + .iter() + .map(|field| field.id) + .collect::>(); + let tasks = files + .iter() + .map(|data_file| { + FileScanTask::builder() + .with_file_size_in_bytes(data_file.file_size_in_bytes()) + .with_start(0) + .with_length(data_file.file_size_in_bytes()) + .with_record_count(Some(data_file.record_count())) + .with_data_file_path(data_file.file_path().to_string()) + .with_data_file_format(data_file.file_format()) + .with_schema(schema.clone()) + .with_project_field_ids(project_field_ids.clone()) + .with_case_sensitive(true) + .build() + }) + .collect::>(); + let task_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream; + let batches = table + .reader_builder() + .build() + .read(task_stream)? + .stream() + .try_collect::>() + .await?; + + let mut rows = Vec::new(); + for batch in batches { + let ids = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + let values = batch + .column_by_name("value") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?; + rows.extend((0..ids.len()).map(|row| (ids.value(row), values.value(row)))); + } + rows.sort_unstable_by_key(|(id, _)| *id); + Ok(rows) + } + + #[tokio::test] + async fn cow_rewrite_keep_all_produces_no_changes() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 2, 3]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(KeepAll)) + .rewrite() + .await?; + + assert!(!result.has_changes()); + assert_eq!(result.removed_data_files.len(), 0); + assert_eq!(result.added_data_files.len(), 0); + assert_eq!(result.unchanged_data_files.len(), 1); + assert_eq!(result.stats.candidate_files, 1); + assert_eq!(result.stats.unchanged_files, 1); + assert_eq!(result.stats.input_rows, 3); + // Nothing was written, so rows emitted for the unchanged file are not + // counted as output. + assert_eq!(result.stats.output_rows, 0); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_requires_rewriter() -> Result<()> { + let fixture = test_table_with_ids(vec![1]).await?; + + let err = CowRewriteBuilder::new(&fixture.table) + .rewrite() + .await + .expect_err("missing rewriter should fail"); + + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_delete_rows_removes_old_file_and_adds_replacement() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 2, 3, 4]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 4); + assert_eq!(result.stats.output_rows, 2); + + let ids = read_ids(&fixture.table, &result.added_data_files).await?; + assert_eq!(ids, vec![1, 3]); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_update_rows_rewrites_file_with_updated_values() -> Result<()> { + let fixture = + test_table_with_id_value_rows(vec![(1, 10), (2, 20), (3, 30), (4, 40)]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(IncrementValueForEvenIds)) + .rewrite() + .await?; + + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 4); + assert_eq!(result.stats.output_rows, 4); + + let rows = read_ids_and_values(&fixture.table, &result.added_data_files).await?; + assert_eq!(rows, vec![(1, 10), (2, 30), (3, 30), (4, 50)]); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_full_file_delete_removes_old_file_without_replacement() -> Result<()> { + let fixture = test_table_with_ids(vec![2, 4]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 0); + assert_eq!(result.stats.input_rows, 2); + assert_eq!(result.stats.output_rows, 0); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_removes_file_fully_covered_by_position_deletes() -> Result<()> { + let mut fixture = TableTestFixture::new(); + let positions = (0..300).collect::>(); + fixture.setup_multi_row_group_manifest(&positions).await; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(KeepAll)) + .rewrite() + .await?; + + assert_eq!(result.stats.candidate_files, 1); + assert_eq!(result.stats.rewritten_files, 1); + assert_eq!(result.stats.input_rows, 0); + assert_eq!(result.stats.output_rows, 0); + assert_eq!(result.removed_data_files.len(), 1); + assert!(result.added_data_files.is_empty()); + assert!(result.unchanged_data_files.is_empty()); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_delete_no_matching_rows_keeps_old_file() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 3]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert!(!result.has_changes()); + assert_eq!(result.removed_data_files.len(), 0); + assert_eq!(result.added_data_files.len(), 0); + assert_eq!(result.unchanged_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 2); + assert_eq!(result.stats.output_rows, 0); + + Ok(()) + } + + /// Drops every batch while reporting `changed: false`, violating the + /// documented contract. The orchestrator must derive the change from + /// `output: None` itself, otherwise the file would be kept as "unchanged" + /// while its rows were meant to be dropped. + struct SilentFullDrop; + + impl CowBatchRewriter for SilentFullDrop { + fn rewrite_batch(&self, _batch: RecordBatch) -> Result { + Ok(CowBatchRewrite { + output: None, + changed: false, + }) + } + } + + #[tokio::test] + async fn cow_rewrite_none_output_implies_change() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 2]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(SilentFullDrop)) + .rewrite() + .await?; + + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 0); + assert!(result.unchanged_data_files.is_empty()); + assert_eq!(result.stats.rewritten_files, 1); + assert_eq!(result.stats.input_rows, 2); + assert_eq!(result.stats.output_rows, 0); + assert_eq!(result.stats.changed_batches, 1); + + Ok(()) + } + + /// Returns `output: None` with `changed: false` for the first batch, then + /// flags the second batch as changed. The replacement must contain exactly + /// the second batch's rows — the silently dropped first batch must not + /// resurrect, and the original file must be removed. + struct DropFirstBatchSilently { + batches_seen: std::sync::atomic::AtomicUsize, + } + + impl CowBatchRewriter for DropFirstBatchSilently { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let seen = self + .batches_seen + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if seen == 0 { + Ok(CowBatchRewrite { + output: None, + changed: false, + }) + } else { + Ok(CowBatchRewrite { + output: Some(batch), + changed: true, + }) + } + } + } + + #[tokio::test] + async fn cow_rewrite_silent_drop_before_change_loses_no_rows() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 2, 3, 4]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_batch_size(2) + .with_rewriter(Arc::new(DropFirstBatchSilently { + batches_seen: std::sync::atomic::AtomicUsize::new(0), + })) + .rewrite() + .await?; + + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 4); + assert_eq!(result.stats.output_rows, 2); + + let ids = read_ids(&fixture.table, &result.added_data_files).await?; + assert_eq!(ids, vec![3, 4]); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_kept_prefix_survives_later_full_batch_delete() -> Result<()> { + let fixture = test_table_with_ids(vec![1, 3, 2, 4]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_batch_size(2) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.stats.input_rows, 4); + assert_eq!(result.stats.output_rows, 2); + assert_eq!(result.added_data_files.len(), 1); + assert!(result.unchanged_data_files.is_empty()); + assert_eq!( + read_ids(&fixture.table, &result.added_data_files).await?, + vec![1, 3] + ); + + Ok(()) + } + + #[tokio::test] + async fn cow_rewrite_uses_unique_replacement_paths_for_multiple_source_files() -> Result<()> { + let fixture = test_table_with_id_batches(vec![vec![1, 2], vec![3, 4]]).await?; + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteEvenIds)) + .rewrite() + .await?; + + assert_eq!(result.stats.candidate_files, 2); + assert_eq!(result.removed_data_files.len(), 2); + assert_eq!(result.added_data_files.len(), 2); + + let added_paths = result + .added_data_files + .iter() + .map(|file| file.file_path().to_string()) + .collect::>(); + let removed_paths = result + .removed_data_files + .iter() + .map(|file| file.file_path().to_string()) + .collect::>(); + + assert_eq!(added_paths.len(), result.added_data_files.len()); + assert!(added_paths.is_disjoint(&removed_paths)); + + let ids = read_ids(&fixture.table, &result.added_data_files).await?; + assert_eq!(ids, vec![1, 3]); + + Ok(()) + } + + #[test] + fn cow_batch_rewriter_is_object_safe() { + let _rewriter: Arc = Arc::new(KeepAll); + } + + #[test] + fn cow_rewrite_result_reports_no_changes() { + let result = crate::cow_rewrite::CowRewriteResult { + removed_data_files: vec![], + added_data_files: vec![], + unchanged_data_files: vec![], + stats: crate::cow_rewrite::CowRewriteStats { + candidate_files: 0, + rewritten_files: 0, + unchanged_files: 0, + input_rows: 0, + output_rows: 0, + changed_batches: 0, + }, + }; + + assert!(!result.has_changes()); + assert_eq!(result.stats.candidate_files, 0); + } + + /// `DeleteIfModThree` keeps rows whose `id` is not divisible by 3. + struct DeleteIfModThree; + + impl CowBatchRewriter for DeleteIfModThree { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let ids = batch + .column_by_name("id") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?; + + let keep = + BooleanArray::from_iter((0..ids.len()).map(|row| Some(ids.value(row) % 3 != 0))); + let filtered = arrow_select::filter::filter_record_batch(&batch, &keep) + .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?; + + Ok(CowBatchRewrite { + changed: filtered.num_rows() != batch.num_rows(), + output: (filtered.num_rows() > 0).then_some(filtered), + }) + } + } + + /// After adding an optional column (`value`) to the schema, a COW rewrite + /// of the pre-existing data must not fail. The replacement writer must use + /// the schema the batches were read in (the snapshot schema), not the + /// table's evolved current schema, otherwise the parquet writer rejects + /// the input batch whose column set does not include `value`. + #[tokio::test] + async fn cow_rewrite_after_optional_column_add() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog: Arc = Arc::new( + MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?, + ); + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("evolved".to_string()) + .schema(schema) + .build(), + ) + .await?; + + // Write the original data file with the {id}-only schema. + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![ + 1, 2, 3, 4, + ])) as ArrayRef])?; + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&*catalog).await?; + + // Evolve the schema: add an optional `value` column. + let tx = Transaction::new(&table); + let tx = tx + .update_schema() + .add_column(AddColumn::optional( + "value", + Type::Primitive(PrimitiveType::Int), + )) + .apply(tx)?; + let table = tx.commit(&*catalog).await?; + + // The current schema now has {id, value} but the current snapshot's + // schema is still the original {id} schema. + let current_snapshot = table.metadata().current_snapshot().unwrap(); + assert_ne!( + table.metadata().current_schema_id(), + current_snapshot.schema_id().unwrap() + ); + + // Before the fix this fails when the parquet writer rejects the {id}-only + // batch while configured with the {id, value} current schema. + let result = CowRewriteBuilder::new(&table) + .with_predicate(crate::expr::Predicate::AlwaysTrue) + .with_rewriter(Arc::new(DeleteIfModThree)) + .rewrite() + .await?; + + // id 3 is divisible by 3 and is dropped; remaining ids are 1, 2, 4. + assert!(result.has_changes()); + assert_eq!(result.removed_data_files.len(), 1); + assert_eq!(result.added_data_files.len(), 1); + + let ids = read_ids(&table, &result.added_data_files).await?; + assert_eq!(ids, vec![1, 2, 4]); + + Ok(()) + } + + /// Builds a two-file table whose files have disjoint id ranges so predicate + /// metrics pruning can exclude one file entirely. + async fn two_file_table_disjoint_ids() -> Result { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("disjoint".to_string()) + .schema(schema) + .properties(HashMap::from([( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "1".to_string(), + )])) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + // First batch ids 1..=4, second batch ids 100..=103. Force a split via + // tiny target file size so each batch lands in its own data file. + let input = vec![vec![1, 2, 3, 4], vec![100, 101, 102, 103]] + .into_iter() + .map(|ids| { + Ok(RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(ids)) as ArrayRef, + ])?) + }); + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(input), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx.fast_append().add_data_files(data_files).apply(tx)?; + let table = tx.commit(&catalog).await?; + + Ok(CowRewriteFixture { + _temp_dir: temp_dir, + table, + }) + } + + /// Predicate-based planning must exclude files whose metrics cannot match + /// the predicate. With `id > 50`, the file holding ids 1..=4 must never be + /// planned as a candidate. + #[tokio::test] + async fn cow_rewrite_predicate_prunes_non_matching_files() -> Result<()> { + let fixture = two_file_table_disjoint_ids().await?; + + let predicate = crate::expr::Reference::new("id").greater_than(crate::spec::Datum::int(50)); + + let result = CowRewriteBuilder::new(&fixture.table) + .with_predicate(predicate) + .with_rewriter(Arc::new(KeepAll)) + .rewrite() + .await?; + + // Only the high-id file should be a candidate. + assert_eq!(result.stats.candidate_files, 1); + // KeepAll does not change anything, so the single candidate is unchanged. + assert_eq!(result.removed_data_files.len(), 0); + assert_eq!(result.added_data_files.len(), 0); + assert_eq!(result.unchanged_data_files.len(), 1); + + // The surviving candidate must be the high-id file (ids 100..=103). + let ids = read_ids(&fixture.table, &result.unchanged_data_files).await?; + assert_eq!(ids, vec![100, 101, 102, 103]); + + Ok(()) + } + + /// Rewrites every batch without altering rows, so every candidate file is + /// replaced. Useful for exercising the replacement-writing path itself. + struct TouchEveryBatch; + + impl CowBatchRewriter for TouchEveryBatch { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + Ok(CowBatchRewrite { + output: Some(batch), + changed: true, + }) + } + } + + /// An end-to-end rewrite on an identity-partitioned table must preserve + /// each source file's partition: the replacement file carries the same + /// partition values and partition spec id, and lands under the same + /// partition directory — including the null partition. + #[tokio::test] + async fn cow_rewrite_preserves_partitions_end_to_end() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "value", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone())) + .add_partition_field("value", "value", Transform::Identity)? + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("partitioned".to_string()) + .schema(schema) + .partition_spec(partition_spec.into_unbound()) + .build(), + ) + .await?; + + // Two source files: ids 1, 2 in the `value=1` partition and ids 3, 4 + // in the null partition. + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("value", DataType::Int32, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + let current_schema = table.metadata().current_schema().clone(); + let default_spec = table.metadata().default_partition_spec().as_ref().clone(); + + let batch_one = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 1])), + ])?; + let one_partition = crate::spec::PartitionKey::new( + default_spec.clone(), + current_schema.clone(), + Struct::from_iter([Some(Literal::int(1))]), + ); + let files_one = super::writer::write_replacement_batches( + &table, + current_schema.clone(), + Some(one_partition), + futures::stream::iter(vec![Ok(batch_one)]), + ) + .await?; + + let batch_null = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![3, 4])) as ArrayRef, + Arc::new(Int32Array::from(vec![None::, None])), + ])?; + let null_partition = crate::spec::PartitionKey::new( + default_spec.clone(), + current_schema.clone(), + Struct::from_iter([None::]), + ); + let files_null = super::writer::write_replacement_batches( + &table, + current_schema.clone(), + Some(null_partition), + futures::stream::iter(vec![Ok(batch_null)]), + ) + .await?; + + let tx = Transaction::new(&table); + let tx = tx + .fast_append() + .add_data_files([files_one, files_null].concat()) + .apply(tx)?; + let table = tx.commit(&catalog).await?; + + let result = CowRewriteBuilder::new(&table) + .with_rewriter(Arc::new(TouchEveryBatch)) + .rewrite() + .await?; + + assert_eq!(result.stats.candidate_files, 2); + assert_eq!(result.stats.rewritten_files, 2); + assert_eq!(result.removed_data_files.len(), 2); + assert_eq!(result.added_data_files.len(), 2); + assert!(result.unchanged_data_files.is_empty()); + + // Partition values survive the rewrite unchanged. + let one_struct = Struct::from_iter([Some(Literal::int(1))]); + let null_struct = Struct::from_iter([None::]); + let mut removed_partitions = result + .removed_data_files + .iter() + .map(|file| file.partition().clone()) + .collect::>(); + let mut added_partitions = result + .added_data_files + .iter() + .map(|file| file.partition().clone()) + .collect::>(); + removed_partitions.sort_by_key(|partition| partition.is_null_at_index(0)); + added_partitions.sort_by_key(|partition| partition.is_null_at_index(0)); + assert_eq!(removed_partitions, added_partitions); + assert_eq!(added_partitions[0], one_struct); + assert_eq!(added_partitions[1], null_struct); + + for file in &result.added_data_files { + assert_eq!(file.partition_spec_id, 0); + if file.partition() == &one_struct { + assert!(file.file_path().contains("value=1"), "{}", file.file_path()); + } else { + assert!( + file.file_path().contains("value=null"), + "{}", + file.file_path() + ); + } + } + + // Replacement content matches the source rows. + let ids = read_ids(&table, &result.added_data_files).await?; + assert_eq!(ids, vec![1, 2, 3, 4]); + + Ok(()) + } + + /// Mutates the identity-partitioned `value` column while flagging the + /// batch as changed — a violation of the partition-preservation contract. + /// In debug builds the partition guard must trip instead of letting the + /// replacement file record a partition that disagrees with its rows. + #[cfg(debug_assertions)] + struct MutatePartitionColumn; + + #[cfg(debug_assertions)] + impl CowBatchRewriter for MutatePartitionColumn { + fn rewrite_batch(&self, batch: RecordBatch) -> Result { + let values = batch + .column_by_name("value") + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?; + let updated = + Int32Array::from_iter((0..values.len()).map(|row| Some(values.value(row) + 100))); + let output = RecordBatch::try_new(batch.schema(), vec![ + batch.column(0).clone(), + Arc::new(updated), + ]) + .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?; + + Ok(CowBatchRewrite { + output: Some(output), + changed: true, + }) + } + } + + #[cfg(debug_assertions)] + #[tokio::test] + #[should_panic(expected = "must preserve the source file's partition values")] + async fn cow_rewrite_partition_column_mutation_trips_debug_guard() { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await + .unwrap(); + let namespace = NamespaceIdent::new("ns".to_string()); + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap(); + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "value", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone())) + .add_partition_field("value", "value", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("partition_guard".to_string()) + .schema(schema) + .partition_spec(partition_spec.into_unbound()) + .build(), + ) + .await + .unwrap(); + + // One source file whose rows all sit in the `value=1` partition. + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("value", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 1])), + ]) + .unwrap(); + let partition_key = crate::spec::PartitionKey::new( + table.metadata().default_partition_spec().as_ref().clone(), + table.metadata().current_schema().clone(), + Struct::from_iter([Some(Literal::int(1))]), + ); + let data_files = super::writer::write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + Some(partition_key), + futures::stream::iter(vec![Ok(batch)]), + ) + .await + .unwrap(); + + let tx = Transaction::new(&table); + let tx = tx + .fast_append() + .add_data_files(data_files) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // The rewriter bumps every `value` by 100, so the output batch + // computes to a different partition than the source file records. + // The debug guard must panic here; without it the rewrite would + // silently commit a replacement file under `value=1` holding rows + // with `value=101`. + let _ = CowRewriteBuilder::new(&table) + .with_rewriter(Arc::new(MutatePartitionColumn)) + .rewrite() + .await; + } +} diff --git a/crates/iceberg/src/cow_rewrite/plan.rs b/crates/iceberg/src/cow_rewrite/plan.rs new file mode 100644 index 0000000000..8e96e9ab4b --- /dev/null +++ b/crates/iceberg/src/cow_rewrite/plan.rs @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use futures::TryStreamExt; + +use crate::scan::FileScanTask; +use crate::spec::DataFile; + +/// A data file selected for COW rewrite. +/// +/// This is a read-only output type: the planning entry points that produce +/// candidates are crate-internal and reached through +/// [`crate::cow_rewrite::CowRewriteBuilder`], and the fields are exposed +/// through accessors only. Follow-up commit-adapter work (overwrite and +/// row-delta actions) consumes planned candidates through those accessors; +/// construction from outside the crate is intentionally not exposed yet and +/// can be added when an adapter actually needs it. +#[derive(Debug, Clone)] +pub struct CowRewriteFile { + /// Original data file from the manifest entry. + pub(crate) old_data_file: DataFile, + /// Full-read task for visible rows in the original file. + pub(crate) scan_task: FileScanTask, +} + +impl CowRewriteFile { + /// Original data file from the manifest entry. + pub fn old_data_file(&self) -> &DataFile { + &self.old_data_file + } + + /// Full-read task for visible rows in the original file. + pub fn scan_task(&self) -> &FileScanTask { + &self.scan_task + } +} + +/// Plan data files that may need copy-on-write rewrite. +pub(crate) async fn plan_cow_rewrite_files( + table: &crate::table::Table, + predicate: Option, + snapshot_id: Option, + case_sensitive: bool, +) -> crate::Result> { + let mut scan_builder = table + .scan() + .select_all() + .with_case_sensitive(case_sensitive); + + if let Some(predicate) = predicate { + scan_builder = scan_builder.with_filter(predicate); + } + + if let Some(snapshot_id) = snapshot_id { + scan_builder = scan_builder.snapshot_id(snapshot_id); + } + + scan_builder + .build()? + .plan_cow_rewrite_files() + .await? + .try_collect() + .await +} + +#[cfg(test)] +mod tests { + use futures::TryStreamExt; + + use crate::Result; + use crate::expr::Predicate; + use crate::test_utils::scan::TableTestFixture; + + #[tokio::test] + async fn cow_planner_returns_old_file_and_full_read_task() -> Result<()> { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + let mut files = + super::plan_cow_rewrite_files(&fixture.table, Some(Predicate::AlwaysTrue), None, true) + .await?; + + assert_eq!(files.len(), 2); + + files.sort_by_key(|file| file.old_data_file.file_path().to_string()); + assert_eq!( + files[0].old_data_file.file_path(), + format!("{}/1.parquet", fixture.table_location) + ); + assert_eq!( + files[1].old_data_file.file_path(), + format!("{}/3.parquet", fixture.table_location) + ); + + for file in files { + assert_eq!( + file.old_data_file.file_path(), + file.scan_task.data_file_path() + ); + assert!(file.scan_task.predicate().is_none()); + assert_eq!( + Some(file.old_data_file.record_count()), + file.scan_task.record_count() + ); + assert_eq!(0, file.scan_task.start()); + assert_eq!( + file.old_data_file.file_size_in_bytes(), + file.scan_task.length() + ); + } + + Ok(()) + } + + #[tokio::test] + async fn cow_planner_preserves_delete_files() -> Result<()> { + let mut fixture = TableTestFixture::new(); + fixture.setup_deadlock_manifests().await; + + let scan = fixture + .table + .scan() + .select_all() + .with_concurrency_limit(1) + .build()?; + + let files = tokio::time::timeout(std::time::Duration::from_secs(5), async { + scan.plan_cow_rewrite_files() + .await? + .try_collect::>() + .await + }) + .await + .expect("COW planning should not deadlock")?; + + assert_eq!(files.len(), 10); + for file in files { + assert_eq!(file.scan_task.deletes().len(), 1); + assert_eq!( + file.scan_task.deletes()[0].file_path(), + format!("{}/del.parquet", fixture.table_location) + ); + } + + Ok(()) + } +} diff --git a/crates/iceberg/src/cow_rewrite/rewriter.rs b/crates/iceberg/src/cow_rewrite/rewriter.rs new file mode 100644 index 0000000000..4ed5e3402c --- /dev/null +++ b/crates/iceberg/src/cow_rewrite/rewriter.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow_array::RecordBatch; + +use crate::Result; + +/// Result of rewriting a single record batch. +#[derive(Debug)] +pub struct CowBatchRewrite { + /// Rewritten output batch, or `None` when the input batch is fully removed. + /// + /// Output batches must use the same schema as their input batch — the + /// planned snapshot's schema, which may be older than the table's current + /// schema. Rewriters must also preserve each source file's partition + /// values: this primitive writes replacements into the source file's + /// partition and does not repartition rows. + pub output: Option, + /// Whether the rewrite changed the input batch contents. + /// + /// Set this to `true` whenever `output` differs from the input batch, + /// including filtered rows, updated values, reordered rows, or `None`. + pub changed: bool, +} + +/// Rewrites record batches for copy-on-write operations. +/// +/// `rewrite_batch` is synchronous: it runs on the async runtime thread that +/// drives the read/write pipeline, so implementations must not perform +/// blocking work — async I/O such as catalog enrichment or cross-table +/// lookups is not supported in this contract. The method stays sync (rather +/// than returning a boxed future) so the trait remains object safe for +/// `Arc`. +pub trait CowBatchRewriter: Send + Sync { + /// Rewrites a record batch and reports whether it changed. + /// + /// `output: None` means the batch is fully removed, which is itself a + /// change: the orchestrator treats it as changed regardless of the + /// `changed` flag, so a rewriter cannot accidentally keep dropped rows + /// alive by reporting `changed: false` alongside a `None` output. + fn rewrite_batch(&self, batch: RecordBatch) -> Result; +} diff --git a/crates/iceberg/src/cow_rewrite/writer.rs b/crates/iceberg/src/cow_rewrite/writer.rs new file mode 100644 index 0000000000..ce235e889b --- /dev/null +++ b/crates/iceberg/src/cow_rewrite/writer.rs @@ -0,0 +1,447 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::str::FromStr; + +use uuid::Uuid; + +use crate::Result; +#[cfg(test)] +use crate::spec::DataFile; +use crate::spec::{DataFileFormat, PartitionKey, SchemaRef}; +use crate::table::Table; +use crate::writer::IcebergWriterBuilder; +use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; +use crate::writer::file_writer::ParquetWriterBuilder; +use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, LocationGenerator, + ObjectStorageLocationGenerator, +}; +use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + +/// Builds a boxed replacement-data-file writer. +/// +/// `write_schema` is the schema the input batches are encoded in. It must match +/// the schema the rows were read in (the planned snapshot's schema), not the +/// table's possibly-evolved current schema; otherwise the parquet writer will +/// reject batches that lack columns added after the source files were written. +/// +/// Building the writer is cheap: no physical file is opened until the first +/// batch is written, so it is safe to construct one optimistically and only +/// write to it once a source file is known to have changed. +/// +/// The location layout follows `write.object-storage.enabled`: tables that opt +/// into the object-storage layout get hash-entropy paths from +/// [`ObjectStorageLocationGenerator`], everything else gets the default layout. +pub(crate) async fn build_replacement_writer( + table: &Table, + write_schema: SchemaRef, + partition_key: Option, +) -> Result> { + let table_props = table.metadata().table_properties(); + // The replacement writer only produces parquet today; refuse to run on a + // table configured for another format instead of silently writing parquet + // against `write.format.default`. + let write_format_default = table_props.write_format_default()?; + if DataFileFormat::from_str(&write_format_default)? != DataFileFormat::Parquet { + return Err(crate::Error::new( + crate::ErrorKind::FeatureUnsupported, + format!("File format {write_format_default} is not supported for COW rewrite yet!"), + )); + } + let mut parquet_builder = + ParquetWriterBuilder::from_table_properties(&table_props, write_schema)?; + // Replacement files must not silently downgrade an encrypted table to + // plaintext: source rows were read through decryption, so re-encrypt the + // rewritten rows when the table carries an encryption manager. + if let Some(encryption_manager) = table.encryption_manager() { + parquet_builder = parquet_builder.with_encryption_manager(encryption_manager.clone()); + } + + if table_props.write_object_storage_enabled()? { + let location_generator = ObjectStorageLocationGenerator::new(table.metadata())?; + build_data_file_writer(table, parquet_builder, location_generator, partition_key).await + } else { + let location_generator = DefaultLocationGenerator::new(table.metadata())?; + build_data_file_writer(table, parquet_builder, location_generator, partition_key).await + } +} + +async fn build_data_file_writer( + table: &Table, + parquet_builder: ParquetWriterBuilder, + location_generator: L, + partition_key: Option, +) -> Result> { + let file_name_generator = DefaultFileNameGenerator::new( + format!("cow-rewrite-{}", Uuid::now_v7()), + None, + DataFileFormat::Parquet, + ); + let table_props = table.metadata().table_properties(); + let rolling_builder = RollingFileWriterBuilder::new( + parquet_builder, + table_props.write_target_file_size_bytes()?, + table.file_io().clone(), + location_generator, + file_name_generator, + ); + let data_writer_builder = DataFileWriterBuilder::new(rolling_builder); + Ok(Box::new(data_writer_builder.build(partition_key).await?)) +} + +/// Writes replacement record batches as Iceberg data files. +/// +/// This is a convenience wrapper around [`build_replacement_writer`] that +/// consumes a stream end-to-end. Empty input or all-zero-row batches produce no +/// data files. +#[cfg(test)] +pub(crate) async fn write_replacement_batches( + table: &Table, + write_schema: SchemaRef, + partition_key: Option, + mut batches: S, +) -> Result> +where + S: futures::Stream> + Unpin, +{ + use futures::TryStreamExt as _; + + let mut writer = build_replacement_writer(table, write_schema, partition_key).await?; + + let mut wrote_rows = false; + while let Some(batch) = batches.try_next().await? { + if batch.num_rows() == 0 { + continue; + } + wrote_rows = true; + writer.write(batch).await?; + } + + if !wrote_rows { + return Ok(vec![]); + } + + writer.close().await +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{ArrayRef, Int32Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use tempfile::TempDir; + + use super::write_replacement_batches; + use crate::io::LocalFsStorageFactory; + use crate::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; + use crate::spec::{ + DataContentType, NestedField, PartitionKey, PrimitiveType, Schema, Struct, Transform, Type, + }; + use crate::{Catalog, CatalogBuilder, NamespaceIdent, Result, TableCreation}; + + #[tokio::test] + async fn cow_replacement_writer_outputs_data_files() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_writer".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![ + 1, 2, 3, + ])) as ArrayRef])?; + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + assert_eq!(data_files.len(), 1); + assert_eq!(data_files[0].record_count(), 3); + assert_eq!(data_files[0].content_type(), DataContentType::Data); + + Ok(()) + } + + #[tokio::test] + async fn cow_replacement_writer_skips_empty_input() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_empty_writer".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![]), + ) + .await?; + + assert!(data_files.is_empty()); + + Ok(()) + } + + #[tokio::test] + async fn cow_replacement_writer_skips_zero_row_batches() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_zero_row_writer".to_string()) + .schema(schema) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = + RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(Vec::::new())) as ArrayRef, + ])?; + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + assert!(data_files.is_empty()); + + Ok(()) + } + + /// With `write.object-storage.enabled` the replacement file must land in + /// the hash-entropy object storage layout instead of the flat default one. + #[tokio::test] + async fn cow_replacement_writer_honors_object_storage_layout() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_object_storage_writer".to_string()) + .schema(schema) + .properties(HashMap::from([( + crate::spec::TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED + .to_string(), + "true".to_string(), + )])) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![ + 1, 2, 3, + ])) as ArrayRef])?; + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + None, + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + assert_eq!(data_files.len(), 1); + // Object storage layout: `{data_location}/{entropy dirs}/{file name}`, + // where the entropy dirs are binary strings of 4, 4, 4, and 8 chars. + let path = data_files[0].file_path(); + let after_data = path + .split("/data/") + .nth(1) + .unwrap_or_else(|| panic!("expected /data/ in {path}")); + let segments = after_data.split('/').collect::>(); + assert_eq!(segments.len(), 5, "{path}"); + for (segment, len) in segments[..4].iter().zip([4, 4, 4, 8]) { + assert_eq!(segment.len(), len, "{path}"); + assert!(segment.chars().all(|c| c == '0' || c == '1'), "{path}"); + } + assert!(segments[4].starts_with("cow-rewrite-"), "{path}"); + + Ok(()) + } + + #[tokio::test] + async fn cow_replacement_writer_preserves_partition_key() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display()); + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]), + ) + .await?; + let namespace = NamespaceIdent::new("ns".to_string()); + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?; + let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone())) + .add_partition_field("id", "id", Transform::Identity)? + .build()?; + let table = catalog + .create_table( + &namespace, + TableCreation::builder() + .name("cow_partitioned_writer".to_string()) + .schema(schema) + .partition_spec(partition_spec.clone().into_unbound()) + .build(), + ) + .await?; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![ + 1, 1, 1, + ])) as ArrayRef])?; + let partition_key = PartitionKey::new( + table.metadata().default_partition_spec().as_ref().clone(), + table.metadata().current_schema().clone(), + Struct::from_iter([Some(crate::spec::Literal::int(1))]), + ); + + let data_files = write_replacement_batches( + &table, + table.metadata().current_schema().clone(), + Some(partition_key), + futures::stream::iter(vec![Ok(batch)]), + ) + .await?; + + assert_eq!(data_files.len(), 1); + assert_eq!(data_files[0].partition_spec_id, 0); + assert_eq!( + data_files[0].partition(), + &Struct::from_iter([Some(crate::spec::Literal::int(1))]) + ); + + Ok(()) + } +} diff --git a/crates/iceberg/src/delete_file_index.rs b/crates/iceberg/src/delete_file_index.rs index d8f7a872e1..01b4773f62 100644 --- a/crates/iceberg/src/delete_file_index.rs +++ b/crates/iceberg/src/delete_file_index.rs @@ -23,9 +23,12 @@ use futures::StreamExt; use futures::channel::mpsc::{Sender, channel}; use tokio::sync::Notify; -use crate::runtime::spawn; +use crate::error::invalid_data; +use crate::metadata_columns::RESERVED_FIELD_ID_DELETE_FILE_PATH; +use crate::runtime::Runtime; use crate::scan::{DeleteFileContext, FileScanTaskDeleteFile}; -use crate::spec::{DataContentType, DataFile, Struct}; +use crate::spec::{DataContentType, DataFile, DataFileFormat, PrimitiveLiteral, Struct}; +use crate::{Error, Result}; /// Index of delete files #[derive(Debug, Clone)] @@ -35,25 +38,72 @@ pub(crate) struct DeleteFileIndex { #[derive(Debug)] enum DeleteFileIndexState { + // Arc, not Box: a waiter clones this out and awaits `notified_owned()` after dropping the + // read lock (a borrowed `notified()` future can't outlive the guard it was created under). + // If multiple callers arrive while still Populating, each clones its own handle to the same + // Notify, so one `notify_waiters()` call wakes all of them. Populating(Arc), - Populated(PopulatedDeleteFileIndex), + // Boxed because PopulatedDeleteFileIndex is large relative to the other variants; there is + // exactly one owner (this enum, behind the RwLock), so this needs heap indirection, not + // shared ownership. + Populated(Box), + // Boxed for the same reason: Error is large enough to trip the same size check, and is never + // cloned out of the lock, only read as `&Error` via deref. + Failed(Box), } #[derive(Debug)] struct PopulatedDeleteFileIndex { - #[allow(dead_code)] - global_deletes: Vec>, + global_equality_deletes: Vec>, eq_deletes_by_partition: HashMap>>, pos_deletes_by_partition: HashMap>>, - // TODO: do we need this? - // pos_deletes_by_path: HashMap>>, + pos_deletes_by_path: HashMap>>, + // V3 deletion vectors, keyed by the data file they apply to (referenced_data_file). At most + // one exists per data file per snapshot, and when one applies it supersedes any position + // delete files for that data file, partition-scoped or path-scoped alike. + dvs_by_referenced_data_file: HashMap>, +} + +/// Determines the single data file referenced by a position delete file, if any. +/// +/// Checks the explicit `referenced_data_file` field first. Falls back to the +/// lower and upper bounds of the delete file's `file_path` column: when both +/// bounds are present and equal, the delete file references exactly one data +/// file. Unequal or missing bounds mean the delete file may reference multiple +/// data files, so no single path can be inferred. A bound that is not a string +/// literal is treated the same way. +fn referenced_data_file(data_file: &DataFile) -> Option { + if let Some(path) = data_file.referenced_data_file() { + return Some(path); + } + + let lower_bound = data_file + .lower_bounds() + .get(&RESERVED_FIELD_ID_DELETE_FILE_PATH)?; + let upper_bound = data_file + .upper_bounds() + .get(&RESERVED_FIELD_ID_DELETE_FILE_PATH)?; + + if lower_bound != upper_bound { + return None; + } + + match lower_bound.literal() { + PrimitiveLiteral::String(path) => Some(path.clone()), + _ => None, + } +} - // TODO: Deletion Vector support +/// Rebuilds an owned `Error` from the boxed `Error` cached in `DeleteFileIndexState::Failed`. +/// `Error` isn't `Clone`, so each waiter gets its own copy of the kind and message rather than +/// sharing the original's backtrace and source chain. +fn clone_failed_index_error(err: &Error) -> Error { + Error::new(err.kind(), err.message().to_string()) } impl DeleteFileIndex { /// create a new `DeleteFileIndex` along with the sender that populates it with delete files - pub(crate) fn new() -> (DeleteFileIndex, Sender) { + pub(crate) fn new(runtime: Runtime) -> (DeleteFileIndex, Sender) { // TODO: what should the channel limit be? let (tx, rx) = channel(10); let notify = Arc::new(Notify::new()); @@ -62,16 +112,20 @@ impl DeleteFileIndex { ))); let delete_file_stream = rx.boxed(); - spawn({ + runtime.io().spawn({ let state = state.clone(); async move { - let delete_files = delete_file_stream.collect::>().await; + let delete_files: Vec = + delete_file_stream.collect::>().await; - let populated_delete_file_index = PopulatedDeleteFileIndex::new(delete_files); + let new_state = match PopulatedDeleteFileIndex::new(delete_files) { + Ok(index) => DeleteFileIndexState::Populated(Box::new(index)), + Err(err) => DeleteFileIndexState::Failed(Box::new(err)), + }; { let mut guard = state.write().unwrap(); - *guard = DeleteFileIndexState::Populated(populated_delete_file_index); + *guard = new_state; } notify.notify_waiters(); } @@ -81,29 +135,44 @@ impl DeleteFileIndex { } /// Gets all the delete files that apply to the specified data file. + /// + /// Fails if building the index found a spec violation, such as multiple deletion vectors + /// referencing the same data file, or if a matched deletion vector's sequence number + /// violates the spec relative to `seq_num`. pub(crate) async fn get_deletes_for_data_file( &self, data_file: &DataFile, seq_num: Option, - ) -> Vec { - let notifier = { + ) -> Result> { + // Create the `Notified` while holding the read lock. The read lock ensures that + // when we go inside it, either the state is already at Populated or it is still + // at Populating AND `notify_waiters()` has not been called yet. Any `Notified` + // created before the invocation of `notify_waiters()` will be notified by it + // even if `await` has not been called on it yet. + let notified = { let guard = self.state.read().unwrap(); - match *guard { - DeleteFileIndexState::Populating(ref notifier) => notifier.clone(), - DeleteFileIndexState::Populated(ref index) => { + match &*guard { + DeleteFileIndexState::Populating(notifier) => notifier.clone().notified_owned(), + DeleteFileIndexState::Populated(index) => { return index.get_deletes_for_data_file(data_file, seq_num); } + DeleteFileIndexState::Failed(err) => { + return Err(clone_failed_index_error(err)); + } } }; - notifier.notified().await; + notified.await; let guard = self.state.read().unwrap(); match guard.deref() { DeleteFileIndexState::Populated(index) => { index.get_deletes_for_data_file(data_file, seq_num) } - _ => unreachable!("Cannot be any other state than loaded"), + DeleteFileIndexState::Failed(err) => Err(clone_failed_index_error(err)), + DeleteFileIndexState::Populating(_) => { + unreachable!("Cannot still be Populating after being notified") + } } } } @@ -112,99 +181,1232 @@ impl PopulatedDeleteFileIndex { /// Creates a new populated delete file index from a list of delete file contexts, which /// allows for fast lookup when determining which delete files apply to a given data file. /// - /// 1. The partition information is extracted from each delete file's manifest entry. - /// 2. If the partition is empty and the delete file is not a positional delete, - /// it is added to the `global_deletes` vector - /// 3. Otherwise, the delete file is added to one of two hash maps based on its content type. - fn new(files: Vec) -> PopulatedDeleteFileIndex { + /// 1. A V3 deletion vector (a `PositionDeletes` entry stored as `Puffin`) is indexed by the + /// `referenced_data_file` field, which the spec requires for deletion vectors. + /// Fails if two deletion vectors reference the same data file: the spec allows at most + /// one deletion vector per data file per snapshot. + /// 2. Other position deletes that reference a single data file, either through the + /// `referenced_data_file` field or through equal `file_path` column bounds, + /// are indexed by that data file's path. + /// 3. All other position deletes are indexed by the partition extracted from + /// their manifest entry. + /// 4. Equality deletes stored with an unpartitioned spec are applied as global + /// deletes, per the spec. All other equality deletes are indexed by partition. + fn new(files: Vec) -> Result { let mut eq_deletes_by_partition: HashMap>> = HashMap::default(); let mut pos_deletes_by_partition: HashMap>> = HashMap::default(); + let mut pos_deletes_by_path: HashMap>> = + HashMap::default(); + let mut dvs_by_referenced_data_file: HashMap> = + HashMap::default(); - let mut global_deletes: Vec> = vec![]; + let mut global_equality_deletes: Vec> = vec![]; - files.into_iter().for_each(|ctx| { + for ctx in files { let arc_ctx = Arc::new(ctx); - let partition = arc_ctx.manifest_entry.data_file().partition(); + let data_file = arc_ctx.manifest_entry.data_file(); + let partition = data_file.partition(); - // The spec states that "Equality delete files stored with an unpartitioned spec are applied as global deletes". - if partition.fields().is_empty() { - // TODO: confirm we're good to skip here if we encounter a pos del - if arc_ctx.manifest_entry.content_type() != DataContentType::PositionDeletes { - global_deletes.push(arc_ctx); - return; - } - } + match arc_ctx.manifest_entry.content_type() { + DataContentType::PositionDeletes => { + // A deletion vector is a position delete stored as a Puffin blob. The file + // format is what distinguishes it from a position delete parquet file. + if data_file.file_format() == DataFileFormat::Puffin { + // referenced_data_file is the map key the index needs; the offset/size/ + // record_count invariants are enforced centrally when the task is built. + let path = data_file.referenced_data_file().ok_or_else(|| { + invalid_data!( + "deletion vector {} is missing referenced_data_file", + data_file.file_path() + ) + })?; - let destination_map = match arc_ctx.manifest_entry.content_type() { - DataContentType::PositionDeletes => &mut pos_deletes_by_partition, - DataContentType::EqualityDeletes => &mut eq_deletes_by_partition, - _ => unreachable!(), - }; + if let Some(existing) = + dvs_by_referenced_data_file.insert(path.clone(), arc_ctx) + { + let inserted = &dvs_by_referenced_data_file[&path]; + return Err(invalid_data!( + "found multiple deletion vectors for data file {path}: {} and {}", + existing.manifest_entry.file_path(), + inserted.manifest_entry.file_path() + )); + } + continue; + } - destination_map - .entry(partition.clone()) - .and_modify(|entry| { - entry.push(arc_ctx.clone()); - }) - .or_insert(vec![arc_ctx.clone()]); - }); + if let Some(path) = referenced_data_file(data_file) { + pos_deletes_by_path.entry(path).or_default().push(arc_ctx); + } else { + pos_deletes_by_partition + .entry(partition.clone()) + .or_default() + .push(arc_ctx); + } + } + DataContentType::EqualityDeletes => { + // The spec states that "Equality delete files stored with an unpartitioned spec are applied as global deletes". + if partition.fields().is_empty() { + global_equality_deletes.push(arc_ctx); + } else { + eq_deletes_by_partition + .entry(partition.clone()) + .or_default() + .push(arc_ctx); + } + } + _ => unreachable!(), + } + } - PopulatedDeleteFileIndex { - global_deletes, + Ok(PopulatedDeleteFileIndex { + global_equality_deletes, eq_deletes_by_partition, pos_deletes_by_partition, - } + pos_deletes_by_path, + dvs_by_referenced_data_file, + }) } /// Determine all the delete files that apply to the provided `DataFile`. + /// + /// Fails if a matched deletion vector's partition or data sequence number is inconsistent + /// with the data file's: a data file's path is permanently tied to one partition, and the + /// spec guarantees a DV is only ever written at or after the sequence number of the data + /// file it applies to, so either violation means the delete manifest is inconsistent, not + /// that the DV simply doesn't apply. fn get_deletes_for_data_file( &self, data_file: &DataFile, seq_num: Option, - ) -> Vec { + ) -> Result> { let mut results = vec![]; - self.global_deletes - .iter() - // filter that returns true if the provided delete file's sequence number is **greater than or equal to** `seq_num` - .filter(|&delete| { + for delete in self.global_equality_deletes.iter().filter(|&delete| { + // filter that returns true if the provided delete file's sequence number is **greater than** `seq_num` + seq_num + .map(|seq_num| delete.manifest_entry.sequence_number() > Some(seq_num)) + .unwrap_or_else(|| true) + }) { + results.push(FileScanTaskDeleteFile::try_from(delete.as_ref())?); + } + + if let Some(deletes) = self.eq_deletes_by_partition.get(data_file.partition()) { + for delete in deletes.iter().filter(|&delete| { + // filter that returns true if the provided delete file's sequence number is **greater than** `seq_num` seq_num - .map(|seq_num| delete.manifest_entry.sequence_number() >= Some(seq_num)) + .map(|seq_num| delete.manifest_entry.sequence_number() > Some(seq_num)) .unwrap_or_else(|| true) - }) - .for_each(|delete| results.push(delete.as_ref().into())); + && data_file.partition_spec_id == delete.partition_spec_id + }) { + results.push(FileScanTaskDeleteFile::try_from(delete.as_ref())?); + } + } - if let Some(deletes) = self.eq_deletes_by_partition.get(data_file.partition()) { - deletes + // A deletion vector supersedes all position delete files for its data file, per the spec: + // "readers ignore any position delete files that would otherwise match it, because the DV + // subsumes them". An exact path match on referenced_data_file is sufficient proof of + // applicability, the same as for pos_deletes_by_path below, so this is checked before + // (and instead of) pos_deletes_by_partition and pos_deletes_by_path. + if let Some(dv) = self.dvs_by_referenced_data_file.get(data_file.file_path()) { + let dv_data_file = dv.manifest_entry.data_file(); + // A file path belongs to exactly one partition for its lifetime, so an exact path + // match already implies partition equality; this checks that the manifest agrees, + // per the spec's explicit partition-equality condition for deletion vectors. + if data_file.partition() != dv_data_file.partition() + || data_file.partition_spec_id != dv.partition_spec_id + { + return Err(invalid_data!( + "deletion vector {} references data file {} but its partition (spec {}, {:?}) does not match the data file's partition (spec {}, {:?})", + dv.manifest_entry.file_path(), + data_file.file_path(), + dv.partition_spec_id, + dv_data_file.partition(), + data_file.partition_spec_id, + data_file.partition() + )); + } + + if let Some(seq_num) = seq_num { + let dv_seq = dv.manifest_entry.sequence_number(); + if dv_seq < Some(seq_num) { + return Err(invalid_data!( + "deletion vector {} has data sequence number {dv_seq:?}, which must be >= the data file's sequence number {seq_num}", + dv.manifest_entry.file_path() + )); + } + } + results.push(FileScanTaskDeleteFile::try_from(dv.as_ref())?); + return Ok(results); + } + + if let Some(deletes) = self.pos_deletes_by_partition.get(data_file.partition()) { + for delete in deletes .iter() - // filter that returns true if the provided delete file's sequence number is **greater than** `seq_num` + // filter that returns true if the provided delete file's sequence number is **greater than or equal to** `seq_num` .filter(|&delete| { seq_num - .map(|seq_num| delete.manifest_entry.sequence_number() > Some(seq_num)) + .map(|seq_num| delete.manifest_entry.sequence_number() >= Some(seq_num)) .unwrap_or_else(|| true) + && data_file.partition_spec_id == delete.partition_spec_id }) - .for_each(|delete| results.push(delete.as_ref().into())); + { + results.push(FileScanTaskDeleteFile::try_from(delete.as_ref())?); + } } - // TODO: the spec states that: - // "The data file's file_path is equal to the delete file's referenced_data_file if it is non-null". - // we're not yet doing that here. The referenced data file's name will also be present in the positional - // delete file's file path column. - if let Some(deletes) = self.pos_deletes_by_partition.get(data_file.partition()) { - deletes + // Position deletes indexed by the exact path of the data file they reference. + // An exact path match is sufficient proof that the delete applies, so no + // partition spec id check is performed. + if let Some(deletes) = self.pos_deletes_by_path.get(data_file.file_path()) { + for delete in deletes .iter() // filter that returns true if the provided delete file's sequence number is **greater than or equal to** `seq_num` .filter(|&delete| { seq_num .map(|seq_num| delete.manifest_entry.sequence_number() >= Some(seq_num)) - .unwrap_or_else(|| true) + .unwrap_or(true) }) - .for_each(|delete| results.push(delete.as_ref().into())); + { + results.push(FileScanTaskDeleteFile::try_from(delete.as_ref())?); + } } - results + Ok(results) + } +} + +#[cfg(test)] +mod tests { + use uuid::Uuid; + + use super::*; + use crate::ErrorKind; + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Datum, Literal, ManifestEntry, + ManifestStatus, Struct, + }; + + #[test] + fn test_delete_file_index_unpartitioned() { + let deletes: Vec = vec![ + build_added_manifest_entry(4, &build_unpartitioned_eq_delete()), + build_added_manifest_entry(6, &build_unpartitioned_eq_delete()), + build_added_manifest_entry(5, &build_unpartitioned_pos_delete()), + build_added_manifest_entry(6, &build_unpartitioned_pos_delete()), + ]; + + let delete_file_paths: Vec = deletes + .iter() + .map(|file| file.file_path().to_string()) + .collect(); + + let delete_contexts: Vec = deletes + .into_iter() + .map(|entry| DeleteFileContext { + manifest_entry: entry.into(), + partition_spec_id: 0, + }) + .collect(); + + let delete_file_index = PopulatedDeleteFileIndex::new(delete_contexts).unwrap(); + + let data_file = build_unpartitioned_data_file(); + + // All deletes apply to sequence 0 + let delete_files_to_apply_for_seq_0 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap(); + assert_eq!(delete_files_to_apply_for_seq_0.len(), 4); + + // All deletes apply to sequence 3 + let delete_files_to_apply_for_seq_3 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(3)) + .unwrap(); + assert_eq!(delete_files_to_apply_for_seq_3.len(), 4); + + // Last 3 deletes apply to sequence 4 + let delete_files_to_apply_for_seq_4 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(4)) + .unwrap(); + let actual_paths_to_apply_for_seq_4: Vec = delete_files_to_apply_for_seq_4 + .into_iter() + .map(|file| file.file_path().to_string()) + .collect(); + + assert_eq!( + actual_paths_to_apply_for_seq_4, + delete_file_paths[delete_file_paths.len() - 3..] + ); + + // Last 3 deletes apply to sequence 5 + let delete_files_to_apply_for_seq_5 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(5)) + .unwrap(); + let actual_paths_to_apply_for_seq_5: Vec = delete_files_to_apply_for_seq_5 + .into_iter() + .map(|file| file.file_path().to_string()) + .collect(); + assert_eq!( + actual_paths_to_apply_for_seq_5, + delete_file_paths[delete_file_paths.len() - 3..] + ); + + // Only the last position delete applies to sequence 6 + let delete_files_to_apply_for_seq_6 = delete_file_index + .get_deletes_for_data_file(&data_file, Some(6)) + .unwrap(); + let actual_paths_to_apply_for_seq_6: Vec = delete_files_to_apply_for_seq_6 + .into_iter() + .map(|file| file.file_path().to_string()) + .collect(); + assert_eq!( + actual_paths_to_apply_for_seq_6, + delete_file_paths[delete_file_paths.len() - 1..] + ); + + // The 2 global equality deletes should match against any partitioned file + let partitioned_file = + build_partitioned_data_file(&Struct::from_iter([Some(Literal::long(100))]), 1); + + let delete_files_to_apply_for_partitioned_file = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(0)) + .unwrap(); + let actual_paths_to_apply_for_partitioned_file: Vec = + delete_files_to_apply_for_partitioned_file + .into_iter() + .map(|file| file.file_path().to_string()) + .collect(); + assert_eq!( + actual_paths_to_apply_for_partitioned_file, + delete_file_paths[..2] + ); + } + + #[test] + fn test_delete_file_index_partitioned() { + let partition_one = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let deletes: Vec = vec![ + build_added_manifest_entry(4, &build_partitioned_eq_delete(&partition_one, spec_id)), + build_added_manifest_entry(6, &build_partitioned_eq_delete(&partition_one, spec_id)), + build_added_manifest_entry(5, &build_partitioned_pos_delete(&partition_one, spec_id)), + build_added_manifest_entry(6, &build_partitioned_pos_delete(&partition_one, spec_id)), + ]; + + let delete_file_paths: Vec = deletes + .iter() + .map(|file| file.file_path().to_string()) + .collect(); + + let delete_contexts: Vec = deletes + .into_iter() + .map(|entry| DeleteFileContext { + manifest_entry: entry.into(), + partition_spec_id: spec_id, + }) + .collect(); + + let delete_file_index = PopulatedDeleteFileIndex::new(delete_contexts).unwrap(); + + let partitioned_file = + build_partitioned_data_file(&Struct::from_iter([Some(Literal::long(100))]), spec_id); + + // All deletes apply to sequence 0 + let delete_files_to_apply_for_seq_0 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(0)) + .unwrap(); + assert_eq!(delete_files_to_apply_for_seq_0.len(), 4); + + // All deletes apply to sequence 3 + let delete_files_to_apply_for_seq_3 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(3)) + .unwrap(); + assert_eq!(delete_files_to_apply_for_seq_3.len(), 4); + + // Last 3 deletes apply to sequence 4 + let delete_files_to_apply_for_seq_4 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(4)) + .unwrap(); + let actual_paths_to_apply_for_seq_4: Vec = delete_files_to_apply_for_seq_4 + .into_iter() + .map(|file| file.file_path().to_string()) + .collect(); + + assert_eq!( + actual_paths_to_apply_for_seq_4, + delete_file_paths[delete_file_paths.len() - 3..] + ); + + // Last 3 deletes apply to sequence 5 + let delete_files_to_apply_for_seq_5 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(5)) + .unwrap(); + let actual_paths_to_apply_for_seq_5: Vec = delete_files_to_apply_for_seq_5 + .into_iter() + .map(|file| file.file_path().to_string()) + .collect(); + assert_eq!( + actual_paths_to_apply_for_seq_5, + delete_file_paths[delete_file_paths.len() - 3..] + ); + + // Only the last position delete applies to sequence 6 + let delete_files_to_apply_for_seq_6 = delete_file_index + .get_deletes_for_data_file(&partitioned_file, Some(6)) + .unwrap(); + let actual_paths_to_apply_for_seq_6: Vec = delete_files_to_apply_for_seq_6 + .into_iter() + .map(|file| file.file_path().to_string()) + .collect(); + assert_eq!( + actual_paths_to_apply_for_seq_6, + delete_file_paths[delete_file_paths.len() - 1..] + ); + + // Data file with different partition tuples does not match any delete files + let partitioned_second_file = + build_partitioned_data_file(&Struct::from_iter([Some(Literal::long(200))]), 1); + let delete_files_to_apply_for_different_partition = delete_file_index + .get_deletes_for_data_file(&partitioned_second_file, Some(0)) + .unwrap(); + let actual_paths_to_apply_for_different_partition: Vec = + delete_files_to_apply_for_different_partition + .into_iter() + .map(|file| file.file_path().to_string()) + .collect(); + assert!(actual_paths_to_apply_for_different_partition.is_empty()); + + // Data file with same tuple but different spec ID does not match any delete files + let partitioned_different_spec = build_partitioned_data_file(&partition_one, 2); + let delete_files_to_apply_for_different_spec = delete_file_index + .get_deletes_for_data_file(&partitioned_different_spec, Some(0)) + .unwrap(); + let actual_paths_to_apply_for_different_spec: Vec = + delete_files_to_apply_for_different_spec + .into_iter() + .map(|file| file.file_path().to_string()) + .collect(); + assert!(actual_paths_to_apply_for_different_spec.is_empty()); + } + + #[test] + fn test_partition_scoped_deletes_distinguish_signed_zero() { + let neg_zero = Struct::from_iter([Some(Literal::double(-0.0))]); + let pos_zero = Struct::from_iter([Some(Literal::double(0.0))]); + let spec_id = 1; + + let delete_contexts: Vec = [ + build_partitioned_eq_delete(&neg_zero, spec_id), + build_partitioned_pos_delete(&neg_zero, spec_id), + ] + .iter() + .map(|delete| DeleteFileContext { + manifest_entry: build_added_manifest_entry(4, delete).into(), + partition_spec_id: spec_id, + }) + .collect(); + + let delete_file_index = PopulatedDeleteFileIndex::new(delete_contexts).unwrap(); + + let deletes_for_neg_zero = delete_file_index + .get_deletes_for_data_file(&build_partitioned_data_file(&neg_zero, spec_id), Some(0)) + .unwrap(); + assert_eq!(deletes_for_neg_zero.len(), 2); + + // -0.0 and 0.0 are different partitions, as in iceberg-java, so deletes written + // for one do not apply to data files in the other. + let deletes_for_pos_zero = delete_file_index + .get_deletes_for_data_file(&build_partitioned_data_file(&pos_zero, spec_id), Some(0)) + .unwrap(); + assert!(deletes_for_pos_zero.is_empty()); + } + + #[test] + fn test_pos_delete_with_referenced_data_file_applies_only_to_that_file() { + let data_file_a = build_unpartitioned_data_file(); + let data_file_b = build_unpartitioned_data_file(); + + let pos_delete = build_pos_delete_referencing(data_file_a.file_path()); + let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), + partition_spec_id: 0, + }]) + .unwrap(); + + let deletes_for_a = index + .get_deletes_for_data_file(&data_file_a, Some(0)) + .unwrap(); + assert_eq!(deletes_for_a.len(), 1); + assert_eq!(deletes_for_a[0].file_path(), pos_delete.file_path()); + + // The delete references data file A, so it must not apply to data file B + // even though B shares A's partition. + let deletes_for_b = index + .get_deletes_for_data_file(&data_file_b, Some(0)) + .unwrap(); + assert!(deletes_for_b.is_empty()); + } + + #[test] + fn test_pos_delete_with_equal_path_bounds_routes_by_path() { + let data_file_a = build_unpartitioned_data_file(); + let data_file_b = build_unpartitioned_data_file(); + + // No referenced_data_file field; equal file_path column bounds identify + // the single referenced data file instead. + let pos_delete = + build_pos_delete_with_path_bounds(data_file_a.file_path(), data_file_a.file_path()); + let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), + partition_spec_id: 0, + }]) + .unwrap(); + + assert_eq!( + index + .get_deletes_for_data_file(&data_file_a, Some(0)) + .unwrap() + .len(), + 1 + ); + assert!( + index + .get_deletes_for_data_file(&data_file_b, Some(0)) + .unwrap() + .is_empty() + ); + } + + #[test] + fn test_pos_delete_with_unequal_path_bounds_routes_by_partition() { + let data_file_a = build_unpartitioned_data_file(); + let data_file_b = build_unpartitioned_data_file(); + + // Unequal bounds mean the delete file may reference multiple data files, + // so it falls back to partition routing and applies to every data file + // in the partition. + let pos_delete = build_pos_delete_with_path_bounds("a-data.parquet", "z-data.parquet"); + let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), + partition_spec_id: 0, + }]) + .unwrap(); + + assert_eq!( + index + .get_deletes_for_data_file(&data_file_a, Some(0)) + .unwrap() + .len(), + 1 + ); + assert_eq!( + index + .get_deletes_for_data_file(&data_file_b, Some(0)) + .unwrap() + .len(), + 1 + ); + } + + #[test] + fn test_pos_delete_by_path_applies_across_partition_spec_ids() { + let data_file = + build_partitioned_data_file(&Struct::from_iter([Some(Literal::long(100))]), 2); + + // The delete is indexed under a different spec id than the data file's. + // An exact path match is sufficient proof that the delete applies. + let pos_delete = build_pos_delete_referencing(data_file.file_path()); + let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), + partition_spec_id: 0, + }]) + .unwrap(); + + assert_eq!( + index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .len(), + 1 + ); + } + + #[test] + fn test_pos_delete_by_path_sequence_number_filter() { + let data_file = build_unpartitioned_data_file(); + + let pos_delete = build_pos_delete_referencing(data_file.file_path()); + let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), + partition_spec_id: 0, + }]) + .unwrap(); + + // Position deletes apply when the delete's sequence number is greater + // than or equal to the data file's. + assert_eq!( + index + .get_deletes_for_data_file(&data_file, Some(4)) + .unwrap() + .len(), + 1 + ); + assert_eq!( + index + .get_deletes_for_data_file(&data_file, Some(5)) + .unwrap() + .len(), + 1 + ); + assert!( + index + .get_deletes_for_data_file(&data_file, Some(6)) + .unwrap() + .is_empty() + ); + // Without a sequence number, the delete applies unconditionally. + assert_eq!( + index + .get_deletes_for_data_file(&data_file, None) + .unwrap() + .len(), + 1 + ); + } + + #[test] + fn test_path_and_partition_pos_deletes_compose_for_one_data_file() { + let data_file = build_unpartitioned_data_file(); + + // One path-keyed pos delete, one partition-keyed pos delete, and one + // global equality delete, all applying to the same data file. The + // result must contain all three. + let path_delete = build_pos_delete_referencing(data_file.file_path()); + let partition_delete = build_unpartitioned_pos_delete(); + let eq_delete = build_unpartitioned_eq_delete(); + + let index = PopulatedDeleteFileIndex::new(vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &path_delete).into(), + partition_spec_id: 0, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &partition_delete).into(), + partition_spec_id: 0, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &eq_delete).into(), + partition_spec_id: 0, + }, + ]) + .unwrap(); + + let mut actual_paths: Vec = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .into_iter() + .map(|delete| delete.file_path().to_string()) + .collect(); + actual_paths.sort(); + + let mut expected_paths = vec![ + path_delete.file_path().to_string(), + partition_delete.file_path().to_string(), + eq_delete.file_path().to_string(), + ]; + expected_paths.sort(); + + assert_eq!(actual_paths, expected_paths); + } + + #[test] + fn test_multiple_pos_deletes_for_the_same_referenced_path() { + let data_file = build_unpartitioned_data_file(); + + let first_delete = build_pos_delete_referencing(data_file.file_path()); + let second_delete = build_pos_delete_referencing(data_file.file_path()); + + let index = PopulatedDeleteFileIndex::new(vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &first_delete).into(), + partition_spec_id: 0, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(6, &second_delete).into(), + partition_spec_id: 0, + }, + ]) + .unwrap(); + + let mut actual_paths: Vec = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .into_iter() + .map(|delete| delete.file_path().to_string()) + .collect(); + actual_paths.sort(); + + let mut expected_paths = vec![ + first_delete.file_path().to_string(), + second_delete.file_path().to_string(), + ]; + expected_paths.sort(); + + assert_eq!(actual_paths, expected_paths); + } + + #[test] + fn test_partitioned_pos_delete_with_referenced_path_routes_by_path() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let other_partition = Struct::from_iter([Some(Literal::long(200))]); + + // The data file lives in a different partition than the delete file. + let data_file = build_partitioned_data_file(&other_partition, 1); + let same_partition_neighbor = build_partitioned_data_file(&partition, 1); + + // A partitioned pos delete with a referenced path must route by path: + // it applies to the referenced file in another partition, and not to + // a different file in the delete's own partition. + let mut pos_delete = build_pos_delete_referencing(data_file.file_path()); + pos_delete.partition = partition.clone(); + pos_delete.partition_spec_id = 1; + + let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), + partition_spec_id: 1, + }]) + .unwrap(); + + let deletes_for_referenced = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap(); + assert_eq!(deletes_for_referenced.len(), 1); + assert_eq!( + deletes_for_referenced[0].file_path(), + pos_delete.file_path() + ); + + assert!( + index + .get_deletes_for_data_file(&same_partition_neighbor, Some(0)) + .unwrap() + .is_empty() + ); + } + + #[test] + fn test_pos_delete_with_one_sided_path_bounds_routes_by_partition() { + let data_file = build_unpartitioned_data_file(); + let other_data_file = build_unpartitioned_data_file(); + + // Only a lower bound is present. No single referenced file can be + // inferred, so the delete falls back to partition routing and applies + // to every data file in the partition. + let mut pos_delete = build_pos_delete_with_path_bounds("a-data.parquet", "unused"); + pos_delete.upper_bounds = HashMap::default(); + + let index = PopulatedDeleteFileIndex::new(vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_delete).into(), + partition_spec_id: 0, + }]) + .unwrap(); + + assert_eq!( + index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .len(), + 1 + ); + assert_eq!( + index + .get_deletes_for_data_file(&other_data_file, Some(0)) + .unwrap() + .len(), + 1 + ); + } + + fn build_pos_delete_referencing(path: &str) -> DataFile { + DataFileBuilder::default() + .file_path(format!("{}-pos-delete.parquet", Uuid::new_v4())) + .file_format(DataFileFormat::Parquet) + .content(DataContentType::PositionDeletes) + .record_count(1) + .referenced_data_file(Some(path.to_string())) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(100) + .build() + .unwrap() + } + + fn build_pos_delete_with_path_bounds(lower: &str, upper: &str) -> DataFile { + DataFileBuilder::default() + .file_path(format!("{}-pos-delete.parquet", Uuid::new_v4())) + .file_format(DataFileFormat::Parquet) + .content(DataContentType::PositionDeletes) + .record_count(1) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(100) + .lower_bounds(HashMap::from([( + RESERVED_FIELD_ID_DELETE_FILE_PATH, + Datum::string(lower), + )])) + .upper_bounds(HashMap::from([( + RESERVED_FIELD_ID_DELETE_FILE_PATH, + Datum::string(upper), + )])) + .build() + .unwrap() + } + + #[test] + fn test_deletion_vector_context_carries_coordinates() { + // A deletion vector is a PositionDeletes entry stored as a Puffin blob, located by + // content_offset / content_size_in_bytes and scoped by referenced_data_file. Those + // three fields, and the record count its bitmap is validated against, must survive the + // conversion into a FileScanTaskDeleteFile so the loader can find and apply the blob. + let dv = DataFileBuilder::default() + .file_path("s3://bucket/data/part-0.parquet-deletes.puffin".to_string()) + .file_format(DataFileFormat::Puffin) + .content(DataContentType::PositionDeletes) + .record_count(3) + .referenced_data_file(Some("s3://bucket/data/part-0.parquet".to_string())) + .content_offset(Some(4)) + .content_size_in_bytes(Some(40)) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(44) + .build() + .unwrap(); + + let ctx = DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: 0, + }; + + let task = FileScanTaskDeleteFile::try_from(&ctx).unwrap(); + assert_eq!(task.file_type(), DataContentType::PositionDeletes); + assert_eq!(task.content_offset(), Some(4)); + assert_eq!(task.content_size_in_bytes(), Some(40)); + assert_eq!(task.record_count(), Some(3)); + assert_eq!( + task.referenced_data_file(), + Some("s3://bucket/data/part-0.parquet") + ); + } + + #[test] + fn test_deletion_vector_supersedes_position_deletes() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_path = dv.file_path().to_string(); + let pos_del = build_partitioned_pos_delete(&partition, spec_id); + + let contexts = vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: spec_id, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_del).into(), + partition_spec_id: spec_id, + }, + ]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let applied: Vec = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .into_iter() + .map(|f| f.file_path().to_string()) + .collect(); + + // Only the deletion vector applies; the partition-scoped position delete file, which + // would otherwise also match, is superseded. + assert_eq!(applied, vec![dv_path]); + } + + #[test] + fn test_deletion_vector_with_stale_sequence_number_is_rejected() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv = build_deletion_vector(data_file.file_path(), &partition, spec_id); + + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(3, &dv).into(), + partition_spec_id: spec_id, + }]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + // The DV's own sequence number (3) is less than the data file's (5): the spec guarantees + // a DV is never written before the data file it applies to, so this is an inconsistent + // manifest rather than a case where the DV simply doesn't apply. + let err = index + .get_deletes_for_data_file(&data_file, Some(5)) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("must be >= the data file's sequence number") + ); + } + + #[test] + fn test_deletion_vector_with_mismatched_partition_is_rejected() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let other_partition = Struct::from_iter([Some(Literal::long(200))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + // Malformed: the DV's referenced_data_file matches data_file's path exactly, but the + // DV's own partition disagrees, a state that cannot arise from a valid writer since a + // file path is permanently tied to one partition. + let dv = build_deletion_vector(data_file.file_path(), &other_partition, spec_id); + + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: spec_id, + }]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let err = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("does not match the data file's partition") + ); + } + + #[test] + fn test_deletion_vector_with_mismatched_partition_spec_is_rejected() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let data_file = build_partitioned_data_file(&partition, 1); + + // Same partition value, but the DV's context was populated under a different partition + // spec id than the data file's: also a manifest inconsistency, since a file path is + // permanently tied to one partition spec. + let dv = build_deletion_vector(data_file.file_path(), &partition, 1); + + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: 2, + }]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let err = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("does not match the data file's partition") + ); + } + + #[test] + fn test_deletion_vector_supersedes_path_scoped_position_delete() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_path = dv.file_path().to_string(); + let pos_del = build_pos_delete_referencing(data_file.file_path()); + + let contexts = vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: spec_id, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &pos_del).into(), + partition_spec_id: 0, + }, + ]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let applied: Vec = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .into_iter() + .map(|f| f.file_path().to_string()) + .collect(); + + // Only the deletion vector applies; the path-scoped position delete file, which would + // otherwise also match by exact referenced_data_file path, is superseded. + assert_eq!(applied, vec![dv_path]); + } + + #[test] + fn test_deletion_vector_coexists_with_equality_delete() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_path = dv.file_path().to_string(); + let eq_del = build_unpartitioned_eq_delete(); + let eq_del_path = eq_del.file_path().to_string(); + + let contexts = vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv).into(), + partition_spec_id: spec_id, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &eq_del).into(), + partition_spec_id: 0, + }, + ]; + + let index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let mut applied: Vec = index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap() + .into_iter() + .map(|f| f.file_path().to_string()) + .collect(); + applied.sort(); + + // A deletion vector only supersedes position deletes, not equality deletes: both apply. + let mut expected = vec![dv_path, eq_del_path]; + expected.sort(); + assert_eq!(applied, expected); + } + + #[test] + fn test_deletion_vector_index_rejects_missing_referenced_data_file() { + let malformed_dv = DataFileBuilder::default() + .file_path("deletes.puffin".to_string()) + .file_format(DataFileFormat::Puffin) + .content(DataContentType::PositionDeletes) + .record_count(1) + .content_offset(Some(4)) + .content_size_in_bytes(Some(40)) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(60) + .build() + .unwrap(); + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &malformed_dv).into(), + partition_spec_id: 0, + }]; + + let err = PopulatedDeleteFileIndex::new(contexts).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("missing referenced_data_file")); + } + + #[test] + fn test_deletion_vector_index_rejects_missing_coordinates() { + // The missing-coordinate check fires in `get_deletes_for_data_file`, where the + // manifest entry is converted into a task. `PopulatedDeleteFileIndex::new` only + // needs `referenced_data_file` as its map key, so a DV that carries it but omits + // `content_offset` survives construction and is rejected when the data file is + // resolved against it. + let data_file = DataFileBuilder::default() + .file_path("target-data.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .content(DataContentType::Data) + .record_count(100) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(100) + .build() + .unwrap(); + + let malformed_dv = DataFileBuilder::default() + .file_path("deletes.puffin".to_string()) + .file_format(DataFileFormat::Puffin) + .content(DataContentType::PositionDeletes) + .record_count(1) + .referenced_data_file(Some("target-data.parquet".to_string())) + .content_size_in_bytes(Some(40)) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(60) + .build() + .unwrap(); + let contexts = vec![DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &malformed_dv).into(), + partition_spec_id: 0, + }]; + + let delete_file_index = PopulatedDeleteFileIndex::new(contexts).unwrap(); + let err = delete_file_index + .get_deletes_for_data_file(&data_file, Some(0)) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("missing content_offset")); + } + + #[test] + fn test_multiple_deletion_vectors_for_same_data_file_is_rejected() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv_1 = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_2 = build_deletion_vector(data_file.file_path(), &partition, spec_id); + + let contexts = vec![ + DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv_1).into(), + partition_spec_id: spec_id, + }, + DeleteFileContext { + manifest_entry: build_added_manifest_entry(6, &dv_2).into(), + partition_spec_id: spec_id, + }, + ]; + + let err = PopulatedDeleteFileIndex::new(contexts).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("multiple deletion vectors")); + } + + #[tokio::test] + async fn test_delete_file_index_propagates_multiple_dv_error_to_waiters() { + let partition = Struct::from_iter([Some(Literal::long(100))]); + let spec_id = 1; + let data_file = build_partitioned_data_file(&partition, spec_id); + + let dv_1 = build_deletion_vector(data_file.file_path(), &partition, spec_id); + let dv_2 = build_deletion_vector(data_file.file_path(), &partition, spec_id); + + let (index, mut tx) = DeleteFileIndex::new(Runtime::current()); + tx.try_send(DeleteFileContext { + manifest_entry: build_added_manifest_entry(5, &dv_1).into(), + partition_spec_id: spec_id, + }) + .unwrap(); + tx.try_send(DeleteFileContext { + manifest_entry: build_added_manifest_entry(6, &dv_2).into(), + partition_spec_id: spec_id, + }) + .unwrap(); + drop(tx); + + let err = index + .get_deletes_for_data_file(&data_file, Some(0)) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("multiple deletion vectors")); + + // A second caller, arriving after the index has already settled into the Failed state, + // must see the same error rather than panicking on an unexpected state. + let err = index + .get_deletes_for_data_file(&data_file, Some(0)) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + // A V3 deletion vector: a PositionDeletes entry stored as a Puffin blob, scoped to a single + // data file via referenced_data_file. + fn build_deletion_vector( + referenced_data_file: &str, + partition: &Struct, + spec_id: i32, + ) -> DataFile { + DataFileBuilder::default() + .file_path(format!("{}-deletes.puffin", Uuid::new_v4())) + .file_format(DataFileFormat::Puffin) + .content(DataContentType::PositionDeletes) + .record_count(1) + .referenced_data_file(Some(referenced_data_file.to_string())) + .content_offset(Some(4)) + .content_size_in_bytes(Some(40)) + .partition(partition.clone()) + .partition_spec_id(spec_id) + .file_size_in_bytes(60) + .build() + .unwrap() + } + + fn build_unpartitioned_eq_delete() -> DataFile { + build_partitioned_eq_delete(&Struct::empty(), 0) + } + + fn build_partitioned_eq_delete(partition: &Struct, spec_id: i32) -> DataFile { + DataFileBuilder::default() + .file_path(format!("{}_equality_delete.parquet", Uuid::new_v4())) + .file_format(DataFileFormat::Parquet) + .content(DataContentType::EqualityDeletes) + .equality_ids(Some(vec![1])) + .record_count(1) + .partition(partition.clone()) + .partition_spec_id(spec_id) + .file_size_in_bytes(100) + .build() + .unwrap() + } + + fn build_unpartitioned_pos_delete() -> DataFile { + build_partitioned_pos_delete(&Struct::empty(), 0) + } + + fn build_partitioned_pos_delete(partition: &Struct, spec_id: i32) -> DataFile { + DataFileBuilder::default() + .file_path(format!("{}-pos-delete.parquet", Uuid::new_v4())) + .file_format(DataFileFormat::Parquet) + .content(DataContentType::PositionDeletes) + .record_count(1) + .partition(partition.clone()) + .partition_spec_id(spec_id) + .file_size_in_bytes(100) + .build() + .unwrap() + } + + fn build_unpartitioned_data_file() -> DataFile { + DataFileBuilder::default() + .file_path(format!("{}-data.parquet", Uuid::new_v4())) + .file_format(DataFileFormat::Parquet) + .content(DataContentType::Data) + .record_count(100) + .partition(Struct::empty()) + .partition_spec_id(0) + .file_size_in_bytes(100) + .build() + .unwrap() + } + + fn build_partitioned_data_file(partition_value: &Struct, spec_id: i32) -> DataFile { + DataFileBuilder::default() + .file_path(format!("{}-data.parquet", Uuid::new_v4())) + .file_format(DataFileFormat::Parquet) + .content(DataContentType::Data) + .record_count(100) + .partition(partition_value.clone()) + .partition_spec_id(spec_id) + .file_size_in_bytes(100) + .build() + .unwrap() + } + + fn build_added_manifest_entry(data_seq_number: i64, file: &DataFile) -> ManifestEntry { + ManifestEntry::builder() + .status(ManifestStatus::Added) + .sequence_number(data_seq_number) + .data_file(file.clone()) + .build() } } diff --git a/crates/iceberg/src/delete_vector.rs b/crates/iceberg/src/delete_vector.rs index f382bf079e..f39140ccce 100644 --- a/crates/iceberg/src/delete_vector.rs +++ b/crates/iceberg/src/delete_vector.rs @@ -15,32 +15,51 @@ // specific language governing permissions and limitations // under the License. +//! Deletion vectors: the deleted row positions of one data file, as a Puffin +//! `deletion-vector-v1` blob decodes to. Public so an engine that reads +//! Parquet with a reader of its own can still decode the vectors Iceberg +//! writes. + use std::ops::BitOrAssign; -use roaring::RoaringTreemap; +use bytes::Buf; use roaring::bitmap::Iter; use roaring::treemap::BitmapIter; +use roaring::{RoaringBitmap, RoaringTreemap}; +use crate::error::invalid_data; use crate::{Error, ErrorKind, Result}; +/// Magic bytes prefixing a serialized `deletion-vector-v1` bitmap, per the Iceberg Puffin spec. +/// Iceberg-Java stores these as the little-endian int 1681511377 (0x6439D3D1). +const DV_MAGIC: [u8; 4] = [0xD1, 0xD3, 0x39, 0x64]; +const DV_LENGTH_PREFIX_BYTES: usize = 4; +const DV_MAGIC_BYTES: usize = 4; +const DV_CRC_BYTES: usize = 4; +const DV_MIN_BLOB_BYTES: usize = DV_LENGTH_PREFIX_BYTES + DV_MAGIC_BYTES + DV_CRC_BYTES; + +/// The deleted row positions of one data file: a set of 64-bit positions, +/// kept as a [`RoaringTreemap`]. #[derive(Debug, Default)] pub struct DeleteVector { inner: RoaringTreemap, } impl DeleteVector { - #[allow(unused)] + /// Wraps `roaring_treemap`, whose members are the deleted positions. pub fn new(roaring_treemap: RoaringTreemap) -> DeleteVector { DeleteVector { inner: roaring_treemap, } } - pub fn iter(&self) -> DeleteVectorIterator { + /// Iterates the deleted positions in ascending order. + pub fn iter(&self) -> DeleteVectorIterator<'_> { let outer = self.inner.bitmaps(); DeleteVectorIterator { outer, inner: None } } + /// Marks `pos` as deleted, returning whether it was not already. pub fn insert(&mut self, pos: u64) -> bool { self.inner.insert(pos) } @@ -52,7 +71,6 @@ impl DeleteVector { /// # Errors /// /// Returns an error if the precondition is not met. - #[allow(dead_code)] pub fn insert_positions(&mut self, positions: &[u64]) -> Result { if let Err(err) = self.inner.append(positions.iter().copied()) { return Err(Error::new( @@ -64,22 +82,165 @@ impl DeleteVector { Ok(positions.len()) } - #[allow(unused)] + /// How many positions are deleted. pub fn len(&self) -> u64 { self.inner.len() } + + /// Whether no position is deleted. + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + /// Returns the deleted positions as ascending, non-overlapping half-open ranges. + /// + /// Adjacent ranges may not be merged (a run is split at each 2^32 boundary), so callers + /// shouldn't assume a range is maximal. A range holding only `u64::MAX` is dropped, because a + /// half-open `u64` range can't hold it and no Parquet row can be there. + pub(crate) fn ranges(&self) -> DeleteVectorRanges<'_> { + DeleteVectorRanges { + bitmaps: self.inner.bitmaps(), + high_bits: 0, + bitmap_iter: None, + buffered: None, + } + } + + /// Parses a `deletion-vector-v1` Puffin blob into a `DeleteVector`. + /// + /// The layout, defined by the Iceberg Puffin spec and matching Iceberg-Java's + /// `BitmapPositionDeleteIndex`, is: + /// + /// ```text + /// [length: u32 big-endian][magic: D1 D3 39 64][vector][crc: u32 big-endian] + /// ``` + /// + /// `length` counts the magic and vector bytes (not itself or the CRC). The CRC-32 is + /// computed over the magic and vector. `vector` is a roaring bitmap in the portable 64-bit + /// format: a directory of 32-bit key / 32-bit roaring bitmap pairs, ordered by unsigned + /// comparison of the keys, one bitmap per key. + /// + /// # Errors + /// + /// Returns [`ErrorKind::DataInvalid`] if the blob is shorter than the minimum, the length + /// prefix or CRC does not match, the magic is wrong, the roaring bitmap count exceeds the + /// portable format's maximum, the roaring directory's keys are not ordered by unsigned + /// comparison, or the roaring payload fails to decode. + pub fn deserialize(blob: &[u8]) -> Result { + if blob.len() < DV_MIN_BLOB_BYTES { + return Err(invalid_data!( + "deletion-vector-v1 blob is {} bytes, shorter than the {DV_MIN_BLOB_BYTES}-byte minimum", + blob.len() + )); + } + + // The magic and vector, i.e. the bytes covered by both the length prefix and the CRC. + let body = &blob[DV_LENGTH_PREFIX_BYTES..blob.len() - DV_CRC_BYTES]; + verify_length_prefix(&blob[..DV_LENGTH_PREFIX_BYTES], body)?; + + // Verify the CRC before interpreting any bytes so a corrupt blob yields a single clear + // error rather than an opaque roaring decode failure. + verify_crc(body, &blob[blob.len() - DV_CRC_BYTES..])?; + + let (magic, vector) = body.split_at(DV_MAGIC_BYTES); + verify_magic(magic)?; + + let inner = decode_roaring_directory(vector)?; + + Ok(DeleteVector { inner }) + } +} + +fn verify_length_prefix(mut prefix: &[u8], body: &[u8]) -> Result<()> { + let declared_len = prefix.try_get_u32().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + "failed to read the deletion-vector-v1 length prefix", + ) + .with_source(e) + })? as usize; + if declared_len != body.len() { + return Err(invalid_data!( + "deletion-vector-v1 length prefix is {declared_len}, expected {}", + body.len() + )); + } + Ok(()) +} + +fn verify_crc(body: &[u8], mut crc_bytes: &[u8]) -> Result<()> { + let stored_crc = crc_bytes.try_get_u32().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + "failed to read the deletion-vector-v1 CRC", + ) + .with_source(e) + })?; + let computed_crc = crc32fast::hash(body); + if computed_crc != stored_crc { + return Err(invalid_data!( + "deletion-vector-v1 CRC mismatch: computed {computed_crc:#010x}, stored {stored_crc:#010x}" + )); + } + Ok(()) +} + +fn verify_magic(magic: &[u8]) -> Result<()> { + if magic != DV_MAGIC { + return Err(invalid_data!( + "deletion-vector-v1 magic mismatch: {magic:02x?}, expected {DV_MAGIC:02x?}" + )); + } + Ok(()) } -// Ideally, we'd just wrap `roaring::RoaringTreemap`'s iterator, `roaring::treemap::Iter` here. -// But right now, it does not have a corresponding implementation of `roaring::bitmap::Iter::advance_to`, -// which is very handy in ArrowReader::build_deletes_row_selection. -// There is a PR open on roaring to add this (https://github.com/RoaringBitmap/roaring-rs/pull/314) -// and if that gets merged then we can simplify `DeleteVectorIterator` here, refactoring `advance_to` -// to just a wrapper around the underlying iterator's method. +// The Puffin spec defines the roaring directory as the bitmaps "ordered by unsigned comparison +// of the 32-bit keys", with one bitmap per key. Walk it ourselves (rather than +// `RoaringTreemap::deserialize_from`, which stores keys in a `BTreeMap` via a plain insert and +// would silently accept a stream with duplicate or out-of-order keys, discarding the earlier +// bitmap on a duplicate) so a non-conformant blob is rejected instead of decoded into a value +// that doesn't match what was actually written. +fn decode_roaring_directory(mut reader: &[u8]) -> Result { + let bitmap_count = reader.try_get_u64_le().map_err(|e| { + invalid_data!("failed to decode deletion-vector-v1 roaring payload").with_source(e) + })?; + // The roaring portable format restricts the bitmap count to [0, 2^32 - 1] (it is stored as a + // u64 with the upper 32 bits reserved as zero padding). + if bitmap_count > u32::MAX as u64 { + return Err(invalid_data!( + "deletion-vector-v1 roaring bitmap count {bitmap_count} exceeds the {}-key maximum", + u32::MAX + )); + } + + let mut bitmaps = Vec::new(); + let mut last_key: Option = None; + for _ in 0..bitmap_count { + let key = reader.try_get_u32_le().map_err(|e| { + invalid_data!("failed to decode deletion-vector-v1 roaring payload").with_source(e) + })?; + if let Some(last) = last_key + && key <= last + { + return Err(invalid_data!( + "deletion-vector-v1 roaring keys must be ordered by unsigned comparison, got key {key} after {last}" + )); + } + last_key = Some(key); + + let bitmap = RoaringBitmap::deserialize_from(&mut reader).map_err(|e| { + invalid_data!("failed to decode deletion-vector-v1 roaring payload").with_source(e) + })?; + bitmaps.push((key, bitmap)); + } + + // `bitmaps` is already sorted by key, but `roaring` has no constructor that accepts + // pre-sorted pairs without re-sorting them; revisit if that changes upstream. + Ok(RoaringTreemap::from_bitmaps(bitmaps)) +} + +/// Iterates the deleted positions of a [`DeleteVector`] in ascending order. pub struct DeleteVectorIterator<'a> { - // NB: `BitMapIter` was only exposed publicly in https://github.com/RoaringBitmap/roaring-rs/pull/316 - // which is not yet released. As a consequence our Cargo.toml temporarily uses a git reference for - // the roaring dependency. outer: BitmapIter<'a>, inner: Option>, } @@ -93,10 +254,10 @@ impl Iterator for DeleteVectorIterator<'_> { type Item = u64; fn next(&mut self) -> Option { - if let Some(inner) = &mut self.inner { - if let Some(inner_next) = inner.bitmap_iter.next() { - return Some(u64::from(inner.high_bits) << 32 | u64::from(inner_next)); - } + if let Some(inner) = &mut self.inner + && let Some(inner_next) = inner.bitmap_iter.next() + { + return Some(u64::from(inner.high_bits) << 32 | u64::from(inner_next)); } if let Some((high_bits, next_bitmap)) = self.outer.next() { @@ -113,26 +274,167 @@ impl Iterator for DeleteVectorIterator<'_> { } impl DeleteVectorIterator<'_> { + /// Skips the positions below `pos`, so the next one yielded is the first + /// at or after it. A `pos` below the next position does nothing. pub fn advance_to(&mut self, pos: u64) { let hi = (pos >> 32) as u32; let lo = pos as u32; - let Some(ref mut inner) = self.inner else { + if let Some(inner) = &mut self.inner { + if inner.high_bits > hi { + return; + } + if inner.high_bits == hi { + inner.bitmap_iter.advance_to(lo); + return; + } + } + + // `pos` is in a later bitmap, or nothing has been yielded yet. + self.inner = None; + for (high_bits, bitmap) in self.outer.by_ref() { + if high_bits < hi { + continue; + } + let mut bitmap_iter = bitmap.iter(); + if high_bits == hi { + bitmap_iter.advance_to(lo); + } + self.inner = Some(DeleteVectorIteratorInner { + high_bits, + bitmap_iter, + }); return; - }; + } + } +} - while inner.high_bits < hi { - let Some((next_hi, next_bitmap)) = self.outer.next() else { - return; +/// Number of consecutive positions [`DeleteVectorRanges`] walks one by one before it asks the +/// bitmap for the rest of the run. +/// +/// `roaring`'s `next_range` binary-searches the rest of an array container on every call, which +/// costs more than `next` when most runs are a single position, so it only pays off for long +/// runs. Tuned with `roaring` 0.11.4 by reading a 20M-row file with deletes in runs of 20, 100 +/// and longer, and as scattered single rows and pairs: 4 was best on runs, compared with 8, 16 +/// and 32, and no slower on scattered deletes. +const RUN_PROBE_LEN: u32 = 4; + +/// What [`DeleteVectorRanges`] has taken from its bitmap iterator but not returned yet. +enum Buffered { + /// Start of the next range. + Pos(u32), + /// A complete range, inclusive end. + Range(u32, u32), +} + +/// Iterator over a [`DeleteVector`]'s deleted positions as half-open ranges, see +/// [`DeleteVector::ranges`]. +pub(crate) struct DeleteVectorRanges<'a> { + bitmaps: BitmapIter<'a>, + /// High 32 bits of the positions in `bitmap_iter` and `buffered`. + high_bits: u32, + bitmap_iter: Option>, + buffered: Option, +} + +impl DeleteVectorRanges<'_> { + /// Returns `None` only for a range holding nothing but `u64::MAX`, see [`DeleteVector::ranges`]. + fn full_range(&self, start: u32, end_inclusive: u32) -> Option<(u64, u64)> { + let base = u64::from(self.high_bits) << 32; + let start = base | u64::from(start); + let end = (base | u64::from(end_inclusive)).saturating_add(1); + (start < end).then_some((start, end)) + } + + /// Skips every deleted position below `pos`. A range still buffered here that starts below + /// `pos` and ends after it is cut to start at `pos`; a range already returned by `next` is + /// the caller's to cut. + pub(crate) fn skip_to(&mut self, pos: u64) { + let hi = (pos >> 32) as u32; + let lo = pos as u32; + + if self.bitmap_iter.is_some() && self.high_bits > hi { + return; + } + if let Some(iter) = self.bitmap_iter.as_mut() + && self.high_bits == hi + { + self.buffered = match self.buffered.take() { + Some(Buffered::Pos(start)) if start < lo => None, + Some(Buffered::Range(_, end)) if end < lo => None, + Some(Buffered::Range(start, end)) if start < lo => Some(Buffered::Range(lo, end)), + buffered => buffered, }; + iter.advance_to(lo); + return; + } - *inner = DeleteVectorIteratorInner { - high_bits: next_hi, - bitmap_iter: next_bitmap.iter(), + // `pos` is in a later bitmap: drop everything buffered from the current one. + self.bitmap_iter = None; + self.buffered = None; + for (high_bits, bitmap) in self.bitmaps.by_ref() { + if high_bits < hi { + continue; } + let mut iter = bitmap.iter(); + if high_bits == hi { + iter.advance_to(lo); + } + self.high_bits = high_bits; + self.bitmap_iter = Some(iter); + return; } + } +} + +impl Iterator for DeleteVectorRanges<'_> { + type Item = (u64, u64); + + fn next(&mut self) -> Option { + loop { + let pending = match self.buffered.take() { + Some(Buffered::Range(start, end)) => return self.full_range(start, end), + Some(Buffered::Pos(start)) => Some(start), + None => None, + }; - inner.bitmap_iter.advance_to(lo); + if let Some(iter) = self.bitmap_iter.as_mut() + && let Some(start) = pending.or_else(|| iter.next()) + { + // Walk short runs with `next`, and ask for the rest of a long run at once. + let mut end = start; + let mut walked = 0; + loop { + if walked == RUN_PROBE_LEN { + match iter.next_range() { + Some(run) if Some(*run.start()) == end.checked_add(1) => { + end = *run.end(); + } + Some(run) => { + self.buffered = Some(Buffered::Range(*run.start(), *run.end())) + } + None => {} + } + break; + } + match iter.next() { + Some(next) if Some(next) == end.checked_add(1) => { + end = next; + walked += 1; + } + next => { + self.buffered = next.map(Buffered::Pos); + break; + } + } + } + return self.full_range(start, end); + } + + let (high_bits, bitmap) = self.bitmaps.next()?; + self.high_bits = high_bits; + self.bitmap_iter = Some(bitmap.iter()); + } } } @@ -142,10 +444,287 @@ impl BitOrAssign for DeleteVector { } } +// Reproduces Iceberg-Java's `deletion-vector-v1` framing so tests can round-trip through +// `deserialize` without a Java writer, and so other test modules can build blob fixtures. +// Cross-implementation golden fixtures produced by Iceberg-Java are tracked separately; this +// only checks that our decode matches our encode. +#[cfg(test)] +pub(crate) fn frame_dv_blob(vector: &[u8]) -> Vec { + let body_len = DV_MAGIC_BYTES + vector.len(); + let mut blob = Vec::with_capacity(DV_LENGTH_PREFIX_BYTES + body_len + DV_CRC_BYTES); + blob.extend_from_slice(&(body_len as u32).to_be_bytes()); + blob.extend_from_slice(&DV_MAGIC); + blob.extend_from_slice(vector); + let crc = crc32fast::hash(&blob[DV_LENGTH_PREFIX_BYTES..]); + blob.extend_from_slice(&crc.to_be_bytes()); + blob +} + +/// Deterministic pseudo-random sets mixing single positions, short and long runs, runs across +/// 2^16 and 2^32 boundaries, and empty bitmaps between used ones. +#[cfg(test)] +pub(crate) fn mixed_delete_vectors() -> Vec { + use rand::rngs::StdRng; + use rand::{Rng, SeedableRng}; + + let mut rng = StdRng::seed_from_u64(7); + let mut vectors = vec![DeleteVector::default()]; + for case in 0..40u64 { + let mut dv = DeleteVector::default(); + let base = if case % 4 == 0 { (1u64 << 32) - 40 } else { 0 }; + let mut pos = base + rng.random_range(0..50); + for _ in 0..rng.random_range(0..60) { + let run = match rng.random_range(0..5) { + 0 => 1, + 1 => 2, + // one past the probe length: the rest comes from `next_range` + 2 => u64::from(RUN_PROBE_LEN) + 1, + 3 => rng.random_range(1..=40), + _ => rng.random_range(1..=70_000), + }; + dv.inner.insert_range(pos..pos + run); + pos += run + rng.random_range(1..=30); + } + vectors.push(dv); + } + let mut boundaries = DeleteVector::default(); + boundaries.inner.insert_range((1 << 16) - 3..(1 << 16) + 3); + boundaries + .inner + .insert_range((1 << 32) - 20..(1 << 32) + 20); + boundaries.inner.insert((2 << 32) + 7); + vectors.push(boundaries); + // High bits 0, 3 and 7, so skipping walks past empty bitmaps. + let mut gaps = DeleteVector::default(); + gaps.inner.insert_range(2..12); + gaps.inner.insert_range((3 << 32) + 5..(3 << 32) + 70_000); + gaps.inner.insert((7 << 32) + 1); + vectors.push(gaps); + vectors +} + #[cfg(test)] mod tests { + use rand::rngs::StdRng; + use rand::{Rng, SeedableRng}; + use super::*; + #[test] + fn test_ranges_cover_exactly_the_deleted_positions() { + for dv in mixed_delete_vectors() { + let positions: Vec = dv.iter().collect(); + let mut expanded = Vec::new(); + let mut last_end = None; + for (start, end) in dv.ranges() { + assert!(start < end, "empty range {start}..{end}"); + assert!( + last_end.is_none_or(|last| last <= start), + "ranges out of order" + ); + last_end = Some(end); + expanded.extend(start..end); + } + assert_eq!(expanded, positions); + } + } + + #[test] + fn test_ranges_skip_to_matches_model() { + let mut rng = StdRng::seed_from_u64(7); + + for dv in mixed_delete_vectors() { + let deleted: Vec = dv.iter().collect(); + let max = deleted.last().copied().unwrap_or(0) + 10; + // `deleted[next..]` has been neither returned nor skipped, and every position below + // `done` has. + let mut next = 0; + let mut done = 0u64; + let mut ranges = dv.ranges(); + loop { + match rng.random_range(0..6) { + 0 | 1 => { + let target = done.max(deleted.get(next).copied().unwrap_or(max)) + + rng.random_range(0..200); + ranges.skip_to(target); + next = deleted.partition_point(|&p| p < target); + done = target; + continue; + } + // A target at or below `done`, near it or anywhere below: does nothing. + 2 => { + let back = if rng.random_bool(0.5) { + done.saturating_sub(rng.random_range(0..200)) + } else { + rng.random_range(0..=done) + }; + ranges.skip_to(back); + continue; + } + _ => {} + } + match ranges.next() { + Some((start, end)) => { + assert!(start < end, "empty range {start}..{end}"); + let len = (end - start) as usize; + assert_eq!( + deleted + .get(next..next + len) + .map(|run| (run[0], run[len - 1])), + Some((start, end - 1)), + "returned {start}..{end}" + ); + next += len; + done = end; + } + None => { + assert_eq!(next, deleted.len(), "missed {:?}", &deleted[next..]); + break; + } + } + } + } + } + + #[test] + fn test_advance_to_matches_model() { + let mut rng = StdRng::seed_from_u64(11); + + for dv in mixed_delete_vectors() { + let deleted: Vec = dv.iter().collect(); + let max = deleted.last().copied().unwrap_or(0) + 10; + let mut next = 0; + let mut done = 0u64; + let mut iter = dv.iter(); + loop { + match rng.random_range(0..6) { + 0 => { + let target = done.max(deleted.get(next).copied().unwrap_or(max)) + + rng.random_range(0..200); + iter.advance_to(target); + next = deleted.partition_point(|&p| p < target); + done = target; + continue; + } + 1 => { + let back = if rng.random_bool(0.5) { + done.saturating_sub(rng.random_range(0..200)) + } else { + rng.random_range(0..=done) + }; + iter.advance_to(back); + continue; + } + _ => {} + } + let pos = iter.next(); + assert_eq!(pos, deleted.get(next).copied()); + match pos { + Some(pos) => { + next += 1; + done = pos + 1; + } + None => break, + } + } + } + } + + #[test] + fn test_ranges_skip_to_inside_buffered_run() { + // After a run of exactly RUN_PROBE_LEN + 1, the following run is already buffered. + let probe = u64::from(RUN_PROBE_LEN); + let mut dv = DeleteVector::default(); + dv.inner.insert_range(0..probe + 1); + dv.inner.insert_range(probe + 3..probe + 13); + let mut ranges = dv.ranges(); + assert_eq!(ranges.next(), Some((0, probe + 1))); + ranges.skip_to(probe + 8); + assert_eq!(ranges.next(), Some((probe + 8, probe + 13))); + assert_eq!(ranges.next(), None); + } + + #[test] + fn test_ranges_join_runs_across_2_16() { + // `roaring`'s `next_range` joins a run across containers, so these come back whole. + // Starting near the boundary also covers the positions walked before `next_range`. + for start in [(1 << 16) - 100, (1 << 16) - 2] { + let mut dv = DeleteVector::default(); + dv.inner.insert_range(start..(3 << 16) + 5); + assert_eq!(dv.ranges().collect::>(), vec![( + start, + (3 << 16) + 5 + )]); + } + } + + #[test] + fn test_ranges_end_at_u64_max() { + let dv = DeleteVector::new(RoaringTreemap::from_iter([5, u64::MAX - 2, u64::MAX - 1])); + assert_eq!(dv.ranges().collect::>(), vec![ + (5, 6), + (u64::MAX - 2, u64::MAX) + ]); + + // `u64::MAX` itself can't end a half-open range, and no row can be there. + let dv = DeleteVector::new(RoaringTreemap::from_iter([5, u64::MAX - 1, u64::MAX])); + assert_eq!(dv.ranges().collect::>(), vec![ + (5, 6), + (u64::MAX - 1, u64::MAX) + ]); + let dv = DeleteVector::new(RoaringTreemap::from_iter([u64::MAX])); + assert_eq!(dv.ranges().next(), None); + } + + // The cases from https://github.com/apache/iceberg-rust/issues/3400, for `skip_to` and for + // `advance_to`. + #[test] + fn test_ranges_skip_to_missing_later_bitmap() { + let dv = DeleteVector::new(RoaringTreemap::from_iter([1u64, 5, 9])); + let mut ranges = dv.ranges(); + assert_eq!(ranges.next(), Some((1, 2))); + ranges.skip_to((1 << 32) + 5); + assert_eq!(ranges.next(), None); + } + + #[test] + fn test_ranges_skip_to_below_current_bitmap() { + let high = 1u64 << 32; + let dv = DeleteVector::new(RoaringTreemap::from_iter([high + 1, high + 2, high + 9])); + let mut ranges = dv.ranges(); + assert_eq!(ranges.next(), Some((high + 1, high + 3))); + ranges.skip_to(10); + assert_eq!(ranges.next(), Some((high + 9, high + 10))); + assert_eq!(ranges.next(), None); + } + + #[test] + fn test_advance_to_missing_later_bitmap() { + let dv = DeleteVector::new(RoaringTreemap::from_iter([5u64, 7])); + let mut iter = dv.iter(); + assert_eq!(iter.next(), Some(5)); + iter.advance_to((1 << 32) + 10); + assert_eq!(iter.next(), None); + } + + #[test] + fn test_advance_to_below_current_bitmap() { + let high = 1u64 << 32; + let dv = DeleteVector::new(RoaringTreemap::from_iter([high + 10, high + 20])); + let mut iter = dv.iter(); + assert_eq!(iter.next(), Some(high + 10)); + iter.advance_to(50); + assert_eq!(iter.collect::>(), vec![high + 20]); + } + + #[test] + fn test_advance_to_before_first_next() { + let dv = DeleteVector::new(RoaringTreemap::from_iter([5u64, 200])); + let mut iter = dv.iter(); + iter.advance_to(100); + assert_eq!(iter.collect::>(), vec![200]); + } + #[test] fn test_insertion_and_iteration() { let mut dv = DeleteVector::default(); @@ -198,4 +777,144 @@ mod tests { let res = dv.insert_positions(&positions); assert!(res.is_err()); } + + fn encode_dv_blob(dv: &DeleteVector) -> Vec { + let mut vector = Vec::with_capacity(dv.inner.serialized_size()); + dv.inner.serialize_into(&mut vector).unwrap(); + frame_dv_blob(&vector) + } + + fn dv_of(positions: impl IntoIterator) -> DeleteVector { + let mut dv = DeleteVector::default(); + for pos in positions { + dv.insert(pos); + } + dv + } + + fn sorted(dv: &DeleteVector) -> Vec { + let mut positions: Vec = dv.iter().collect(); + positions.sort_unstable(); + positions + } + + #[test] + fn test_deserialize_roundtrip_empty() { + let blob = encode_dv_blob(&DeleteVector::default()); + assert_eq!(DeleteVector::deserialize(&blob).unwrap().len(), 0); + } + + #[test] + fn test_deserialize_roundtrip_small() { + let positions = [0u64, 5, 100, 1000]; + let dv = DeleteVector::deserialize(&encode_dv_blob(&dv_of(positions))).unwrap(); + assert_eq!(sorted(&dv), positions); + } + + #[test] + fn test_deserialize_roundtrip_spanning_64bit_keys() { + let positions = [1u64, 1 << 33, (1 << 33) + 5, 1 << 34]; + let dv = DeleteVector::deserialize(&encode_dv_blob(&dv_of(positions))).unwrap(); + assert_eq!(sorted(&dv), positions); + } + + // Java run-optimizes every deletion vector before writing, so real blobs carry RUN + // containers, which use the SERIAL_COOKIE roaring layout. Force that layout so decode + // exercises the run-container path rather than only array and bitmap containers. + #[test] + fn test_deserialize_roundtrip_run_optimized() { + let mut dv = dv_of(0..10_000); + assert!( + dv.inner.optimize(), + "expected a dense range to run-length encode" + ); + let decoded = DeleteVector::deserialize(&encode_dv_blob(&dv)).unwrap(); + assert_eq!(decoded.len(), 10_000); + let positions = sorted(&decoded); + assert_eq!(positions.first(), Some(&0)); + assert_eq!(positions.last(), Some(&9_999)); + } + + #[test] + fn test_deserialize_rejects_short_blob() { + let err = DeleteVector::deserialize(&[0u8; DV_MIN_BLOB_BYTES - 1]).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_deserialize_rejects_bad_magic() { + let mut blob = encode_dv_blob(&dv_of([1])); + blob[DV_LENGTH_PREFIX_BYTES] ^= 0xFF; + // Recompute the CRC so the magic check, not the CRC check, is what fails. + let end = blob.len() - DV_CRC_BYTES; + let crc = crc32fast::hash(&blob[DV_LENGTH_PREFIX_BYTES..end]); + blob[end..].copy_from_slice(&crc.to_be_bytes()); + let err = DeleteVector::deserialize(&blob).unwrap_err(); + assert!(err.message().contains("magic mismatch"), "got: {err}"); + } + + #[test] + fn test_deserialize_rejects_bad_crc() { + let mut blob = encode_dv_blob(&dv_of([1, 2, 3])); + let end = blob.len() - DV_CRC_BYTES; + blob[end] ^= 0xFF; + let err = DeleteVector::deserialize(&blob).unwrap_err(); + assert!(err.message().contains("CRC mismatch"), "got: {err}"); + } + + #[test] + fn test_deserialize_rejects_length_prefix_mismatch() { + let mut blob = encode_dv_blob(&dv_of([1])); + let declared = u32::from_be_bytes(blob[..DV_LENGTH_PREFIX_BYTES].try_into().unwrap()); + blob[..DV_LENGTH_PREFIX_BYTES].copy_from_slice(&(declared + 1).to_be_bytes()); + let err = DeleteVector::deserialize(&blob).unwrap_err(); + assert!(err.message().contains("length prefix"), "got: {err}"); + } + + // Crafts a raw roaring treemap directory (bitmap count header + key/bitmap entries) so tests + // can exercise key-ordering violations that `dv_of`/`encode_dv_blob` can never produce, since + // `DeleteVector::insert` always keeps keys unique and ascending. + fn raw_roaring_vector(entries: &[(u32, &[u32])]) -> Vec { + let mut vector = Vec::new(); + vector.extend_from_slice(&(entries.len() as u64).to_le_bytes()); + for (key, positions) in entries { + let mut bitmap = RoaringBitmap::new(); + for &pos in *positions { + bitmap.insert(pos); + } + vector.extend_from_slice(&key.to_le_bytes()); + bitmap.serialize_into(&mut vector).unwrap(); + } + vector + } + + // `RoaringTreemap::deserialize_from` stores keys in a `BTreeMap` via a plain insert, so + // without our own ordering check, decoding this would silently keep only the second bitmap + // for key 5 (position 2) and drop the first (position 1). + #[test] + fn test_deserialize_rejects_duplicate_keys() { + let vector = raw_roaring_vector(&[(5, &[1]), (5, &[2])]); + let err = DeleteVector::deserialize(&frame_dv_blob(&vector)).unwrap_err(); + assert!(err.message().contains("unsigned comparison"), "got: {err}"); + } + + // The Puffin spec requires the roaring directory's keys to be "ordered by unsigned + // comparison"; a stream with unique but out-of-order keys is not a conformant blob even + // though `BTreeMap` would happily reorder it into a correct-looking result. + #[test] + fn test_deserialize_rejects_out_of_order_keys() { + let vector = raw_roaring_vector(&[(5, &[1]), (3, &[2])]); + let err = DeleteVector::deserialize(&frame_dv_blob(&vector)).unwrap_err(); + assert!(err.message().contains("unsigned comparison"), "got: {err}"); + } + + // The roaring portable format stores the bitmap count as a u64 with the upper 32 bits + // reserved as zero padding, restricting it to [0, 2^32 - 1]; a value above that is not a + // conformant blob, regardless of whether any key/bitmap entries follow. + #[test] + fn test_deserialize_rejects_bitmap_count_overflow() { + let vector = (u32::MAX as u64 + 1).to_le_bytes().to_vec(); + let err = DeleteVector::deserialize(&frame_dv_blob(&vector)).unwrap_err(); + assert!(err.message().contains("exceeds the"), "got: {err}"); + } } diff --git a/crates/iceberg/src/encryption/crypto.rs b/crates/iceberg/src/encryption/crypto.rs new file mode 100644 index 0000000000..27ae9a336f --- /dev/null +++ b/crates/iceberg/src/encryption/crypto.rs @@ -0,0 +1,492 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Core cryptographic operations for Iceberg encryption. + +use std::str::FromStr; + +use aes_gcm::aead::generic_array::typenum::U12; +use aes_gcm::aead::rand_core::RngCore; +use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng, Payload}; +use aes_gcm::{Aes128Gcm, Aes256Gcm, AesGcm, Nonce}; + +/// AES-192-GCM with 96-bit nonce. Not provided by `aes-gcm` but constructible +/// from the underlying primitives, same as `Aes128Gcm` and `Aes256Gcm`. +type Aes192Gcm = AesGcm; + +use crate::error::invalid_data; +use crate::sensitive::SensitiveBytes; +use crate::{Error, ErrorKind, Result}; + +/// Supported AES key sizes for AES-GCM encryption. +/// +/// The Iceberg spec supports 128, 192, and 256-bit keys for AES-GCM. +/// See: +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum AesKeySize { + /// 128-bit AES key (16 bytes). Default per the Iceberg spec. + #[default] + Bits128 = 128, + /// 192-bit AES key (24 bytes) + Bits192 = 192, + /// 256-bit AES key (32 bytes) + Bits256 = 256, +} + +impl AesKeySize { + /// Returns the key length in bytes for this key size. + pub fn key_length(&self) -> usize { + match self { + Self::Bits128 => 16, + Self::Bits192 => 24, + Self::Bits256 => 32, + } + } + + /// Returns the key size for a given DEK length in bytes. + /// + /// Matches Java's `encryption.data-key-length` property semantics: + /// 16 → 128-bit, 24 → 192-bit, 32 → 256-bit. + pub fn from_key_length(len: usize) -> Result { + match len { + 16 => Ok(Self::Bits128), + 24 => Ok(Self::Bits192), + 32 => Ok(Self::Bits256), + _ => Err(invalid_data!( + "Invalid data key length: {len} (must be 16, 24, or 32)" + )), + } + } +} + +impl FromStr for AesKeySize { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "128" | "AES_GCM_128" | "AES128_GCM" => Ok(Self::Bits128), + "192" | "AES_GCM_192" | "AES192_GCM" => Ok(Self::Bits192), + "256" | "AES_GCM_256" | "AES256_GCM" => Ok(Self::Bits256), + _ => Err(invalid_data!("Invalid AES key size: {s}")), + } + } +} + +/// A secure encryption key that zeroes its memory on drop. +/// +/// The `Debug` impl is safe to expose: the inner [`SensitiveBytes`] redacts +/// the key material, printing only its length. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SecureKey { + key: SensitiveBytes, + key_size: AesKeySize, +} + +impl SecureKey { + /// Creates a new secure key with the specified key size. + /// + /// # Errors + /// Returns an error if the key length doesn't match the key size requirements. + pub fn new(key: &[u8]) -> Result { + let key_size = AesKeySize::from_key_length(key.len())?; + Ok(Self { + key: SensitiveBytes::new(key), + key_size, + }) + } + + /// Generates a new random key for the specified key size. + pub fn generate(key_size: AesKeySize) -> Self { + let mut key = vec![0u8; key_size.key_length()]; + OsRng.fill_bytes(&mut key); + Self { + key: SensitiveBytes::new(key), + key_size, + } + } + + /// Returns the AES key size. + pub fn key_size(&self) -> AesKeySize { + self.key_size + } + + /// Returns the key bytes. + pub fn as_bytes(&self) -> &[u8] { + self.key.as_bytes() + } +} + +impl TryFrom for SecureKey { + type Error = Error; + + fn try_from(key: SensitiveBytes) -> Result { + let key_size = AesKeySize::from_key_length(key.len())?; + Ok(Self { key, key_size }) + } +} + +/// AES-GCM cipher for encrypting and decrypting data. +pub struct AesGcmCipher { + key: SensitiveBytes, + key_size: AesKeySize, +} + +impl AesGcmCipher { + /// AES-GCM nonce length in bytes (96 bits). + pub const NONCE_LEN: usize = 12; + /// AES-GCM authentication tag length in bytes (128 bits). + pub const TAG_LEN: usize = 16; + + /// Creates a new cipher with the specified key. + pub fn new(key: SecureKey) -> Self { + Self { + key: SensitiveBytes::new(key.as_bytes()), + key_size: key.key_size(), + } + } + + /// Encrypts data using AES-GCM. + /// + /// # Arguments + /// * `plaintext` - The data to encrypt + /// * `aad` - Additional authenticated data (optional) + /// + /// # Returns + /// The encrypted data in the format: `[12-byte nonce][ciphertext][16-byte auth tag]` + /// This matches the Java implementation format for compatibility. + pub fn encrypt(&self, plaintext: &[u8], aad: Option<&[u8]>) -> Result> { + match self.key_size { + AesKeySize::Bits128 => { + encrypt_aes_gcm::(self.key.as_bytes(), plaintext, aad) + } + AesKeySize::Bits192 => { + encrypt_aes_gcm::(self.key.as_bytes(), plaintext, aad) + } + AesKeySize::Bits256 => { + encrypt_aes_gcm::(self.key.as_bytes(), plaintext, aad) + } + } + } + + /// Decrypts data using AES-GCM. + /// + /// # Arguments + /// * `ciphertext` - The encrypted data with format: [12-byte nonce][encrypted data][16-byte auth tag] + /// * `aad` - Additional authenticated data (must match encryption) + /// + /// # Returns + /// The decrypted plaintext. + pub fn decrypt(&self, ciphertext: &[u8], aad: Option<&[u8]>) -> Result> { + if ciphertext.len() < Self::NONCE_LEN + Self::TAG_LEN { + return Err(invalid_data!( + "Ciphertext too short: expected at least {} bytes, got {}", + Self::NONCE_LEN + Self::TAG_LEN, + ciphertext.len() + )); + } + + match self.key_size { + AesKeySize::Bits128 => { + decrypt_aes_gcm::(self.key.as_bytes(), ciphertext, aad) + } + AesKeySize::Bits192 => { + decrypt_aes_gcm::(self.key.as_bytes(), ciphertext, aad) + } + AesKeySize::Bits256 => { + decrypt_aes_gcm::(self.key.as_bytes(), ciphertext, aad) + } + } + } +} + +fn encrypt_aes_gcm(key_bytes: &[u8], plaintext: &[u8], aad: Option<&[u8]>) -> Result> +where C: Aead + AeadCore + KeyInit { + let cipher = C::new_from_slice(key_bytes) + .map_err(|e| invalid_data!("Invalid AES key").with_source(anyhow::anyhow!(e)))?; + let nonce = C::generate_nonce(&mut OsRng); + + let ciphertext = if let Some(aad) = aad { + cipher.encrypt(&nonce, Payload { + msg: plaintext, + aad, + }) + } else { + cipher.encrypt(&nonce, plaintext.as_ref()) + } + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "AES-GCM encryption failed") + .with_source(anyhow::anyhow!(e)) + })?; + + // Prepend nonce to ciphertext (Java compatible format) + let mut result = Vec::with_capacity(nonce.len() + ciphertext.len()); + result.extend_from_slice(&nonce); + result.extend_from_slice(&ciphertext); + Ok(result) +} + +fn decrypt_aes_gcm(key_bytes: &[u8], ciphertext: &[u8], aad: Option<&[u8]>) -> Result> +where C: Aead + AeadCore + KeyInit { + let cipher = C::new_from_slice(key_bytes) + .map_err(|e| invalid_data!("Invalid AES key").with_source(anyhow::anyhow!(e)))?; + + let nonce = Nonce::from_slice(&ciphertext[..AesGcmCipher::NONCE_LEN]); + let encrypted_data = &ciphertext[AesGcmCipher::NONCE_LEN..]; + + let plaintext = if let Some(aad) = aad { + cipher.decrypt(nonce, Payload { + msg: encrypted_data, + aad, + }) + } else { + cipher.decrypt(nonce, encrypted_data) + } + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "AES-GCM decryption failed") + .with_source(anyhow::anyhow!(e)) + })?; + + Ok(plaintext) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_aes_key_size() { + assert_eq!(AesKeySize::Bits128.key_length(), 16); + assert_eq!(AesKeySize::Bits192.key_length(), 24); + assert_eq!(AesKeySize::Bits256.key_length(), 32); + + assert_eq!( + AesKeySize::from_key_length(16).unwrap(), + AesKeySize::Bits128 + ); + assert_eq!( + AesKeySize::from_key_length(24).unwrap(), + AesKeySize::Bits192 + ); + assert_eq!( + AesKeySize::from_key_length(32).unwrap(), + AesKeySize::Bits256 + ); + assert!(AesKeySize::from_key_length(8).is_err()); + + for len in [0, 8, 15, 20, 33] { + let err = AesKeySize::from_key_length(len).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid, "for length {len}"); + assert_eq!( + err.message(), + format!("Invalid data key length: {len} (must be 16, 24, or 32)") + ); + } + + assert_eq!(AesKeySize::from_str("128").unwrap(), AesKeySize::Bits128); + assert_eq!( + AesKeySize::from_str("AES_GCM_128").unwrap(), + AesKeySize::Bits128 + ); + assert_eq!( + AesKeySize::from_str("AES_GCM_256").unwrap(), + AesKeySize::Bits256 + ); + for size in ["", "127", "AES_GCM_512", "INVALID"] { + let err = AesKeySize::from_str(size).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid, "for size {size}"); + assert_eq!(err.message(), format!("Invalid AES key size: {size}")); + } + } + + #[test] + fn test_secure_key() { + // Test key generation + let key1 = SecureKey::generate(AesKeySize::Bits128); + assert_eq!(key1.as_bytes().len(), 16); + assert_eq!(key1.key_size(), AesKeySize::Bits128); + + // Test key creation with validation + let valid_key = [0u8; 16]; + assert!(SecureKey::new(valid_key.as_slice()).is_ok()); + + let invalid_key = [0u8; 33]; + assert!(SecureKey::new(invalid_key.as_slice()).is_err()); + } + + #[test] + fn test_aes128_gcm_encryption_roundtrip() { + let key = SecureKey::generate(AesKeySize::Bits128); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Hello, Iceberg encryption!"; + let aad = b"additional authenticated data"; + + // Test without AAD + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + assert!(ciphertext.len() > plaintext.len() + 12); // nonce + tag + assert_ne!(&ciphertext[12..], plaintext); // encrypted portion differs + + let decrypted = cipher.decrypt(&ciphertext, None).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with AAD + let ciphertext = cipher.encrypt(plaintext, Some(aad)).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, Some(aad)).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with wrong AAD fails + assert!(cipher.decrypt(&ciphertext, Some(b"wrong aad")).is_err()); + } + + #[test] + fn test_aes192_gcm_encryption_roundtrip() { + let key = SecureKey::generate(AesKeySize::Bits192); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Hello, Iceberg encryption!"; + let aad = b"additional authenticated data"; + + // Test without AAD + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, None).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with AAD + let ciphertext = cipher.encrypt(plaintext, Some(aad)).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, Some(aad)).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with wrong AAD fails + assert!(cipher.decrypt(&ciphertext, Some(b"wrong aad")).is_err()); + } + + #[test] + fn test_aes256_gcm_encryption_roundtrip() { + let key = SecureKey::generate(AesKeySize::Bits256); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Hello, Iceberg encryption!"; + let aad = b"additional authenticated data"; + + // Test without AAD + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, None).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with AAD + let ciphertext = cipher.encrypt(plaintext, Some(aad)).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, Some(aad)).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with wrong AAD fails + assert!(cipher.decrypt(&ciphertext, Some(b"wrong aad")).is_err()); + } + + #[test] + fn test_cross_key_size_incompatibility() { + let plaintext = b"Cross-key test"; + + let key128 = SecureKey::generate(AesKeySize::Bits128); + let key256 = SecureKey::generate(AesKeySize::Bits256); + + let cipher128 = AesGcmCipher::new(key128); + let cipher256 = AesGcmCipher::new(key256); + + // Ciphertext from 128-bit key should not decrypt with 256-bit key + let ciphertext = cipher128.encrypt(plaintext, None).unwrap(); + assert!(cipher256.decrypt(&ciphertext, None).is_err()); + } + + #[test] + fn test_encryption_with_empty_plaintext() { + let key = SecureKey::generate(AesKeySize::Bits128); + let cipher = AesGcmCipher::new(key); + + let plaintext = b""; + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + + // Even empty plaintext produces nonce + tag + assert_eq!(ciphertext.len(), 12 + 16); // 12-byte nonce + 16-byte tag + + let decrypted = cipher.decrypt(&ciphertext, None).unwrap(); + assert_eq!(decrypted, plaintext); + } + + #[test] + fn test_decryption_with_tampered_ciphertext() { + let key = SecureKey::generate(AesKeySize::Bits128); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Sensitive data"; + let mut ciphertext = cipher.encrypt(plaintext, None).unwrap(); + + // Tamper with the encrypted portion (after the nonce) + if ciphertext.len() > 12 { + ciphertext[12] ^= 0xFF; + } + + // Decryption should fail due to authentication tag mismatch + assert!(cipher.decrypt(&ciphertext, None).is_err()); + } + + #[test] + fn test_different_keys_produce_different_ciphertexts() { + let key1 = SecureKey::generate(AesKeySize::Bits128); + let key2 = SecureKey::generate(AesKeySize::Bits128); + + let cipher1 = AesGcmCipher::new(key1); + let cipher2 = AesGcmCipher::new(key2); + + let plaintext = b"Same plaintext"; + + let ciphertext1 = cipher1.encrypt(plaintext, None).unwrap(); + let ciphertext2 = cipher2.encrypt(plaintext, None).unwrap(); + + // Different keys should produce different ciphertexts (comparing the encrypted portion) + // Note: The nonces will also be different, but we're mainly interested in the encrypted data + assert_ne!(&ciphertext1[12..], &ciphertext2[12..]); + } + + #[test] + fn test_ciphertext_format_java_compatible() { + // Test that our ciphertext format matches Java's: [12-byte nonce][ciphertext][16-byte tag] + let key = SecureKey::generate(AesKeySize::Bits128); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Test data"; + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + + // Format should be: [12-byte nonce][encrypted_data + 16-byte GCM tag] + assert_eq!( + ciphertext.len(), + 12 + plaintext.len() + 16, + "Ciphertext should be nonce + plaintext + tag length" + ); + + // Verify we can decrypt by extracting nonce from the beginning + let nonce = &ciphertext[..12]; + assert_eq!(nonce.len(), 12, "Nonce should be 12 bytes"); + + // The rest is encrypted data + tag + let encrypted_with_tag = &ciphertext[12..]; + assert_eq!( + encrypted_with_tag.len(), + plaintext.len() + 16, + "Encrypted portion should be plaintext length + 16-byte tag" + ); + } +} diff --git a/crates/iceberg/src/encryption/io.rs b/crates/iceberg/src/encryption/io.rs new file mode 100644 index 0000000000..17fc1a1f23 --- /dev/null +++ b/crates/iceberg/src/encryption/io.rs @@ -0,0 +1,411 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Encrypted file wrappers for InputFile / OutputFile. + +use std::sync::Arc; + +use bytes::Bytes; + +use super::crypto::AesGcmCipher; +use super::key_metadata::StandardKeyMetadata; +use super::stream::{AesGcmFileRead, AesGcmFileWrite, MIN_STREAM_LENGTH}; +use crate::Result; +use crate::error::invalid_data; +use crate::io::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; + +/// An AGS1 stream-encrypted input file wrapping a plain [`InputFile`]. +/// +/// Transparently decrypts on read. +pub struct EncryptedInputFile { + inner: InputFile, + key_metadata: StandardKeyMetadata, +} + +impl EncryptedInputFile { + /// Creates a new encrypted input file. + pub fn new(inner: InputFile, key_metadata: StandardKeyMetadata) -> Self { + Self { + inner, + key_metadata, + } + } + + /// Absolute path of the file. + pub fn location(&self) -> &str { + self.inner.location() + } + + /// Check if file exists. + pub async fn exists(&self) -> Result { + self.inner.exists().await + } + + /// Returns file metadata from the declared encrypted length without performing I/O. + /// + /// The returned size is the **plaintext** size. + pub fn metadata(&self) -> Result { + let plaintext_size = AesGcmFileRead::calculate_plaintext_length(self.encrypted_length()?)?; + Ok(FileMetadata { + size: plaintext_size, + }) + } + + /// Read and returns whole content of file (decrypted plaintext). + pub async fn read(&self) -> Result { + let meta = self.metadata()?; + let reader = self.reader().await?; + reader.read(0..meta.size).await + } + + /// Creates a reader that transparently decrypts on each read. + pub async fn reader(&self) -> Result> { + let encrypted_length = self.encrypted_length()?; + let raw_reader = self.inner.reader().await?; + let cipher = build_cipher(&self.key_metadata)?; + let aad_prefix: Box<[u8]> = self.key_metadata.aad_prefix().unwrap_or_default().into(); + let decrypting = AesGcmFileRead::new(raw_reader, cipher, aad_prefix, encrypted_length)?; + Ok(Box::new(decrypting)) + } + + // A storage stat would hide truncation; require the original length from key metadata. + fn encrypted_length(&self) -> Result { + let length = self.key_metadata.file_length().ok_or_else(|| { + invalid_data!("AGS1 key metadata is missing the encrypted file length") + })?; + if length < u64::from(MIN_STREAM_LENGTH) { + return Err(invalid_data!( + "Invalid encrypted file length: {length} is less than {MIN_STREAM_LENGTH}" + )); + } + Ok(length) + } + + /// Returns a reference to the file's key metadata. + pub fn key_metadata(&self) -> &StandardKeyMetadata { + &self.key_metadata + } + + /// Consumes self and returns the underlying plain input file. + pub fn into_inner(self) -> InputFile { + self.inner + } +} + +impl std::fmt::Debug for EncryptedInputFile { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EncryptedInputFile") + .field("path", &self.inner.location()) + .finish_non_exhaustive() + } +} + +/// An AGS1 stream-encrypted output file wrapping a plain [`OutputFile`]. +/// +/// Transparently encrypts on write. +pub struct EncryptedOutputFile { + inner: OutputFile, + key_metadata: StandardKeyMetadata, +} + +impl EncryptedOutputFile { + /// Creates a new encrypted output file. + pub fn new(inner: OutputFile, key_metadata: StandardKeyMetadata) -> Self { + Self { + inner, + key_metadata, + } + } + + /// Returns key metadata using the encrypted size returned by [`FileWrite::close`] or [`Self::write`]. + pub fn key_metadata_with_saved_file_metadata( + &self, + file_metadata: &FileMetadata, + ) -> StandardKeyMetadata { + self.key_metadata + .clone() + .with_file_length(file_metadata.size) + } + + /// Absolute path of the file. + pub fn location(&self) -> &str { + self.inner.location() + } + + /// Creates a writer that transparently encrypts on each write. + pub async fn writer(&self) -> Result> { + let raw_writer = self.inner.writer().await?; + let cipher = build_cipher(&self.key_metadata)?; + let aad_prefix: Box<[u8]> = self.key_metadata.aad_prefix().unwrap_or_default().into(); + Ok(Box::new(AesGcmFileWrite::new( + raw_writer, cipher, aad_prefix, + ))) + } + + /// Write bytes to the file and return its encrypted size. + pub async fn write(&self, bs: Bytes) -> Result { + let mut writer = self.writer().await?; + writer.write(bs).await?; + writer.close().await + } + + /// Deletes the underlying file. + pub async fn delete(&self) -> Result<()> { + self.inner.delete().await + } + + /// Consumes self and returns the underlying plain output file. + pub fn into_inner(self) -> OutputFile { + self.inner + } +} + +impl std::fmt::Debug for EncryptedOutputFile { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EncryptedOutputFile") + .field("path", &self.inner.location()) + .finish_non_exhaustive() + } +} + +fn build_cipher(metadata: &StandardKeyMetadata) -> Result> { + let key = metadata.encryption_key().clone(); + Ok(Arc::new(AesGcmCipher::new(key))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ErrorKind; + use crate::encryption::stream::{ + CIPHER_BLOCK_SIZE, GCM_STREAM_HEADER_LENGTH, PLAIN_BLOCK_SIZE, + }; + use crate::io::FileIO; + + fn key_metadata() -> StandardKeyMetadata { + StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!") + } + + #[tokio::test] + async fn test_write_read_roundtrip() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/io_roundtrip.bin"; + let plaintext = b"Hello from EncryptedInputFile/EncryptedOutputFile!"; + + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + let file_metadata = output.write(Bytes::from(plaintext.to_vec())).await.unwrap(); + + let input = EncryptedInputFile::new( + fileio.new_input(path).unwrap(), + output.key_metadata_with_saved_file_metadata(&file_metadata), + ); + let content = input.read().await.unwrap(); + assert_eq!(&content[..], plaintext); + } + + #[tokio::test] + async fn test_metadata_returns_plaintext_size() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/io_metadata.bin"; + let plaintext = b"some bytes to measure"; + + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + let file_metadata = output.write(Bytes::from(plaintext.to_vec())).await.unwrap(); + + let raw_size = fileio + .new_input(path) + .unwrap() + .metadata() + .await + .unwrap() + .size; + assert!( + raw_size > plaintext.len() as u64, + "encrypted file should be larger than plaintext (header + nonce + tag)" + ); + + // A missing path proves the size comes from the key metadata rather than a stat call. + let input = EncryptedInputFile::new( + fileio.new_input("memory:///does-not-exist").unwrap(), + output.key_metadata_with_saved_file_metadata(&file_metadata), + ); + let meta = input.metadata().unwrap(); + assert_eq!(meta.size, plaintext.len() as u64); + } + + #[tokio::test] + async fn test_missing_file_length_is_rejected() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/missing_length.bin"; + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + output.write(Bytes::from_static(b"data")).await.unwrap(); + let input = EncryptedInputFile::new(fileio.new_input(path).unwrap(), key_metadata()); + + for err in [ + input.metadata().err().unwrap(), + input.reader().await.err().unwrap(), + input.read().await.unwrap_err(), + ] { + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("missing the encrypted file length") + ); + } + } + + #[tokio::test] + async fn test_invalid_file_length_is_rejected() { + let fileio = FileIO::new_with_memory(); + for length in [ + 0, + u64::from(GCM_STREAM_HEADER_LENGTH), + u64::from(MIN_STREAM_LENGTH - 1), + ] { + let input = EncryptedInputFile::new( + fileio + .new_input("memory:///test/invalid_length.bin") + .unwrap(), + key_metadata().with_file_length(length), + ); + assert_eq!( + input.metadata().err().unwrap().kind(), + ErrorKind::DataInvalid + ); + assert_eq!( + input.reader().await.err().unwrap().kind(), + ErrorKind::DataInvalid + ); + } + } + + #[tokio::test] + async fn test_oversized_file_length_is_rejected() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/oversized_length.bin"; + let plaintext = Bytes::from_static(b"some bytes to measure"); + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + let file_metadata = output.write(plaintext.clone()).await.unwrap(); + + // A declared length is trusted without a stat, so an inflated one is only caught once a + // read runs off the end of the real file. Both a minimal overstatement and one spanning a + // whole extra block must fail rather than silently return short plaintext. + for excess in [1, u64::from(CIPHER_BLOCK_SIZE)] { + let input = EncryptedInputFile::new( + fileio.new_input(path).unwrap(), + key_metadata().with_file_length(file_metadata.size + excess), + ); + + let inflated_size = input.metadata().unwrap().size; + assert!(inflated_size > plaintext.len() as u64); + + assert_eq!( + input.read().await.unwrap_err().kind(), + ErrorKind::DataInvalid + ); + + // Not even the bytes that genuinely are on disk can be read back. + let reader = input.reader().await.unwrap(); + assert_eq!( + reader + .read(0..plaintext.len() as u64) + .await + .unwrap_err() + .kind(), + ErrorKind::DataInvalid + ); + } + } + + #[tokio::test] + async fn test_truncated_file_is_rejected() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/truncated.bin"; + let plaintext = Bytes::from(vec![42; 2 * PLAIN_BLOCK_SIZE as usize + 17]); + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + let file_metadata = output.write(plaintext.clone()).await.unwrap(); + let metadata = key_metadata().with_file_length(file_metadata.size); + let ciphertext = fileio.new_input(path).unwrap().read().await.unwrap(); + let truncated_length = (GCM_STREAM_HEADER_LENGTH + CIPHER_BLOCK_SIZE) as usize; + fileio + .new_output(path) + .unwrap() + .write(ciphertext.slice(..truncated_length)) + .await + .unwrap(); + + let input = EncryptedInputFile::new(fileio.new_input(path).unwrap(), metadata); + assert_eq!(input.metadata().unwrap().size, plaintext.len() as u64); + let reader = input.reader().await.unwrap(); + assert_eq!( + reader.read(0..u64::from(PLAIN_BLOCK_SIZE)).await.unwrap(), + plaintext.slice(..PLAIN_BLOCK_SIZE as usize) + ); + assert_eq!( + input.read().await.unwrap_err().kind(), + ErrorKind::DataInvalid + ); + } + + #[tokio::test] + async fn test_truncated_empty_file_is_rejected() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/truncated_empty.bin"; + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + let file_metadata = output.write(Bytes::new()).await.unwrap(); + let metadata = key_metadata().with_file_length(file_metadata.size); + let ciphertext = fileio.new_input(path).unwrap().read().await.unwrap(); + fileio + .new_output(path) + .unwrap() + .write(ciphertext.slice(..GCM_STREAM_HEADER_LENGTH as usize)) + .await + .unwrap(); + let input = EncryptedInputFile::new(fileio.new_input(path).unwrap(), metadata); + assert_eq!( + input.read().await.unwrap_err().kind(), + ErrorKind::DataInvalid + ); + } + + #[tokio::test] + async fn test_close_returns_encrypted_size() { + let fileio = FileIO::new_with_memory(); + let path = "memory:///test/streaming.bin"; + let output = EncryptedOutputFile::new(fileio.new_output(path).unwrap(), key_metadata()); + for plaintext in [ + Bytes::from(vec![42; PLAIN_BLOCK_SIZE as usize + 17]), + Bytes::new(), + ] { + let mut writer = output.writer().await.unwrap(); + for chunk in plaintext.chunks(1024) { + writer.write(Bytes::copy_from_slice(chunk)).await.unwrap(); + } + let metadata = writer.close().await.unwrap(); + let size = fileio + .new_input(path) + .unwrap() + .metadata() + .await + .unwrap() + .size; + assert_eq!(metadata.size, size); + } + } +} diff --git a/crates/iceberg/src/encryption/key_metadata.rs b/crates/iceberg/src/encryption/key_metadata.rs new file mode 100644 index 0000000000..b3222eec53 --- /dev/null +++ b/crates/iceberg/src/encryption/key_metadata.rs @@ -0,0 +1,431 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Avro-serialized key metadata format compatible with Java's +//! `org.apache.iceberg.encryption.StandardKeyMetadata`. + +use std::fmt; + +use aes_gcm::aead::OsRng; +use aes_gcm::aead::rand_core::RngCore; + +use super::{AesKeySize, SecureKey}; +use crate::error::invalid_data; +use crate::{Error, ErrorKind, Result}; + +/// Standard key metadata for Iceberg table encryption. +/// +/// Contains the Data Encryption Key (DEK), AAD prefix, and optional file +/// length. Byte-compatible with Java's `StandardKeyMetadata` via Avro +/// serialization. +/// +/// Wire format: `[version byte (0x01)] [Avro binary datum]` +#[derive(Clone, PartialEq, Eq)] +pub struct StandardKeyMetadata { + encryption_key: SecureKey, + aad_prefix: Option>, + file_length: Option, +} + +impl fmt::Debug for StandardKeyMetadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StandardKeyMetadata") + .field("encryption_key", &self.encryption_key) + .field( + "aad_prefix", + &self + .aad_prefix + .as_ref() + .map(|b| format!("[{} bytes]", b.len())), + ) + .field("file_length", &self.file_length) + .finish() + } +} + +impl StandardKeyMetadata { + /// Creates a new `StandardKeyMetadata` from raw key bytes. + pub fn try_new(encryption_key: &[u8]) -> Result { + Ok(Self::from(SecureKey::new(encryption_key)?)) + } + + /// Generates a `StandardKeyMetadata` carrying a fresh random DEK of + /// `key_size` together with a fresh random AAD prefix. + pub(crate) fn generate(key_size: AesKeySize) -> Self { + Self::from(SecureKey::generate(key_size)).with_aad_prefix(&generate_aad_prefix()) + } + + /// Adds an AAD prefix. + pub fn with_aad_prefix(mut self, aad_prefix: &[u8]) -> Self { + self.aad_prefix = Some(aad_prefix.into()); + self + } + + /// Sets the encrypted file length in bytes, required for AGS1 truncation protection. + pub fn with_file_length(mut self, length: u64) -> Self { + self.file_length = Some(length); + self + } + + /// Returns the plaintext Data Encryption Key. + pub fn encryption_key(&self) -> &SecureKey { + &self.encryption_key + } + + /// Returns the AAD prefix. + pub fn aad_prefix(&self) -> Option<&[u8]> { + self.aad_prefix.as_deref() + } + + /// Returns the optional encrypted file length in bytes; AGS1 readers require it to be set. + pub fn file_length(&self) -> Option { + self.file_length + } + + /// Encodes to Java-compatible format: `[0x01] [Avro binary datum]` + pub fn encode(&self) -> Result> { + _serde::StandardKeyMetadataV1::try_from(self)?.encode() + } + + /// Decodes from Java-compatible format. + pub fn decode(bytes: &[u8]) -> Result { + _serde::StandardKeyMetadataV1::decode(bytes).and_then(Self::try_from) + } +} + +impl From for StandardKeyMetadata { + /// Creates a `StandardKeyMetadata` from an already-validated key. + fn from(encryption_key: SecureKey) -> Self { + Self { + encryption_key, + aad_prefix: None, + file_length: None, + } + } +} + +/// AAD prefix length in bytes. +const AAD_PREFIX_LENGTH: usize = 16; + +fn generate_aad_prefix() -> Box<[u8]> { + let mut prefix = vec![0u8; AAD_PREFIX_LENGTH]; + OsRng.fill_bytes(&mut prefix); + prefix.into_boxed_slice() +} + +mod _serde { + use std::io::Cursor; + use std::sync::{Arc, LazyLock}; + + use apache_avro::reader::datum::GenericDatumReader; + use apache_avro::writer::datum::GenericDatumWriter; + use apache_avro::{Schema as AvroSchema, from_value, to_value}; + use serde::{Deserialize, Serialize}; + + use super::*; + use crate::avro::schema_to_avro_schema; + use crate::spec::{NestedField, PrimitiveType, Schema, Type}; + + pub(super) const V1: u8 = 1; + + /// Avro schema for StandardKeyMetadata V1, derived from Iceberg schema. + pub(super) static AVRO_SCHEMA_V1: LazyLock = LazyLock::new(|| { + let schema = Schema::builder() + .with_fields(vec![ + Arc::new(NestedField::required( + 0, + "encryption_key", + Type::Primitive(PrimitiveType::Binary), + )), + Arc::new(NestedField::optional( + 1, + "aad_prefix", + Type::Primitive(PrimitiveType::Binary), + )), + Arc::new(NestedField::optional( + 2, + "file_length", + Type::Primitive(PrimitiveType::Long), + )), + ]) + .build() + .expect("Failed to build StandardKeyMetadata Iceberg schema"); + + schema_to_avro_schema("StandardKeyMetadata", &schema) + .expect("Failed to convert StandardKeyMetadata to Avro schema") + }); + + /// Serde struct for Avro serialization of [`StandardKeyMetadata`] V1. + /// Field names must match [`AVRO_SCHEMA_V1`] exactly. + #[derive(Serialize, Deserialize)] + pub(super) struct StandardKeyMetadataV1 { + pub encryption_key: serde_bytes::ByteBuf, + pub aad_prefix: Option, + pub file_length: Option, + } + + impl StandardKeyMetadataV1 { + pub(super) fn encode(&self) -> Result> { + let value = to_value(self) + .and_then(|v| v.resolve(&AVRO_SCHEMA_V1)) + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "Failed to encode key metadata") + .with_source(e) + })?; + + let datum = GenericDatumWriter::builder(&AVRO_SCHEMA_V1) + .build() + .and_then(|writer| writer.write_value_to_vec(value)) + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "Failed to encode key metadata") + .with_source(e) + })?; + + let mut result = Vec::with_capacity(1 + datum.len()); + result.push(V1); + result.extend_from_slice(&datum); + Ok(result.into_boxed_slice()) + } + + pub(super) fn decode(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(invalid_data!("Empty key metadata buffer")); + } + + let version = bytes[0]; + if version != V1 { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported key metadata version: {version} (supported: {V1})"), + )); + } + + let mut reader = Cursor::new(&bytes[1..]); + let value = GenericDatumReader::builder(&AVRO_SCHEMA_V1) + .build() + .and_then(|datum_reader| datum_reader.read_value(&mut reader)) + .map_err(|e| invalid_data!("Failed to decode key metadata").with_source(e))?; + + from_value(&value) + .map_err(|e| invalid_data!("Failed to decode key metadata fields").with_source(e)) + } + } + + impl TryFrom<&StandardKeyMetadata> for StandardKeyMetadataV1 { + type Error = Error; + + fn try_from(metadata: &StandardKeyMetadata) -> Result { + let file_length = metadata + .file_length + .map(i64::try_from) + .transpose() + .map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + "Key metadata file length exceeds the Avro long range", + ) + .with_source(e) + })?; + Ok(Self { + encryption_key: serde_bytes::ByteBuf::from(metadata.encryption_key.as_bytes()), + aad_prefix: metadata + .aad_prefix + .as_ref() + .map(|b| serde_bytes::ByteBuf::from(b.as_ref())), + file_length, + }) + } + } + + impl TryFrom for StandardKeyMetadata { + type Error = Error; + + fn try_from(v1: StandardKeyMetadataV1) -> Result { + let encryption_key = SecureKey::new(&v1.encryption_key).map_err(|e| { + invalid_data!("Invalid encryption key in key metadata").with_source(e) + })?; + Ok(Self { + encryption_key, + aad_prefix: v1.aad_prefix.map(|b| b.into_vec().into_boxed_slice()), + file_length: v1.file_length.map(u64::try_from).transpose().map_err(|e| { + invalid_data!("Negative file length in key metadata").with_source(e) + })?, + }) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_roundtrip() { + let key = b"0123456789012345"; + let aad = b"1234567890123456"; + + let metadata = StandardKeyMetadata::try_new(key) + .unwrap() + .with_aad_prefix(aad); + let serialized = metadata.encode().unwrap(); + let parsed = StandardKeyMetadata::decode(&serialized).unwrap(); + + assert_eq!(parsed.encryption_key().as_bytes(), key); + assert_eq!(parsed.aad_prefix(), Some(aad.as_slice())); + assert_eq!(parsed.file_length(), None); + } + + #[test] + fn test_roundtrip_with_length() { + let key = b"0123456789012345"; + let aad = b"1234567890123456"; + + let file_length = 100_000; + let metadata = StandardKeyMetadata::try_new(key) + .unwrap() + .with_aad_prefix(aad) + .with_file_length(file_length); + let serialized = metadata.encode().unwrap(); + let parsed = StandardKeyMetadata::decode(&serialized).unwrap(); + + assert_eq!(parsed.encryption_key().as_bytes(), key); + assert_eq!(parsed.aad_prefix(), Some(aad.as_slice())); + assert_eq!(parsed.file_length(), Some(file_length)); + } + + #[test] + fn test_unsupported_version() { + let result = StandardKeyMetadata::decode(&[0x02]); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert_eq!( + err.message(), + "Unsupported key metadata version: 2 (supported: 1)" + ); + } + + #[test] + fn test_empty_buffer() { + let result = StandardKeyMetadata::decode(&[]); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_roundtrip_without_aad() { + let key = b"0123456789012345"; + let metadata = StandardKeyMetadata::try_new(key).unwrap(); + let serialized = metadata.encode().unwrap(); + let parsed = StandardKeyMetadata::decode(&serialized).unwrap(); + + assert_eq!(parsed.encryption_key().as_bytes(), key); + assert_eq!(parsed.aad_prefix(), None); + } + + #[test] + fn test_new_rejects_invalid_key_length() { + // 24-byte (AES-192) and 32-byte (AES-256) keys are accepted. + for len in [16usize, 24, 32] { + assert!(StandardKeyMetadata::try_new(&vec![0u8; len]).is_ok()); + } + + // Invalid lengths are rejected at construction, so an invalid + // `StandardKeyMetadata` can never exist. + for len in [0usize, 4, 15, 20, 33] { + assert!(StandardKeyMetadata::try_new(&vec![0u8; len]).is_err()); + } + } + + #[test] + fn test_encode_rejects_file_length_above_long_max() { + let metadata = StandardKeyMetadata::try_new(&[0u8; 16]) + .unwrap() + .with_file_length(i64::MAX as u64 + 1); + + let err = metadata.encode().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_decode_rejects_negative_file_length() { + let serialized = _serde::StandardKeyMetadataV1 { + encryption_key: serde_bytes::ByteBuf::from(vec![0u8; 16]), + aad_prefix: None, + file_length: Some(-1), + } + .encode() + .unwrap(); + + let err = StandardKeyMetadata::decode(&serialized).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_decode_rejects_invalid_key_length() { + // Craft wire bytes carrying an invalid-length DEK directly via the + // serde struct (bypassing the validated public constructors) to prove + // `decode` still rejects malformed key material off the wire. + for len in [0usize, 4, 15, 20, 33] { + let serialized = _serde::StandardKeyMetadataV1 { + encryption_key: serde_bytes::ByteBuf::from(vec![0u8; len]), + aad_prefix: None, + file_length: None, + } + .encode() + .unwrap(); + + let err = StandardKeyMetadata::decode(&serialized).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("Invalid encryption key in key metadata") + ); + } + } + + #[test] + fn test_decode_tolerates_trailing_bytes() { + // Deliberate, not an oversight: Java's RawDecoder reads a single datum without + // checking for EOF, and Iceberg depends on that leniency — `file_length` was + // appended to SCHEMA_V1 without bumping the version byte. Requiring EOF here + // would stop older readers tolerating fields a newer writer appended. + let key = b"0123456789012345"; + let aad = b"1234567890123456"; + let file_length = 1024; + + let serialized = StandardKeyMetadata::try_new(key) + .unwrap() + .with_aad_prefix(aad) + .with_file_length(file_length) + .encode() + .unwrap(); + + // Arbitrary junk, then bytes shaped like a further optional field appended by a + // newer writer (union tag "present", then a 4-byte value). + for trailing in [b"\xde\xad\xbe\xef".as_slice(), b"\x02\x08more".as_slice()] { + let mut extended = serialized.to_vec(); + extended.extend_from_slice(trailing); + + let parsed = StandardKeyMetadata::decode(&extended).unwrap(); + + assert_eq!(parsed.encryption_key().as_bytes(), key); + assert_eq!(parsed.aad_prefix(), Some(aad.as_slice())); + assert_eq!(parsed.file_length(), Some(file_length)); + } + } +} diff --git a/crates/iceberg/src/encryption/kms/client.rs b/crates/iceberg/src/encryption/kms/client.rs new file mode 100644 index 0000000000..85cd511758 --- /dev/null +++ b/crates/iceberg/src/encryption/kms/client.rs @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key management client trait for encryption key operations. +//! +//! Mirrors the Java `KeyManagementClient` interface from the Apache Iceberg spec. + +use async_trait::async_trait; + +use crate::Result; +use crate::encryption::SensitiveBytes; + +/// Result of a server-side key generation operation. +/// +/// Returned by [`KeyManagementClient::generate_key`] when the KMS supports +/// atomic key generation and wrapping. +pub struct GeneratedKey { + key: SensitiveBytes, + wrapped_key: Vec, +} + +impl GeneratedKey { + /// Creates a new `GeneratedKey` from plaintext key bytes and wrapped key bytes. + pub fn new(key: SensitiveBytes, wrapped_key: Vec) -> Self { + Self { key, wrapped_key } + } + + /// Returns the plaintext key bytes. Zeroized on drop, redacted in Debug. + pub fn key(&self) -> &SensitiveBytes { + &self.key + } + + /// Returns the wrapped (encrypted) key bytes. + pub fn wrapped_key(&self) -> &[u8] { + &self.wrapped_key + } +} + +/// Pluggable interface for key management systems (AWS KMS, Azure Key Vault, etc.). +#[async_trait] +pub trait KeyManagementClient: Send + Sync + std::fmt::Debug { + /// Wrap (encrypt) a key using a wrapping key managed by the KMS. + async fn wrap_key(&self, key: &[u8], wrapping_key_id: &str) -> Result>; + + /// Unwrap (decrypt) a previously wrapped key. + async fn unwrap_key(&self, wrapped_key: &[u8], wrapping_key_id: &str) + -> Result; + + /// Whether this KMS supports server-side key generation. + /// + /// If `true`, callers can use [`generate_key`](Self::generate_key) for atomic + /// key generation and wrapping, which is more secure than generating a key + /// locally and then wrapping it. + fn supports_key_generation(&self) -> bool; + + /// Generate a new key and wrap it atomically on the server side. + /// + /// This is only supported when [`supports_key_generation`](Self::supports_key_generation) + /// returns `true`. + async fn generate_key(&self, wrapping_key_id: &str) -> Result; +} + +#[async_trait] +impl + Send + Sync + std::fmt::Debug> KeyManagementClient for T { + async fn wrap_key(&self, key: &[u8], wrapping_key_id: &str) -> Result> { + self.as_ref().wrap_key(key, wrapping_key_id).await + } + + async fn unwrap_key( + &self, + wrapped_key: &[u8], + wrapping_key_id: &str, + ) -> Result { + self.as_ref().unwrap_key(wrapped_key, wrapping_key_id).await + } + + fn supports_key_generation(&self) -> bool { + self.as_ref().supports_key_generation() + } + + async fn generate_key(&self, wrapping_key_id: &str) -> Result { + self.as_ref().generate_key(wrapping_key_id).await + } +} diff --git a/crates/iceberg/src/encryption/kms/factory.rs b/crates/iceberg/src/encryption/kms/factory.rs new file mode 100644 index 0000000000..b957b5a5f6 --- /dev/null +++ b/crates/iceberg/src/encryption/kms/factory.rs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Factory trait for creating [`KeyManagementClient`] instances. + +use std::collections::HashMap; +use std::fmt::Debug; +use std::sync::Arc; + +use async_trait::async_trait; + +use super::KeyManagementClient; +use crate::Result; + +/// Factory for creating a [`KeyManagementClient`] from catalog properties. +/// +/// Replaces Java's reflection-based `encryption.kms-impl` + `initialize(properties)` +/// pattern. Users provide an implementation of this trait to the catalog builder via +/// [`CatalogBuilder::with_kms_client_factory`](crate::CatalogBuilder::with_kms_client_factory). +/// +/// The catalog calls [`create_kms_client`](Self::create_kms_client) **once** during +/// catalog initialization with the catalog's properties. The resulting client is +/// shared across all tables in the catalog and passed to each table's +/// [`EncryptionManager`](crate::encryption::EncryptionManager) via +/// `TableBuilder::kms_client(...)`. +#[async_trait] +pub trait KmsClientFactory: Debug + Send + Sync { + /// Create a [`KeyManagementClient`] from catalog properties. + /// + /// Called once during catalog initialization. Properties may include + /// KMS endpoint, region, credentials, or any backend-specific + /// configuration needed to construct the client. + async fn create_kms_client( + &self, + properties: &HashMap, + ) -> Result>; +} diff --git a/crates/iceberg/src/encryption/kms/memory.rs b/crates/iceberg/src/encryption/kms/memory.rs new file mode 100644 index 0000000000..3950b11fbf --- /dev/null +++ b/crates/iceberg/src/encryption/kms/memory.rs @@ -0,0 +1,398 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! In-memory KMS implementation for testing and development. +//! +//! **WARNING**: This implementation is NOT suitable for production use. +//! Keys are stored in memory only and will be lost when the process exits. + +use std::collections::HashMap; +use std::fmt; +use std::sync::{Arc, RwLock}; + +use async_trait::async_trait; + +use super::KeyManagementClient; +use super::factory::KmsClientFactory; +use crate::encryption::{AesGcmCipher, AesKeySize, SecureKey, SensitiveBytes}; +use crate::error::{invalid_data, lock_error}; +use crate::{Error, ErrorKind, Result}; + +/// In-memory KMS for testing. Not suitable for production use. +/// +/// ``` +/// use iceberg::encryption::KeyManagementClient; +/// use iceberg::encryption::kms::MemoryKeyManagementClient; +/// +/// # async fn example() -> iceberg::Result<()> { +/// let kms = MemoryKeyManagementClient::new(); +/// kms.add_master_key("my-master-key")?; +/// +/// let dek = vec![0u8; 16]; +/// let wrapped = kms.wrap_key(&dek, "my-master-key").await?; +/// let unwrapped = kms.unwrap_key(&wrapped, "my-master-key").await?; +/// assert_eq!(dek.as_slice(), unwrapped.as_bytes()); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Default)] +pub struct MemoryKeyManagementClient { + master_keys: Arc>>, + master_key_size: AesKeySize, +} + +impl fmt::Debug for MemoryKeyManagementClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("MemoryKeyManagementClient") + .field("master_key_size", &self.master_key_size) + .field("key_count", &self.key_count()) + .finish() + } +} + +impl MemoryKeyManagementClient { + /// Creates a new in-memory KMS with 128-bit AES keys. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new in-memory KMS with the specified master key size. + pub fn with_master_key_size(master_key_size: AesKeySize) -> Self { + Self { + master_keys: Arc::new(RwLock::new(HashMap::new())), + master_key_size, + } + } + + /// Adds a randomly generated master key with the given ID. + pub fn add_master_key(&self, key_id: impl Into) -> Result<()> { + let key = SecureKey::generate(self.master_key_size); + self.insert_key(key_id.into(), SensitiveBytes::new(key.as_bytes())) + } + + /// Adds a master key with explicit key bytes. + /// + /// Use this to seed the KMS with known key material, e.g. for + /// cross-language integration tests where both Java and Rust must + /// share the same master key bytes. + pub fn add_master_key_bytes( + &self, + key_id: impl Into, + key_bytes: SensitiveBytes, + ) -> Result<()> { + Self::check_key_length(&key_bytes)?; + self.insert_key(key_id.into(), key_bytes) + } + + /// Check the key length is valid by constructing a SecureKey. + fn check_key_length(key_bytes: &SensitiveBytes) -> Result<()> { + SecureKey::new(key_bytes.as_bytes())?; + Ok(()) + } + + fn insert_key(&self, key_id: String, key: SensitiveBytes) -> Result<()> { + let mut keys = self.master_keys.write().map_err(lock_error)?; + + if keys.contains_key(&key_id) { + return Err(invalid_data!("Master key already exists: {key_id}")); + } + + keys.insert(key_id, key); + Ok(()) + } + + fn get_master_key(&self, key_id: &str) -> Result { + let keys = self.master_keys.read().map_err(lock_error)?; + + keys.get(key_id) + .cloned() + .ok_or_else(|| invalid_data!("Master key not found: {key_id}")) + } + + /// Number of master keys stored. + pub fn key_count(&self) -> usize { + self.master_keys.read().map(|keys| keys.len()).unwrap_or(0) + } + + /// Whether a master key with the given ID exists. + pub fn has_key(&self, key_id: &str) -> bool { + self.master_keys + .read() + .map(|keys| keys.contains_key(key_id)) + .unwrap_or(false) + } +} + +/// Factory for creating [`MemoryKeyManagementClient`] instances. +/// +/// The factory owns the master-key table and its key size; seed it with +/// [`add_master_key`](Self::add_master_key) / +/// [`add_master_key_bytes`](Self::add_master_key_bytes), then every client it +/// produces via [`create_kms_client`](KmsClientFactory::create_kms_client) +/// shares that same table. Useful for testing encryption flows without a real +/// KMS backend. +#[derive(Debug, Clone, Default)] +pub struct MemoryKmsClientFactory { + master_keys: Arc>>, + master_key_size: AesKeySize, +} + +impl MemoryKmsClientFactory { + /// Creates a new factory with 128-bit AES master keys. + pub fn new() -> Self { + Self::default() + } + + /// Creates a new factory whose clients use the given master key size. + pub fn with_master_key_size(master_key_size: AesKeySize) -> Self { + Self { + master_keys: Arc::new(RwLock::new(HashMap::new())), + master_key_size, + } + } + + /// A client view over this factory's shared master-key table. + fn client(&self) -> MemoryKeyManagementClient { + MemoryKeyManagementClient { + master_keys: Arc::clone(&self.master_keys), + master_key_size: self.master_key_size, + } + } + + /// Adds a randomly generated master key with the given ID. + pub fn add_master_key(&self, key_id: impl Into) -> Result<()> { + self.client().add_master_key(key_id) + } + + /// Adds a master key with explicit key bytes. + /// + /// Use this to seed the factory with known key material, e.g. for + /// cross-language integration tests where both Java and Rust must + /// share the same master key bytes. + pub fn add_master_key_bytes( + &self, + key_id: impl Into, + key_bytes: SensitiveBytes, + ) -> Result<()> { + self.client().add_master_key_bytes(key_id, key_bytes) + } +} + +#[async_trait] +impl KmsClientFactory for MemoryKmsClientFactory { + async fn create_kms_client( + &self, + _properties: &HashMap, + ) -> Result> { + Ok(Arc::new(self.client())) + } +} + +#[async_trait] +impl KeyManagementClient for MemoryKeyManagementClient { + async fn wrap_key(&self, key: &[u8], wrapping_key_id: &str) -> Result> { + let master_key_bytes = self.get_master_key(wrapping_key_id)?; + let master_key = SecureKey::new(master_key_bytes.as_bytes())?; + let cipher = AesGcmCipher::new(master_key); + + cipher.encrypt(key, None) + } + + async fn unwrap_key( + &self, + wrapped_key: &[u8], + wrapping_key_id: &str, + ) -> Result { + let master_key_bytes = self.get_master_key(wrapping_key_id)?; + let master_key = SecureKey::new(master_key_bytes.as_bytes())?; + let cipher = AesGcmCipher::new(master_key); + + Ok(SensitiveBytes::new(cipher.decrypt(wrapped_key, None)?)) + } + + fn supports_key_generation(&self) -> bool { + false + } + + async fn generate_key(&self, _wrapping_key_id: &str) -> Result { + Err(Error::new( + ErrorKind::FeatureUnsupported, + "MemoryKeyManagementClient does not support server-side key generation", + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_wrap_unwrap_roundtrip() { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + let dek = vec![0u8; 16]; + + let wrapped = kms.wrap_key(&dek, "master-1").await.unwrap(); + let unwrapped = kms.unwrap_key(&wrapped, "master-1").await.unwrap(); + assert_eq!(unwrapped.as_bytes(), dek.as_slice()); + } + + #[tokio::test] + async fn test_wrap_unknown_key_fails() { + let kms = MemoryKeyManagementClient::new(); + let dek = vec![0u8; 16]; + + let result = kms.wrap_key(&dek, "nonexistent").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_wrong_master_key_fails_unwrap() { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + kms.add_master_key("master-2").unwrap(); + let dek = vec![0u8; 16]; + + let wrapped = kms.wrap_key(&dek, "master-1").await.unwrap(); + + let result = kms.unwrap_key(&wrapped, "master-2").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_does_not_support_key_generation() { + let kms = MemoryKeyManagementClient::new(); + assert!(!kms.supports_key_generation()); + + let result = kms.generate_key("master-1").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_multiple_master_keys() { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + kms.add_master_key("master-2").unwrap(); + let dek1 = vec![1u8; 16]; + let dek2 = vec![2u8; 16]; + + let wrapped1 = kms.wrap_key(&dek1, "master-1").await.unwrap(); + let wrapped2 = kms.wrap_key(&dek2, "master-2").await.unwrap(); + + let unwrapped1 = kms.unwrap_key(&wrapped1, "master-1").await.unwrap(); + let unwrapped2 = kms.unwrap_key(&wrapped2, "master-2").await.unwrap(); + + assert_eq!(unwrapped1.as_bytes(), dek1.as_slice()); + assert_eq!(unwrapped2.as_bytes(), dek2.as_slice()); + } + + #[tokio::test] + async fn test_add_master_key() { + let kms = MemoryKeyManagementClient::new(); + + kms.add_master_key("my-key").unwrap(); + assert!(kms.has_key("my-key")); + assert_eq!(kms.key_count(), 1); + + let result = kms.add_master_key("my-key"); + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_add_master_key_bytes() { + let kms = MemoryKeyManagementClient::new(); + let key_bytes = SensitiveBytes::new([42u8; 16]); + + kms.add_master_key_bytes("my-key", key_bytes).unwrap(); + assert!(kms.has_key("my-key")); + + let dek = vec![7u8; 16]; + let wrapped = kms.wrap_key(&dek, "my-key").await.unwrap(); + let unwrapped = kms.unwrap_key(&wrapped, "my-key").await.unwrap(); + assert_eq!(unwrapped.as_bytes(), dek.as_slice()); + } + + #[tokio::test] + async fn test_add_master_key_bytes_invalid_length() { + let kms = MemoryKeyManagementClient::new(); + + let result = kms.add_master_key_bytes("my-key", SensitiveBytes::new([0u8; 7])); + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_with_master_key_size() { + let kms = MemoryKeyManagementClient::with_master_key_size(AesKeySize::Bits256); + kms.add_master_key("master-256").unwrap(); + + let dek = vec![0u8; 16]; + let wrapped = kms.wrap_key(&dek, "master-256").await.unwrap(); + let unwrapped = kms.unwrap_key(&wrapped, "master-256").await.unwrap(); + assert_eq!(unwrapped.as_bytes(), dek.as_slice()); + } + + #[tokio::test] + async fn test_clone_shares_state() { + let kms1 = MemoryKeyManagementClient::new(); + let kms2 = kms1.clone(); + + kms1.add_master_key("shared-key").unwrap(); + assert!(kms2.has_key("shared-key")); + } + + #[tokio::test] + async fn test_factory_seeds_produced_clients() { + // Seed the factory, then every client it produces sees those keys and + // can wrap/unwrap with them. + let factory = MemoryKmsClientFactory::new(); + factory + .add_master_key_bytes("master-1", SensitiveBytes::new([9u8; 16])) + .unwrap(); + + let client = factory.create_kms_client(&HashMap::new()).await.unwrap(); + + let dek = vec![3u8; 16]; + let wrapped = client.wrap_key(&dek, "master-1").await.unwrap(); + let unwrapped = client.unwrap_key(&wrapped, "master-1").await.unwrap(); + assert_eq!(unwrapped.as_bytes(), dek.as_slice()); + } + + #[tokio::test] + async fn test_factory_seeding_after_client_creation_is_visible() { + // The factory owns the shared table, so keys added after a client is + // produced are still visible to that client. + let factory = MemoryKmsClientFactory::new(); + let client = factory.create_kms_client(&HashMap::new()).await.unwrap(); + + factory.add_master_key("late-key").unwrap(); + + let dek = vec![1u8; 16]; + assert!(client.wrap_key(&dek, "late-key").await.is_ok()); + } + + #[tokio::test] + async fn test_factory_with_master_key_size() { + let factory = MemoryKmsClientFactory::with_master_key_size(AesKeySize::Bits256); + factory.add_master_key("master-256").unwrap(); + + let client = factory.create_kms_client(&HashMap::new()).await.unwrap(); + let dek = vec![0u8; 16]; + let wrapped = client.wrap_key(&dek, "master-256").await.unwrap(); + let unwrapped = client.unwrap_key(&wrapped, "master-256").await.unwrap(); + assert_eq!(unwrapped.as_bytes(), dek.as_slice()); + } +} diff --git a/crates/iceberg/src/encryption/kms/mod.rs b/crates/iceberg/src/encryption/kms/mod.rs new file mode 100644 index 0000000000..0b4f331d13 --- /dev/null +++ b/crates/iceberg/src/encryption/kms/mod.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key Management System trait and implementations. +//! +//! This module provides the [`KeyManagementClient`] trait for pluggable KMS +//! integration and implementations for different key management systems. + +mod client; +mod factory; +mod memory; + +pub use client::{GeneratedKey, KeyManagementClient}; +pub use factory::KmsClientFactory; +pub use memory::{MemoryKeyManagementClient, MemoryKmsClientFactory}; diff --git a/crates/iceberg/src/encryption/manager.rs b/crates/iceberg/src/encryption/manager.rs new file mode 100644 index 0000000000..47ff981603 --- /dev/null +++ b/crates/iceberg/src/encryption/manager.rs @@ -0,0 +1,690 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Encryption manager for file-level encryption and two-layer envelope key management. +//! +//! [`EncryptionManager`] provides file-level `decrypt` / `encrypt` +//! operations matching Java's `org.apache.iceberg.encryption.EncryptionManager`, +//! using envelope encryption: +//! - A master key (in KMS) wraps a Key Encryption Key (KEK) +//! - The KEK wraps Data Encryption Keys (DEKs) locally + +use std::collections::HashMap; +use std::fmt; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use chrono::Utc; +use moka::future::Cache; +use uuid::Uuid; + +const MILLIS_IN_DAY: i64 = 24 * 60 * 60 * 1000; + +use super::crypto::{AesGcmCipher, AesKeySize, SecureKey}; +use super::io::EncryptedOutputFile; +use super::key_metadata::StandardKeyMetadata; +use super::kms::KeyManagementClient; +use crate::error::invalid_data; +use crate::io::OutputFile; +use crate::sensitive::SensitiveBytes; +use crate::spec::{EncryptedKey, FormatVersion, TableMetadataRef}; +use crate::{Error, ErrorKind, Result}; + +/// Property key for the KEK creation timestamp (milliseconds since epoch). +/// Matches Java's `StandardEncryptionManager.KEY_TIMESTAMP`. +pub const KEK_CREATED_AT_PROPERTY: &str = "KEY_TIMESTAMP"; + +/// Default KEK lifespan in days, per NIST SP 800-57. +const DEFAULT_KEK_LIFESPAN_DAYS: i64 = 730; + +/// Default cache TTL for unwrapped KEKs. +const DEFAULT_CACHE_TTL: Duration = Duration::from_secs(3600); + +/// File-level encryption manager using two-layer envelope encryption. +/// +/// Uses an async cache for unwrapped KEK bytes to avoid repeated KMS calls. +#[derive(typed_builder::TypedBuilder)] +#[builder(mutators( + /// Add an encryption key (KEK or wrapped key metadata entry). + pub fn add_encryption_key(&mut self, key: EncryptedKey) { + self.encryption_keys + .write() + .expect("encryption_keys lock poisoned") + .insert(key.key_id().to_string(), key); + } + /// Set all encryption keys from table metadata. + pub fn encryption_keys(&mut self, keys: HashMap) { + self.encryption_keys = RwLock::new(keys); + } +))] +pub struct EncryptionManager { + kms_client: Arc, + #[builder( + default = Cache::builder().time_to_live(DEFAULT_CACHE_TTL).build(), + setter(skip) + )] + kek_cache: Cache, + /// AES key size for DEK generation. Defaults to 128-bit. + #[builder(default = AesKeySize::default())] + key_size: AesKeySize, + /// Master key ID from table property `encryption.key-id`. + #[builder(setter(into))] + table_key_id: String, + /// All encryption keys from table metadata (KEKs and wrapped key metadata entries). + /// Newly created KEKs and wrapped manifest-list entries are inserted here so + /// callers can snapshot the full set at commit time via [`EncryptionManager::encryption_keys`]. + #[builder(default = RwLock::new(HashMap::new()), via_mutators)] + encryption_keys: RwLock>, +} + +impl fmt::Debug for EncryptionManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("EncryptionManager") + .field("key_size", &self.key_size) + .field("table_key_id", &self.table_key_id) + .finish_non_exhaustive() + } +} + +impl EncryptionManager { + /// Attempt to construct an [`EncryptionManager`] from table metadata. + /// + /// Returns `Ok(None)` if the format version is below v3 or the + /// `encryption.key-id` property is not set. Returns an error if the + /// property is set but no [`KeyManagementClient`] was provided. + pub(crate) fn from_table_metadata( + kms_client: Option<&Arc>, + metadata: &TableMetadataRef, + ) -> Result>> { + if metadata.format_version() < FormatVersion::V3 { + return Ok(None); + } + + let table_properties = metadata.table_properties(); + let encryption_key_id = table_properties.encryption_key_id()?; + let Some(table_key_id) = encryption_key_id.as_deref() else { + if kms_client.is_some() { + tracing::warn!( + "KeyManagementClient provided but table does not have encryption.key-id set" + ); + } + return Ok(None); + }; + + let kms_client = kms_client.ok_or_else(|| { + Error::new( + ErrorKind::PreconditionFailed, + "Table has encryption.key-id set but no KeyManagementClient was provided to TableBuilder", + ) + })?; + + let em = EncryptionManager::builder() + .kms_client(Arc::clone(kms_client)) + .table_key_id(table_key_id) + .encryption_keys(metadata.encryption_keys.clone()) + .key_size(table_properties.data_encryption_key_size()?) + .build(); + Ok(Some(Arc::new(em))) + } + + /// Generate key metadata for a single file: a fresh DEK of the table's + /// configured key length together with a fresh AAD prefix. + pub fn generate_key_metadata(&self) -> StandardKeyMetadata { + StandardKeyMetadata::generate(self.key_size) + } + + /// Encrypt a file with AGS1 stream encryption. + /// + /// Returns an [`EncryptedOutputFile`] that transparently encrypts on + /// write, along with key metadata for later decryption. + pub fn encrypt(&self, raw_output: OutputFile) -> EncryptedOutputFile { + EncryptedOutputFile::new(raw_output, self.generate_key_metadata()) + } + + /// Wrap a manifest list key metadata with a KEK for storage in table metadata. + /// + /// Stores the resulting wrapped entry (and any newly created KEK) in the + /// manager's internal `encryption_keys` map. Callers persist the full set + /// at commit time via the manager's `encryption_keys`. + /// + /// Returns the `key_id` of the wrapped entry, which should be recorded on + /// the snapshot as `encryption_key_id` so readers can locate it later. + pub async fn encrypt_manifest_list_key_metadata( + &self, + key_metadata: &StandardKeyMetadata, + ) -> Result { + let kek = match self.find_active_kek()? { + Some(existing) => existing, + None => self.create_kek().await?, + }; + + let kek_bytes = self.unwrap_key_encryption_key(&kek).await?; + + // Use the KEK timestamp as AAD to prevent timestamp tampering attacks. + let aad = Self::kek_timestamp_aad(&kek)?; + let serialized = key_metadata.encode()?; + let wrapped_metadata = self.wrap_dek_with_kek(&serialized, &kek_bytes, Some(aad))?; + + let wrapped_key = EncryptedKey::builder() + .key_id(Uuid::new_v4().to_string()) + .encrypted_key_metadata(wrapped_metadata) + .encrypted_by_id(kek.key_id()) + .build(); + + let wrapped_key_id = wrapped_key.key_id().to_string(); + self.insert_encryption_key(wrapped_key); + Ok(wrapped_key_id) + } + + /// Decrypt a manifest list key metadata previously wrapped via + /// [`Self::encrypt_manifest_list_key_metadata`]. + /// + /// Looks up the entry by `encryption_key_id` (typically read from the + /// snapshot) in the manager's `encryption_keys` map. + pub async fn decrypt_manifest_list_key_metadata( + &self, + encryption_key_id: &str, + ) -> Result { + let encrypted_key = self + .encryption_keys + .read() + .expect("encryption_keys lock poisoned") + .get(encryption_key_id) + .cloned() + .ok_or_else(|| invalid_data!("Encryption key '{encryption_key_id}' not found"))?; + + let kek_key_id = encrypted_key.encrypted_by_id().ok_or_else(|| { + invalid_data!( + "EncryptedKey '{}' has no encrypted_by_id", + encrypted_key.key_id() + ) + })?; + + let bytes = self + .decrypt_dek(kek_key_id, encrypted_key.encrypted_key_metadata()) + .await?; + + StandardKeyMetadata::decode(bytes.as_bytes()) + } + + /// Borrow the encryption keys held by this manager. + /// + /// Use at commit time to persist newly created KEKs and wrapped + /// manifest-list entries into `TableMetadata.encryption_keys`. + pub fn with_encryption_keys(&self, f: F) -> R + where F: FnOnce(&HashMap) -> R { + let keys = self + .encryption_keys + .read() + .expect("encryption_keys lock poisoned"); + f(&keys) + } + + fn insert_encryption_key(&self, key: EncryptedKey) { + self.encryption_keys + .write() + .expect("encryption_keys lock poisoned") + .insert(key.key_id().to_string(), key); + } + + /// Create a new KEK, wrapped by the table's master key, and store it in + /// the manager's `encryption_keys` map. + async fn create_kek(&self) -> Result { + let (plaintext_kek, wrapped_kek) = if self.kms_client.supports_key_generation() { + let result = self.kms_client.generate_key(&self.table_key_id).await?; + (result.key().clone(), result.wrapped_key().to_vec()) + } else { + let plaintext_key = SecureKey::generate(self.key_size); + let wrapped = self + .kms_client + .wrap_key(plaintext_key.as_bytes(), &self.table_key_id) + .await?; + + (SensitiveBytes::new(plaintext_key.as_bytes()), wrapped) + }; + + let key_id = Uuid::new_v4().to_string(); + let now_ms = Utc::now().timestamp_millis(); + + let mut properties = HashMap::new(); + properties.insert(KEK_CREATED_AT_PROPERTY.to_string(), now_ms.to_string()); + + self.kek_cache.insert(key_id.clone(), plaintext_kek).await; + + let kek = EncryptedKey::builder() + .key_id(key_id) + .encrypted_key_metadata(wrapped_kek) + .encrypted_by_id(&self.table_key_id) + .properties(properties) + .build(); + + self.insert_encryption_key(kek.clone()); + Ok(kek) + } + + /// Check whether a KEK has exceeded its configured lifespan (730 days per NIST SP 800-57). + fn is_kek_expired(&self, kek: &EncryptedKey) -> bool { + let created_at_ms = match kek + .properties() + .get(KEK_CREATED_AT_PROPERTY) + .and_then(|ts| ts.parse::().ok()) + { + Some(ts) => ts, + None => return true, // No timestamp -> treat as expired + }; + + let now_ms = Utc::now().timestamp_millis(); + let lifespan_ms = DEFAULT_KEK_LIFESPAN_DAYS * MILLIS_IN_DAY; + (now_ms - created_at_ms) >= lifespan_ms + } + + /// Find the latest non-expired KEK for the table's master key. + fn find_active_kek(&self) -> Result> { + let keys = self + .encryption_keys + .read() + .expect("encryption_keys lock poisoned"); + Ok(keys + .values() + .filter(|kek| { + kek.encrypted_by_id() + .map(|id| id == self.table_key_id) + .unwrap_or(false) + && !self.is_kek_expired(kek) + }) + .max_by_key(|kek| { + kek.properties() + .get(KEK_CREATED_AT_PROPERTY) + .and_then(|ts| ts.parse::().ok()) + .unwrap_or(0) + }) + .cloned()) + } + + /// Unwrap a KEK using the KMS, with caching to avoid repeated calls. + async fn unwrap_key_encryption_key(&self, kek: &EncryptedKey) -> Result { + let cache_key = kek.key_id().to_string(); + + if let Some(cached) = self.kek_cache.get(&cache_key).await { + return Ok(cached); + } + + let master_key_id = kek + .encrypted_by_id() + .ok_or_else(|| invalid_data!("KEK '{}' has no encrypted_by_id", kek.key_id()))?; + + let plaintext = self + .kms_client + .unwrap_key(kek.encrypted_key_metadata(), master_key_id) + .await?; + + self.kek_cache.insert(cache_key, plaintext.clone()).await; + + Ok(plaintext) + } + + /// Decrypt a wrapped DEK using the KEK identified by `kek_key_id`, + /// looked up in the manager's own `encryption_keys` map. + async fn decrypt_dek(&self, kek_key_id: &str, wrapped_dek: &[u8]) -> Result { + let kek = self + .encryption_keys + .read() + .expect("encryption_keys lock poisoned") + .get(kek_key_id) + .cloned() + .ok_or_else(|| invalid_data!("KEK not found in encryption keys: {kek_key_id}"))?; + + // KEK timestamp as AAD prevents timestamp tampering. + let aad = Self::kek_timestamp_aad(&kek)?; + + let kek_bytes = self.unwrap_key_encryption_key(&kek).await?; + self.unwrap_dek_with_kek(wrapped_dek, &kek_bytes, Some(aad)) + .map_err(|e| { + Error::new( + e.kind(), + format!("Failed to unwrap key metadata with KEK '{kek_key_id}'"), + ) + .with_source(e) + }) + } + + /// Extract the KEK timestamp for use as AAD. Returns an error if missing. + fn kek_timestamp_aad(kek: &EncryptedKey) -> Result<&[u8]> { + kek.properties() + .get(KEK_CREATED_AT_PROPERTY) + .map(|ts| ts.as_bytes()) + .ok_or_else(|| { + invalid_data!( + "KEK '{}' is missing required '{}' property", + kek.key_id(), + KEK_CREATED_AT_PROPERTY + ) + }) + } + + /// Wrap a DEK with a KEK using local AES-GCM. + fn wrap_dek_with_kek( + &self, + dek: &[u8], + kek: &SensitiveBytes, + aad: Option<&[u8]>, + ) -> Result> { + let key = SecureKey::try_from(kek.clone())?; + let cipher = AesGcmCipher::new(key); + cipher.encrypt(dek, aad) + } + + /// Unwrap a DEK with a KEK using local AES-GCM. + fn unwrap_dek_with_kek( + &self, + wrapped_dek: &[u8], + kek: &SensitiveBytes, + aad: Option<&[u8]>, + ) -> Result { + let key = SecureKey::try_from(kek.clone())?; + let cipher = AesGcmCipher::new(key); + cipher.decrypt(wrapped_dek, aad).map(SensitiveBytes::new) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::encryption::EncryptedInputFile; + use crate::encryption::kms::MemoryKeyManagementClient; + + fn create_test_kms() -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + Arc::new(kms) + } + + fn create_test_manager() -> EncryptionManager { + EncryptionManager::builder() + .kms_client(create_test_kms()) + .table_key_id("master-1") + .build() + } + + #[tokio::test] + async fn test_create_kek() { + let mgr = create_test_manager(); + let kek = mgr.create_kek().await.unwrap(); + + assert!(!kek.key_id().is_empty()); + assert!(!kek.encrypted_key_metadata().is_empty()); + assert_eq!(kek.encrypted_by_id(), Some("master-1")); + assert!(kek.properties().contains_key(KEK_CREATED_AT_PROPERTY)); + } + + fn sample_key_metadata() -> StandardKeyMetadata { + StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!") + } + + #[tokio::test] + async fn test_wrap_unwrap_key_metadata_roundtrip() { + let mgr = create_test_manager(); + let plaintext = sample_key_metadata(); + + let key_id = mgr + .encrypt_manifest_list_key_metadata(&plaintext) + .await + .unwrap(); + + // First wrap should create a new KEK and the wrapped entry — both stored on the manager + assert_eq!(mgr.with_encryption_keys(|k| k.len()), 2); + + let decrypted = mgr + .decrypt_manifest_list_key_metadata(&key_id) + .await + .unwrap(); + assert_eq!(decrypted, plaintext); + } + + #[tokio::test] + async fn test_kek_reuse_when_not_expired() { + let mgr = create_test_manager(); + + // First wrap creates a new KEK + wrapped entry (2 keys) + let _id1 = mgr + .encrypt_manifest_list_key_metadata(&sample_key_metadata()) + .await + .unwrap(); + let kek_id = mgr.with_encryption_keys(|keys| { + assert_eq!(keys.len(), 2); + keys.values() + .find(|k| k.encrypted_by_id() == Some("master-1")) + .unwrap() + .key_id() + .to_string() + }); + + // Second wrap should reuse the existing KEK (only adds 1 new wrapped entry) + let id2 = mgr + .encrypt_manifest_list_key_metadata(&sample_key_metadata()) + .await + .unwrap(); + let entry2 = mgr.with_encryption_keys(|keys| { + assert_eq!(keys.len(), 3); + keys.get(&id2).cloned().unwrap() + }); + assert_eq!(entry2.encrypted_by_id(), Some(kek_id.as_str())); + } + + #[tokio::test] + async fn test_kek_rotation_when_expired() { + let kms = create_test_kms(); + + // Create a KEK with a timestamp 3 years in the past (exceeds 730-day lifespan) + let three_years_ago_ms = Utc::now().timestamp_millis() - (3 * 365 * MILLIS_IN_DAY); + let mut properties = HashMap::new(); + properties.insert( + KEK_CREATED_AT_PROPERTY.to_string(), + three_years_ago_ms.to_string(), + ); + + // Wrap a real KEK so unwrap works if needed + let kek_key = SecureKey::generate(AesKeySize::Bits128); + let wrapped = kms.wrap_key(kek_key.as_bytes(), "master-1").await.unwrap(); + + let old_kek = EncryptedKey::builder() + .key_id("expired-kek") + .encrypted_key_metadata(wrapped) + .encrypted_by_id("master-1") + .properties(properties) + .build(); + + // Build manager with the expired KEK + let mgr = EncryptionManager::builder() + .kms_client(kms) + .table_key_id("master-1") + .add_encryption_key(old_kek.clone()) + .build(); + + // Wrap should rotate to a new KEK since the existing one is expired + let new_entry_id = mgr + .encrypt_manifest_list_key_metadata(&sample_key_metadata()) + .await + .unwrap(); + let entry = mgr + .with_encryption_keys(|keys| keys.get(&new_entry_id).cloned()) + .unwrap(); + let used_kek_id = entry.encrypted_by_id().unwrap(); + assert_ne!(used_kek_id, old_kek.key_id()); + } + + #[tokio::test] + async fn test_is_kek_expired_no_timestamp() { + let mgr = create_test_manager(); + + // KEK without a created-at timestamp -> treated as expired + let kek = EncryptedKey::builder() + .key_id("no-ts") + .encrypted_key_metadata(vec![0u8; 32]) + .build(); + + assert!(mgr.is_kek_expired(&kek)); + } + + #[tokio::test] + async fn test_decrypt_with_unknown_key_id() { + let mgr = create_test_manager(); + let result = mgr.decrypt_manifest_list_key_metadata("nonexistent").await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_kek_cache_hit() { + let mgr = create_test_manager(); + + // First wrap caches the plaintext KEK during create_kek(). + let key_id = mgr + .encrypt_manifest_list_key_metadata(&sample_key_metadata()) + .await + .unwrap(); + + // Decrypt unwraps the KEK; with the cache populated this should not hit KMS again. + let _ = mgr + .decrypt_manifest_list_key_metadata(&key_id) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_unwrap_fails_when_kek_missing_timestamp() { + let mgr = create_test_manager(); + + // Wrap some metadata to get a valid encrypted entry stored on the manager + let entry_id = mgr + .encrypt_manifest_list_key_metadata(&sample_key_metadata()) + .await + .unwrap(); + + // Find the KEK that wrapped the entry and replace it with a copy that + // is missing the KEY_TIMESTAMP property, simulating a malformed table. + let mut keys = mgr.with_encryption_keys(|k| k.clone()); + let kek_id = keys + .get(&entry_id) + .unwrap() + .encrypted_by_id() + .unwrap() + .to_string(); + let kek = keys.remove(&kek_id).unwrap(); + let kek_no_ts = EncryptedKey::builder() + .key_id(kek.key_id()) + .encrypted_key_metadata(kek.encrypted_key_metadata()) + .encrypted_by_id(kek.encrypted_by_id().unwrap()) + .build(); + keys.insert(kek_no_ts.key_id().to_string(), kek_no_ts); + + let mgr = EncryptionManager::builder() + .kms_client(create_test_kms()) + .table_key_id("master-1") + .encryption_keys(keys) + .build(); + + let result = mgr.decrypt_manifest_list_key_metadata(&entry_id).await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains(KEK_CREATED_AT_PROPERTY), + "error should mention the missing property: {err}" + ); + } + + #[tokio::test] + async fn test_unwrap_fails_when_kek_timestamp_tampered() { + let mgr = create_test_manager(); + + // Wrap metadata normally + let entry_id = mgr + .encrypt_manifest_list_key_metadata(&sample_key_metadata()) + .await + .unwrap(); + + // Tamper with the KEK timestamp (change the AAD) + let mut keys = mgr.with_encryption_keys(|k| k.clone()); + let kek_id = keys + .get(&entry_id) + .unwrap() + .encrypted_by_id() + .unwrap() + .to_string(); + let kek = keys.remove(&kek_id).unwrap(); + let mut tampered_properties = kek.properties().clone(); + tampered_properties.insert(KEK_CREATED_AT_PROPERTY.to_string(), "9999999".to_string()); + let tampered_kek = EncryptedKey::builder() + .key_id(kek.key_id()) + .encrypted_key_metadata(kek.encrypted_key_metadata()) + .encrypted_by_id(kek.encrypted_by_id().unwrap()) + .properties(tampered_properties) + .build(); + keys.insert(tampered_kek.key_id().to_string(), tampered_kek); + + let mgr = EncryptionManager::builder() + .kms_client(create_test_kms()) + .table_key_id("master-1") + .encryption_keys(keys) + .build(); + + // Unwrap should fail because the AAD (timestamp) doesn't match what was used to wrap + let result = mgr.decrypt_manifest_list_key_metadata(&entry_id).await; + assert!( + result.is_err(), + "tampered timestamp should cause decryption failure" + ); + } + + #[tokio::test] + async fn test_encrypt_decrypt_roundtrip() { + use crate::io::FileIO; + + let io = FileIO::new_with_memory(); + let path = "memory:///test/encrypt_roundtrip.bin"; + + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + let mgr = EncryptionManager::builder() + .kms_client(Arc::new(kms) as Arc) + .table_key_id("master-1") + .build(); + + let output = io.new_output(path).unwrap(); + let encrypted_output = mgr.encrypt(output); + + let plaintext = b"Hello, encrypted Iceberg round-trip!"; + let file_metadata = encrypted_output + .write(bytes::Bytes::from(plaintext.to_vec())) + .await + .unwrap(); + + let serialized_metadata = encrypted_output + .key_metadata_with_saved_file_metadata(&file_metadata) + .encode() + .unwrap(); + + let input = io.new_input(path).unwrap(); + let parsed_metadata = StandardKeyMetadata::decode(&serialized_metadata).unwrap(); + let decrypted_file = EncryptedInputFile::new(input, parsed_metadata); + + let content = decrypted_file.read().await.unwrap(); + assert_eq!(&content[..], plaintext); + } +} diff --git a/crates/iceberg/src/encryption/mod.rs b/crates/iceberg/src/encryption/mod.rs new file mode 100644 index 0000000000..73e7527ce4 --- /dev/null +++ b/crates/iceberg/src/encryption/mod.rs @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Encryption module for Apache Iceberg. +//! +//! This module provides core cryptographic primitives and key management +//! for encrypting and decrypting data in Iceberg tables. + +mod crypto; +pub(crate) mod io; +pub(crate) mod key_metadata; +pub mod kms; +mod manager; +mod stream; + +pub use crypto::{AesGcmCipher, AesKeySize, SecureKey}; +pub use io::{EncryptedInputFile, EncryptedOutputFile}; +pub use key_metadata::StandardKeyMetadata; +pub use kms::{GeneratedKey, KeyManagementClient}; +pub use manager::EncryptionManager; +pub use stream::{AesGcmFileRead, AesGcmFileWrite}; + +pub use crate::sensitive::SensitiveBytes; diff --git a/crates/iceberg/src/encryption/stream.rs b/crates/iceberg/src/encryption/stream.rs new file mode 100644 index 0000000000..14c0d38446 --- /dev/null +++ b/crates/iceberg/src/encryption/stream.rs @@ -0,0 +1,1313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AGS1 stream encryption/decryption for Iceberg. +//! +//! Implements the block-based AES-GCM stream format used by Iceberg for +//! encrypting manifest lists and manifest files. The format is +//! byte-compatible with Java's `AesGcmInputStream` / `AesGcmOutputStream`. +//! +//! # AGS1 File Format +//! +//! ```text +//! ┌─────────────────────────────────────────────┐ +//! │ Header (8 bytes) │ +//! │ Magic: "AGS1" (4 bytes, ASCII) │ +//! │ Plain block size: u32 LE (4 bytes) │ +//! │ Default: 1,048,576 (1 MiB) │ +//! ├─────────────────────────────────────────────┤ +//! │ Block 0 │ +//! │ Nonce (12 bytes) │ +//! │ Ciphertext (up to plain_block_size bytes) │ +//! │ GCM Tag (16 bytes) │ +//! ├─────────────────────────────────────────────┤ +//! │ Block 1..N (same structure) │ +//! ├─────────────────────────────────────────────┤ +//! │ Final block (may be shorter) │ +//! └─────────────────────────────────────────────┘ +//! ``` +//! +//! Each block's AAD is: `aad_prefix || block_index (4 bytes, LE)`. + +use std::ops::Range; +use std::sync::Arc; + +use bytes::{Bytes, BytesMut}; + +use super::AesGcmCipher; +use crate::error::invalid_data; +use crate::io::{FileMetadata, FileRead, FileWrite}; +use crate::{Error, ErrorKind, Result}; + +/// Default plaintext block size (1 MiB), matching Java's `Ciphers.PLAIN_BLOCK_SIZE`. +pub const PLAIN_BLOCK_SIZE: u32 = 1024 * 1024; + +/// AES-GCM nonce length in bytes. +pub const NONCE_LENGTH: u32 = 12; + +/// AES-GCM authentication tag length in bytes. +pub const GCM_TAG_LENGTH: u32 = 16; + +/// Cipher block size = plaintext block size + nonce + GCM tag. +pub const CIPHER_BLOCK_SIZE: u32 = PLAIN_BLOCK_SIZE + NONCE_LENGTH + GCM_TAG_LENGTH; + +/// AGS1 stream magic bytes. +pub const GCM_STREAM_MAGIC: [u8; 4] = *b"AGS1"; + +/// AGS1 stream header length (4-byte magic + 4-byte block size). +pub const GCM_STREAM_HEADER_LENGTH: u32 = 8; + +/// Minimum valid AGS1 stream length (header + one empty block). +pub(crate) const MIN_STREAM_LENGTH: u32 = GCM_STREAM_HEADER_LENGTH + NONCE_LENGTH + GCM_TAG_LENGTH; + +/// Constructs the per-block AAD for AGS1 stream encryption. +/// +/// Format: `aad_prefix || block_index (4 bytes, little-endian)` +/// +/// This matches Java's `Ciphers.streamBlockAAD()`. +pub(crate) fn stream_block_aad(aad_prefix: &[u8], block_index: u32) -> Vec { + let index_bytes = block_index.to_le_bytes(); + if aad_prefix.is_empty() { + index_bytes.to_vec() + } else { + let mut aad = Vec::with_capacity(aad_prefix.len() + 4); + aad.extend_from_slice(aad_prefix); + aad.extend_from_slice(&index_bytes); + aad + } +} + +/// Transparent decryption of AGS1 stream-encrypted files. +/// +/// Implements the [`FileRead`] trait, providing random-access reads over +/// encrypted data. Each `read()` call determines which encrypted blocks +/// overlap the requested plaintext range, reads and decrypts them, then +/// returns the requested plaintext bytes. +/// +/// # Usage +/// +/// ```ignore +/// // (ignored: requires async runtime and concrete FileRead/FileWrite impls) +/// let reader = AesGcmFileRead::new( +/// inner_reader, // Box for the encrypted file +/// cipher, // Arc with the DEK +/// aad_prefix.to_vec(), +/// encrypted_file_length, +/// )?; +/// +/// // Read plaintext bytes transparently +/// let plaintext = reader.read(0..1024).await?; +/// ``` +pub struct AesGcmFileRead { + /// The underlying encrypted file reader. + inner: Box, + /// The AES-GCM cipher holding the DEK. + cipher: Arc, + /// AAD prefix from the key metadata. + aad_prefix: Box<[u8]>, + /// Total plaintext stream size in bytes. + plain_stream_size: u64, + /// Total number of encrypted blocks. + num_blocks: u64, + /// Size of the last cipher block (may be smaller than `CIPHER_BLOCK_SIZE`). + last_cipher_block_size: u32, +} + +impl AesGcmFileRead { + /// Creates a new `AesGcmFileRead` for decrypting an AGS1 stream. + /// + /// Computes the plaintext size and block layout from the encrypted file + /// length. No I/O is performed; header validation happens implicitly + /// when blocks are decrypted (GCM authentication will fail on corrupt data). + /// + /// # Arguments + /// + /// * `inner` - Reader for the underlying encrypted file + /// * `cipher` - AES-GCM cipher initialized with the file's DEK + /// * `aad_prefix` - AAD prefix from the file's `StandardKeyMetadata` + /// * `encrypted_file_length` - Total byte length of the encrypted file + pub fn new( + inner: Box, + cipher: Arc, + aad_prefix: Box<[u8]>, + encrypted_file_length: u64, + ) -> Result { + if encrypted_file_length < u64::from(MIN_STREAM_LENGTH) { + return Err(invalid_data!( + "Invalid encrypted file length: {encrypted_file_length} is less than {MIN_STREAM_LENGTH}" + )); + } + let plain_stream_size = Self::calculate_plaintext_length(encrypted_file_length)?; + let stream_length = encrypted_file_length - GCM_STREAM_HEADER_LENGTH as u64; + + let num_full_blocks = stream_length / CIPHER_BLOCK_SIZE as u64; + let cipher_bytes_in_last_block = (stream_length % CIPHER_BLOCK_SIZE as u64) as u32; + let full_blocks_only = cipher_bytes_in_last_block == 0; + + let num_blocks = if full_blocks_only { + num_full_blocks + } else { + num_full_blocks + 1 + }; + + if num_blocks > u32::MAX as u64 { + return Err(invalid_data!( + "AGS1 format supports at most {} blocks (~4 TiB per file), but file requires {num_blocks} blocks", + u32::MAX + )); + } + + let last_cipher_block_size = if full_blocks_only { + CIPHER_BLOCK_SIZE + } else { + cipher_bytes_in_last_block + }; + + Ok(Self { + inner, + cipher, + aad_prefix, + plain_stream_size, + num_blocks, + last_cipher_block_size, + }) + } + + /// Returns the plaintext stream size in bytes. + pub fn plaintext_length(&self) -> u64 { + self.plain_stream_size + } + + /// Calculates the plaintext length from an encrypted file's total length. + /// + /// This is a static calculation matching Java's + /// `AesGcmInputStream.calculatePlaintextLength()`. + pub fn calculate_plaintext_length(encrypted_file_length: u64) -> Result { + if encrypted_file_length < GCM_STREAM_HEADER_LENGTH as u64 { + return Err(invalid_data!( + "Encrypted file too short: {encrypted_file_length} bytes (minimum {GCM_STREAM_HEADER_LENGTH})" + )); + } + + let stream_length = encrypted_file_length - GCM_STREAM_HEADER_LENGTH as u64; + + if stream_length == 0 { + return Ok(0); + } + + let num_full_blocks = stream_length / CIPHER_BLOCK_SIZE as u64; + let cipher_bytes_in_last_block = stream_length % CIPHER_BLOCK_SIZE as u64; + let full_blocks_only = cipher_bytes_in_last_block == 0; + + let plain_bytes_in_last_block = if full_blocks_only { + 0 + } else { + if cipher_bytes_in_last_block < (NONCE_LENGTH + GCM_TAG_LENGTH) as u64 { + return Err(invalid_data!( + "Truncated encrypted file: last block is {} bytes (minimum {})", + cipher_bytes_in_last_block, + NONCE_LENGTH + GCM_TAG_LENGTH + )); + } + cipher_bytes_in_last_block - NONCE_LENGTH as u64 - GCM_TAG_LENGTH as u64 + }; + + Ok(num_full_blocks * PLAIN_BLOCK_SIZE as u64 + plain_bytes_in_last_block) + } + + /// Returns the encrypted byte offset for a given block index. + fn encrypted_block_offset(block_index: u64) -> u64 { + block_index * CIPHER_BLOCK_SIZE as u64 + GCM_STREAM_HEADER_LENGTH as u64 + } + + /// Returns the cipher block size for a given block index. + fn cipher_block_size(&self, block_index: u64) -> u32 { + if block_index == self.num_blocks - 1 { + self.last_cipher_block_size + } else { + CIPHER_BLOCK_SIZE + } + } +} + +#[async_trait::async_trait] +impl FileRead for AesGcmFileRead { + /// Reads and decrypts a plaintext byte range from the encrypted AGS1 stream. + /// + /// The caller specifies a range in **plaintext** coordinates (e.g. "bytes 0..1024 + /// of the original file"). This method translates that into the encrypted file + /// layout and performs the following steps: + /// + /// 1. **Map to blocks** — divides the plaintext range by `PLAIN_BLOCK_SIZE` to + /// find which encrypted blocks (`first_block..=last_block`) contain the + /// requested data. + /// + /// 2. **Single I/O read** — calculates the contiguous byte range in the + /// encrypted file that covers all needed blocks (including the 8-byte AGS1 + /// header offset, 12-byte nonces, and 16-byte GCM tags) and fetches them in + /// one call to the inner `FileRead`. + /// + /// 3. **Decrypt per block** — iterates over each cipher block in the response, + /// decrypts it with AES-GCM using the per-block AAD (`aad_prefix || block_index`), + /// and slices out only the plaintext bytes that overlap the requested range. + /// + /// 4. **Assemble result** — concatenates the slices into a single `Bytes` buffer + /// matching exactly `range.end - range.start` bytes. + /// + /// Because each block is independently encrypted with its own nonce and AAD, + /// arbitrary random-access reads are supported without decrypting the entire + /// file. GCM authentication is verified per-block, so any tampering is detected + /// at the granularity of individual blocks. + async fn read(&self, range: Range) -> Result { + // An empty stream still has one block whose GCM tag must be verified, so an empty read + // cannot short-circuit there: that is what detects a file truncated to just its header. + if range.start == range.end && self.plain_stream_size != 0 { + return Ok(Bytes::new()); + } + + if range.start > range.end { + return Err(invalid_data!( + "Invalid read range: start ({}) is greater than end ({})", + range.start, + range.end + )); + } + + if range.end > self.plain_stream_size { + return Err(invalid_data!( + "Read range {}..{} exceeds plaintext size {}", + range.start, + range.end, + self.plain_stream_size + )); + } + + let first_block = range.start / PLAIN_BLOCK_SIZE as u64; + let last_block = range.end.saturating_sub(1) / PLAIN_BLOCK_SIZE as u64; + + // Read all needed encrypted blocks in a single I/O call + let encrypted_start = Self::encrypted_block_offset(first_block); + let encrypted_end = + Self::encrypted_block_offset(last_block) + self.cipher_block_size(last_block) as u64; + + let all_encrypted = self.inner.read(encrypted_start..encrypted_end).await?; + if all_encrypted.len() as u64 != encrypted_end - encrypted_start { + return Err(invalid_data!( + "Invalid encrypted read length: expected {} bytes, got {}", + encrypted_end - encrypted_start, + all_encrypted.len() + )); + } + + // Decrypt each block and extract the requested plaintext range + let result_len = (range.end - range.start) as usize; + let mut result = BytesMut::with_capacity(result_len); + let mut encrypted_offset = 0usize; + + for block_idx in first_block..=last_block { + let block_size = self.cipher_block_size(block_idx) as usize; + let cipher_block = &all_encrypted[encrypted_offset..encrypted_offset + block_size]; + encrypted_offset += block_size; + + let aad = stream_block_aad(&self.aad_prefix, block_idx as u32); + let decrypted = self.cipher.decrypt(cipher_block, Some(&aad))?; + + // Calculate which slice of this decrypted block we need + let block_plain_start = block_idx * PLAIN_BLOCK_SIZE as u64; + let slice_start = if block_idx == first_block { + (range.start - block_plain_start) as usize + } else { + 0 + }; + let slice_end = if block_idx == last_block { + (range.end - block_plain_start) as usize + } else { + decrypted.len() + }; + + result.extend_from_slice(&decrypted[slice_start..slice_end]); + } + + Ok(result.freeze()) + } +} + +/// Transparent encryption of AGS1 stream-encrypted files. +/// +/// Implements the [`FileWrite`] trait, buffering plaintext and emitting +/// encrypted AGS1 blocks. This is the streaming write counterpart to +/// [`AesGcmFileRead`]. +/// +/// # Usage +/// +/// ```ignore +/// // (ignored: requires async runtime and concrete FileRead/FileWrite impls) +/// let writer = AesGcmFileWrite::new( +/// inner_writer, // Box for the output file +/// cipher, // Arc with the DEK +/// aad_prefix.to_vec(), +/// ); +/// +/// writer.write(plaintext_chunk).await?; +/// writer.close().await?; +/// ``` +pub struct AesGcmFileWrite { + /// The underlying output writer. + inner: Box, + /// The AES-GCM cipher holding the DEK. + cipher: Arc, + /// AAD prefix from the key metadata. + aad_prefix: Box<[u8]>, + /// Plaintext buffer accumulating data before block encryption. + buffer: Vec, + /// Current block index for AAD construction. + block_index: u32, + /// Whether the AGS1 header has been written. + header_written: bool, + /// Whether close() has been called. + closed: bool, + /// Whether the writer is in a poisoned state due to a failed inner write. + /// Once poisoned, all subsequent operations are rejected because the inner + /// writer may have received partial data. + poisoned: bool, +} + +impl AesGcmFileWrite { + /// Creates a new `AesGcmFileWrite` for encrypting to AGS1 format. + /// + /// No I/O is performed until `write()` or `close()` is called. + pub fn new( + inner: Box, + cipher: Arc, + aad_prefix: impl Into>, + ) -> Self { + Self { + inner, + cipher, + aad_prefix: aad_prefix.into(), + buffer: Vec::new(), + block_index: 0, + header_written: false, + closed: false, + poisoned: false, + } + } + + /// Writes the AGS1 header (magic + plain block size) to the inner writer. + async fn write_header(&mut self) -> Result<()> { + let mut header = Vec::with_capacity(GCM_STREAM_HEADER_LENGTH as usize); + header.extend_from_slice(&GCM_STREAM_MAGIC); + header.extend_from_slice(&PLAIN_BLOCK_SIZE.to_le_bytes()); + if let Err(e) = self.inner.write(Bytes::from(header)).await { + self.poisoned = true; + return Err(e); + } + self.header_written = true; + Ok(()) + } + + /// Encrypts a plaintext block and writes it to the inner writer. + async fn encrypt_and_write_block(&mut self, block_data: &[u8]) -> Result<()> { + let aad = stream_block_aad(&self.aad_prefix, self.block_index); + let encrypted = self.cipher.encrypt(block_data, Some(&aad))?; + if let Err(e) = self.inner.write(Bytes::from(encrypted)).await { + self.poisoned = true; + return Err(e); + } + self.block_index = self.block_index.checked_add(1).ok_or_else(|| { + invalid_data!( + "AGS1 block index overflow: file exceeds the maximum supported size (~4 TiB)" + ) + })?; + Ok(()) + } + + /// Encrypts the first `PLAIN_BLOCK_SIZE` bytes of the buffer in-place + /// and drains them, avoiding a 1 MiB temporary copy. + async fn encrypt_and_drain_block(&mut self) -> Result<()> { + let aad = stream_block_aad(&self.aad_prefix, self.block_index); + let encrypted = self + .cipher + .encrypt(&self.buffer[..PLAIN_BLOCK_SIZE as usize], Some(&aad))?; + if let Err(e) = self.inner.write(Bytes::from(encrypted)).await { + self.poisoned = true; + return Err(e); + } + self.block_index = self.block_index.checked_add(1).ok_or_else(|| { + invalid_data!( + "AGS1 block index overflow: file exceeds the maximum supported size (~4 TiB)" + ) + })?; + self.buffer.drain(..PLAIN_BLOCK_SIZE as usize); + Ok(()) + } +} + +#[async_trait::async_trait] +impl FileWrite for AesGcmFileWrite { + async fn write(&mut self, bs: Bytes) -> Result<()> { + if self.closed { + return Err(Error::new( + ErrorKind::Unexpected, + "Cannot write to a closed AesGcmFileWrite", + )); + } + if self.poisoned { + return Err(Error::new( + ErrorKind::Unexpected, + "AesGcmFileWrite is in a poisoned state due to a previous write failure", + )); + } + + if !self.header_written { + self.write_header().await?; + } + + self.buffer.extend_from_slice(&bs); + + // Flush full blocks + while self.buffer.len() >= PLAIN_BLOCK_SIZE as usize { + self.encrypt_and_drain_block().await?; + } + + Ok(()) + } + + async fn close(&mut self) -> Result { + if self.closed { + return Err(Error::new( + ErrorKind::Unexpected, + "AesGcmFileWrite already closed", + )); + } + if self.poisoned { + return Err(Error::new( + ErrorKind::Unexpected, + "AesGcmFileWrite is in a poisoned state due to a previous write failure", + )); + } + + if !self.header_written { + self.write_header().await?; + } + + // Write the final block if there's remaining data, or if this is an empty file + // (block_index == 0). Skip writing a spurious empty block when the plaintext was + // exactly block-aligned (buffer empty, blocks already written). + if !self.buffer.is_empty() || self.block_index == 0 { + let final_block = std::mem::take(&mut self.buffer); + self.encrypt_and_write_block(&final_block).await?; + } + self.closed = true; + + self.inner.close().await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Encrypts plaintext into AGS1 format for testing. + /// + /// Mirrors Java's `AesGcmOutputStream` behavior: + /// - Always writes header + at least one block (even for empty input) + /// - Full blocks are `PLAIN_BLOCK_SIZE` bytes; last block may be shorter + fn encrypt_ags1(plaintext: &[u8], cipher: &AesGcmCipher, aad_prefix: &[u8]) -> Vec { + let mut result = Vec::new(); + + // Write header: "AGS1" + PLAIN_BLOCK_SIZE (LE) + result.extend_from_slice(&GCM_STREAM_MAGIC); + result.extend_from_slice(&PLAIN_BLOCK_SIZE.to_le_bytes()); + + // Write blocks + let mut offset = 0; + let mut block_index = 0u32; + + loop { + let remaining = plaintext.len() - offset; + let block_size = std::cmp::min(remaining, PLAIN_BLOCK_SIZE as usize); + + // Block 0 is always written (even if empty); subsequent empty blocks are skipped + if block_size == 0 && block_index > 0 { + break; + } + + let block_data = &plaintext[offset..offset + block_size]; + let aad = stream_block_aad(aad_prefix, block_index); + let encrypted = cipher.encrypt(block_data, Some(&aad)).unwrap(); + result.extend_from_slice(&encrypted); + + offset += block_size; + block_index += 1; + + // A partial block is always the last + if block_size < PLAIN_BLOCK_SIZE as usize { + break; + } + } + + result + } + + /// Helper to create an AesGcmCipher from raw key bytes. + fn make_cipher(key: &[u8]) -> AesGcmCipher { + use super::super::SecureKey; + let secure_key = SecureKey::new(key).unwrap(); + AesGcmCipher::new(secure_key) + } + + /// Helper to create an in-memory FileRead from bytes. + fn memory_reader(data: Vec) -> Box { + Box::new(MemoryFileRead(Bytes::from(data))) + } + + /// Simple in-memory FileRead for tests. + struct MemoryFileRead(Bytes); + + #[async_trait::async_trait] + impl FileRead for MemoryFileRead { + async fn read(&self, range: Range) -> Result { + let start = range.start as usize; + let end = range.end as usize; + if end > self.0.len() { + return Err(invalid_data!( + "Range {}..{} out of bounds for {} bytes", + start, + end, + self.0.len() + )); + } + Ok(self.0.slice(start..end)) + } + } + + #[tokio::test] + async fn test_empty_file_roundtrip() { + let key = b"0123456789abcdef"; + let aad_prefix = b"test-aad-prefix!"; + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(b"", &cipher, aad_prefix); + + // Verify minimum length: header(8) + nonce(12) + tag(16) = 36 + assert_eq!(encrypted.len(), MIN_STREAM_LENGTH as usize); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), 0); + + // Reading empty range should return empty bytes + let result = reader.read(0..0).await.unwrap(); + assert!(result.is_empty()); + } + + #[tokio::test] + async fn test_short_ciphertext_read_is_rejected() { + struct ShortRead; + + #[async_trait::async_trait] + impl FileRead for ShortRead { + async fn read(&self, range: Range) -> Result { + let len = (range.end - range.start).saturating_sub(1) as usize; + Ok(Bytes::from(vec![0; len])) + } + } + + let reader = AesGcmFileRead::new( + Box::new(ShortRead), + Arc::new(make_cipher(b"0123456789abcdef")), + Box::default(), + u64::from(MIN_STREAM_LENGTH) + 10, + ) + .unwrap(); + let err = reader.read(0..10).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("Invalid encrypted read length")); + } + + #[tokio::test] + async fn test_oversized_declared_length_is_rejected() { + // Object stores clamp reads at EOF rather than failing, unlike the in-memory FileIO. + struct ClampingRead(Bytes); + + #[async_trait::async_trait] + impl FileRead for ClampingRead { + async fn read(&self, range: Range) -> Result { + let start = (range.start as usize).min(self.0.len()); + let end = (range.end as usize).min(self.0.len()); + Ok(self.0.slice(start..end)) + } + } + + let key = b"0123456789abcdef"; + let aad_prefix = b"test-aad-prefix!"; + let plaintext = b"some bytes to measure"; + let encrypted = write_through_ags1(plaintext, key, aad_prefix).await; + + // A declared length larger than the real file must be rejected rather than yielding + // truncated plaintext, whether it overstates by one byte or by a whole extra block. + for excess in [1, u64::from(CIPHER_BLOCK_SIZE)] { + let reader = AesGcmFileRead::new( + Box::new(ClampingRead(Bytes::from(encrypted.clone()))), + Arc::new(make_cipher(key)), + aad_prefix.to_vec().into_boxed_slice(), + encrypted.len() as u64 + excess, + ) + .unwrap(); + + let err = reader + .read(0..plaintext.len() as u64) + .await + .expect_err("an inflated declared length must not read back as plaintext"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("Invalid encrypted read length")); + } + } + + #[tokio::test] + async fn test_small_file_roundtrip() { + let key = b"0123456789abcdef"; + let aad_prefix = b"test-aad-prefix!"; + let plaintext = b"Hello, Iceberg encryption!"; + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(plaintext, &cipher, aad_prefix); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), plaintext.len() as u64); + + // Read entire file + let result = reader.read(0..plaintext.len() as u64).await.unwrap(); + assert_eq!(&result[..], plaintext); + } + + #[tokio::test] + async fn test_partial_read() { + let key = b"0123456789abcdef"; + let aad_prefix = b"aad-prefix-here!"; + let plaintext = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(plaintext, &cipher, aad_prefix); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + // Read a slice from the middle + let result = reader.read(10..20).await.unwrap(); + assert_eq!(&result[..], &plaintext[10..20]); + + // Read first byte + let result = reader.read(0..1).await.unwrap(); + assert_eq!(&result[..], &plaintext[0..1]); + + // Read last byte + let last = plaintext.len() as u64; + let result = reader.read(last - 1..last).await.unwrap(); + assert_eq!(&result[..], &plaintext[plaintext.len() - 1..]); + } + + #[tokio::test] + async fn test_multi_block_roundtrip() { + let key = b"0123456789abcdef"; + let aad_prefix = b"multi-block-aad!"; + + // 1.5 blocks of data + let size = PLAIN_BLOCK_SIZE as usize + PLAIN_BLOCK_SIZE as usize / 2; + let plaintext: Vec = (0..size).map(|i| (i % 256) as u8).collect(); + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(&plaintext, &cipher, aad_prefix); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), plaintext.len() as u64); + + // Read entire file + let result = reader.read(0..plaintext.len() as u64).await.unwrap(); + assert_eq!(&result[..], &plaintext[..]); + } + + #[tokio::test] + async fn test_cross_block_read() { + let key = b"0123456789abcdef"; + let aad_prefix = b"cross-block-aad!"; + + // 2.5 blocks of data + let size = PLAIN_BLOCK_SIZE as usize * 2 + PLAIN_BLOCK_SIZE as usize / 2; + let plaintext: Vec = (0..size).map(|i| (i % 256) as u8).collect(); + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(&plaintext, &cipher, aad_prefix); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + // Read across block boundary (last 100 bytes of block 0 + first 100 bytes of block 1) + let boundary = PLAIN_BLOCK_SIZE as u64; + let result = reader.read(boundary - 100..boundary + 100).await.unwrap(); + assert_eq!( + &result[..], + &plaintext[(boundary - 100) as usize..(boundary + 100) as usize] + ); + + // Read across two block boundaries (spans blocks 0, 1, and 2) + let result = reader.read(boundary - 50..boundary * 2 + 50).await.unwrap(); + assert_eq!( + &result[..], + &plaintext[(boundary - 50) as usize..(boundary * 2 + 50) as usize] + ); + } + + #[tokio::test] + async fn test_exact_block_size() { + let key = b"0123456789abcdef"; + let aad_prefix = b"exact-block-aad!"; + + // Exactly 1 block + let plaintext: Vec = (0..PLAIN_BLOCK_SIZE as usize) + .map(|i| (i % 256) as u8) + .collect(); + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(&plaintext, &cipher, aad_prefix); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), PLAIN_BLOCK_SIZE as u64); + + let result = reader.read(0..PLAIN_BLOCK_SIZE as u64).await.unwrap(); + assert_eq!(&result[..], &plaintext[..]); + } + + #[tokio::test] + async fn test_block_size_plus_one() { + let key = b"0123456789abcdef"; + let aad_prefix = b"block-plus-one!!"; + + // 1 block + 1 byte + let size = PLAIN_BLOCK_SIZE as usize + 1; + let plaintext: Vec = (0..size).map(|i| (i % 256) as u8).collect(); + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(&plaintext, &cipher, aad_prefix); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), size as u64); + + // Read the last byte (in block 1) + let result = reader.read(size as u64 - 1..size as u64).await.unwrap(); + assert_eq!(result[0], plaintext[size - 1]); + + // Read all + let result = reader.read(0..size as u64).await.unwrap(); + assert_eq!(&result[..], &plaintext[..]); + } + + #[tokio::test] + async fn test_block_size_minus_one() { + let key = b"0123456789abcdef"; + let aad_prefix = b"block-minus-one!"; + + // 1 block - 1 byte + let size = PLAIN_BLOCK_SIZE as usize - 1; + let plaintext: Vec = (0..size).map(|i| (i % 256) as u8).collect(); + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(&plaintext, &cipher, aad_prefix); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), size as u64); + + let result = reader.read(0..size as u64).await.unwrap(); + assert_eq!(&result[..], &plaintext[..]); + } + + #[tokio::test] + async fn test_wrong_aad_fails() { + let key = b"0123456789abcdef"; + let aad_prefix = b"correct-aad-here"; + let plaintext = b"sensitive data here"; + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(plaintext, &cipher, aad_prefix); + + // Try to decrypt with wrong AAD + let mut bad_aad = aad_prefix.to_vec(); + bad_aad[0] ^= 0xFF; + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + bad_aad.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + let result = reader.read(0..plaintext.len() as u64).await; + assert!(result.is_err(), "Decryption with wrong AAD should fail"); + } + + #[tokio::test] + async fn test_wrong_key_fails() { + let key = b"0123456789abcdef"; + let wrong_key = b"fedcba9876543210"; + let aad_prefix = b"test-aad-prefix!"; + let plaintext = b"sensitive data"; + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(plaintext, &cipher, aad_prefix); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(wrong_key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + let result = reader.read(0..plaintext.len() as u64).await; + assert!(result.is_err(), "Decryption with wrong key should fail"); + } + + #[tokio::test] + async fn test_out_of_bounds_read() { + let key = b"0123456789abcdef"; + let aad_prefix = b"test-aad-prefix!"; + let plaintext = b"short data"; + let cipher = make_cipher(key); + + let encrypted = encrypt_ags1(plaintext, &cipher, aad_prefix); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + let result = reader.read(0..plaintext.len() as u64 + 1).await; + assert!(result.is_err(), "Reading past end should fail"); + } + + #[tokio::test] + async fn test_calculate_plaintext_length() { + // Empty file: header only (not valid per Java, but handled) + assert_eq!( + AesGcmFileRead::calculate_plaintext_length(GCM_STREAM_HEADER_LENGTH as u64).unwrap(), + 0 + ); + + // Empty file with one empty block: header(8) + nonce(12) + tag(16) = 36 + assert_eq!( + AesGcmFileRead::calculate_plaintext_length(MIN_STREAM_LENGTH as u64).unwrap(), + 0 + ); + + // One full block: header(8) + cipher_block(1048604) = 1048612 + let one_full = GCM_STREAM_HEADER_LENGTH as u64 + CIPHER_BLOCK_SIZE as u64; + assert_eq!( + AesGcmFileRead::calculate_plaintext_length(one_full).unwrap(), + PLAIN_BLOCK_SIZE as u64 + ); + + // One full block + 1 byte: need partial second block + // Second block = nonce(12) + 1 byte ciphertext + tag(16) = 29 + let one_full_plus_one = one_full + NONCE_LENGTH as u64 + 1 + GCM_TAG_LENGTH as u64; + assert_eq!( + AesGcmFileRead::calculate_plaintext_length(one_full_plus_one).unwrap(), + PLAIN_BLOCK_SIZE as u64 + 1 + ); + } + + #[tokio::test] + async fn test_stream_block_aad() { + // With prefix + let aad = stream_block_aad(b"prefix", 0); + assert_eq!(&aad[..6], b"prefix"); + assert_eq!(&aad[6..], &0u32.to_le_bytes()); + + let aad = stream_block_aad(b"prefix", 1); + assert_eq!(&aad[..6], b"prefix"); + assert_eq!(&aad[6..], &1u32.to_le_bytes()); + + // Without prefix + let aad = stream_block_aad(b"", 42); + assert_eq!(&aad[..], &42u32.to_le_bytes()); + } + + #[test] + fn test_encrypted_file_too_short() { + for length in 0..MIN_STREAM_LENGTH { + let result = AesGcmFileRead::new( + memory_reader(vec![0; length as usize]), + Arc::new(make_cipher(b"0123456789abcdef")), + [].into(), + u64::from(length), + ); + let err = result + .err() + .expect("a stream must contain an authenticated block"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("Invalid encrypted file length")); + } + } + + // --- AesGcmFileWrite tests --- + + /// Shared-buffer FileWrite for testing AesGcmFileWrite output. + struct SharedMemoryWrite { + buffer: Arc>>, + } + + /// FileWrite that fails after a configured number of successful writes. + struct FailingFileWrite { + writes_before_failure: usize, + write_count: usize, + } + + #[async_trait::async_trait] + impl FileWrite for FailingFileWrite { + async fn write(&mut self, _bs: Bytes) -> Result<()> { + if self.write_count >= self.writes_before_failure { + return Err(Error::new(ErrorKind::Unexpected, "simulated write failure")); + } + self.write_count += 1; + Ok(()) + } + + // A poisoned AesGcmFileWrite must reject close() before delegating, so reaching this is a + // bug. Returning an error rather than panicking surfaces it as a failed assertion on the + // expected "poisoned" message. + async fn close(&mut self) -> Result { + Err(Error::new( + ErrorKind::Unexpected, + "FailingFileWrite::close called unexpectedly", + )) + } + } + + #[async_trait::async_trait] + impl FileWrite for SharedMemoryWrite { + async fn write(&mut self, bs: Bytes) -> Result<()> { + self.buffer.lock().unwrap().extend_from_slice(&bs); + Ok(()) + } + + async fn close(&mut self) -> Result { + Ok(FileMetadata { + size: self.buffer.lock().unwrap().len() as u64, + }) + } + } + + /// Helper: one-shot encrypt through AesGcmFileWrite, return encrypted bytes. + async fn write_through_ags1(plaintext: &[u8], key: &[u8], aad_prefix: &[u8]) -> Vec { + let buffer = Arc::new(std::sync::Mutex::new(Vec::new())); + let inner: Box = Box::new(SharedMemoryWrite { + buffer: buffer.clone(), + }); + let cipher = Arc::new(make_cipher(key)); + let mut writer = AesGcmFileWrite::new(inner, cipher, aad_prefix.to_vec()); + + writer.write(Bytes::from(plaintext.to_vec())).await.unwrap(); + let metadata = writer.close().await.unwrap(); + + let encrypted = buffer.lock().unwrap().clone(); + assert_eq!( + metadata.size, + encrypted.len() as u64, + "close() must report the full ciphertext length" + ); + encrypted + } + + #[tokio::test] + async fn test_write_empty_roundtrip() { + let key = b"0123456789abcdef"; + let aad_prefix = b"test-aad-prefix!"; + + let encrypted = write_through_ags1(b"", key, aad_prefix).await; + + // Should produce header + one empty encrypted block + assert_eq!(encrypted.len(), MIN_STREAM_LENGTH as usize); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), 0); + } + + #[tokio::test] + async fn test_write_small_roundtrip() { + let key = b"0123456789abcdef"; + let aad_prefix = b"test-aad-prefix!"; + let plaintext = b"Hello, Iceberg encryption!"; + + let encrypted = write_through_ags1(plaintext, key, aad_prefix).await; + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), plaintext.len() as u64); + let result = reader.read(0..plaintext.len() as u64).await.unwrap(); + assert_eq!(&result[..], plaintext); + } + + #[tokio::test] + async fn test_write_multi_block_roundtrip() { + let key = b"0123456789abcdef"; + let aad_prefix = b"multi-block-aad!"; + + // 1.5 blocks of data + let size = PLAIN_BLOCK_SIZE as usize + PLAIN_BLOCK_SIZE as usize / 2; + let plaintext: Vec = (0..size).map(|i| (i % 256) as u8).collect(); + + let encrypted = write_through_ags1(&plaintext, key, aad_prefix).await; + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), plaintext.len() as u64); + let result = reader.read(0..plaintext.len() as u64).await.unwrap(); + assert_eq!(&result[..], &plaintext[..]); + } + + #[tokio::test] + async fn test_write_cross_block_accumulation() { + let key = b"0123456789abcdef"; + let aad_prefix = b"cross-block-aad!"; + + let buffer = Arc::new(std::sync::Mutex::new(Vec::new())); + let inner: Box = Box::new(SharedMemoryWrite { + buffer: buffer.clone(), + }); + let cipher = Arc::new(make_cipher(key)); + let mut writer = AesGcmFileWrite::new(inner, cipher, aad_prefix.to_vec()); + + // Write 1.5 blocks in 1000-byte chunks + let total_size = PLAIN_BLOCK_SIZE as usize + PLAIN_BLOCK_SIZE as usize / 2; + let plaintext: Vec = (0..total_size).map(|i| (i % 256) as u8).collect(); + let chunk_size = 1000; + for chunk in plaintext.chunks(chunk_size) { + writer.write(Bytes::from(chunk.to_vec())).await.unwrap(); + } + writer.close().await.unwrap(); + + let encrypted = buffer.lock().unwrap().clone(); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), plaintext.len() as u64); + let result = reader.read(0..plaintext.len() as u64).await.unwrap(); + assert_eq!(&result[..], &plaintext[..]); + } + + #[tokio::test] + async fn test_write_exact_block_size() { + let key = b"0123456789abcdef"; + let aad_prefix = b"exact-block-aad!"; + + // Exactly 1 block + let plaintext: Vec = (0..PLAIN_BLOCK_SIZE as usize) + .map(|i| (i % 256) as u8) + .collect(); + + let encrypted = write_through_ags1(&plaintext, key, aad_prefix).await; + + let reader = AesGcmFileRead::new( + memory_reader(encrypted.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), PLAIN_BLOCK_SIZE as u64); + let result = reader.read(0..PLAIN_BLOCK_SIZE as u64).await.unwrap(); + assert_eq!(&result[..], &plaintext[..]); + } + + #[tokio::test] + async fn test_write_block_aligned_no_spurious_empty_block() { + let key = b"0123456789abcdef"; + let aad_prefix = b"block-align-aad!"; + + // Write exactly one block of plaintext — close() should NOT add + // a trailing empty encrypted block (28 bytes: 12-byte nonce + 16-byte tag). + let plaintext: Vec = (0..PLAIN_BLOCK_SIZE as usize) + .map(|i| (i % 256) as u8) + .collect(); + + let encrypted_via_writer = write_through_ags1(&plaintext, key, aad_prefix).await; + let encrypted_via_reference = encrypt_ags1(&plaintext, &make_cipher(key), aad_prefix); + + // Both should be the same length — no extra 28-byte empty block + assert_eq!( + encrypted_via_writer.len(), + encrypted_via_reference.len(), + "Writer output should match reference encryption length (no spurious trailing block)" + ); + + // Verify roundtrip + let reader = AesGcmFileRead::new( + memory_reader(encrypted_via_writer.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted_via_writer.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), PLAIN_BLOCK_SIZE as u64); + let result = reader.read(0..PLAIN_BLOCK_SIZE as u64).await.unwrap(); + assert_eq!(&result[..], &plaintext[..]); + } + + #[tokio::test] + async fn test_write_two_blocks_aligned_no_spurious_empty_block() { + let key = b"0123456789abcdef"; + let aad_prefix = b"2blk-align-aad!!"; + + // Exactly 2 blocks + let size = PLAIN_BLOCK_SIZE as usize * 2; + let plaintext: Vec = (0..size).map(|i| (i % 256) as u8).collect(); + + let encrypted_via_writer = write_through_ags1(&plaintext, key, aad_prefix).await; + let encrypted_via_reference = encrypt_ags1(&plaintext, &make_cipher(key), aad_prefix); + + assert_eq!( + encrypted_via_writer.len(), + encrypted_via_reference.len(), + "Writer output should match reference encryption length (no spurious trailing block)" + ); + + let reader = AesGcmFileRead::new( + memory_reader(encrypted_via_writer.clone()), + Arc::new(make_cipher(key)), + aad_prefix.as_slice().into(), + encrypted_via_writer.len() as u64, + ) + .unwrap(); + + assert_eq!(reader.plaintext_length(), size as u64); + let result = reader.read(0..size as u64).await.unwrap(); + assert_eq!(&result[..], &plaintext[..]); + } + + #[tokio::test] + async fn test_write_poisoned_after_inner_write_failure() { + let cipher = Arc::new(make_cipher(b"0123456789abcdef")); + // Fail on the second write (first write is the header, second is block data) + let inner: Box = Box::new(FailingFileWrite { + writes_before_failure: 1, + write_count: 0, + }); + let mut writer = AesGcmFileWrite::new(inner, cipher, b"aad-prefix-here!".to_vec()); + + // First write triggers header (succeeds) + block encrypt+write (fails) + let data = vec![0u8; PLAIN_BLOCK_SIZE as usize]; + let result = writer.write(Bytes::from(data)).await; + assert!(result.is_err()); + + // Subsequent write should be rejected as poisoned + let result = writer.write(Bytes::from(b"more data".to_vec())).await; + assert!(result.is_err()); + assert!( + result.unwrap_err().to_string().contains("poisoned"), + "expected poisoned error" + ); + + // Close should also be rejected + let err = writer.close().await.err().expect("close should fail"); + assert!( + err.to_string().contains("poisoned"), + "expected poisoned error on close" + ); + } +} diff --git a/crates/iceberg/src/error.rs b/crates/iceberg/src/error.rs index 6ab3a78c8b..2f5c92894d 100644 --- a/crates/iceberg/src/error.rs +++ b/crates/iceberg/src/error.rs @@ -18,6 +18,7 @@ use std::backtrace::{Backtrace, BacktraceStatus}; use std::fmt; use std::fmt::{Debug, Display, Formatter}; +use std::sync::PoisonError; use chrono::{DateTime, TimeZone as _, Utc}; @@ -384,12 +385,6 @@ define_from_err!( "Failure in conversion with avro" ); -define_from_err!( - opendal::Error, - ErrorKind::Unexpected, - "Failure in doing io operation" -); - define_from_err!( url::ParseError, ErrorKind::DataInvalid, @@ -408,12 +403,6 @@ define_from_err!( "Failed to parse json string" ); -define_from_err!( - rust_decimal::Error, - ErrorKind::DataInvalid, - "Failed to convert decimal literal to rust decimal" -); - define_from_err!( parquet::errors::ParquetError, ErrorKind::Unexpected, @@ -434,6 +423,11 @@ define_from_err!( define_from_err!(std::io::Error, ErrorKind::Unexpected, "IO Operation failed"); +/// Converts a [`PoisonError`] from a poisoned lock into an [`Error`]. +pub(crate) fn lock_error(e: PoisonError) -> Error { + Error::new(ErrorKind::Unexpected, format!("Lock poisoned: {e}")) +} + /// Converts a timestamp in milliseconds to `DateTime`, handling errors. /// /// # Arguments @@ -475,6 +469,47 @@ macro_rules! ensure_data_valid { }; } +/// Helper macro to construct an [`ErrorKind::DataInvalid`] error. +/// +/// This is a shorthand for `Error::new(ErrorKind::DataInvalid, ...)`, the most +/// common error constructed in this crate. It returns the [`Error`] value (it +/// does *not* return from the enclosing function), so it composes with `?`, +/// `.map_err(...)`, `.ok_or_else(...)`, and explicit `return Err(...)`. +/// +/// The message may be a plain expression or a format string with arguments. +/// +/// Unlike the public [`ensure_data_valid!`], this macro is deliberately +/// crate-internal — adding `#[macro_export]` would commit it to the public API. +/// +/// # Examples +/// +/// The `use` path below is crate-internal and only resolves inside this crate. +/// +/// ```ignore +/// use crate::error::invalid_data; +/// +/// // As an expression +/// let err = invalid_data!("unexpected value: {value}"); +/// +/// // With `.ok_or_else` +/// let field = fields.get(id).ok_or_else(|| invalid_data!("missing field {id}"))?; +/// +/// // Attaching a source error +/// let n: i32 = s.parse().map_err(|e| invalid_data!("not an int: {s}").with_source(e))?; +/// ``` +macro_rules! invalid_data { + // Bare literals stay in `format!` so inline captures like `{id}` still interpolate. + ($fmt: literal $(, $($arg:tt)*)?) => { + $crate::error::Error::new($crate::error::ErrorKind::DataInvalid, format!($fmt $(, $($arg)*)?)) + }; + ($msg: expr $(,)?) => { + $crate::error::Error::new($crate::error::ErrorKind::DataInvalid, $msg) + }; +} + +// Crate-internal macro: re-exported so other modules can `use crate::error::invalid_data;`. +pub(crate) use invalid_data; + #[cfg(test)] mod tests { use anyhow::anyhow; @@ -538,6 +573,36 @@ Source: networking error ) } + #[test] + fn test_invalid_data_macro() { + // Plain literal. + let err = invalid_data!("something is wrong"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), "something is wrong"); + + // Literal with an inline capture. + let value = 42; + let err = invalid_data!("unexpected value: {value}"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), "unexpected value: 42"); + + // Literal with trailing arguments. + let err = invalid_data!("unexpected value: {}", value); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), "unexpected value: 42"); + + // Non-literal message expression. + let computed = String::from("computed"); + let err = invalid_data!(computed); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), "computed"); + + // Composes with builder methods like `.with_source`. + let err = invalid_data!("wrapping").with_source(anyhow!("root cause")); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.source.is_some()); + } + /// Backtrace contains build information, so we just assert the header of error content. #[test] fn test_error_debug_with_backtrace() { diff --git a/crates/iceberg/src/expr/predicate.rs b/crates/iceberg/src/expr/predicate.rs index 123cfada5f..f8ea896395 100644 --- a/crates/iceberg/src/expr/predicate.rs +++ b/crates/iceberg/src/expr/predicate.rs @@ -27,7 +27,7 @@ use fnv::FnvHashSet; use itertools::Itertools; use serde::{Deserialize, Serialize}; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::expr::visitors::bound_predicate_visitor::visit as visit_bound; use crate::expr::visitors::predicate_visitor::visit; use crate::expr::visitors::rewrite_not::RewriteNotVisitor; @@ -53,8 +53,8 @@ impl<'de, T: Deserialize<'de>, const N: usize> Deserialize<'de> for LogicalExpre where D: serde::Deserializer<'de> { let inputs = Vec::>::deserialize(deserializer)?; Ok(LogicalExpression::new( - array_init::from_iter(inputs.into_iter()).ok_or_else(|| { - serde::de::Error::custom(format!("Failed to deserialize LogicalExpression: the len of inputs is not match with the len of LogicalExpression {}",N)) + array_init::from_iter(inputs).ok_or_else(|| { + serde::de::Error::custom(format!("Failed to deserialize LogicalExpression: the len of inputs is not match with the len of LogicalExpression {N}")) })?, )) } @@ -310,7 +310,7 @@ impl Bind for SetExpression { impl Display for SetExpression { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut literal_strs = self.literals.iter().map(|l| format!("{}", l)); + let mut literal_strs = self.literals.iter().map(|l| format!("{l}")); write!(f, "{} {} ({})", self.term, self.op, literal_strs.join(", ")) } @@ -398,12 +398,9 @@ impl Bind for Predicate { } &PredicateOperator::IsNan | &PredicateOperator::NotNan => { if !bound_expr.term.field().field_type.is_floating_type() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Expecting floating point type, but found {}", - bound_expr.term.field().field_type - ), + return Err(invalid_data!( + "Expecting floating point type, but found {}", + bound_expr.term.field().field_type )); } } @@ -529,13 +526,13 @@ impl Display for Predicate { write!(f, "NOT ({})", expr.inputs()[0]) } Predicate::Unary(expr) => { - write!(f, "{}", expr) + write!(f, "{expr}") } Predicate::Binary(expr) => { - write!(f, "{}", expr) + write!(f, "{expr}") } Predicate::Set(expr) => { - write!(f, "{}", expr) + write!(f, "{expr}") } } } @@ -805,13 +802,13 @@ impl Display for BoundPredicate { write!(f, "NOT ({})", expr.inputs()[0]) } BoundPredicate::Unary(expr) => { - write!(f, "{}", expr) + write!(f, "{expr}") } BoundPredicate::Binary(expr) => { - write!(f, "{}", expr) + write!(f, "{expr}") } BoundPredicate::Set(expr) => { - write!(f, "{}", expr) + write!(f, "{expr}") } } } diff --git a/crates/iceberg/src/expr/term.rs b/crates/iceberg/src/expr/term.rs index ecdb88912b..41970e1efe 100644 --- a/crates/iceberg/src/expr/term.rs +++ b/crates/iceberg/src/expr/term.rs @@ -22,12 +22,12 @@ use std::fmt::{Display, Formatter}; use fnv::FnvHashSet; use serde::{Deserialize, Serialize}; +use crate::error::invalid_data; use crate::expr::accessor::{StructAccessor, StructAccessorRef}; use crate::expr::{ BinaryExpression, Bind, Predicate, PredicateOperator, SetExpression, UnaryExpression, }; use crate::spec::{Datum, NestedField, NestedFieldRef, SchemaRef}; -use crate::{Error, ErrorKind}; /// Unbound term before binding to a schema. pub type Term = Reference; @@ -313,22 +313,15 @@ impl Bind for Reference { let field = if case_sensitive { schema.field_by_name(&self.name) } else { - schema.field_by_name_case_insensitive(&self.name) + schema.field_by_name_case_insensitive_checked(&self.name)? }; - let field = field.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Field {} not found in schema", self.name), - ) - })?; + let field = + field.ok_or_else(|| invalid_data!("Field {} not found in schema", self.name))?; - let accessor = schema.accessor_by_field_id(field.id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Accessor for Field {} not found", self.name), - ) - })?; + let accessor = schema + .accessor_by_field_id(field.id) + .ok_or_else(|| invalid_data!("Accessor for Field {} not found", self.name))?; Ok(BoundReference::new( self.name.clone(), @@ -435,6 +428,23 @@ mod tests { assert_eq!(expected_ref, reference); } + #[test] + fn test_bind_reference_case_insensitive_ambiguous() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", PrimitiveType::Int).into(), + NestedField::optional(2, "ID", PrimitiveType::Int).into(), + ]) + .build() + .unwrap(), + ); + + let error = Reference::new("Id").bind(schema, false).unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::DataInvalid); + assert!(error.message().contains("case-insensitively"), "{error}"); + } + #[test] fn test_bind_reference_failure() { let schema = table_schema_simple(); diff --git a/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs new file mode 100644 index 0000000000..3dfc8ec4a9 --- /dev/null +++ b/crates/iceberg/src/expr/visitors/bloom_filter_evaluator.rs @@ -0,0 +1,1615 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Evaluates predicates against Parquet bloom filters to determine whether +//! a row group can be skipped. + +use std::collections::{HashMap, HashSet}; + +use fnv::FnvHashSet; +use parquet::basic::Type as PhysicalType; +use parquet::bloom_filter::Sbbf; +use parquet::data_type::ByteArray; + +use crate::Result; +use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; +use crate::expr::{BoundPredicate, BoundReference}; +use crate::spec::decimal_utils::decimal_to_fixed_length_bytes_exact; +use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType, Type}; + +const ROW_GROUP_MIGHT_MATCH: Result = Ok(true); +const ROW_GROUP_CANT_MATCH: Result = Ok(false); + +/// A column's bloom filter for one row group, together with the file's physical +/// encoding of that column. A probe must be encoded the way the writer encoded +/// the values it inserted, so the encoding travels with the filter. +pub(crate) struct ColumnBloomFilter { + sbbf: Sbbf, + physical_type: PhysicalType, + /// `type_length` from the file's column descriptor. Only meaningful for + /// `FIXED_LEN_BYTE_ARRAY`. + type_length: i32, +} + +impl ColumnBloomFilter { + pub(crate) fn new(sbbf: Sbbf, physical_type: PhysicalType, type_length: i32) -> Self { + Self { + sbbf, + physical_type, + type_length, + } + } +} + +pub(crate) struct BloomFilterEvaluator<'a> { + /// Maps Iceberg field_id -> bloom filter for this row group + bloom_filters: &'a HashMap, +} + +impl BloomFilterEvaluator<'_> { + /// Evaluate the predicate against the provided bloom filters. + /// Returns `false` if the row group definitely does not match, + /// `true` if it might match. + pub(crate) fn eval( + filter: &BoundPredicate, + bloom_filters: &HashMap, + ) -> Result { + if bloom_filters.is_empty() { + return ROW_GROUP_MIGHT_MATCH; + } + + let mut evaluator = BloomFilterEvaluator { bloom_filters }; + visit(&mut evaluator, filter) + } + + fn check_datum(&self, reference: &BoundReference, datum: &Datum) -> bool { + let field_id = reference.field().id; + let Some(column) = self.bloom_filters.get(&field_id) else { + // No bloom filter for this column — conservatively might match + return true; + }; + + check_in_bloom_filter(column, datum) + } +} + +/// Collects field IDs that appear in `eq` or `in` predicates — the only +/// predicate types that benefit from bloom filter checks. +/// +/// Each node returns the field IDs its subtree contributes, mirroring +/// [`BloomFilterEvaluator`]'s structure so that a field is collected only where +/// the evaluator can act on it. In particular `not` discards its subtree: the +/// evaluator's `not` returns might-match regardless, so a filter fetched for a +/// field under a `NOT` could never prune and would be pure wasted I/O. +/// +/// Boolean fields are skipped for the same reason: with only two possible values, +/// a column chunk's min/max statistics already determine membership exactly, so a +/// bloom filter cannot prune anything statistics did not, and reading one costs a +/// round trip to learn nothing. +pub(crate) fn collect_bloom_filter_field_ids(predicate: &BoundPredicate) -> Result> { + visit(&mut BloomFilterFieldIdCollector, predicate) +} + +struct BloomFilterFieldIdCollector; + +/// Field IDs worth fetching a bloom filter for: none if probing the column could +/// never prune. +fn probeable_field_id(reference: &BoundReference) -> HashSet { + if matches!( + reference.field().field_type.as_ref(), + Type::Primitive(PrimitiveType::Boolean) + ) { + return HashSet::new(); + } + HashSet::from([reference.field().id]) +} + +impl BoundPredicateVisitor for BloomFilterFieldIdCollector { + type T = HashSet; + + fn always_true(&mut self) -> Result { + Ok(HashSet::new()) + } + + fn always_false(&mut self) -> Result { + Ok(HashSet::new()) + } + + fn and(&mut self, mut lhs: Self::T, rhs: Self::T) -> Result { + lhs.extend(rhs); + Ok(lhs) + } + + fn or(&mut self, mut lhs: Self::T, rhs: Self::T) -> Result { + lhs.extend(rhs); + Ok(lhs) + } + + fn not(&mut self, _inner: Self::T) -> Result { + Ok(HashSet::new()) + } + + fn is_null(&mut self, _r: &BoundReference, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn not_null(&mut self, _r: &BoundReference, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn is_nan(&mut self, _r: &BoundReference, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn not_nan(&mut self, _r: &BoundReference, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn less_than( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn less_than_or_eq( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn greater_than( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn greater_than_or_eq( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn eq(&mut self, r: &BoundReference, _l: &Datum, _p: &BoundPredicate) -> Result { + Ok(probeable_field_id(r)) + } + + fn not_eq(&mut self, _r: &BoundReference, _l: &Datum, _p: &BoundPredicate) -> Result { + Ok(HashSet::new()) + } + + fn starts_with( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn not_starts_with( + &mut self, + _r: &BoundReference, + _l: &Datum, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } + + fn r#in( + &mut self, + r: &BoundReference, + _literals: &FnvHashSet, + _p: &BoundPredicate, + ) -> Result { + Ok(probeable_field_id(r)) + } + + fn not_in( + &mut self, + _r: &BoundReference, + _literals: &FnvHashSet, + _p: &BoundPredicate, + ) -> Result { + Ok(HashSet::new()) + } +} + +/// Check whether a datum value might be present in the bloom filter. +/// +/// The value must be checked using the same physical encoding the Parquet +/// writer used when inserting into the bloom filter. We use the actual +/// physical type from the column metadata to ensure correctness regardless +/// of which writer produced the file. +fn check_in_bloom_filter(column: &ColumnBloomFilter, datum: &Datum) -> bool { + let ColumnBloomFilter { + sbbf, + physical_type, + type_length, + } = column; + let physical_type = *physical_type; + + match datum.literal() { + // Parquet does not define bloom filter semantics for BOOLEAN — parquet-java + // only hashes int/long/float/double/binary. arrow-rs does write them, but min/max + // statistics already decide membership for a two-valued domain, so there is nothing to gain. + PrimitiveLiteral::Boolean(_) => true, + // A promoted column (int -> long, float -> double) keeps its original + // physical width in files written before the promotion, and the writer + // hashed that width, so probe at the file's width rather than the + // predicate's. + PrimitiveLiteral::Int(v) => match physical_type { + PhysicalType::INT32 => sbbf.check(v), + PhysicalType::INT64 => sbbf.check(&i64::from(*v)), + _ => true, + }, + PrimitiveLiteral::Long(v) => match physical_type { + PhysicalType::INT64 => sbbf.check(v), + PhysicalType::INT32 => match i32::try_from(*v) { + Ok(narrowed) => sbbf.check(&narrowed), + // Too wide for an INT32 column to hold, so the value is provably + // absent and pruning here would be sound; kept conservative for now. + Err(_) => true, + }, + _ => true, + }, + PrimitiveLiteral::Float(v) => match physical_type { + PhysicalType::FLOAT => sbbf.check(&v.0), + PhysicalType::DOUBLE => sbbf.check(&f64::from(v.0)), + _ => true, + }, + PrimitiveLiteral::Double(v) => match physical_type { + PhysicalType::DOUBLE => sbbf.check(&v.0), + PhysicalType::FLOAT => { + let narrowed = v.0 as f32; + // Only an exactly representable value can equal a widened f32. + if f64::from(narrowed) == v.0 { + sbbf.check(&narrowed) + } else { + true + } + } + _ => true, + }, + PrimitiveLiteral::String(v) => sbbf.check(v.as_str()), + PrimitiveLiteral::Binary(v) => sbbf.check(v.as_slice()), + PrimitiveLiteral::Int128(v) => { + // Decimal: dispatch based on the actual Parquet physical type + // from the file, not inferred from precision. + match physical_type { + // Narrow only when the mantissa round-trips; a truncated copy would + // hash to an unrelated slot. + PhysicalType::INT32 => match i32::try_from(*v) { + Ok(narrowed) => sbbf.check(&narrowed), + Err(_) => true, + }, + PhysicalType::INT64 => match i64::try_from(*v) { + Ok(narrowed) => sbbf.check(&narrowed), + Err(_) => true, + }, + PhysicalType::FIXED_LEN_BYTE_ARRAY => { + // Encode to the file's declared length, not one derived from + // the Iceberg precision: a widened precision would change the + // length and miss every entry the writer inserted. + match usize::try_from(*type_length) + .ok() + .and_then(|len| decimal_to_fixed_length_bytes_exact(*v, len)) + { + Some(bytes) => sbbf.check(&ByteArray::from(bytes)), + // Unusable length, or a value too large for the column to + // hold — conservatively might match. + None => true, + } + } + // Known gap: BYTE_ARRAY decimals are valid in Parquet (though not in + // Iceberg's Appendix A) and some older Spark writers emit them. Not + // probed because BYTE_ARRAY carries no `type_length` and sign-extension + // padding is writer-dependent, so a single-length probe would miss + // padded entries and prune row groups that do hold the value. A sound + // version ORs a check over every length from minimal..=16. + _ => true, // Conservatively might match + } + } + PrimitiveLiteral::UInt128(v) => { + // UUID: stored as FIXED_LEN_BYTE_ARRAY(16), big-endian + let bytes = v.to_be_bytes(); + sbbf.check(bytes.as_slice()) + } + PrimitiveLiteral::AboveMax | PrimitiveLiteral::BelowMin => true, + } +} + +impl BoundPredicateVisitor for BloomFilterEvaluator<'_> { + type T = bool; + + fn always_true(&mut self) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn always_false(&mut self) -> Result { + ROW_GROUP_CANT_MATCH + } + + fn and(&mut self, lhs: Self::T, rhs: Self::T) -> Result { + Ok(lhs && rhs) + } + + fn or(&mut self, lhs: Self::T, rhs: Self::T) -> Result { + Ok(lhs || rhs) + } + + fn not(&mut self, _inner: Self::T) -> Result { + // Bloom filters are not invertible — we cannot prove presence, + // so NOT of any result must conservatively return "might match". + ROW_GROUP_MIGHT_MATCH + } + + fn is_null( + &mut self, + _reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn not_null( + &mut self, + _reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn is_nan( + &mut self, + _reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn not_nan( + &mut self, + _reference: &BoundReference, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn less_than( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn less_than_or_eq( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn greater_than( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn greater_than_or_eq( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn eq( + &mut self, + reference: &BoundReference, + literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + if self.check_datum(reference, literal) { + ROW_GROUP_MIGHT_MATCH + } else { + ROW_GROUP_CANT_MATCH + } + } + + fn not_eq( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn starts_with( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn not_starts_with( + &mut self, + _reference: &BoundReference, + _literal: &Datum, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } + + fn r#in( + &mut self, + reference: &BoundReference, + literals: &FnvHashSet, + _predicate: &BoundPredicate, + ) -> Result { + let field_id = reference.field().id; + let Some(column) = self.bloom_filters.get(&field_id) else { + return ROW_GROUP_MIGHT_MATCH; + }; + + // If ANY literal might be present, the row group might match + for literal in literals { + if check_in_bloom_filter(column, literal) { + return ROW_GROUP_MIGHT_MATCH; + } + } + + // All literals are definitely absent + ROW_GROUP_CANT_MATCH + } + + fn not_in( + &mut self, + _reference: &BoundReference, + _literals: &FnvHashSet, + _predicate: &BoundPredicate, + ) -> Result { + ROW_GROUP_MIGHT_MATCH + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::ops::Not; + + use parquet::basic::Type as PhysicalType; + use parquet::bloom_filter::Sbbf; + use parquet::data_type::ByteArray; + + use super::{BloomFilterEvaluator, ColumnBloomFilter, collect_bloom_filter_field_ids}; + use crate::expr::{Bind, BoundPredicate, Reference}; + use crate::spec::decimal_utils::decimal_to_fixed_length_bytes_exact; + use crate::spec::{Datum, NestedField, PrimitiveType, Schema, Type}; + + fn create_test_schema() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap() + } + + fn create_bloom_filter_with_values_i32(values: &[i32]) -> Sbbf { + let mut sbbf = Sbbf::new_with_ndv_fpp(values.len() as u64, 0.01).unwrap(); + for v in values { + sbbf.insert(v); + } + sbbf + } + + fn create_bloom_filter_with_values_str(values: &[&str]) -> Sbbf { + let mut sbbf = Sbbf::new_with_ndv_fpp(values.len() as u64, 0.01).unwrap(); + for v in values { + sbbf.insert(*v); + } + sbbf + } + + #[test] + fn test_eq_value_present() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::int(2)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Row group should might-match when value is present"); + } + + #[test] + fn test_eq_value_absent() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::int(999)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "Row group should not match when value is absent from bloom filter" + ); + } + + #[test] + fn test_eq_no_bloom_filter_for_column() { + let schema = create_test_schema(); + let bloom_filters = HashMap::new(); // No bloom filters + + let predicate = Reference::new("id") + .equal_to(Datum::int(1)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Row group should might-match when no bloom filter available" + ); + } + + #[test] + fn test_in_all_absent() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .is_in([Datum::int(100), Datum::int(200), Datum::int(300)]) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "Row group should not match when all IN values are absent" + ); + } + + #[test] + fn test_in_some_present() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .is_in([Datum::int(2), Datum::int(200)]) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Row group should might-match when at least one IN value is present" + ); + } + + #[test] + fn test_and_one_absent() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([ + ( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + ), + ( + 2, + ColumnBloomFilter::new( + create_bloom_filter_with_values_str(&["alice", "bob"]), + PhysicalType::BYTE_ARRAY, + 0, + ), + ), + ]); + + // id = 999 AND name = 'alice' + // id=999 is absent, so AND should be false + let predicate = Reference::new("id") + .equal_to(Datum::int(999)) + .and(Reference::new("name").equal_to(Datum::string("alice"))) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "AND should be false when one operand is definitely absent" + ); + } + + #[test] + fn test_or_one_present() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + // id = 999 OR id = 2 + // id=2 is present, so OR should be true + let predicate = Reference::new("id") + .equal_to(Datum::int(999)) + .or(Reference::new("id").equal_to(Datum::int(2))) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "OR should be true when one operand might match"); + } + + #[test] + fn test_not_always_might_match() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + // NOT(id = 999) — even though 999 is absent, NOT should still return true + let predicate = Reference::new("id") + .equal_to(Datum::int(999)) + .not() + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "NOT should always return might-match"); + } + + // --- Field ID collection --- + // + // The collector decides which bloom filters get fetched, one round trip per + // column per row group, so anything it reports that the evaluator cannot act + // on is wasted I/O. + + fn collected_ids(predicate: BoundPredicate) -> Vec { + let mut ids: Vec = collect_bloom_filter_field_ids(&predicate) + .unwrap() + .into_iter() + .collect(); + ids.sort_unstable(); + ids + } + + #[test] + fn test_collects_eq_and_in_field_ids() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .equal_to(Datum::int(1)) + .and(Reference::new("name").is_in([Datum::string("alice")])) + .bind(schema.into(), true) + .unwrap(); + + assert_eq!(collected_ids(predicate), vec![1, 2]); + } + + /// `not` discards its subtree, so `eq`/`in` beneath a `NOT` are not collected. + /// The evaluator's `not` returns might-match regardless, so fetching those + /// filters could never prune. + #[test] + fn test_does_not_collect_eq_under_not() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .equal_to(Datum::int(1)) + .not() + .bind(schema.into(), true) + .unwrap(); + + assert!(collected_ids(predicate).is_empty()); + } + + #[test] + fn test_does_not_collect_in_under_not() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .is_in([Datum::int(1), Datum::int(2)]) + .not() + .bind(schema.into(), true) + .unwrap(); + + assert!(collected_ids(predicate).is_empty()); + } + + /// A `NOT` must not suppress collection for its siblings. + #[test] + fn test_collects_sibling_of_not() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .equal_to(Datum::int(1)) + .not() + .and(Reference::new("name").equal_to(Datum::string("alice"))) + .bind(schema.into(), true) + .unwrap(); + + assert_eq!(collected_ids(predicate), vec![2]); + } + + /// A boolean column's min/max statistics already determine membership exactly, + /// so fetching a bloom filter for one can never prune and is pure wasted I/O. + #[test] + fn test_does_not_collect_boolean_field_ids() { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "flag", Type::Primitive(PrimitiveType::Boolean)).into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let predicate = Reference::new("flag") + .equal_to(Datum::bool(true)) + .and(Reference::new("flag").is_in([Datum::bool(false)])) + .and(Reference::new("id").equal_to(Datum::int(1))) + .bind(schema.into(), true) + .unwrap(); + + assert_eq!( + collected_ids(predicate), + vec![2], + "only the non-boolean field should be collected" + ); + } + + /// Even given a filter, a boolean probe stays conservative: Parquet does not + /// define bloom filter semantics for BOOLEAN, so the encoding is not portable. + #[test] + fn test_boolean_probe_never_prunes() { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "flag", Type::Primitive(PrimitiveType::Boolean)).into(), + ]) + .build() + .unwrap(); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&true); + let filters = HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::BOOLEAN, 0))]); + + // `false` was never inserted, yet the row group must survive. + let predicate = Reference::new("flag") + .equal_to(Datum::bool(false)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!(result, "boolean probes must never prune"); + } + + /// Range and `not_eq` predicates cannot be probed, so they contribute nothing. + #[test] + fn test_does_not_collect_unprobeable_operators() { + let schema = create_test_schema(); + let predicate = Reference::new("id") + .less_than(Datum::int(1)) + .and(Reference::new("id").not_equal_to(Datum::int(2))) + .and(Reference::new("name").is_not_null()) + .bind(schema.into(), true) + .unwrap(); + + assert!(collected_ids(predicate).is_empty()); + } + + #[test] + fn test_range_predicates_always_might_match() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .less_than(Datum::int(0)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Range predicates should always return might-match"); + } + + #[test] + fn test_string_eq_present() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 2, + ColumnBloomFilter::new( + create_bloom_filter_with_values_str(&["alice", "bob"]), + PhysicalType::BYTE_ARRAY, + 0, + ), + )]); + + let predicate = Reference::new("name") + .equal_to(Datum::string("alice")) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Should might-match when string is in bloom filter"); + } + + #[test] + fn test_string_eq_absent() { + let schema = create_test_schema(); + let bloom_filters = HashMap::from([( + 2, + ColumnBloomFilter::new( + create_bloom_filter_with_values_str(&["alice", "bob"]), + PhysicalType::BYTE_ARRAY, + 0, + ), + )]); + + let predicate = Reference::new("name") + .equal_to(Datum::string("charlie")) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "Should not match when string is absent from bloom filter" + ); + } + + // --- Decimal tests --- + + fn create_decimal_schema(precision: u32, scale: u32) -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "amount", + Type::Primitive(PrimitiveType::Decimal { precision, scale }), + ) + .into(), + ]) + .build() + .unwrap() + } + + /// Decimal with precision <= 9 is stored as INT32 in Parquet. + /// The bloom filter contains i32 values (the unscaled mantissa). + #[test] + fn test_decimal_int32_present() { + let schema = create_decimal_schema(9, 2); + + // Parquet stores decimal(9,2) as INT32 with unscaled value + // Value "123.45" has mantissa 12345 + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i32); + sbbf.insert(&67890_i32); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT32, 0))]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(12345, 2), + 9, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Decimal INT32 value present should might-match"); + } + + #[test] + fn test_decimal_int32_absent() { + let schema = create_decimal_schema(9, 2); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i32); + sbbf.insert(&67890_i32); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT32, 0))]); + + // Value "999.99" has mantissa 99999, not in the bloom filter + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(99999, 2), + 9, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "Decimal INT32 value absent should not match"); + } + + /// Decimal with precision 10-18 is stored as INT64 in Parquet. + #[test] + fn test_decimal_int64_present() { + let schema = create_decimal_schema(15, 2); + + // "1234567890123.45" has mantissa 123456789012345 + let mantissa: i64 = 123456789012345; + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&mantissa); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0))]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa as i128, 2), + 15, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Decimal INT64 value present should might-match"); + } + + #[test] + fn test_decimal_int64_absent() { + let schema = create_decimal_schema(15, 2); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&123456789012345_i64); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0))]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(999999999999999, 2), + 15, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "Decimal INT64 value absent should not match"); + } + + /// Decimal with precision 19+ is stored as FIXED_LEN_BYTE_ARRAY in Parquet. + #[test] + fn test_decimal_fixed_bytes_present() { + let schema = create_decimal_schema(25, 2); + + // Large mantissa that requires FIXED_LEN_BYTE_ARRAY + let mantissa: i128 = 12345678901234567890; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 11).unwrap(); + let type_length = bytes.len() as i32; + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, type_length), + )]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa, 2), + 25, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Decimal FIXED_LEN_BYTE_ARRAY value present should might-match" + ); + } + + #[test] + fn test_decimal_fixed_bytes_absent() { + let schema = create_decimal_schema(25, 2); + + let mantissa: i128 = 12345678901234567890; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 11).unwrap(); + let type_length = bytes.len() as i32; + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, type_length), + )]); + + // Different value not in the bloom filter + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale( + 99999999999999999999, + 2, + ), + 25, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + !result, + "Decimal FIXED_LEN_BYTE_ARRAY value absent should not match" + ); + } + + fn create_uuid_schema() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "u", Type::Primitive(PrimitiveType::Uuid)).into(), + ]) + .build() + .unwrap() + } + + /// The probe hands the 16 big-endian bytes to `check` as a slice while the writer + /// inserts them as a `ByteArray`; both hash the same bytes, so a present UUID must + /// still might-match. + #[test] + fn test_uuid_fixed_bytes_present() { + let uuid = uuid::Uuid::parse_str("f79c3e09-677c-4bbd-a479-3f349cb785e7").unwrap(); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(uuid.as_u128().to_be_bytes().to_vec())); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, 16), + )]); + + let predicate = Reference::new("u") + .equal_to(Datum::uuid(uuid)) + .bind(create_uuid_schema().into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "UUID present should might-match"); + } + + #[test] + fn test_uuid_fixed_bytes_absent() { + let present = uuid::Uuid::parse_str("f79c3e09-677c-4bbd-a479-3f349cb785e7").unwrap(); + let absent = uuid::Uuid::parse_str("00000000-0000-4000-8000-000000000001").unwrap(); + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(present.as_u128().to_be_bytes().to_vec())); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, 16), + )]); + + let predicate = Reference::new("u") + .equal_to(Datum::uuid(absent)) + .bind(create_uuid_schema().into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "UUID absent should not match"); + } + + /// Negative decimal values should also work correctly. + #[test] + fn test_decimal_negative_int32() { + let schema = create_decimal_schema(9, 2); + + // "-123.45" has mantissa -12345 + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&(-12345_i32)); + + let bloom_filters = + HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT32, 0))]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(-12345, 2), + 9, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "Negative decimal INT32 present should might-match"); + } + + #[test] + fn test_decimal_negative_fixed_bytes() { + let schema = create_decimal_schema(25, 2); + + let mantissa: i128 = -12345678901234567890; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 11).unwrap(); + let type_length = bytes.len() as i32; + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, type_length), + )]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa, 2), + 25, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Negative decimal FIXED_LEN_BYTE_ARRAY present should might-match" + ); + } + + /// A Parquet writer other than arrow-rs (e.g. Spark, Java Parquet) might + /// choose FIXED_LEN_BYTE_ARRAY for a low-precision decimal rather than INT32. + /// The evaluator must use the physical type from the file metadata, not assume + /// a mapping based on precision. + #[test] + fn test_decimal_low_precision_stored_as_fixed_len_byte_array() { + let schema = create_decimal_schema(5, 2); + + // Simulate a file where decimal(5,2) was stored as FIXED_LEN_BYTE_ARRAY + let mantissa: i128 = 12345; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 3).unwrap(); + let type_length = bytes.len() as i32; + + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new(sbbf, PhysicalType::FIXED_LEN_BYTE_ARRAY, type_length), + )]); + + let predicate = Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa, 2), + 5, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "Should match when physical type is FIXED_LEN_BYTE_ARRAY even for low-precision decimal" + ); + } + + /// Builds the (schema, bloom filter) pair for a `decimal` column that the file + /// stored at `file_type_length` bytes while the table schema now declares + /// `schema_precision`, as a precision-widening evolution produces. + fn widened_decimal_case( + mantissa: i128, + file_type_length: usize, + schema_precision: u32, + ) -> (Schema, HashMap) { + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, file_type_length).unwrap(); + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + let filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + sbbf, + PhysicalType::FIXED_LEN_BYTE_ARRAY, + file_type_length as i32, + ), + )]); + + (create_decimal_schema(schema_precision, 2), filters) + } + + fn decimal_eq_predicate(schema: Schema, mantissa: i128, precision: u32) -> BoundPredicate { + Reference::new("amount") + .equal_to( + Datum::decimal_with_precision( + crate::spec::decimal_utils::decimal_from_i128_with_scale(mantissa, 2), + precision, + ) + .unwrap(), + ) + .bind(schema.into(), true) + .unwrap() + } + + /// Widening a decimal's precision does not rewrite existing files, so the + /// column keeps its original `type_length`. Deriving the probe length from the + /// widened precision changed the encoding and missed every entry the writer + /// inserted, silently pruning row groups holding matching rows. + #[test] + fn test_decimal_widened_precision_present_is_not_pruned() { + let mantissa: i128 = 12345678901234567890; + // File written as decimal(20,2) -> 9 bytes; schema since widened to 38. + let (schema, filters) = widened_decimal_case(mantissa, 9, 38); + let predicate = decimal_eq_predicate(schema, mantissa, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!( + result, + "widened decimal precision must still find the value at the file's length" + ); + } + + #[test] + fn test_decimal_widened_precision_absent_still_prunes() { + let mantissa: i128 = 12345678901234567890; + let (schema, filters) = widened_decimal_case(mantissa, 9, 38); + let predicate = decimal_eq_predicate(schema, 999999999999999999, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!(!result, "genuinely absent value must still prune"); + } + + /// A value too wide for the column's `type_length` cannot be present, but + /// truncating it would probe an unrelated slot, so stay conservative. + #[test] + fn test_decimal_value_wider_than_column_does_not_prune() { + let (schema, filters) = widened_decimal_case(1234, 2, 38); + let predicate = decimal_eq_predicate(schema, i64::MAX as i128, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!(result, "value too wide for the column must not prune"); + } + + /// The `INT32` / `INT64` counterparts of the case above: after a precision + /// widening the bound literal can carry a mantissa wider than the file's + /// physical column, and a truncated probe would hash an unrelated slot. + #[test] + fn test_decimal_wider_than_int32_column_does_not_prune() { + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i32); + let filters = HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT32, 0))]); + + // File written as decimal(9,2) -> INT32; schema since widened to decimal(38,2). + let predicate = decimal_eq_predicate(create_decimal_schema(38, 2), i64::MAX as i128, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!( + result, + "mantissa too wide for an INT32 column must not prune" + ); + } + + #[test] + fn test_decimal_wider_than_int64_column_does_not_prune() { + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i64); + let filters = HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0))]); + + let predicate = + decimal_eq_predicate(create_decimal_schema(38, 2), i128::from(i64::MAX) + 1, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!( + result, + "mantissa too wide for an INT64 column must not prune" + ); + } + + /// A mantissa that does fit must still prune when genuinely absent, so the + /// `try_from` guard above does not disable pruning outright. + #[test] + fn test_decimal_within_int64_column_still_prunes() { + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&12345_i64); + let filters = HashMap::from([(1, ColumnBloomFilter::new(sbbf, PhysicalType::INT64, 0))]); + + let predicate = decimal_eq_predicate(create_decimal_schema(38, 2), 999_999, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!(!result, "absent in-range mantissa must still prune"); + } + + /// A zero or oversized `type_length` is unusable; never prune on it. + #[test] + fn test_decimal_unusable_type_length_does_not_prune() { + let mantissa: i128 = 1234; + let bytes = decimal_to_fixed_length_bytes_exact(mantissa, 4).unwrap(); + let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); + sbbf.insert(&ByteArray::from(bytes)); + + for bogus_length in [0, -1, 17] { + let filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(), + PhysicalType::FIXED_LEN_BYTE_ARRAY, + bogus_length, + ), + )]); + let predicate = decimal_eq_predicate(create_decimal_schema(38, 2), mantissa, 38); + + let result = BloomFilterEvaluator::eval(&predicate, &filters).unwrap(); + assert!( + result, + "type_length {bogus_length} is unusable and must not prune" + ); + } + } + + fn single_field_schema(name: &str, ty: PrimitiveType) -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, name, Type::Primitive(ty)).into(), + ]) + .build() + .unwrap() + } + + /// After an `int` -> `long` promotion the predicate carries a `long`, but files + /// written before the promotion store the column as `INT32` and hashed it at + /// that width. Probing at the predicate's width made every lookup miss and + /// silently pruned row groups holding matching rows. + #[test] + fn test_promoted_int_to_long_present_is_not_pruned() { + let schema = single_field_schema("id", PrimitiveType::Long); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[100, 150, 199]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::long(150)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "long predicate against an INT32 column must find the promoted value" + ); + } + + #[test] + fn test_promoted_int_to_long_absent_still_prunes() { + let schema = single_field_schema("id", PrimitiveType::Long); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[100, 150, 199]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::long(4242)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "genuinely absent value must still prune"); + } + + /// A `long` predicate outside `i32` range cannot be present in an `INT32` + /// column; stay conservative rather than probe a truncated value. + #[test] + fn test_long_out_of_int32_range_does_not_prune() { + let schema = single_field_schema("id", PrimitiveType::Long); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_i32(&[1, 2, 3]), + PhysicalType::INT32, + 0, + ), + )]); + + let predicate = Reference::new("id") + .equal_to(Datum::long(i64::from(i32::MAX) + 1)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "out-of-range long must not prune"); + } + + fn create_bloom_filter_with_values_f32(values: &[f32]) -> Sbbf { + let mut sbbf = Sbbf::new_with_ndv_fpp(values.len() as u64, 0.01).unwrap(); + for v in values { + sbbf.insert(v); + } + sbbf + } + + /// The `float` -> `double` counterpart of the `int` -> `long` promotion above. + #[test] + fn test_promoted_float_to_double_present_is_not_pruned() { + let schema = single_field_schema("val", PrimitiveType::Double); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_f32(&[1.5, 2.25, 4.0]), + PhysicalType::FLOAT, + 0, + ), + )]); + + let predicate = Reference::new("val") + .equal_to(Datum::double(1.5)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!( + result, + "double predicate against a FLOAT column must find the promoted value" + ); + } + + #[test] + fn test_promoted_float_to_double_absent_still_prunes() { + let schema = single_field_schema("val", PrimitiveType::Double); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_f32(&[1.5, 2.25, 4.0]), + PhysicalType::FLOAT, + 0, + ), + )]); + + let predicate = Reference::new("val") + .equal_to(Datum::double(9.75)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(!result, "genuinely absent value must still prune"); + } + + /// A double with no exact `f32` representation cannot equal any widened + /// `f32` in the column, so probing it would be meaningless; stay conservative. + #[test] + fn test_double_not_representable_as_f32_does_not_prune() { + let schema = single_field_schema("val", PrimitiveType::Double); + let bloom_filters = HashMap::from([( + 1, + ColumnBloomFilter::new( + create_bloom_filter_with_values_f32(&[1.5, 2.25]), + PhysicalType::FLOAT, + 0, + ), + )]); + + let predicate = Reference::new("val") + .equal_to(Datum::double(0.1)) + .bind(schema.into(), true) + .unwrap(); + + let result = BloomFilterEvaluator::eval(&predicate, &bloom_filters).unwrap(); + assert!(result, "non-representable double must not prune"); + } +} diff --git a/crates/iceberg/src/expr/visitors/bound_predicate_visitor.rs b/crates/iceberg/src/expr/visitors/bound_predicate_visitor.rs index 22f6d08ad3..1d7816dcf7 100644 --- a/crates/iceberg/src/expr/visitors/bound_predicate_visitor.rs +++ b/crates/iceberg/src/expr/visitors/bound_predicate_visitor.rs @@ -185,7 +185,7 @@ pub(crate) fn visit( PredicateOperator::IsNan => visitor.is_nan(expr.term(), predicate), PredicateOperator::NotNan => visitor.not_nan(expr.term(), predicate), op => { - panic!("Unexpected op for unary predicate: {}", &op) + panic!("Unexpected op for unary predicate: {}", op) } }, BoundPredicate::Binary(expr) => { @@ -209,7 +209,7 @@ pub(crate) fn visit( visitor.not_starts_with(reference, literal, predicate) } op => { - panic!("Unexpected op for binary predicate: {}", &op) + panic!("Unexpected op for binary predicate: {}", op) } } } @@ -220,7 +220,7 @@ pub(crate) fn visit( PredicateOperator::In => visitor.r#in(reference, literals, predicate), PredicateOperator::NotIn => visitor.not_in(reference, literals, predicate), op => { - panic!("Unexpected op for set predicate: {}", &op) + panic!("Unexpected op for set predicate: {}", op) } } } diff --git a/crates/iceberg/src/expr/visitors/expression_evaluator.rs b/crates/iceberg/src/expr/visitors/expression_evaluator.rs index 3675ce355f..d9c2e1b571 100644 --- a/crates/iceberg/src/expr/visitors/expression_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/expression_evaluator.rs @@ -280,11 +280,12 @@ mod tests { .with_spec_id(1) .add_unbound_field( UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("a".to_string()) .field_id(1) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ) .unwrap() .build() @@ -330,7 +331,7 @@ mod tests { } fn create_data_file_float() -> DataFile { - let partition = Struct::from_iter([Some(Literal::float(1.0))]); + let partition = Struct::from_iter([Some(Literal::float(1.0_f32))]); DataFile { content: DataContentType::Data, @@ -346,7 +347,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -374,7 +375,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -393,12 +394,12 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::LessThan, Reference::new("a"), - Datum::float(1.0), + Datum::float(1.0_f32), )) .or(Predicate::Binary(BinaryExpression::new( PredicateOperator::GreaterThanOrEq, Reference::new("a"), - Datum::float(0.4), + Datum::float(0.4_f32), ))) .bind(schema.clone(), case_sensitive)?; @@ -422,12 +423,12 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::LessThan, Reference::new("a"), - Datum::float(1.1), + Datum::float(1.1_f32), )) .and(Predicate::Binary(BinaryExpression::new( PredicateOperator::GreaterThanOrEq, Reference::new("a"), - Datum::float(0.4), + Datum::float(0.4_f32), ))) .bind(schema.clone(), case_sensitive)?; @@ -451,7 +452,11 @@ mod tests { let predicate = Predicate::Set(SetExpression::new( PredicateOperator::NotIn, Reference::new("a"), - FnvHashSet::from_iter([Datum::float(0.9), Datum::float(1.2), Datum::float(2.4)]), + FnvHashSet::from_iter([ + Datum::float(0.9_f32), + Datum::float(1.2_f32), + Datum::float(2.4_f32), + ]), )) .bind(schema.clone(), case_sensitive)?; @@ -475,7 +480,11 @@ mod tests { let predicate = Predicate::Set(SetExpression::new( PredicateOperator::In, Reference::new("a"), - FnvHashSet::from_iter([Datum::float(1.0), Datum::float(1.2), Datum::float(2.4)]), + FnvHashSet::from_iter([ + Datum::float(1.0_f32), + Datum::float(1.2_f32), + Datum::float(2.4_f32), + ]), )) .bind(schema.clone(), case_sensitive)?; @@ -547,7 +556,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::NotEq, Reference::new("a"), - Datum::float(0.9), + Datum::float(0.9_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -571,7 +580,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::Eq, Reference::new("a"), - Datum::float(1.0), + Datum::float(1.0_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -587,6 +596,76 @@ mod tests { Ok(()) } + #[test] + fn test_expr_eq_signed_zero() -> Result<()> { + let case_sensitive = true; + let (partition_spec, schema) = create_partition_spec(PrimitiveType::Float)?; + + let data_file = DataFile { + partition: Struct::from_iter([Some(Literal::float(-0.0_f32))]), + ..create_data_file_float() + }; + + // -0.0 and 0.0 are different values, as in iceberg-java's evaluator. + for (literal, expected) in [(0.0_f32, false), (-0.0_f32, true)] { + let predicate = Predicate::Binary(BinaryExpression::new( + PredicateOperator::Eq, + Reference::new("a"), + Datum::float(literal), + )) + .bind(schema.clone(), case_sensitive)?; + + let expression_evaluator = create_expression_evaluator( + partition_spec.clone(), + &schema, + &predicate, + case_sensitive, + )?; + + assert_eq!(expression_evaluator.eval(&data_file)?, expected); + } + + Ok(()) + } + + #[test] + fn test_expr_in_signed_zero() -> Result<()> { + let case_sensitive = true; + let (partition_spec, schema) = create_partition_spec(PrimitiveType::Float)?; + + let data_file = DataFile { + partition: Struct::from_iter([Some(Literal::float(-0.0_f32))]), + ..create_data_file_float() + }; + + // -0.0 and 0.0 are different values, so `IN` keeps both and matches only its own zero. + for (literals, expected) in [ + ([0.0_f32, 1.0_f32], false), + ([-0.0_f32, 1.0_f32], true), + ([0.0_f32, -0.0_f32], true), + ] { + let predicate = Predicate::Set(SetExpression::new( + PredicateOperator::In, + Reference::new("a"), + FnvHashSet::from_iter(literals.map(Datum::float)), + )) + .bind(schema.clone(), case_sensitive)?; + + assert!(matches!(&predicate, BoundPredicate::Set(expr) if expr.literals().len() == 2)); + + let expression_evaluator = create_expression_evaluator( + partition_spec.clone(), + &schema, + &predicate, + case_sensitive, + )?; + + assert_eq!(expression_evaluator.eval(&data_file)?, expected); + } + + Ok(()) + } + #[test] fn test_expr_greater_than_or_eq() -> Result<()> { let case_sensitive = true; @@ -595,7 +674,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::GreaterThanOrEq, Reference::new("a"), - Datum::float(1.0), + Datum::float(1.0_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -619,7 +698,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::GreaterThan, Reference::new("a"), - Datum::float(0.9), + Datum::float(0.9_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -643,7 +722,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::LessThanOrEq, Reference::new("a"), - Datum::float(1.0), + Datum::float(1.0_f32), )) .bind(schema.clone(), case_sensitive)?; @@ -667,7 +746,7 @@ mod tests { let predicate = Predicate::Binary(BinaryExpression::new( PredicateOperator::LessThan, Reference::new("a"), - Datum::float(1.1), + Datum::float(1.1_f32), )) .bind(schema.clone(), case_sensitive)?; diff --git a/crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs index 2b65cf12aa..37546fa2cb 100644 --- a/crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/inclusive_metrics_evaluator.rs @@ -15,8 +15,11 @@ // specific language governing permissions and limitations // under the License. +use std::borrow::Cow; + use fnv::FnvHashSet; +use super::scalar_bound; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::{BoundPredicate, BoundReference}; use crate::spec::{DataFile, Datum, PrimitiveLiteral}; @@ -65,11 +68,11 @@ impl<'a> InclusiveMetricsEvaluator<'a> { } fn lower_bound(&self, field_id: i32) -> Option<&Datum> { - self.data_file.lower_bounds.get(&field_id) + scalar_bound(self.data_file.lower_bounds.get(&field_id)) } fn upper_bound(&self, field_id: i32) -> Option<&Datum> { - self.data_file.upper_bounds.get(&field_id) + scalar_bound(self.data_file.upper_bounds.get(&field_id)) } fn contains_nans_only(&self, field_id: i32) -> bool { @@ -120,6 +123,7 @@ impl<'a> InclusiveMetricsEvaluator<'a> { }; if let Some(bound) = bound { + let bound = &*Self::promote_bound(bound, reference); if cmp_fn(bound, datum) { return ROWS_MIGHT_MATCH; } @@ -129,6 +133,19 @@ impl<'a> InclusiveMetricsEvaluator<'a> { ROWS_MIGHT_MATCH } + + /// Widens a stored bound to the reference's type so a bound written under a + /// narrower type (e.g. an `int` column later promoted to `long`) compares + /// against a predicate bound to the wider type. Borrows the bound when nothing + /// is promoted, so unpromoted bounds (e.g. strings/binaries) are not copied. + fn promote_bound<'b>(bound: &'b Datum, reference: &BoundReference) -> Cow<'b, Datum> { + reference + .field() + .field_type + .as_primitive_type() + .and_then(|primitive_type| bound.promote_to(primitive_type)) + .map_or(Cow::Borrowed(bound), Cow::Owned) + } } impl BoundPredicateVisitor for InclusiveMetricsEvaluator<'_> { @@ -259,6 +276,7 @@ impl BoundPredicateVisitor for InclusiveMetricsEvaluator<'_> { } if let Some(lower_bound) = self.lower_bound(field_id) { + let lower_bound = &*Self::promote_bound(lower_bound, reference); if lower_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -269,6 +287,7 @@ impl BoundPredicateVisitor for InclusiveMetricsEvaluator<'_> { } if let Some(upper_bound) = self.upper_bound(field_id) { + let upper_bound = &*Self::promote_bound(upper_bound, reference); if upper_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -437,28 +456,21 @@ impl BoundPredicateVisitor for InclusiveMetricsEvaluator<'_> { return ROWS_MIGHT_MATCH; } - if let Some(lower_bound) = self.lower_bound(field_id) { - if lower_bound.is_nan() { - // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. - return ROWS_MIGHT_MATCH; - } - - if !literals.iter().any(|datum| datum.ge(lower_bound)) { - // if all values are less than lower bound, rows cannot match. - return ROWS_CANNOT_MATCH; - } - } - - if let Some(upper_bound) = self.upper_bound(field_id) { - if upper_bound.is_nan() { - // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. - return ROWS_MIGHT_MATCH; - } - - if !literals.iter().any(|datum| datum.le(upper_bound)) { - // if all values are greater than upper bound, rows cannot match. - return ROWS_CANNOT_MATCH; - } + let lower_bound = self + .lower_bound(field_id) + .map(|bound| Self::promote_bound(bound, reference)); + let upper_bound = self + .upper_bound(field_id) + .map(|bound| Self::promote_bound(bound, reference)); + + // A NaN bound is unreliable on that side only. Drop it to unbounded + // so a valid bound can still prune. + if super::all_literals_outside_bounds( + super::finite_bound(lower_bound.as_deref()), + super::finite_bound(upper_bound.as_deref()), + literals, + ) { + return ROWS_CANNOT_MATCH; } ROWS_MIGHT_MATCH @@ -521,6 +533,92 @@ mod test { assert!(result); } + fn promotion_schema(primitive: PrimitiveType) -> SchemaRef { + Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::optional( + 1, + "a", + Type::Primitive(primitive), + ))]) + .build() + .unwrap(), + ) + } + + fn data_file_with_bounds(lower: Datum, upper: Datum) -> DataFile { + DataFile { + lower_bounds: HashMap::from([(1, lower)]), + upper_bounds: HashMap::from([(1, upper)]), + ..create_test_data_file() + } + } + + #[test] + fn eval_promotes_int_bounds_for_inequality() { + // A column promoted int->long keeps int-typed column bounds in + // pre-promotion manifests; a predicate bound to the promoted long type + // must still match a file whose range covers it, not silently prune it. + let schema = promotion_schema(PrimitiveType::Long); + let data_file = data_file_with_bounds(Datum::int(5), Datum::int(10)); + let predicate = Predicate::Binary(BinaryExpression::new( + GreaterThan, + Reference::new("a"), + Datum::long(4), + )) + .bind(schema, true) + .unwrap(); + + assert!(InclusiveMetricsEvaluator::eval(&predicate, &data_file, true).unwrap()); + } + + #[test] + fn eval_promotes_int_bounds_for_in() { + let schema = promotion_schema(PrimitiveType::Long); + let data_file = data_file_with_bounds(Datum::int(5), Datum::int(10)); + let predicate = Predicate::Set(SetExpression::new( + In, + Reference::new("a"), + FnvHashSet::from_iter([Datum::long(5), Datum::long(6)]), + )) + .bind(schema, true) + .unwrap(); + + assert!(InclusiveMetricsEvaluator::eval(&predicate, &data_file, true).unwrap()); + } + + #[test] + fn eval_promotes_int_bounds_for_eq() { + // With the bounds widened, `== 5` correctly prunes a file whose range is + // entirely above 5 (int bounds [6, 10]) instead of failing open. + let schema = promotion_schema(PrimitiveType::Long); + let data_file = data_file_with_bounds(Datum::int(6), Datum::int(10)); + let predicate = Predicate::Binary(BinaryExpression::new( + Eq, + Reference::new("a"), + Datum::long(5), + )) + .bind(schema, true) + .unwrap(); + + assert!(!InclusiveMetricsEvaluator::eval(&predicate, &data_file, true).unwrap()); + } + + #[test] + fn eval_promotes_float_bounds_for_inequality() { + let schema = promotion_schema(PrimitiveType::Double); + let data_file = data_file_with_bounds(Datum::float(1.5_f32), Datum::float(2.0_f32)); + let predicate = Predicate::Binary(BinaryExpression::new( + GreaterThan, + Reference::new("a"), + Datum::double(1.0), + )) + .bind(schema, true) + .unwrap(); + + assert!(InclusiveMetricsEvaluator::eval(&predicate, &data_file, true).unwrap()); + } + #[test] fn test_all_nulls() { let result = @@ -1551,6 +1649,59 @@ mod test { ); } + #[test] + fn test_integer_in_straddling_bounds() { + let result = InclusiveMetricsEvaluator::eval( + &r#in_int("id", &[INT_MIN_VALUE - 25, INT_MAX_VALUE + 25]), + &get_test_file_1(), + true, + ) + .unwrap(); + assert!(!result, "Should skip: id in (5, 104), bounds are [30, 79]"); + } + + #[test] + fn test_float_in_nan_upper_bound_prunes_below_lower() { + let result = InclusiveMetricsEvaluator::eval( + &r#in_float("no_nans", &[2.0, 3.0]), + &get_test_file_float_nan_upper(), + true, + ) + .unwrap(); + assert!( + !result, + "Should skip: NaN upper is unbounded, both literals are below lower 4.0" + ); + } + + #[test] + fn test_float_in_nan_lower_bound_prunes_above_upper() { + let result = InclusiveMetricsEvaluator::eval( + &r#in_float("no_nans", &[2.0, 3.0]), + &get_test_file_float_nan_lower(), + true, + ) + .unwrap(); + assert!( + !result, + "Should skip: NaN lower is unbounded, both literals are above upper 1.0" + ); + } + + #[test] + fn test_float_in_nan_lower_bound_does_not_prune_when_a_literal_is_inside_upper() { + let result = InclusiveMetricsEvaluator::eval( + &r#in_float("no_nans", &[2.0, 4.0]), + &get_test_file_float_nan_lower_upper_3(), + true, + ) + .unwrap(); + assert!( + result, + "Should read: NaN lower is unbounded, 2.0 is inside upper 3.0" + ); + } + #[test] fn test_integer_not_in() { let result = InclusiveMetricsEvaluator::eval( @@ -1661,11 +1812,12 @@ mod test { .with_spec_id(1) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("a".to_string()) .field_id(1) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ]) .unwrap() .build() @@ -1880,6 +2032,16 @@ mod test { filter.bind(schema.clone(), true).unwrap() } + fn in_float(reference: &str, float_literals: &[f32]) -> BoundPredicate { + let schema = create_test_schema(); + let filter = Predicate::Set(SetExpression::new( + In, + Reference::new(reference), + FnvHashSet::from_iter(float_literals.iter().copied().map(Datum::float)), + )); + filter.bind(schema.clone(), true).unwrap() + } + fn not_in_int(reference: &str, int_literals: &[i32]) -> BoundPredicate { let schema = create_test_schema(); let filter = Predicate::Set(SetExpression::new( @@ -1995,7 +2157,7 @@ mod test { lower_bounds: Default::default(), upper_bounds: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -2021,7 +2183,7 @@ mod test { lower_bounds: Default::default(), upper_bounds: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -2083,7 +2245,85 @@ mod test { column_sizes: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, + equality_ids: None, + sort_order_id: None, + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + } + } + + fn get_test_file_float_nan_upper() -> DataFile { + DataFile { + content: DataContentType::Data, + file_path: "/test/path".to_string(), + file_format: DataFileFormat::Parquet, + partition: Struct::empty(), + record_count: 10, + file_size_in_bytes: 10, + column_sizes: Default::default(), + value_counts: HashMap::from([(9, 10)]), + null_value_counts: HashMap::from([(9, 0)]), + nan_value_counts: HashMap::from([(9, 0)]), + lower_bounds: HashMap::from([(9, Datum::float(4.0_f32))]), + upper_bounds: HashMap::from([(9, Datum::float(f32::NAN))]), + key_metadata: None, + split_offsets: None, + equality_ids: None, + sort_order_id: None, + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + } + } + + fn get_test_file_float_nan_lower() -> DataFile { + DataFile { + content: DataContentType::Data, + file_path: "/test/path".to_string(), + file_format: DataFileFormat::Parquet, + partition: Struct::empty(), + record_count: 10, + file_size_in_bytes: 10, + column_sizes: Default::default(), + value_counts: HashMap::from([(9, 10)]), + null_value_counts: HashMap::from([(9, 0)]), + nan_value_counts: HashMap::from([(9, 0)]), + lower_bounds: HashMap::from([(9, Datum::float(f32::NAN))]), + upper_bounds: HashMap::from([(9, Datum::float(1.0_f32))]), + key_metadata: None, + split_offsets: None, + equality_ids: None, + sort_order_id: None, + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + } + } + + fn get_test_file_float_nan_lower_upper_3() -> DataFile { + DataFile { + content: DataContentType::Data, + file_path: "/test/path".to_string(), + file_format: DataFileFormat::Parquet, + partition: Struct::empty(), + record_count: 10, + file_size_in_bytes: 10, + column_sizes: Default::default(), + value_counts: HashMap::from([(9, 10)]), + null_value_counts: HashMap::from([(9, 0)]), + nan_value_counts: HashMap::from([(9, 0)]), + lower_bounds: HashMap::from([(9, Datum::float(f32::NAN))]), + upper_bounds: HashMap::from([(9, Datum::float(3.0_f32))]), + key_metadata: None, + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -2093,6 +2333,7 @@ mod test { content_size_in_bytes: None, } } + fn get_test_file_2() -> DataFile { DataFile { content: DataContentType::Data, @@ -2114,7 +2355,7 @@ mod test { column_sizes: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -2146,7 +2387,7 @@ mod test { column_sizes: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -2178,7 +2419,7 @@ mod test { column_sizes: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, diff --git a/crates/iceberg/src/expr/visitors/inclusive_projection.rs b/crates/iceberg/src/expr/visitors/inclusive_projection.rs index d9544e4c47..88508af844 100644 --- a/crates/iceberg/src/expr/visitors/inclusive_projection.rs +++ b/crates/iceberg/src/expr/visitors/inclusive_projection.rs @@ -304,11 +304,12 @@ mod tests { .with_spec_id(1) .add_unbound_field( UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("a".to_string()) .field_id(1) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ) .unwrap() .build() @@ -339,12 +340,15 @@ mod tests { let partition_spec = PartitionSpec::builder(arc_schema.clone()) .with_spec_id(1) - .add_unbound_fields(vec![UnboundPartitionField { - source_id: 2, - name: "year".to_string(), - field_id: Some(1000), - transform: Transform::Year, - }]) + .add_unbound_fields(vec![ + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("year".to_string()) + .transform(Transform::Year) + .build() + .unwrap(), + ]) .unwrap() .build() .unwrap(); @@ -374,12 +378,15 @@ mod tests { let partition_spec = PartitionSpec::builder(arc_schema.clone()) .with_spec_id(1) - .add_unbound_fields(vec![UnboundPartitionField { - source_id: 2, - name: "month".to_string(), - field_id: Some(1000), - transform: Transform::Month, - }]) + .add_unbound_fields(vec![ + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("month".to_string()) + .transform(Transform::Month) + .build() + .unwrap(), + ]) .unwrap() .build() .unwrap(); @@ -409,12 +416,15 @@ mod tests { let partition_spec = PartitionSpec::builder(arc_schema.clone()) .with_spec_id(1) - .add_unbound_fields(vec![UnboundPartitionField { - source_id: 2, - name: "day".to_string(), - field_id: Some(1000), - transform: Transform::Day, - }]) + .add_unbound_fields(vec![ + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("day".to_string()) + .transform(Transform::Day) + .build() + .unwrap(), + ]) .unwrap() .build() .unwrap(); @@ -446,11 +456,12 @@ mod tests { .with_spec_id(1) .add_unbound_field( UnboundPartitionField::builder() - .source_id(3) + .source_ids(vec![3]) .name("name_truncate".to_string()) .field_id(3) .transform(Transform::Truncate(4)) - .build(), + .build() + .unwrap(), ) .unwrap() .build() @@ -486,11 +497,12 @@ mod tests { .with_spec_id(1) .add_unbound_field( UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("a_bucket[7]".to_string()) .field_id(1) .transform(Transform::Bucket(7)) - .build(), + .build() + .unwrap(), ) .unwrap() .build() diff --git a/crates/iceberg/src/expr/visitors/manifest_evaluator.rs b/crates/iceberg/src/expr/visitors/manifest_evaluator.rs index 903c9ea4a7..b9140d591d 100644 --- a/crates/iceberg/src/expr/visitors/manifest_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/manifest_evaluator.rs @@ -108,34 +108,30 @@ const IN_PREDICATE_LIMIT: usize = 200; impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { type T = bool; - fn always_true(&mut self) -> crate::Result { + fn always_true(&mut self) -> Result { ROWS_MIGHT_MATCH } - fn always_false(&mut self) -> crate::Result { + fn always_false(&mut self) -> Result { ROWS_CANNOT_MATCH } - fn and(&mut self, lhs: bool, rhs: bool) -> crate::Result { + fn and(&mut self, lhs: bool, rhs: bool) -> Result { Ok(lhs && rhs) } - fn or(&mut self, lhs: bool, rhs: bool) -> crate::Result { + fn or(&mut self, lhs: bool, rhs: bool) -> Result { Ok(lhs || rhs) } - fn not(&mut self, _: bool) -> crate::Result { + fn not(&mut self, _: bool) -> Result { Err(Error::new( ErrorKind::Unexpected, "not operator is not supported in partition filter", )) } - fn is_null( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> crate::Result { + fn is_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { Ok(self.field_summary_for_reference(reference).contains_null) } @@ -143,7 +139,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { &mut self, reference: &BoundReference, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field = self.field_summary_for_reference(reference); // contains_null encodes whether at least one partition value is null, @@ -155,16 +151,12 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { } } - fn is_nan( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> crate::Result { + fn is_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { let field = self.field_summary_for_reference(reference); - if let Some(contains_nan) = field.contains_nan { - if !contains_nan { - return ROWS_CANNOT_MATCH; - } + if let Some(contains_nan) = field.contains_nan + && !contains_nan + { + return ROWS_CANNOT_MATCH; } if ManifestFilterVisitor::are_all_null(field, &reference.field().field_type) { @@ -174,11 +166,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { ROWS_MIGHT_MATCH } - fn not_nan( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> crate::Result { + fn not_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { let field = self.field_summary_for_reference(reference); if let Some(contains_nan) = field.contains_nan { // check if all values are nan @@ -194,14 +182,14 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field = self.field_summary_for_reference(reference); match &field.lower_bound { Some(bound_bytes) => { let bound = ManifestFilterVisitor::bytes_to_datum( bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if datum <= &bound { ROWS_CANNOT_MATCH @@ -218,13 +206,13 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field = self.field_summary_for_reference(reference); match &field.lower_bound { Some(bound_bytes) => { let bound = ManifestFilterVisitor::bytes_to_datum( bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if datum < &bound { ROWS_CANNOT_MATCH @@ -241,13 +229,13 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field = self.field_summary_for_reference(reference); match &field.upper_bound { Some(bound_bytes) => { let bound = ManifestFilterVisitor::bytes_to_datum( bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if datum >= &bound { ROWS_CANNOT_MATCH @@ -264,13 +252,13 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field = self.field_summary_for_reference(reference); match &field.upper_bound { Some(bound_bytes) => { let bound = ManifestFilterVisitor::bytes_to_datum( bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if datum > &bound { ROWS_CANNOT_MATCH @@ -287,7 +275,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field = self.field_summary_for_reference(reference); if field.lower_bound.is_none() || field.upper_bound.is_none() { @@ -297,7 +285,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { if let Some(lower_bound_bytes) = &field.lower_bound { let lower_bound = ManifestFilterVisitor::bytes_to_datum( lower_bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if &lower_bound > datum { return ROWS_CANNOT_MATCH; @@ -307,7 +295,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { if let Some(upper_bound_bytes) = &field.upper_bound { let upper_bound = ManifestFilterVisitor::bytes_to_datum( upper_bound_bytes, - *reference.field().field_type.clone(), + &reference.field().field_type, ); if &upper_bound < datum { return ROWS_CANNOT_MATCH; @@ -322,7 +310,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { _reference: &BoundReference, _datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { // because the bounds are not necessarily a min or max value, this cannot be answered using // them. notEq(col, X) with (X, Y) doesn't guarantee that X is a value in col. ROWS_MIGHT_MATCH @@ -333,7 +321,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field = self.field_summary_for_reference(reference); if field.lower_bound.is_none() || field.upper_bound.is_none() { @@ -368,7 +356,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field = self.field_summary_for_reference(reference); if field.contains_null || field.lower_bound.is_none() || field.upper_bound.is_none() { @@ -389,16 +377,16 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { return ROWS_MIGHT_MATCH; } - if prefix.as_bytes().eq(&lower_bound[..prefix_len]) { - if let Some(upper_bound) = &field.upper_bound { - // if upper is shorter than the prefix then upper can't start with the prefix - if prefix_len > upper_bound.len() { - return ROWS_MIGHT_MATCH; - } + if prefix.as_bytes().eq(&lower_bound[..prefix_len]) + && let Some(upper_bound) = &field.upper_bound + { + // if upper is shorter than the prefix then upper can't start with the prefix + if prefix_len > upper_bound.len() { + return ROWS_MIGHT_MATCH; + } - if prefix.as_bytes().eq(&upper_bound[..prefix_len]) { - return ROWS_CANNOT_MATCH; - } + if prefix.as_bytes().eq(&upper_bound[..prefix_len]) { + return ROWS_CANNOT_MATCH; } } } @@ -411,8 +399,11 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { reference: &BoundReference, literals: &FnvHashSet, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field = self.field_summary_for_reference(reference); + // A missing lower_bound covers all-null, all-NaN, and mixed null+NaN + // summaries (Iceberg spec). Those cases prune here and never reach + // all_literals_outside_bounds. if field.lower_bound.is_none() { return ROWS_CANNOT_MATCH; } @@ -421,24 +412,16 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { return ROWS_MIGHT_MATCH; } - if let Some(lower_bound) = &field.lower_bound { - let lower_bound = ManifestFilterVisitor::bytes_to_datum( - lower_bound, - *reference.field().clone().field_type, - ); - if literals.iter().all(|datum| &lower_bound > datum) { - return ROWS_CANNOT_MATCH; - } - } + let lower_bound = field.lower_bound.as_ref().map(|bound| { + ManifestFilterVisitor::bytes_to_datum(bound, reference.field().field_type.as_ref()) + }); + let upper_bound = field.upper_bound.as_ref().map(|bound| { + ManifestFilterVisitor::bytes_to_datum(bound, reference.field().field_type.as_ref()) + }); - if let Some(upper_bound) = &field.upper_bound { - let upper_bound = ManifestFilterVisitor::bytes_to_datum( - upper_bound, - *reference.field().clone().field_type, - ); - if literals.iter().all(|datum| &upper_bound < datum) { - return ROWS_CANNOT_MATCH; - } + if super::all_literals_outside_bounds(lower_bound.as_ref(), upper_bound.as_ref(), literals) + { + return ROWS_CANNOT_MATCH; } ROWS_MIGHT_MATCH @@ -449,7 +432,7 @@ impl BoundPredicateVisitor for ManifestFilterVisitor<'_> { _reference: &BoundReference, _literals: &FnvHashSet, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { // because the bounds are not necessarily a min or max value, this cannot be answered using // them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col. ROWS_MIGHT_MATCH @@ -479,14 +462,14 @@ impl ManifestFilterVisitor<'_> { all_null } - fn datum_as_str<'a>(bound: &'a Datum, err_msg: &str) -> crate::Result<&'a String> { + fn datum_as_str<'a>(bound: &'a Datum, err_msg: &str) -> Result<&'a String> { let PrimitiveLiteral::String(bound) = bound.literal() else { return Err(Error::new(ErrorKind::Unexpected, err_msg)); }; Ok(bound) } - fn bytes_to_datum(bytes: &ByteBuf, t: Type) -> Datum { + fn bytes_to_datum(bytes: &ByteBuf, t: &Type) -> Datum { let p = t.as_primitive_type().unwrap(); Datum::try_from_bytes(bytes, p.clone()).unwrap() } @@ -624,8 +607,8 @@ mod test { FieldSummary { contains_null: true, contains_nan: None, - lower_bound: Some(Datum::float(0.0).to_bytes().unwrap()), - upper_bound: Some(Datum::float(20.0).to_bytes().unwrap()), + lower_bound: Some(Datum::float(0.0_f32).to_bytes().unwrap()), + upper_bound: Some(Datum::float(20.0_f32).to_bytes().unwrap()), }, // all_nulls_double FieldSummary { @@ -659,8 +642,8 @@ mod test { FieldSummary { contains_null: false, contains_nan: Some(false), - lower_bound: Some(Datum::float(0.0).to_bytes().unwrap()), - upper_bound: Some(Datum::float(20.0).to_bytes().unwrap()), + lower_bound: Some(Datum::float(0.0_f32).to_bytes().unwrap()), + upper_bound: Some(Datum::float(20.0_f32).to_bytes().unwrap()), }, // all_nulls_missing_nan_float FieldSummary { @@ -703,6 +686,7 @@ mod test { deleted_rows_count: None, partitions: Some(partitions), key_metadata: None, + first_row_id: None, } } @@ -1386,6 +1370,30 @@ mod test { Ok(()) } + #[test] + fn test_in_straddling_bounds() -> Result<()> { + let case_sensitive = true; + let schema = create_schema()?; + let manifest_file = create_manifest_file(create_partitions()); + + let filter = Predicate::Set(SetExpression::new( + PredicateOperator::In, + Reference::new("id"), + FnvHashSet::from_iter(vec![ + Datum::int(INT_MIN_VALUE - 25), + Datum::int(INT_MAX_VALUE + 25), + ]), + )) + .bind(schema.clone(), case_sensitive)?; + assert!( + !ManifestEvaluator::builder(filter) + .build() + .eval(&manifest_file)?, + "Should not read: id in (5, 104), summary is [30, 79]" + ); + Ok(()) + } + #[test] fn test_not_in() -> Result<()> { let case_sensitive = true; diff --git a/crates/iceberg/src/expr/visitors/mod.rs b/crates/iceberg/src/expr/visitors/mod.rs index 38332ea6a0..e29c959a17 100644 --- a/crates/iceberg/src/expr/visitors/mod.rs +++ b/crates/iceberg/src/expr/visitors/mod.rs @@ -15,6 +15,11 @@ // specific language governing permissions and limitations // under the License. +use fnv::FnvHashSet; + +use crate::spec::{Datum, PrimitiveType}; + +pub(crate) mod bloom_filter_evaluator; pub(crate) mod bound_predicate_visitor; pub(crate) mod expression_evaluator; pub(crate) mod inclusive_metrics_evaluator; @@ -26,3 +31,136 @@ pub(crate) mod rewrite_not; pub(crate) mod row_group_metrics_evaluator; pub(crate) mod strict_metrics_evaluator; pub(crate) mod strict_projection; + +fn scalar_bound(bound: Option<&Datum>) -> Option<&Datum> { + // Geospatial bounds encode bounding-box coordinates, not ordered column values. + bound.filter(|datum| { + !matches!( + datum.data_type(), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) + ) + }) +} + +/// Returns true if every literal is provably outside the inclusive `[lower, upper]` +/// range, so rows cannot match. Missing bounds are treated as unbounded on that side. +/// +/// Returns false when there are no bounds, or when a literal does not compare +/// with a bound. Incomparable values must not be pruned. +pub(crate) fn all_literals_outside_bounds( + lower: Option<&Datum>, + upper: Option<&Datum>, + literals: &FnvHashSet, +) -> bool { + match (lower, upper) { + (Some(lower), Some(upper)) => literals + .iter() + .all(|datum| datum.lt(lower) || datum.gt(upper)), + (Some(lower), None) => literals.iter().all(|datum| datum.lt(lower)), + (None, Some(upper)) => literals.iter().all(|datum| datum.gt(upper)), + (None, None) => false, + } +} + +/// Drops a NaN bound so that side is treated as unbounded. +/// +/// A NaN min or max is unreliable, but the other bound may still prune. +/// Inclusive evaluators in Java and PyIceberg bail to might-match when +/// either bound is NaN. Dropping only the NaN side is a deliberate +/// divergence: `total_cmp` treats NaN as the maximum, so the remaining +/// finite bound is still a valid prune. +pub(crate) fn finite_bound(bound: Option<&Datum>) -> Option<&Datum> { + bound.filter(|datum| !datum.is_nan()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn floats(vals: &[f32]) -> FnvHashSet { + vals.iter().copied().map(Datum::float).collect() + } + + #[test] + fn both_bounds_require_a_literal_inside_the_range() { + let lower = Datum::float(4.0_f32); + let upper = Datum::float(6.0_f32); + assert!(all_literals_outside_bounds( + Some(&lower), + Some(&upper), + &floats(&[2.0, 8.0]) + )); + assert!(!all_literals_outside_bounds( + Some(&lower), + Some(&upper), + &floats(&[2.0, 5.0]) + )); + } + + #[test] + fn lower_only_prunes_literals_below_the_bound() { + let lower = Datum::float(4.0_f32); + assert!(all_literals_outside_bounds( + Some(&lower), + None, + &floats(&[2.0, 3.0]) + )); + assert!(!all_literals_outside_bounds( + Some(&lower), + None, + &floats(&[2.0, 4.0]) + )); + } + + #[test] + fn upper_only_prunes_literals_above_the_bound() { + let upper = Datum::float(1.0_f32); + assert!(all_literals_outside_bounds( + None, + Some(&upper), + &floats(&[2.0, 3.0]) + )); + assert!(!all_literals_outside_bounds( + None, + Some(&upper), + &floats(&[0.5, 3.0]) + )); + } + + #[test] + fn neither_bound_cannot_prune() { + assert!(!all_literals_outside_bounds( + None, + None, + &floats(&[2.0, 3.0]) + )); + } + + #[test] + fn incomparable_literal_blocks_pruning() { + let lower = Datum::int(1); + let upper = Datum::int(3); + let string_only: FnvHashSet = [Datum::string("a")].into_iter().collect(); + assert!(!all_literals_outside_bounds( + Some(&lower), + Some(&upper), + &string_only + )); + + let mixed: FnvHashSet = [Datum::int(0), Datum::string("a")].into_iter().collect(); + assert!(!all_literals_outside_bounds( + Some(&lower), + Some(&upper), + &mixed + )); + } + + #[test] + fn finite_bound_drops_nan() { + let nan = Datum::float(f32::NAN); + let finite = Datum::float(4.0_f32); + assert!(finite_bound(Some(&nan)).is_none()); + assert_eq!(finite_bound(Some(&finite)), Some(&finite)); + assert!(finite_bound(None).is_none()); + } +} diff --git a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs index ea56c32c66..50276e26f9 100644 --- a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs @@ -23,16 +23,16 @@ use fnv::FnvHashSet; use ordered_float::OrderedFloat; use parquet::arrow::arrow_reader::{RowSelection, RowSelector}; use parquet::file::metadata::RowGroupMetaData; -use parquet::file::page_index::index::Index; +use parquet::file::metadata::page_index::RowGroupPageIndex; +use parquet::file::page_index::column_index::ColumnIndexMetaData; use parquet::file::page_index::offset_index::OffsetIndexMetaData; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::{BoundPredicate, BoundReference}; +use crate::spec::decimal_utils::i128_from_be_bytes; use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType, Schema}; use crate::{Error, ErrorKind, Result}; -type OffsetIndex = Vec; - const IN_PREDICATE_LIMIT: usize = 200; enum MissingColBehavior { @@ -59,8 +59,7 @@ impl PageNullCount { } pub(crate) struct PageIndexEvaluator<'a> { - column_index: &'a [Index], - offset_index: &'a OffsetIndex, + page_index: &'a RowGroupPageIndex, row_group_metadata: &'a RowGroupMetaData, iceberg_field_id_to_parquet_column_index: &'a HashMap, snapshot_schema: &'a Schema, @@ -69,15 +68,13 @@ pub(crate) struct PageIndexEvaluator<'a> { impl<'a> PageIndexEvaluator<'a> { pub(crate) fn new( - column_index: &'a [Index], - offset_index: &'a OffsetIndex, + page_index: &'a RowGroupPageIndex, row_group_metadata: &'a RowGroupMetaData, field_id_map: &'a HashMap, snapshot_schema: &'a Schema, ) -> Self { Self { - column_index, - offset_index, + page_index, row_group_metadata, iceberg_field_id_to_parquet_column_index: field_id_map, snapshot_schema, @@ -92,8 +89,7 @@ impl<'a> PageIndexEvaluator<'a> { /// matching the filter predicate. pub(crate) fn eval( filter: &'a BoundPredicate, - column_index: &'a [Index], - offset_index: &'a OffsetIndex, + page_index: &'a RowGroupPageIndex, row_group_metadata: &'a RowGroupMetaData, field_id_map: &'a HashMap, snapshot_schema: &'a Schema, @@ -103,8 +99,7 @@ impl<'a> PageIndexEvaluator<'a> { } let mut evaluator = Self::new( - column_index, - offset_index, + page_index, row_group_metadata, field_id_map, snapshot_schema, @@ -150,38 +145,46 @@ impl<'a> PageIndexEvaluator<'a> { let Some(field) = self.snapshot_schema.field_by_id(field_id) else { return Err(Error::new( ErrorKind::Unexpected, - format!("Field with id {} missing from snapshot schema", field_id), + format!("Field with id {field_id} missing from snapshot schema"), )); }; let Some(field_type) = field.field_type.as_primitive_type() else { return Err(Error::new( ErrorKind::Unexpected, - format!( - "Field with id {} not convertible to primitive type", - field_id - ), + format!("Field with id {field_id} not convertible to primitive type"), )); }; - let Some(column_index) = self.column_index.get(parquet_column_index) else { - // This should not happen, but we fail soft anyway so that the scan is still - // successful, just a bit slower + let Some(column_index) = self.page_index.column_index(parquet_column_index) else { + // The writer can omit the column index for a column chunk, for example + // when page statistics are disabled. Without it we can't prune pages. return self.select_all_rows(); }; + // Declared byte width of a FIXED_LEN_BYTE_ARRAY column, used to detect + // truncated page-index bounds when decoding decimals. `None` when the + // column metadata is absent or reports no fixed width. + let type_length = self + .row_group_metadata + .columns() + .get(parquet_column_index) + .map(|column| column.column_descr().type_length()) + .filter(|&len| len > 0) + .map(|len| len as usize); + let row_counts = { // Caches row count calculations for columns that appear multiple times in // the predicate match self.row_count_cache.get(&parquet_column_index) { Some(count) => count.clone(), None => { - let Some(offset_index) = self.offset_index.get(parquet_column_index) else { - // if we have a column index, we should always have an offset index. - return Err(Error::new( - ErrorKind::Unexpected, - format!("Missing offset index for field id {}", field_id), - )); + let Some(offset_index) = self.page_index.offset_index(parquet_column_index) + else { + // A page index provider can load column and offset indexes + // independently. Without offsets, we cannot construct a row + // selection, so preserve correctness by skipping page pruning. + return self.select_all_rows(); }; let count = self.calc_row_counts(offset_index); @@ -194,8 +197,10 @@ impl<'a> PageIndexEvaluator<'a> { }; let Some(page_filter) = Self::apply_predicate_to_column_index( + field_id, predicate, field_type, + type_length, column_index, &row_counts, )? @@ -221,9 +226,8 @@ impl<'a> PageIndexEvaluator<'a> { /// Returns a list of row counts per page fn calc_row_counts(&self, offset_index: &OffsetIndexMetaData) -> Vec { let mut remaining_rows = self.row_group_metadata.num_rows() as usize; - let mut row_counts = Vec::with_capacity(self.offset_index.len()); - let page_locations = offset_index.page_locations(); + let mut row_counts = Vec::with_capacity(page_locations.len()); for (idx, page_location) in page_locations.iter().enumerate() { let row_count = if idx < page_locations.len() - 1 { let row_count = (page_locations[idx + 1].first_row_index @@ -241,149 +245,323 @@ impl<'a> PageIndexEvaluator<'a> { } fn apply_predicate_to_column_index( + field_id: i32, predicate: F, field_type: &PrimitiveType, - column_index: &Index, + type_length: Option, + column_index: &ColumnIndexMetaData, row_counts: &[usize], ) -> Result>> where F: Fn(Option, Option, PageNullCount) -> Result, { let result: Result> = match column_index { - Index::NONE => { - return Ok(None); - } - Index::BOOLEAN(idx) => idx - .indexes - .iter() + ColumnIndexMetaData::BOOLEAN(idx) => idx + .min_values_iter() + .zip(idx.max_values_iter()) + .enumerate() .zip(row_counts.iter()) - .map(|(item, &row_count)| { + .map(|((i, (min, max)), &row_count)| { predicate( - item.min.map(|val| { + min.map(|&val| { Datum::new(field_type.clone(), PrimitiveLiteral::Boolean(val)) }), - item.max.map(|val| { + max.map(|&val| { Datum::new(field_type.clone(), PrimitiveLiteral::Boolean(val)) }), - PageNullCount::from_row_and_null_counts(row_count, item.null_count), + PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), ) }) .collect(), - Index::INT32(idx) => idx - .indexes - .iter() + ColumnIndexMetaData::INT32(idx) => idx + .min_values_iter() + .zip(idx.max_values_iter()) + .enumerate() .zip(row_counts.iter()) - .map(|(item, &row_count)| { + .map(|((i, (min, max)), &row_count)| { predicate( - item.min - .map(|val| Datum::new(field_type.clone(), PrimitiveLiteral::Int(val))), - item.max - .map(|val| Datum::new(field_type.clone(), PrimitiveLiteral::Int(val))), - PageNullCount::from_row_and_null_counts(row_count, item.null_count), + min.map(|&val| Self::int32_bound_to_datum(field_type, val)), + max.map(|&val| Self::int32_bound_to_datum(field_type, val)), + PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), ) }) .collect(), - Index::INT64(idx) => idx - .indexes - .iter() + ColumnIndexMetaData::INT64(idx) => idx + .min_values_iter() + .zip(idx.max_values_iter()) + .enumerate() .zip(row_counts.iter()) - .map(|(item, &row_count)| { + .map(|((i, (min, max)), &row_count)| { predicate( - item.min - .map(|val| Datum::new(field_type.clone(), PrimitiveLiteral::Long(val))), - item.max - .map(|val| Datum::new(field_type.clone(), PrimitiveLiteral::Long(val))), - PageNullCount::from_row_and_null_counts(row_count, item.null_count), + min.map(|&val| Self::int64_bound_to_datum(field_type, val)), + max.map(|&val| Self::int64_bound_to_datum(field_type, val)), + PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), ) }) .collect(), - Index::FLOAT(idx) => idx - .indexes - .iter() + ColumnIndexMetaData::FLOAT(idx) => idx + .min_values_iter() + .zip(idx.max_values_iter()) + .enumerate() .zip(row_counts.iter()) - .map(|(item, &row_count)| { + .map(|((i, (min, max)), &row_count)| { predicate( - item.min.map(|val| { + min.map(|&val| { Datum::new( field_type.clone(), PrimitiveLiteral::Float(OrderedFloat::from(val)), ) }), - item.max.map(|val| { + max.map(|&val| { Datum::new( field_type.clone(), PrimitiveLiteral::Float(OrderedFloat::from(val)), ) }), - PageNullCount::from_row_and_null_counts(row_count, item.null_count), + PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), ) }) .collect(), - Index::DOUBLE(idx) => idx - .indexes - .iter() + ColumnIndexMetaData::DOUBLE(idx) => idx + .min_values_iter() + .zip(idx.max_values_iter()) + .enumerate() .zip(row_counts.iter()) - .map(|(item, &row_count)| { + .map(|((i, (min, max)), &row_count)| { predicate( - item.min.map(|val| { + min.map(|&val| { Datum::new( field_type.clone(), PrimitiveLiteral::Double(OrderedFloat::from(val)), ) }), - item.max.map(|val| { + max.map(|&val| { Datum::new( field_type.clone(), PrimitiveLiteral::Double(OrderedFloat::from(val)), ) }), - PageNullCount::from_row_and_null_counts(row_count, item.null_count), - ) - }) - .collect(), - Index::BYTE_ARRAY(idx) => idx - .indexes - .iter() - .zip(row_counts.iter()) - .map(|(item, &row_count)| { - predicate( - item.min.clone().map(|val| { - Datum::new( - field_type.clone(), - PrimitiveLiteral::String( - String::from_utf8(val.data().to_vec()).unwrap(), - ), - ) - }), - item.max.clone().map(|val| { - Datum::new( - field_type.clone(), - PrimitiveLiteral::String( - String::from_utf8(val.data().to_vec()).unwrap(), - ), - ) - }), - PageNullCount::from_row_and_null_counts(row_count, item.null_count), + PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), ) }) .collect(), - Index::FIXED_LEN_BYTE_ARRAY(_) => { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "unsupported 'FIXED_LEN_BYTE_ARRAY' index type in column_index", - )); + ColumnIndexMetaData::BYTE_ARRAY(idx) => { + // Parquet stores Iceberg string and binary bounds as BYTE_ARRAY. + // Any other field type on a BYTE_ARRAY column (e.g. a non-spec + // BYTE_ARRAY decimal) can't be decoded safely, so skip page + // pruning for it rather than risk pruning pages that match. + if !matches!(field_type, PrimitiveType::String | PrimitiveType::Binary) { + tracing::debug!( + field_id, + %field_type, + "Skipping page-index pruning: BYTE_ARRAY column index on a non-string/binary field" + ); + return Ok(None); + } + + let mut page_filter = Vec::with_capacity(row_counts.len()); + for ((i, (min, max)), &row_count) in idx + .min_values_iter() + .zip(idx.max_values_iter()) + .enumerate() + .zip(row_counts.iter()) + { + // A bound that won't decode (e.g. a min/max stat truncated + // mid-UTF-8-sequence) means this column's page index can't + // be trusted, so skip pruning for the whole column rather + // than abort the scan. + let (min, max) = match ( + min.map(|val| Self::byte_array_bound_to_datum(field_type, val)) + .transpose(), + max.map(|val| Self::byte_array_bound_to_datum(field_type, val)) + .transpose(), + ) { + (Ok(min), Ok(max)) => (min, max), + (Err(err), _) | (_, Err(err)) => { + tracing::debug!( + field_id, + %field_type, + page_index = i, + %err, + "Skipping page-index pruning: undecodable BYTE_ARRAY page bound" + ); + return Ok(None); + } + }; + + page_filter.push(predicate( + min, + max, + PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), + )?); + } + + Ok(page_filter) + } + ColumnIndexMetaData::FIXED_LEN_BYTE_ARRAY(idx) => { + // Parquet stores Iceberg decimals with precision > 18 as + // FIXED_LEN_BYTE_ARRAY. Other field types on such a column + // (fixed, uuid) aren't decoded here, so skip page pruning for + // them rather than risk pruning pages that match. + if !matches!(field_type, PrimitiveType::Decimal { .. }) { + tracing::debug!( + field_id, + %field_type, + "Skipping page-index pruning: FIXED_LEN_BYTE_ARRAY column index on a non-decimal field" + ); + return Ok(None); + } + + let Some(type_length) = type_length else { + tracing::debug!( + field_id, + %field_type, + "Skipping page-index pruning: unknown FIXED_LEN_BYTE_ARRAY decimal column width" + ); + return Ok(None); + }; + + let mut page_filter = Vec::with_capacity(row_counts.len()); + for ((i, (min, max)), &row_count) in idx + .min_values_iter() + .zip(idx.max_values_iter()) + .enumerate() + .zip(row_counts.iter()) + { + // An undecodable bound (only a non-conforming writer + // produces one, see fixed_len_byte_array_decimal_bound_to_datum) + // means this column's page index can't be trusted, so skip + // pruning for the whole column rather than abort the scan. + let decode = |val: &[u8]| { + Self::fixed_len_byte_array_decimal_bound_to_datum( + field_type, + val, + type_length, + ) + }; + let (min, max) = match ( + min.map(decode).transpose(), + max.map(decode).transpose(), + ) { + (Ok(min), Ok(max)) => (min, max), + (Err(err), _) | (_, Err(err)) => { + tracing::debug!( + field_id, + %field_type, + page_index = i, + %err, + "Skipping page-index pruning: undecodable FIXED_LEN_BYTE_ARRAY decimal page bound" + ); + return Ok(None); + } + }; + + page_filter.push(predicate( + min, + max, + PageNullCount::from_row_and_null_counts(row_count, idx.null_count(i)), + )?); + } + + Ok(page_filter) } - Index::INT96(_) => { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "unsupported 'INT96' index type in column_index", - )); + // Column index types we can't interpret: skip page pruning rather + // than abort the scan. Row-group filtering and the Arrow row filter + // still apply the predicate. + ColumnIndexMetaData::INT96(_) => { + tracing::debug!( + field_id, + %field_type, + "Skipping page-index pruning: unsupported INT96 column index" + ); + return Ok(None); } }; Ok(Some(result?)) } + /// Converts an `INT32` page bound into a [`Datum`] according to the field's + /// primitive type. + fn int32_bound_to_datum(field_type: &PrimitiveType, val: i32) -> Datum { + match field_type { + PrimitiveType::Decimal { .. } => Datum::new( + field_type.clone(), + PrimitiveLiteral::Int128(i128::from(val)), + ), + _ => Datum::new(field_type.clone(), PrimitiveLiteral::Int(val)), + } + } + + /// Converts an `INT64` page bound into a [`Datum`] according to the field's + /// primitive type. + fn int64_bound_to_datum(field_type: &PrimitiveType, val: i64) -> Datum { + match field_type { + PrimitiveType::Decimal { .. } => Datum::new( + field_type.clone(), + PrimitiveLiteral::Int128(i128::from(val)), + ), + _ => Datum::new(field_type.clone(), PrimitiveLiteral::Long(val)), + } + } + + /// Converts a `BYTE_ARRAY` page bound into a [`Datum`] according to the + /// field's primitive type. Parquet stores Iceberg `string` and `binary` + /// bounds as `BYTE_ARRAY`. + fn byte_array_bound_to_datum(field_type: &PrimitiveType, bytes: &[u8]) -> Result { + match field_type { + PrimitiveType::String => { + let value = std::str::from_utf8(bytes).map_err(|err| { + Error::new(ErrorKind::DataInvalid, "Invalid UTF-8 in string page bound") + .with_source(err) + })?; + Ok(Datum::string(value)) + } + PrimitiveType::Binary => Ok(Datum::binary(bytes.to_vec())), + // Defensive: production callers only invoke this for String/Binary. + _ => Err(Error::new( + ErrorKind::Unexpected, + format!("Unsupported primitive type for BYTE_ARRAY page bound: {field_type}"), + )), + } + } + + /// Converts a `FIXED_LEN_BYTE_ARRAY` decimal page bound into a [`Datum`]. + /// Parquet stores Iceberg decimals with precision > 18 as big-endian + /// two's-complement `FIXED_LEN_BYTE_ARRAY` of width `type_length`. + /// + /// Returns an error for a bound whose width differs from `type_length`. + /// Conforming writers never truncate a decimal column index (a truncated + /// two's-complement prefix would not preserve numeric order), so a short + /// bound comes from a non-conforming writer and would decode to the wrong + /// value. Reject it rather than prune on a bad bound. + fn fixed_len_byte_array_decimal_bound_to_datum( + field_type: &PrimitiveType, + bytes: &[u8], + type_length: usize, + ) -> Result { + if bytes.len() != type_length { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Truncated FIXED_LEN_BYTE_ARRAY decimal page bound: {} bytes (expected {type_length})", + bytes.len() + ), + )); + } + + let value = i128_from_be_bytes(bytes).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Can't convert bytes to i128: {bytes:?}"), + ) + })?; + Ok(Datum::new( + field_type.clone(), + PrimitiveLiteral::Int128(value), + )) + } + fn visit_inequality( &mut self, reference: &BoundReference, @@ -408,7 +586,9 @@ impl<'a> PageIndexEvaluator<'a> { let bound = if use_lower_bound { min } else { max }; if let Some(bound) = bound { - if cmp_fn(&bound, datum) { + // A bound that can't be compared with the literal (e.g. a + // file column narrower than the field type) might match. + if bound.partial_cmp(datum).is_none() || cmp_fn(&bound, datum) { return Ok(true); } @@ -550,16 +730,16 @@ impl BoundPredicateVisitor for PageIndexEvaluator<'_> { return Ok(false); } - if let Some(min) = min { - if min.gt(datum) { - return Ok(false); - } + if let Some(min) = min + && min.gt(datum) + { + return Ok(false); } - if let Some(max) = max { - if max.lt(datum) { - return Ok(false); - } + if let Some(max) = max + && max.lt(datum) + { + return Ok(false); } Ok(true) @@ -741,28 +921,24 @@ impl BoundPredicateVisitor for PageIndexEvaluator<'_> { } match (min, max) { - (Some(min), Some(max)) => { + (Some(min), Some(max)) if literals .iter() - .all(|datum| datum.lt(&min) || datum.gt(&max)) - { - // if all values are outside the bounds, rows cannot match. - return Ok(false); - } + .all(|datum| datum.lt(&min) || datum.gt(&max)) => + { + // if all values are outside the bounds, no rows can match + return Ok(false); } - (Some(min), _) => { - if !literals.iter().any(|datum| datum.ge(&min)) { - // if none of the values are greater than the min bound, rows cant match - return Ok(false); - } + // `all(lt)` rather than `!any(ge)`, so that a bound that + // can't be compared with the literals keeps the page. + (Some(min), _) if literals.iter().all(|datum| datum.lt(&min)) => { + // if no values are within the min bound, no rows can match + return Ok(false); } - (_, Some(max)) => { - if !literals.iter().any(|datum| datum.le(&max)) { - // if all values are greater than upper bound, rows cannot match. - return Ok(false); - } + (_, Some(max)) if literals.iter().all(|datum| datum.gt(&max)) => { + // if no values are within the max bound, no rows can match + return Ok(false); } - _ => {} } @@ -790,56 +966,418 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use parquet::arrow::arrow_reader::RowSelector; - use parquet::basic::{LogicalType as ParquetLogicalType, Type as ParquetPhysicalType}; - use parquet::data_type::ByteArray; - use parquet::file::metadata::{ColumnChunkMetaData, RowGroupMetaData}; - use parquet::file::page_index::index::{Index, NativeIndex, PageIndex}; - use parquet::file::page_index::offset_index::OffsetIndexMetaData; - use parquet::file::statistics::Statistics; - use parquet::format::{BoundaryOrder, PageLocation}; - use parquet::schema::types::{ - ColumnDescriptor, ColumnPath, SchemaDescriptor, Type as parquetSchemaType, + use arrow_array::{ + ArrayRef, Decimal128Array, FixedSizeBinaryArray, Float32Array, Int32Array, + LargeBinaryArray, RecordBatch, StringArray, UInt32Array, + }; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use parquet::arrow::ArrowWriter; + use parquet::arrow::arrow_reader::{ + ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelector, }; - use rand::{Rng, thread_rng}; + use parquet::file::metadata::page_index::{PageIndex, RowGroupPageIndex}; + use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; + use parquet::file::properties::WriterProperties; + use rand::Rng; + use tempfile::NamedTempFile; use super::PageIndexEvaluator; - use crate::expr::{Bind, Reference}; + use crate::expr::{Bind, Predicate, Reference}; + use crate::spec::decimal_utils::decimal_from_i128_with_scale; use crate::spec::{Datum, NestedField, PrimitiveType, Schema, Type}; use crate::{ErrorKind, Result}; + /// Helper function to create a test parquet file with page indexes + /// and return the metadata needed for testing + fn create_test_parquet_file() -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("col_float", DataType::Float32, true), + Field::new("col_string", DataType::Utf8, true), + ])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + let mut batches = vec![]; + + // Batch 1: 1024 rows - strings with AARDVARK, BEAR, BISON + let float_vals: Vec> = vec![None; 1024]; + let mut string_vals = vec![]; + string_vals.push(Some("AARDVARK".to_string())); + for _ in 1..1023 { + string_vals.push(Some("BEAR".to_string())); + } + string_vals.push(Some("BISON".to_string())); + + batches.push( + RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Float32Array::from(float_vals)), + Arc::new(StringArray::from(string_vals)), + ]) + .unwrap(), + ); + + // Batch 2: 1024 rows - all DEER + let float_vals: Vec> = vec![None; 1024]; + let string_vals = vec![Some("DEER".to_string()); 1024]; + + batches.push( + RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Float32Array::from(float_vals)), + Arc::new(StringArray::from(string_vals)), + ]) + .unwrap(), + ); + + // Batch 3: 1024 rows - float 0-10 + let mut float_vals = vec![]; + for i in 0..1024 { + float_vals.push(Some(i as f32 * 10.0 / 1024.0)); + } + let mut string_vals = vec![]; + string_vals.push(Some("GIRAFFE".to_string())); + string_vals.push(None); + for _ in 2..1024 { + string_vals.push(Some("HIPPO".to_string())); + } + + batches.push( + RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Float32Array::from(float_vals)), + Arc::new(StringArray::from(string_vals)), + ]) + .unwrap(), + ); + + // Batch 4: 1024 rows - float 10-20 + let mut float_vals = vec![None]; + for i in 1..1024 { + float_vals.push(Some(10.0 + i as f32 * 10.0 / 1024.0)); + } + let string_vals = vec![Some("HIPPO".to_string()); 1024]; + + batches.push( + RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Float32Array::from(float_vals)), + Arc::new(StringArray::from(string_vals)), + ]) + .unwrap(), + ); + + // Write rows one at a time to give the writer a chance to split into pages + for batch in &batches { + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + + /// Creates a single-column parquet file whose `col_binary` page bounds hold + /// non-UTF-8 bytes, backing the binary path through `BYTE_ARRAY` column + /// indexes. Page 0 is `[0x01]`, page 1 is `[0xff, 0x00]`. + fn create_binary_parquet_file() -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "col_binary", + DataType::LargeBinary, + true, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + // Each page holds 1024 rows of a single value, one page per byte pattern. + let pages: [&[u8]; 2] = [&[0x01], &[0xff, 0x00]]; + + for value in pages { + let array = Arc::new(LargeBinaryArray::from_iter_values(std::iter::repeat_n( + value, 1024, + ))) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + + /// Creates a single-column parquet file with a `FixedSizeBinary(2)` column, + /// producing `FIXED_LEN_BYTE_ARRAY` page bounds. Page 0 is `[0x01, 0x01]`, + /// page 1 is `[0x02, 0x02]`. + fn create_fixed_len_byte_array_parquet_file() -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "col_fixed", + DataType::FixedSizeBinary(2), + true, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + // Each page holds 1024 rows of a single value, one page per byte pattern. + let pages: [&[u8]; 2] = [&[0x01, 0x01], &[0x02, 0x02]]; + + for value in pages { + let array = Arc::new( + FixedSizeBinaryArray::try_from_iter(std::iter::repeat_n(value, 1024)).unwrap(), + ) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + + /// Creates a single-column `FixedSizeBinary(width)` parquet file whose column + /// index is truncated to `truncate_length` bytes (`< width`), producing + /// `FIXED_LEN_BYTE_ARRAY` page bounds narrower than the declared `type_length`. + /// Writes one 1024-row page per value in `pages`; each value must be `width` + /// bytes. + fn create_truncated_flba_file( + width: i32, + truncate_length: usize, + pages: &[&[u8]], + ) -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "col_fixed", + DataType::FixedSizeBinary(width), + true, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .set_column_index_truncate_length(Some(truncate_length)) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + for value in pages { + let array = Arc::new( + FixedSizeBinaryArray::try_from_iter(std::iter::repeat_n(*value, 1024)).unwrap(), + ) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![array]).unwrap(); + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + + /// Creates a single-column `Decimal128(precision, scale)` parquet file. + /// Parquet encodes precision <= 9 as `INT32`, precision 10..=18 as `INT64`, + /// and precision > 18 as `FIXED_LEN_BYTE_ARRAY`, so `precision` selects which + /// page-index encoding is exercised. Writes one 1024-row page per value in + /// `unscaled`, so page bounds partition the value range. + fn create_decimal_parquet_file( + precision: u8, + scale: i8, + unscaled: &[i128], + ) -> Result<(Arc, NamedTempFile)> { + let pages: Vec> = unscaled.iter().copied().map(Some).collect(); + create_decimal_parquet_file_with_null_pages(precision, scale, &pages) + } + + /// Like [`create_decimal_parquet_file`], but each entry in `pages` is one + /// 1024-row page: `Some(v)` writes that repeated unscaled value, `None` + /// writes an all-null page (whose column index carries no min/max bound). + fn create_decimal_parquet_file_with_null_pages( + precision: u8, + scale: i8, + pages: &[Option], + ) -> Result<(Arc, NamedTempFile)> { + let pages: Vec = pages + .iter() + .map(|page| { + Arc::new( + match page { + Some(unscaled) => { + Decimal128Array::from_iter_values(std::iter::repeat_n(*unscaled, 1024)) + } + None => Decimal128Array::from(vec![None; 1024]), + } + .with_precision_and_scale(precision, scale) + .unwrap(), + ) as ArrayRef + }) + .collect(); + create_single_column_parquet_file(&pages) + } + + /// Creates a single-column parquet file named `col` with one page per + /// 1024-row array in `pages`. Used to back an Iceberg field with a + /// narrower physical column, e.g. a `long` field with an `INT32` column + /// written before an `int` -> `long` promotion. + fn create_single_column_parquet_file( + pages: &[ArrayRef], + ) -> Result<(Arc, NamedTempFile)> { + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "col", + pages[0].data_type().clone(), + true, + )])); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let props = WriterProperties::builder() + .set_data_page_row_count_limit(1024) + .set_write_batch_size(512) + .build(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + + for page in pages { + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![page.clone()]).unwrap(); + // Write rows one at a time so the writer splits into per-value pages. + for i in 0..batch.num_rows() { + writer.write(&batch.slice(i, 1)).unwrap(); + } + } + + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata().clone(); + + Ok((metadata, temp_file)) + } + + /// Creates a single-column parquet file with one 1024-row page per value, + /// using `to_array` to build each page. + fn create_parquet_file_from_values( + values: &[T], + to_array: impl Fn(Vec) -> ArrayRef, + ) -> Result<(Arc, NamedTempFile)> { + let pages: Vec = values.iter().map(|&v| to_array(vec![v; 1024])).collect(); + create_single_column_parquet_file(&pages) + } + + /// Get the test metadata components for testing + fn get_test_metadata( + metadata: &ParquetMetaData, + ) -> ( + RowGroupPageIndex, + &parquet::file::metadata::RowGroupMetaData, + ) { + let row_group_metadata = metadata.row_group(0); + let page_index = metadata.page_index_for_row_group(0); + (page_index, row_group_metadata) + } + #[test] fn eval_matches_no_rows_for_empty_row_group() -> Result<()> { - let row_group_metadata = create_row_group_metadata(0, 0, None, 0, None)?; - let (column_index, offset_index) = create_page_index()?; + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("col_float", DataType::Float32, true), + Field::new("col_string", DataType::Utf8, true), + ])); + + let empty_float: ArrayRef = Arc::new(Float32Array::from(Vec::>::new())); + let empty_string: ArrayRef = Arc::new(StringArray::from(Vec::>::new())); + let empty_batch = + RecordBatch::try_new(arrow_schema.clone(), vec![empty_float, empty_string]).unwrap(); + + let temp_file = NamedTempFile::new().unwrap(); + let file = temp_file.reopen().unwrap(); + + let mut writer = ArrowWriter::try_new(file, arrow_schema, None).unwrap(); + writer.write(&empty_batch).unwrap(); + writer.close().unwrap(); + + let file = temp_file.reopen().unwrap(); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); + let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); + let metadata = reader.metadata(); + + if metadata.num_row_groups() == 0 || metadata.row_group(0).num_rows() == 0 { + return Ok(()); + } let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; + let row_group_metadata = metadata.row_group(0); + let page_index = metadata.page_index_for_row_group(0); + let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; - let expected = vec![]; - - assert_eq!(result, expected); + assert_eq!(result.len(), 0); Ok(()) } #[test] fn eval_is_null_select_only_pages_with_nulls() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") @@ -848,17 +1386,16 @@ mod tests { let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; let expected = vec![ - RowSelector::select(1024), - RowSelector::skip(1024), RowSelector::select(2048), + RowSelector::skip(1024), + RowSelector::select(1024), ]; assert_eq!(result, expected); @@ -867,10 +1404,47 @@ mod tests { } #[test] - fn eval_is_not_null_dont_select_pages_with_all_nulls() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; + fn eval_selects_all_rows_when_column_lacks_offset_index() -> Result<()> { + let (metadata, _temp_file) = create_test_parquet_file()?; + let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + + let page_index = metadata.page_index().unwrap(); + let mut builder = page_index + .as_any() + .downcast_ref::() + .unwrap() + .clone() + .into_builder(); + builder.allocate_offset_indexes(1, 2); + builder.put_offset_index(page_index.offset_index(0, 1).unwrap().clone(), 0, 1); + let metadata = metadata + .as_ref() + .clone() + .into_builder() + .set_page_index(Some(Arc::new(builder.build()))) + .build(); + + let filter = Reference::new("col_float") + .is_not_null() + .bind(iceberg_schema_ref.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &metadata.page_index_for_row_group(0), + metadata.row_group(0), + &field_id_map, + iceberg_schema_ref.as_ref(), + )?; + + assert_eq!(result, vec![RowSelector::select(4096)]); + Ok(()) + } + + #[test] + fn eval_is_not_null_dont_select_pages_with_all_nulls() -> Result<()> { + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") @@ -879,14 +1453,13 @@ mod tests { let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; - let expected = vec![RowSelector::skip(1024), RowSelector::select(3072)]; + let expected = vec![RowSelector::skip(2048), RowSelector::select(2048)]; assert_eq!(result, expected); @@ -895,9 +1468,8 @@ mod tests { #[test] fn eval_is_nan_select_all() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") @@ -906,9 +1478,8 @@ mod tests { let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; @@ -922,9 +1493,8 @@ mod tests { #[test] fn eval_not_nan_select_all() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") @@ -933,9 +1503,8 @@ mod tests { let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; @@ -949,9 +1518,8 @@ mod tests { #[test] fn eval_inequality_nan_datum_all_rows_except_all_null_pages() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") @@ -960,14 +1528,13 @@ mod tests { let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; - let expected = vec![RowSelector::skip(1024), RowSelector::select(3072)]; + let expected = vec![RowSelector::skip(2048), RowSelector::select(2048)]; assert_eq!(result, expected); @@ -976,29 +1543,26 @@ mod tests { #[test] fn eval_inequality_pages_containing_value_except_all_null_pages() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .less_than(Datum::float(5.0)) + .less_than(Datum::float(5.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; let expected = vec![ - RowSelector::skip(1024), + RowSelector::skip(2048), RowSelector::select(1024), RowSelector::skip(1024), - RowSelector::select(1024), ]; assert_eq!(result, expected); @@ -1008,29 +1572,29 @@ mod tests { #[test] fn eval_eq_pages_containing_value_except_all_null_pages() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(5.0)) + .equal_to(Datum::float(5.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; + // Pages 0-1: all null (skip) + // Page 2: 0-10 (select, might contain 5.0) + // Page 3: 10-20 (skip, min > 5.0) let expected = vec![ - RowSelector::skip(1024), + RowSelector::skip(2048), RowSelector::select(1024), RowSelector::skip(1024), - RowSelector::select(1024), ]; assert_eq!(result, expected); @@ -1040,20 +1604,18 @@ mod tests { #[test] fn eval_not_eq_all_rows() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .not_equal_to(Datum::float(5.0)) + .not_equal_to(Datum::float(5.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; @@ -1066,86 +1628,253 @@ mod tests { } #[test] - fn eval_starts_with_error_float_col() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; + fn eval_inequality_prunes_binary_pages_with_non_utf8_bounds() -> Result<()> { + let (metadata, _temp_file) = create_binary_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); - let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_binary", + Type::Primitive(PrimitiveType::Binary), + true, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); - let filter = Reference::new("col_float") - .starts_with(Datum::float(5.0)) - .bind(iceberg_schema_ref.clone(), false)?; + // Page 0 bounds are [0x01], page 1 bounds are [0xff, 0x00]; only page 1 + // exceeds [0x80]. Decoding [0xff, 0x00] as UTF-8 would panic. + let filter = Reference::new("col_binary") + .greater_than(Datum::binary(vec![0x80u8])) + .bind(iceberg_schema.clone(), false)?; let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, - iceberg_schema_ref.as_ref(), - ); + iceberg_schema.as_ref(), + )?; - assert_eq!(result.unwrap_err().kind(), ErrorKind::Unexpected); + let expected = vec![RowSelector::skip(1024), RowSelector::select(1024)]; + + assert_eq!(result, expected); Ok(()) } #[test] - fn eval_starts_with_pages_containing_value_except_all_null_pages() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - - let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + fn eval_skips_pruning_for_byte_array_decimal_bounds() -> Result<()> { + // A non-spec writer can store a decimal column as BYTE_ARRAY. Those + // bounds can't be decoded safely (Parquet may truncate them, and + // truncation only preserves lexicographic byte order, not decimal + // order), so the evaluator skips page pruning rather than prune pages + // that might match. + let (metadata, _temp_file) = create_binary_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_decimal", + Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }), + true, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); - let filter = Reference::new("col_string") - .starts_with(Datum::string("B")) - .bind(iceberg_schema_ref.clone(), false)?; + // A predicate that would prune every page if the bounds were decoded. + let filter = Reference::new("col_decimal") + .greater_than(Datum::decimal_with_precision( + decimal_from_i128_with_scale(99999, 2), + 10, + )?) + .bind(iceberg_schema.clone(), false)?; let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, - iceberg_schema_ref.as_ref(), + iceberg_schema.as_ref(), )?; - let expected = vec![ - RowSelector::select(512), - RowSelector::skip(3536), - RowSelector::select(48), - ]; - - assert_eq!(result, expected); + // Both 1024-row pages survive: no page pruning is applied. + assert_eq!(result, vec![RowSelector::select(2048)]); Ok(()) } #[test] - fn eval_not_starts_with_pages_containing_value_except_pages_with_min_and_max_equal_to_prefix_and_all_null_pages() - -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; + fn eval_skips_pruning_for_fixed_len_byte_array() -> Result<()> { + // FIXED_LEN_BYTE_ARRAY page indexes back spec-conforming fixed, uuid, + // and decimal(P > 18) columns. Only decimals are decoded; fixed and uuid + // bounds aren't, so the evaluator skips page pruning for them rather than + // abort the scan. + let (metadata, _temp_file) = create_fixed_len_byte_array_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_fixed", + Type::Primitive(PrimitiveType::Fixed(2)), + true, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); - let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + // A predicate that would prune every page if the bounds were decoded. + let filter = Reference::new("col_fixed") + .greater_than(Datum::fixed(vec![0xffu8, 0xff])) + .bind(iceberg_schema.clone(), false)?; - let filter = Reference::new("col_string") - .not_starts_with(Datum::string("DE")) - .bind(iceberg_schema_ref.clone(), false)?; + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + // All rows survive: no page pruning is applied for FIXED_LEN_BYTE_ARRAY. + assert_eq!(result, vec![RowSelector::select(2048)]); + + Ok(()) + } + + #[test] + fn eval_skips_pruning_for_non_utf8_string_bound() -> Result<()> { + // A writer can truncate a string min/max stat mid-UTF-8-sequence, + // leaving a BYTE_ARRAY page bound that isn't valid UTF-8. Decoding it + // as a string fails, so the evaluator skips page pruning for the column + // rather than aborting the whole file read. + let (metadata, _temp_file) = create_binary_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_string", + Type::Primitive(PrimitiveType::String), + true, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); + + // Page 1's bound [0xff, 0x00] isn't valid UTF-8. A predicate that would + // otherwise prune every page must not: the undecodable bound forces a + // skip rather than an error. + let filter = Reference::new("col_string") + .greater_than(Datum::string("zzz")) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + // Both pages survive: no page pruning is applied when a bound won't decode. + assert_eq!(result, vec![RowSelector::select(2048)]); + + Ok(()) + } + + #[test] + fn eval_starts_with_error_float_col() -> Result<()> { + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + + let filter = Reference::new("col_float") + .starts_with(Datum::float(5.0_f32)) + .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema_ref.as_ref(), + ); + + assert_eq!(result.unwrap_err().kind(), ErrorKind::Unexpected); + + Ok(()) + } + + #[test] + fn eval_starts_with_pages_containing_value_except_all_null_pages() -> Result<()> { + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + + // Test starts_with on string column where only some pages match + // Our file has 4 pages: ["AARDVARK".."BISON"], ["DEER"], ["GIRAFFE".."HIPPO"], ["HIPPO"] + // Testing starts_with("B") should select only page 0 + let filter = Reference::new("col_string") + .starts_with(Datum::string("B")) + .bind(iceberg_schema_ref.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; + // Page 0 has "BEAR" and "BISON" (starts with B), rest don't + let expected = vec![RowSelector::select(1024), RowSelector::skip(3072)]; + + assert_eq!(result, expected); + + Ok(()) + } + + #[test] + fn eval_not_starts_with_pages_containing_value_except_pages_with_min_and_max_equal_to_prefix_and_all_null_pages() + -> Result<()> { + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + + // Test not_starts_with where one page has ALL values starting with prefix + // Our file has page 1 with all "DEER" (min="DEER", max="DEER") + // Testing not_starts_with("DE") should skip page 1 where all values start with "DE" + let filter = Reference::new("col_string") + .not_starts_with(Datum::string("DE")) + .bind(iceberg_schema_ref.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema_ref.as_ref(), + )?; + + // Page 0: mixed values (select) + // Page 1: all "DEER" starting with "DE" (skip) + // Pages 2-3: other values not all starting with "DE" (select) let expected = vec![ - RowSelector::select(512), - RowSelector::skip(512), - RowSelector::select(3072), + RowSelector::select(1024), + RowSelector::skip(1024), + RowSelector::select(2048), ]; assert_eq!(result, expected); @@ -1155,22 +1884,22 @@ mod tests { #[test] fn eval_in_length_of_set_above_limit_all_rows() -> Result<()> { - let mut rng = thread_rng(); - - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - + let mut rng = rand::rng(); + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in(std::iter::repeat_with(|| Datum::float(rng.gen_range(0.0..10.0))).take(1000)) + .is_in( + std::iter::repeat_with(|| Datum::float(rng.random_range(0.0_f32..10.0_f32))) + .take(1000), + ) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; @@ -1184,30 +1913,31 @@ mod tests { #[test] fn eval_in_valid_set_size_some_rows() -> Result<()> { - let row_group_metadata = create_row_group_metadata(4096, 1000, None, 1000, None)?; - let (column_index, offset_index) = create_page_index()?; - + let (metadata, _temp_file) = create_test_parquet_file()?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + // Test is_in with multiple values using min/max bounds + // Our file has 4 pages: ["AARDVARK".."BISON"], ["DEER"], ["GIRAFFE".."HIPPO"], ["HIPPO"] + // Testing is_in(["AARDVARK", "GIRAFFE"]) - both are in different pages let filter = Reference::new("col_string") - .is_in([Datum::string("AARDVARK"), Datum::string("ICEBERG")]) + .is_in([Datum::string("AARDVARK"), Datum::string("GIRAFFE")]) .bind(iceberg_schema_ref.clone(), false)?; let result = PageIndexEvaluator::eval( &filter, - &column_index, - &offset_index, - &row_group_metadata, + &page_index, + row_group_metadata, &field_id_map, iceberg_schema_ref.as_ref(), )?; + // Page 0 contains "AARDVARK", page 1 doesn't contain either, page 2 contains "GIRAFFE", page 3 doesn't let expected = vec![ - RowSelector::select(512), - RowSelector::skip(512), - RowSelector::select(2976), - RowSelector::skip(48), - RowSelector::select(48), + RowSelector::select(1024), + RowSelector::skip(1024), + RowSelector::select(1024), + RowSelector::skip(1024), ]; assert_eq!(result, expected); @@ -1215,6 +1945,558 @@ mod tests { Ok(()) } + #[test] + fn eval_inequality_prunes_int32_decimal_pages() -> Result<()> { + // precision 9 -> Parquet INT32 page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(9, 2, &[100, 200, 300, 400])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(9, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. `> 2.50` keeps the pages whose + // upper bound exceeds 2.50 (3.00 and 4.00). + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(250, 2, 9)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(2048) + ]); + + Ok(()) + } + + #[test] + fn eval_in_prunes_int32_decimal_pages() -> Result<()> { + // precision 9 -> Parquet INT32 page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(9, 2, &[100, 200, 300, 400])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(9, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. IN (2.00, 4.00) keeps only the + // pages whose single value is one of the literals. + let filter = Reference::new("col_decimal") + .is_in([decimal_datum(200, 2, 9)?, decimal_datum(400, 2, 9)?]) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(1024), + RowSelector::skip(1024), + RowSelector::select(1024), + ]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_int64_decimal_pages() -> Result<()> { + // precision 18 -> Parquet INT64 page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(18, 2, &[100, 200, 300, 400])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(18, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. `> 2.50` keeps the pages whose + // upper bound exceeds 2.50 (3.00 and 4.00). + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(250, 2, 18)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(2048) + ]); + + Ok(()) + } + + #[test] + fn eval_in_prunes_int64_decimal_pages() -> Result<()> { + // precision 18 -> Parquet INT64 page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(18, 2, &[100, 200, 300, 400])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(18, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. IN (2.00, 4.00) keeps only the + // pages whose single value is one of the literals. + let filter = Reference::new("col_decimal") + .is_in([decimal_datum(200, 2, 18)?, decimal_datum(400, 2, 18)?]) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(1024), + RowSelector::skip(1024), + RowSelector::select(1024), + ]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_negative_decimal_pages_at_boundary() -> Result<()> { + // precision 9 -> Parquet INT32 page bounds, spanning negative values. + let (metadata, _temp_file) = create_decimal_parquet_file(9, 2, &[-400, -200, 100, 300])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(9, 2)?; + + // Pages hold -4.00, -2.00, 1.00, 3.00. `>= -2.00` keeps the pages whose + // upper bound is at least -2.00, including page 1 whose bound equals it. + let filter = Reference::new("col_decimal") + .greater_than_or_equal_to(decimal_datum(-200, 2, 9)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(3072) + ]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_fixed_len_byte_array_decimal_pages() -> Result<()> { + // precision 30 -> Parquet FIXED_LEN_BYTE_ARRAY page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(30, 2, &[100, 200, 300, 400])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. `> 2.50` keeps the pages whose + // upper bound exceeds 2.50 (3.00 and 4.00). + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(250, 2, 30)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(2048) + ]); + + Ok(()) + } + + #[test] + fn eval_in_prunes_fixed_len_byte_array_decimal_pages() -> Result<()> { + // precision 30 -> Parquet FIXED_LEN_BYTE_ARRAY page bounds. + let (metadata, _temp_file) = create_decimal_parquet_file(30, 2, &[100, 200, 300, 400])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + // Pages hold 1.00, 2.00, 3.00, 4.00. IN (2.00, 4.00) keeps only the + // pages whose single value is one of the literals. + let filter = Reference::new("col_decimal") + .is_in([decimal_datum(200, 2, 30)?, decimal_datum(400, 2, 30)?]) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(1024), + RowSelector::skip(1024), + RowSelector::select(1024), + ]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_negative_fixed_len_byte_array_decimal_pages() -> Result<()> { + // precision 30 -> Parquet FIXED_LEN_BYTE_ARRAY page bounds, spanning + // negative values. Negative decimals have a 0xff high byte in two's + // complement, so this only prunes correctly if the bounds are compared + // numerically rather than by raw byte order. + let (metadata, _temp_file) = create_decimal_parquet_file(30, 2, &[-400, -200, 100, 300])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + // Pages hold -4.00, -2.00, 1.00, 3.00. `>= -2.00` keeps the pages whose + // upper bound is at least -2.00, including page 1 whose bound equals it. + let filter = Reference::new("col_decimal") + .greater_than_or_equal_to(decimal_datum(-200, 2, 30)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(1024), + RowSelector::select(3072) + ]); + + Ok(()) + } + + #[test] + fn eval_skips_pruning_for_truncated_fixed_len_byte_array_decimal_bound() -> Result<()> { + // A non-Iceberg writer can store a decimal as a plain FIXED_LEN_BYTE_ARRAY + // and truncate its column index (parquet-rs truncates FLBA bounds but not + // decimal/float16 ones). A bound narrower than the column's declared + // type_length can't be decoded as an i128 without changing its value, so + // the evaluator skips page pruning for the column rather than aborting the + // file read. + // + // Page 0 is 0.00, page 1 is 2.55; with full-width bounds `> 1.00` would + // prune page 0. The 13-byte values are truncated to 4-byte column-index + // bounds (!= type_length 13), so neither page is pruned instead. + let page0 = [0u8; 13]; // unscaled 0 -> 0.00 + let mut page1 = [0u8; 13]; + page1[12] = 0xff; // unscaled 255 -> 2.55 + let (metadata, _temp_file) = create_truncated_flba_file(13, 4, &[&page0, &page1])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + + // The Iceberg schema calls the 13-byte column a Decimal(30,2). + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(100, 2, 30)?) // > 1.00 + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + // Truncated bounds -> no page pruning: every row survives. + assert_eq!(result, vec![RowSelector::select(2048)]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_precision_38_fixed_len_byte_array_decimal_pages() -> Result<()> { + // precision 38 is the max Iceberg decimal precision -> 16-byte + // FIXED_LEN_BYTE_ARRAY bounds, the width extreme where decoding fills + // the full i128 with no sign-extension padding. Values near +/-(10^38-1) + // carry significant bits in the high-order bytes, and span both signs. + let max = 10_i128.pow(38) - 1; + let (metadata, _temp_file) = create_decimal_parquet_file(38, 2, &[-max, 0, max])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(38, 2)?; + + // `> 0` keeps only the +max page; the -max and 0 pages are pruned. + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(0, 2, 38)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(1024) + ]); + + Ok(()) + } + + #[test] + fn eval_inequality_prunes_precision_widened_fixed_len_byte_array_decimal_pages() -> Result<()> { + // Precision widening keeps a file's original FIXED_LEN_BYTE_ARRAY width, + // so a decimal(20,2) file's 9-byte bounds are read here as decimal(30,2). + // The guard keys off the file's type_length (9), not the field + // precision's canonical width (13). + let (metadata, _temp_file) = create_decimal_parquet_file(20, 2, &[-400, 100, 300])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + // Read the 9-byte column as decimal(30,2). + let (iceberg_schema, field_id_map) = build_decimal_schema_and_field_map(30, 2)?; + + // Pages hold -4.00, 1.00, 3.00. `> 1.50` keeps only the 3.00 page. + let filter = Reference::new("col_decimal") + .greater_than(decimal_datum(150, 2, 30)?) + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::skip(2048), + RowSelector::select(1024) + ]); + + Ok(()) + } + + #[test] + fn eval_is_null_selects_all_null_fixed_len_byte_array_decimal_page() -> Result<()> { + // precision 30 -> FIXED_LEN_BYTE_ARRAY page bounds. Page 0 is all-null, + // so its column index carries no min/max; the arm must pass those absent + // bounds straight to the predicate rather than try to decode them. Pages + // 1-2 hold 2.00 and 3.00. + let (metadata, _temp_file) = + create_decimal_parquet_file_with_null_pages(30, 2, &[None, Some(200), Some(300)])?; + let (page_index, row_group_metadata) = get_test_metadata(&metadata); + + // The column is optional so IS NULL binds to a real predicate (on a + // required field it would simplify to always-false). + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_decimal", + Type::Primitive(PrimitiveType::Decimal { + precision: 30, + scale: 2, + }), + false, + ))]) + .build()?, + ); + let field_id_map = HashMap::from_iter([(1, 0)]); + + // IS NULL keeps the all-null page and prunes the two value pages. + let filter = Reference::new("col_decimal") + .is_null() + .bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![ + RowSelector::select(1024), + RowSelector::skip(2048) + ]); + + Ok(()) + } + + #[test] + fn eval_keeps_int32_pages_of_promoted_long_field() -> Result<()> { + // An `int` -> `long` promotion leaves older files with INT32 page + // bounds, which can't be compared with a `long` literal. + let (metadata, _temp_file) = create_parquet_file_from_values(&[100, 200, 300, 400], |v| { + Arc::new(Int32Array::from(v)) + })?; + + assert_keeps_all_pages(&metadata, PrimitiveType::Long, 4096, [ + Reference::new("col").less_than(Datum::long(250)), + Reference::new("col").less_than_or_equal_to(Datum::long(250)), + Reference::new("col").greater_than(Datum::long(250)), + Reference::new("col").greater_than_or_equal_to(Datum::long(250)), + // A literal outside the `int` range. + Reference::new("col").less_than(Datum::long(i64::from(i32::MAX) + 1)), + Reference::new("col").is_in([Datum::long(200), Datum::long(400)]), + ]) + } + + #[test] + fn eval_keeps_float_pages_of_promoted_double_field() -> Result<()> { + // A `float` -> `double` promotion leaves older files with FLOAT page + // bounds, which can't be compared with a `double` literal. + let (metadata, _temp_file) = + create_parquet_file_from_values(&[1.0_f32, 2.0, 3.0, 4.0], |v| { + Arc::new(Float32Array::from(v)) + })?; + + assert_keeps_all_pages(&metadata, PrimitiveType::Double, 4096, [ + Reference::new("col").less_than(Datum::double(2.5)), + Reference::new("col").less_than_or_equal_to(Datum::double(2.5)), + Reference::new("col").greater_than(Datum::double(2.5)), + Reference::new("col").greater_than_or_equal_to(Datum::double(2.5)), + Reference::new("col").is_in([Datum::double(2.0), Datum::double(4.0)]), + ]) + } + + #[test] + fn eval_keeps_uint32_pages_of_long_field() -> Result<()> { + // Parquet `UINT_32` columns map to `long` but have INT32 page bounds. + // The last page mixes both values, so its raw signed max is smaller + // than its min and a sign-extending widening would prune it. + let mixed: Vec = [100_u32, 3_000_000_000].repeat(512); + let (metadata, _temp_file) = create_single_column_parquet_file(&[ + Arc::new(UInt32Array::from(vec![100_u32; 1024])), + Arc::new(UInt32Array::from(vec![3_000_000_000_u32; 1024])), + Arc::new(UInt32Array::from(mixed)), + ])?; + + let page_index = metadata.page_index_for_row_group(0); + let column_index = page_index.column_index(0).unwrap(); + let parquet::file::page_index::column_index::ColumnIndexMetaData::INT32(idx) = column_index + else { + panic!("expected INT32 column index, got {column_index:?}"); + }; + let bounds: Vec<_> = idx.min_values_iter().zip(idx.max_values_iter()).collect(); + assert_eq!(bounds.len(), 3); + assert_eq!(bounds[2], (Some(&100), Some(&(3_000_000_000_u32 as i32)))); + + assert_keeps_all_pages(&metadata, PrimitiveType::Long, 3072, [ + Reference::new("col").equal_to(Datum::long(3_000_000_000_i64)), + Reference::new("col").greater_than(Datum::long(2_000_000_000)), + Reference::new("col").is_in([ + Datum::long(3_000_000_000_i64), + Datum::long(4_000_000_000_i64), + ]), + ]) + } + + #[test] + fn eval_keeps_pages_with_bounds_incomparable_to_predicate() -> Result<()> { + // INT32 bounds on a `timestamp` field (e.g. a file whose column was + // written as `date`) can't be compared with the predicate literal. + let (metadata, _temp_file) = create_parquet_file_from_values(&[100, 200, 300, 400], |v| { + Arc::new(Int32Array::from(v)) + })?; + + assert_keeps_all_pages(&metadata, PrimitiveType::Timestamp, 4096, [ + Reference::new("col").less_than(Datum::timestamp_micros(250)), + Reference::new("col").less_than_or_equal_to(Datum::timestamp_micros(250)), + Reference::new("col").greater_than(Datum::timestamp_micros(250)), + Reference::new("col").greater_than_or_equal_to(Datum::timestamp_micros(250)), + Reference::new("col").equal_to(Datum::timestamp_micros(250)), + Reference::new("col") + .is_in([Datum::timestamp_micros(200), Datum::timestamp_micros(400)]), + ]) + } + + /// Asserts that each of `filters`, bound to a single `field_type` field + /// over the column in `metadata`, selects all `row_count` rows. Pages + /// whose bounds can't be compared with the predicate might match. + fn assert_keeps_all_pages( + metadata: &ParquetMetaData, + field_type: PrimitiveType, + row_count: usize, + filters: impl IntoIterator, + ) -> Result<()> { + let (page_index, row_group_metadata) = get_test_metadata(metadata); + let (iceberg_schema, field_id_map) = build_single_field_schema_and_field_map(field_type)?; + + for filter in filters { + let filter = filter.bind(iceberg_schema.clone(), false)?; + + let result = PageIndexEvaluator::eval( + &filter, + &page_index, + row_group_metadata, + &field_id_map, + iceberg_schema.as_ref(), + )?; + + assert_eq!(result, vec![RowSelector::select(row_count)], "{filter}"); + } + + Ok(()) + } + + /// Builds a schema holding a single optional field `col` (id 1) of + /// `field_type`, mapped to parquet column 0. + fn build_single_field_schema_and_field_map( + field_type: PrimitiveType, + ) -> Result<(Arc, HashMap)> { + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col", + Type::Primitive(field_type), + false, + ))]) + .build()?, + ); + + Ok((iceberg_schema, HashMap::from_iter([(1, 0)]))) + } + + /// Builds a decimal query [`Datum`] from an `unscaled` mantissa, matching + /// the field's `scale` and `precision`. + fn decimal_datum(unscaled: i128, scale: u32, precision: u32) -> Result { + Datum::decimal_with_precision(decimal_from_i128_with_scale(unscaled, scale), precision) + } + + fn build_decimal_schema_and_field_map( + precision: u32, + scale: u32, + ) -> Result<(Arc, HashMap)> { + let iceberg_schema = Arc::new( + Schema::builder() + .with_fields([Arc::new(NestedField::new( + 1, + "col_decimal", + Type::Primitive(PrimitiveType::Decimal { precision, scale }), + true, + ))]) + .build()?, + ); + + Ok((iceberg_schema, HashMap::from_iter([(1, 0)]))) + } + fn build_iceberg_schema_and_field_map() -> Result<(Arc, HashMap)> { let iceberg_schema = Schema::builder() .with_fields([ @@ -1238,183 +2520,4 @@ mod tests { Ok((iceberg_schema_ref, field_id_map)) } - - fn build_parquet_schema_descriptor() -> Result> { - let field_1 = Arc::new( - parquetSchemaType::primitive_type_builder("col_float", ParquetPhysicalType::FLOAT) - .with_id(Some(1)) - .build()?, - ); - - let field_2 = Arc::new( - parquetSchemaType::primitive_type_builder( - "col_string", - ParquetPhysicalType::BYTE_ARRAY, - ) - .with_id(Some(2)) - .with_logical_type(Some(ParquetLogicalType::String)) - .build()?, - ); - - let group_type = Arc::new( - parquetSchemaType::group_type_builder("all") - .with_id(Some(1000)) - .with_fields(vec![field_1, field_2]) - .build()?, - ); - - let schema_descriptor = SchemaDescriptor::new(group_type); - let schema_descriptor_arc = Arc::new(schema_descriptor); - Ok(schema_descriptor_arc) - } - - fn create_row_group_metadata( - num_rows: i64, - col_1_num_vals: i64, - col_1_stats: Option, - col_2_num_vals: i64, - col_2_stats: Option, - ) -> Result { - let schema_descriptor_arc = build_parquet_schema_descriptor()?; - - let column_1_desc_ptr = Arc::new(ColumnDescriptor::new( - schema_descriptor_arc.column(0).self_type_ptr(), - 1, - 1, - ColumnPath::new(vec!["col_float".to_string()]), - )); - - let column_2_desc_ptr = Arc::new(ColumnDescriptor::new( - schema_descriptor_arc.column(1).self_type_ptr(), - 1, - 1, - ColumnPath::new(vec!["col_string".to_string()]), - )); - - let mut col_1_meta = - ColumnChunkMetaData::builder(column_1_desc_ptr).set_num_values(col_1_num_vals); - if let Some(stats1) = col_1_stats { - col_1_meta = col_1_meta.set_statistics(stats1) - } - - let mut col_2_meta = - ColumnChunkMetaData::builder(column_2_desc_ptr).set_num_values(col_2_num_vals); - if let Some(stats2) = col_2_stats { - col_2_meta = col_2_meta.set_statistics(stats2) - } - - let row_group_metadata = RowGroupMetaData::builder(schema_descriptor_arc) - .set_num_rows(num_rows) - .set_column_metadata(vec![ - col_1_meta.build()?, - // .set_statistics(Statistics::float(None, None, None, 1, false)) - col_2_meta.build()?, - ]) - .build(); - - Ok(row_group_metadata?) - } - - fn create_page_index() -> Result<(Vec, Vec)> { - let idx_float = Index::FLOAT(NativeIndex:: { - indexes: vec![ - PageIndex { - min: None, - max: None, - null_count: Some(1024), - repetition_level_histogram: None, - definition_level_histogram: None, - }, - PageIndex { - min: Some(0.0), - max: Some(10.0), - null_count: Some(0), - repetition_level_histogram: None, - definition_level_histogram: None, - }, - PageIndex { - min: Some(10.0), - max: Some(20.0), - null_count: Some(1), - repetition_level_histogram: None, - definition_level_histogram: None, - }, - PageIndex { - min: None, - max: None, - null_count: None, - repetition_level_histogram: None, - definition_level_histogram: None, - }, - ], - boundary_order: BoundaryOrder(0), // UNORDERED - }); - - let idx_string = Index::BYTE_ARRAY(NativeIndex:: { - indexes: vec![ - PageIndex { - min: Some("AA".into()), - max: Some("DD".into()), - null_count: Some(0), - repetition_level_histogram: None, - definition_level_histogram: None, - }, - PageIndex { - min: Some("DE".into()), - max: Some("DE".into()), - null_count: Some(0), - repetition_level_histogram: None, - definition_level_histogram: None, - }, - PageIndex { - min: Some("DF".into()), - max: Some("UJ".into()), - null_count: Some(1), - repetition_level_histogram: None, - definition_level_histogram: None, - }, - PageIndex { - min: None, - max: None, - null_count: Some(48), - repetition_level_histogram: None, - definition_level_histogram: None, - }, - PageIndex { - min: None, - max: None, - null_count: None, - repetition_level_histogram: None, - definition_level_histogram: None, - }, - ], - boundary_order: BoundaryOrder(0), // UNORDERED - }); - - let page_locs_float = vec![ - PageLocation::new(0, 1024, 0), - PageLocation::new(1024, 1024, 1024), - PageLocation::new(2048, 1024, 2048), - PageLocation::new(3072, 1024, 3072), - ]; - - let page_locs_string = vec![ - PageLocation::new(0, 512, 0), - PageLocation::new(512, 512, 512), - PageLocation::new(1024, 2976, 1024), - PageLocation::new(4000, 48, 4000), - PageLocation::new(4048, 48, 4048), - ]; - - Ok((vec![idx_float, idx_string], vec![ - OffsetIndexMetaData { - page_locations: page_locs_float, - unencoded_byte_array_data_bytes: None, - }, - OffsetIndexMetaData { - page_locations: page_locs_string, - unencoded_byte_array_data_bytes: None, - }, - ])) - } } diff --git a/crates/iceberg/src/expr/visitors/predicate_visitor.rs b/crates/iceberg/src/expr/visitors/predicate_visitor.rs index 61671cc2cd..1fa94e7f80 100644 --- a/crates/iceberg/src/expr/visitors/predicate_visitor.rs +++ b/crates/iceberg/src/expr/visitors/predicate_visitor.rs @@ -169,7 +169,7 @@ pub(crate) fn visit(visitor: &mut V, predicate: &Predicate) PredicateOperator::IsNan => visitor.is_nan(expr.term(), predicate), PredicateOperator::NotNan => visitor.not_nan(expr.term(), predicate), op => { - panic!("Unexpected op for unary predicate: {}", &op) + panic!("Unexpected op for unary predicate: {}", op) } }, Predicate::Binary(expr) => { @@ -193,7 +193,7 @@ pub(crate) fn visit(visitor: &mut V, predicate: &Predicate) visitor.not_starts_with(reference, literal, predicate) } op => { - panic!("Unexpected op for binary predicate: {}", &op) + panic!("Unexpected op for binary predicate: {}", op) } } } @@ -204,7 +204,7 @@ pub(crate) fn visit(visitor: &mut V, predicate: &Predicate) PredicateOperator::In => visitor.r#in(reference, literals, predicate), PredicateOperator::NotIn => visitor.not_in(reference, literals, predicate), op => { - panic!("Unexpected op for set predicate: {}", &op) + panic!("Unexpected op for set predicate: {}", op) } } } diff --git a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs index 0506b33af0..ea3e135959 100644 --- a/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/row_group_metrics_evaluator.rs @@ -117,7 +117,7 @@ impl<'a> RowGroupMetricsEvaluator<'a> { ErrorKind::Unexpected, format!( "Could not find a field with id '{}' in the snapshot schema", - &field_id + field_id ), )); }; @@ -127,7 +127,7 @@ impl<'a> RowGroupMetricsEvaluator<'a> { ErrorKind::Unexpected, format!( "Could not determine the PrimitiveType for field id '{}'", - &field_id + field_id ), )); }; @@ -177,6 +177,13 @@ impl<'a> RowGroupMetricsEvaluator<'a> { }?; if let Some(bound) = bound { + if bound.is_nan() { + // NaN indicates unreliable bounds (e.g. parquet-mr <= 1.9 records a + // NaN that starts a row group as both min and max). + // See the InclusiveMetricsEvaluator docs for more. + return ROW_GROUP_MIGHT_MATCH; + } + if cmp_fn(&bound, datum) { return ROW_GROUP_MIGHT_MATCH; } @@ -207,8 +214,14 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { Ok(lhs || rhs) } - fn not(&mut self, inner: bool) -> Result { - Ok(!inner) + fn not(&mut self, _: bool) -> Result { + // Negating a conservative "might match" bound is unsound (it can turn into + // "can't match" and prune row groups that do match), so NOT must be + // rewritten out of the predicate before it reaches this evaluator. + Err(Error::new( + ErrorKind::Unexpected, + "NOT unsupported at this point. NOT-rewrite should be performed first", + )) } fn is_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { @@ -297,7 +310,11 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { return ROW_GROUP_CANT_MATCH; } - if let Some(lower_bound) = self.min_value(field_id)? { + let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { + return ROW_GROUP_MIGHT_MATCH; + }; + + if let Some(lower_bound) = get_parquet_stat_min_as_datum(&primitive_type, stats)? { if lower_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -307,7 +324,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { } } - if let Some(upper_bound) = self.max_value(field_id)? { + if let Some(upper_bound) = get_parquet_stat_max_as_datum(&primitive_type, stats)? { if upper_bound.is_nan() { // NaN indicates unreliable bounds. // See the InclusiveMetricsEvaluator docs for more. @@ -351,7 +368,11 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { )); }; - if let Some(lower_bound) = self.min_value(field_id)? { + let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { + return ROW_GROUP_MIGHT_MATCH; + }; + + if let Some(lower_bound) = get_parquet_stat_min_as_datum(&primitive_type, stats)? { let PrimitiveLiteral::String(lower_bound) = lower_bound.literal() else { return Err(Error::new( ErrorKind::Unexpected, @@ -369,7 +390,7 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { } } - if let Some(upper_bound) = self.max_value(field_id)? { + if let Some(upper_bound) = get_parquet_stat_max_as_datum(&primitive_type, stats)? { let PrimitiveLiteral::String(upper_bound) = upper_bound.literal() else { return Err(Error::new( ErrorKind::Unexpected, @@ -476,28 +497,20 @@ impl BoundPredicateVisitor for RowGroupMetricsEvaluator<'_> { return ROW_GROUP_MIGHT_MATCH; } - if let Some(lower_bound) = self.min_value(field_id)? { - if lower_bound.is_nan() { - // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. - return ROW_GROUP_MIGHT_MATCH; - } - - if !literals.iter().any(|datum| datum.ge(&lower_bound)) { - // if all values are less than lower bound, rows cannot match. - return ROW_GROUP_CANT_MATCH; - } - } - - if let Some(upper_bound) = self.max_value(field_id)? { - if upper_bound.is_nan() { - // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more. - return ROW_GROUP_MIGHT_MATCH; - } - - if !literals.iter().any(|datum| datum.le(&upper_bound)) { - // if all values are greater than upper bound, rows cannot match. - return ROW_GROUP_CANT_MATCH; - } + let Some((stats, primitive_type)) = self.stats_and_type_for_field_id(field_id)? else { + return ROW_GROUP_MIGHT_MATCH; + }; + let lower_bound = get_parquet_stat_min_as_datum(&primitive_type, stats)?; + let upper_bound = get_parquet_stat_max_as_datum(&primitive_type, stats)?; + + // A NaN bound is unreliable on that side only. Drop it to unbounded + // so a valid bound can still prune. See InclusiveMetricsEvaluator. + if super::all_literals_outside_bounds( + super::finite_bound(lower_bound.as_ref()), + super::finite_bound(upper_bound.as_ref()), + literals, + ) { + return ROW_GROUP_CANT_MATCH; } ROW_GROUP_MIGHT_MATCH @@ -528,12 +541,12 @@ mod tests { use parquet::schema::types::{ ColumnDescriptor, ColumnPath, SchemaDescriptor, Type as parquetSchemaType, }; - use rand::{Rng, thread_rng}; + use rand::Rng; use super::RowGroupMetricsEvaluator; - use crate::Result; - use crate::expr::{Bind, Reference}; + use crate::expr::{Bind, Predicate, Reference}; use crate::spec::{Datum, NestedField, PrimitiveType, Schema, Type}; + use crate::{ErrorKind, Result}; #[test] fn eval_matches_no_rows_for_empty_row_group() -> Result<()> { @@ -542,7 +555,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -564,7 +577,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -579,6 +592,32 @@ mod tests { Ok(()) } + #[test] + fn eval_error_for_not_predicate_requires_rewrite() -> Result<()> { + // The evaluator cannot soundly negate a conservative bound, so a NOT node + // that reaches it (one that was not rewritten away first) is rejected + // rather than silently mispruning a matching row group. + let row_group_metadata = create_row_group_metadata(1, 1, None, 1, None)?; + + let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + + let filter = (!Reference::new("col_float").greater_than(Datum::float(5.0_f32))) + .bind(iceberg_schema_ref.clone(), false)?; + + let err = RowGroupMetricsEvaluator::eval( + &filter, + &row_group_metadata, + &field_id_map, + iceberg_schema_ref.as_ref(), + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::Unexpected); + assert!(err.message().contains("NOT")); + + Ok(()) + } + #[test] fn eval_false_for_meta_all_null_filter_not_null() -> Result<()> { let row_group_metadata = create_row_group_metadata( @@ -700,7 +739,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -760,7 +799,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -793,7 +832,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -826,7 +865,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .greater_than(Datum::float(1.0)) + .greater_than(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -853,7 +892,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -886,7 +925,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -919,7 +958,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -952,7 +991,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -985,7 +1024,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1018,7 +1057,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .equal_to(Datum::float(1.0)) + .equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1051,7 +1090,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .not_equal_to(Datum::float(1.0)) + .not_equal_to(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1105,7 +1144,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .starts_with(Datum::float(1.0)) + .starts_with(Datum::float(1.0_f32)) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1617,7 +1656,7 @@ mod tests { #[test] fn eval_true_for_too_many_literals_filter_is_in() -> Result<()> { - let mut rng = thread_rng(); + let mut rng = rand::rng(); let row_group_metadata = create_row_group_metadata( 1, @@ -1636,7 +1675,10 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in(std::iter::repeat_with(|| Datum::float(rng.gen_range(0.0..10.0))).take(1000)) + .is_in( + std::iter::repeat_with(|| Datum::float(rng.random_range(0.0_f32..10.0_f32))) + .take(1000), + ) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1677,10 +1719,87 @@ mod tests { Ok(()) } + // A NaN with the sign bit set sorts below -Inf under `total_cmp`, so it + // exercises the `>` / `>=` side of the NaN check as well. + const NAN_BOUNDS: [f32; 2] = [f32::NAN, f32::from_bits(0xffc0_0000)]; + + fn eval_float_bounds(filter: &Predicate, min: f32, max: f32) -> Result { + let row_group_metadata = create_row_group_metadata( + 1, + 1, + Some(Statistics::float( + Some(min), + Some(max), + None, + Some(0), + false, + )), + 1, + None, + )?; + + let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + let filter = filter.clone().bind(iceberg_schema_ref.clone(), false)?; + + RowGroupMetricsEvaluator::eval( + &filter, + &row_group_metadata, + &field_id_map, + iceberg_schema_ref.as_ref(), + ) + } + + #[test] + fn eval_true_for_nan_lower_bound_less_than() -> Result<()> { + let filter = Reference::new("col_float").less_than(Datum::float(5.0_f32)); + for nan in NAN_BOUNDS { + assert!(eval_float_bounds(&filter, nan, nan)?); + } + Ok(()) + } + + #[test] + fn eval_true_for_nan_lower_bound_less_than_or_eq() -> Result<()> { + let filter = Reference::new("col_float").less_than_or_equal_to(Datum::float(5.0_f32)); + for nan in NAN_BOUNDS { + assert!(eval_float_bounds(&filter, nan, nan)?); + } + Ok(()) + } + + #[test] + fn eval_true_for_nan_lower_bound_finite_upper_bound_less_than() -> Result<()> { + let lt = Reference::new("col_float").less_than(Datum::float(5.0_f32)); + let lt_eq = Reference::new("col_float").less_than_or_equal_to(Datum::float(5.0_f32)); + for nan in NAN_BOUNDS { + assert!(eval_float_bounds(<, nan, 2.0)?); + assert!(eval_float_bounds(<_eq, nan, 2.0)?); + } + Ok(()) + } + + #[test] + fn eval_true_for_nan_upper_bound_greater_than() -> Result<()> { + let filter = Reference::new("col_float").greater_than(Datum::float(5.0_f32)); + for nan in NAN_BOUNDS { + assert!(eval_float_bounds(&filter, nan, nan)?); + } + Ok(()) + } + + #[test] + fn eval_true_for_nan_upper_bound_greater_than_or_eq() -> Result<()> { + let filter = Reference::new("col_float").greater_than_or_equal_to(Datum::float(5.0_f32)); + for nan in NAN_BOUNDS { + assert!(eval_float_bounds(&filter, nan, nan)?); + } + Ok(()) + } + #[test] - fn eval_true_for_lower_bound_is_nan_filter_is_in() -> Result<()> { - // TODO: should this be false, since the max stat - // is lower than the min val in the set? + fn eval_false_for_lower_bound_is_nan_all_literals_above_upper_is_in() -> Result<()> { + // NaN lower is treated as unbounded. The valid upper bound (1.0) + // still prunes IN (2.0, 3.0). let row_group_metadata = create_row_group_metadata( 1, 1, @@ -1698,7 +1817,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in([Datum::float(2.0), Datum::float(3.0)]) + .is_in([Datum::float(2.0_f32), Datum::float(3.0_f32)]) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1708,7 +1827,7 @@ mod tests { iceberg_schema_ref.as_ref(), )?; - assert!(result); + assert!(!result); Ok(()) } @@ -1725,7 +1844,7 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in([Datum::float(2.0), Datum::float(3.0)]) + .is_in([Datum::float(2.0_f32), Datum::float(3.0_f32)]) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1758,7 +1877,77 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in([Datum::float(2.0), Datum::float(3.0)]) + .is_in([Datum::float(2.0_f32), Datum::float(3.0_f32)]) + .bind(iceberg_schema_ref.clone(), false)?; + + let result = RowGroupMetricsEvaluator::eval( + &filter, + &row_group_metadata, + &field_id_map, + iceberg_schema_ref.as_ref(), + )?; + + assert!(result); + Ok(()) + } + + #[test] + fn eval_false_for_nan_upper_bound_all_literals_below_lower_is_in() -> Result<()> { + // NaN upper is treated as unbounded. The valid lower bound (4.0) + // still prunes IN (2.0, 3.0). + let row_group_metadata = create_row_group_metadata( + 1, + 1, + Some(Statistics::float( + Some(4.0), + Some(f32::NAN), + None, + Some(0), + false, + )), + 1, + None, + )?; + + let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + + let filter = Reference::new("col_float") + .is_in([Datum::float(2.0_f32), Datum::float(3.0_f32)]) + .bind(iceberg_schema_ref.clone(), false)?; + + let result = RowGroupMetricsEvaluator::eval( + &filter, + &row_group_metadata, + &field_id_map, + iceberg_schema_ref.as_ref(), + )?; + + assert!(!result); + Ok(()) + } + + #[test] + fn eval_true_for_nan_lower_bound_literal_inside_upper_is_in() -> Result<()> { + // NaN lower is treated as unbounded. upper = 3.0 with IN (2.0, 4.0) + // must still match because 2.0 is inside the finite upper bound. + let row_group_metadata = create_row_group_metadata( + 1, + 1, + Some(Statistics::float( + Some(f32::NAN), + Some(3.0), + None, + Some(0), + false, + )), + 1, + None, + )?; + + let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + + let filter = Reference::new("col_float") + .is_in([Datum::float(2.0_f32), Datum::float(4.0_f32)]) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( @@ -1791,7 +1980,42 @@ mod tests { let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; let filter = Reference::new("col_float") - .is_in([Datum::float(2.0), Datum::float(3.0)]) + .is_in([Datum::float(2.0_f32), Datum::float(3.0_f32)]) + .bind(iceberg_schema_ref.clone(), false)?; + + let result = RowGroupMetricsEvaluator::eval( + &filter, + &row_group_metadata, + &field_id_map, + iceberg_schema_ref.as_ref(), + )?; + + assert!(!result); + Ok(()) + } + + #[test] + fn eval_false_for_literals_straddling_bounds_is_in() -> Result<()> { + // Bounds are [4.0, 6.0]; IN (2.0, 8.0) straddles the range with no + // literal inside it and must be pruned. + let row_group_metadata = create_row_group_metadata( + 1, + 1, + Some(Statistics::float( + Some(4.0), + Some(6.0), + None, + Some(0), + false, + )), + 1, + None, + )?; + + let (iceberg_schema_ref, field_id_map) = build_iceberg_schema_and_field_map()?; + + let filter = Reference::new("col_float") + .is_in([Datum::float(2.0_f32), Datum::float(8.0_f32)]) .bind(iceberg_schema_ref.clone(), false)?; let result = RowGroupMetricsEvaluator::eval( diff --git a/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs b/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs index e17c44c658..1c8a4a6747 100644 --- a/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/strict_metrics_evaluator.rs @@ -17,25 +17,27 @@ use fnv::FnvHashSet; +use super::scalar_bound; +use crate::Result; +use crate::error::invalid_data; use crate::expr::visitors::bound_predicate_visitor::{BoundPredicateVisitor, visit}; use crate::expr::{BoundPredicate, BoundReference}; use crate::spec::{DataFile, Datum}; -use crate::{Error, ErrorKind, Result}; #[allow(dead_code)] const ROWS_MUST_MATCH: Result = Ok(true); #[allow(dead_code)] const ROWS_MIGHT_NOT_MATCH: Result = Ok(false); -#[allow(dead_code)] /// Evaluates an `Expression` on a `DataFile` to test whether all rows in the file match. -/// +/// /// This evaluation is strict: it returns true if all rows in a file must match the expression. /// For example, if a file's ts column has min X and max Y, this evaluator will return true for ts /// < Y+1 but not for ts < Y-1. /// /// Files are passed to `eval(DataFile)`, which returns true if all rows in the file /// must contain matching rows and false if the file may contain rows that do not match. +#[allow(dead_code)] pub(crate) struct StrictMetricsEvaluator<'a> { data_file: &'a DataFile, } @@ -51,7 +53,7 @@ impl<'a> StrictMetricsEvaluator<'a> { /// see if this `DataFile` contains data that could match /// the scan's filter. #[allow(dead_code)] - pub(crate) fn eval(filter: &'a BoundPredicate, data_file: &'a DataFile) -> crate::Result { + pub(crate) fn eval(filter: &'a BoundPredicate, data_file: &'a DataFile) -> Result { if data_file.record_count == 0 { return ROWS_MUST_MATCH; } @@ -73,11 +75,11 @@ impl<'a> StrictMetricsEvaluator<'a> { } fn lower_bound(&self, field_id: i32) -> Option<&Datum> { - self.data_file.lower_bounds.get(&field_id) + scalar_bound(self.data_file.lower_bounds.get(&field_id)) } fn upper_bound(&self, field_id: i32) -> Option<&Datum> { - self.data_file.upper_bounds.get(&field_id) + scalar_bound(self.data_file.upper_bounds.get(&field_id)) } fn contains_nans_only(&self, field_id: i32) -> bool { @@ -116,7 +118,7 @@ impl<'a> StrictMetricsEvaluator<'a> { datum: &Datum, cmp_fn: fn(&Datum, &Datum) -> bool, use_lower_bound: bool, - ) -> crate::Result { + ) -> Result { let field_id = reference.field().id; if self.may_contain_null(field_id) || self.may_contain_nan(field_id) { @@ -129,10 +131,10 @@ impl<'a> StrictMetricsEvaluator<'a> { self.upper_bound(field_id) }; - if let Some(bound) = bound { - if cmp_fn(bound, datum) { - return ROWS_MUST_MATCH; - } + if let Some(bound) = bound + && cmp_fn(bound, datum) + { + return ROWS_MUST_MATCH; } ROWS_MIGHT_NOT_MATCH @@ -142,34 +144,27 @@ impl<'a> StrictMetricsEvaluator<'a> { impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { type T = bool; - fn always_true(&mut self) -> crate::Result { + fn always_true(&mut self) -> Result { ROWS_MUST_MATCH } - fn always_false(&mut self) -> crate::Result { + fn always_false(&mut self) -> Result { ROWS_MIGHT_NOT_MATCH } - fn and(&mut self, lhs: bool, rhs: bool) -> crate::Result { + fn and(&mut self, lhs: bool, rhs: bool) -> Result { Ok(lhs && rhs) } - fn or(&mut self, lhs: bool, rhs: bool) -> crate::Result { + fn or(&mut self, lhs: bool, rhs: bool) -> Result { Ok(lhs || rhs) } - fn not(&mut self, _inner: bool) -> crate::Result { - Err(Error::new( - ErrorKind::DataInvalid, - "NOT should be rewritten", - )) + fn not(&mut self, _inner: bool) -> Result { + Err(invalid_data!("NOT should be rewritten")) } - fn is_null( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> crate::Result { + fn is_null(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { let field_id = reference.field().id; if self.contains_nulls_only(field_id) { @@ -183,7 +178,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { &mut self, reference: &BoundReference, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field_id = reference.field().id; if let Some(&count) = self.null_count(field_id) { @@ -196,11 +191,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { ROWS_MIGHT_NOT_MATCH } - fn is_nan( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> crate::Result { + fn is_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { let field_id = reference.field().id; let contains_only = self.contains_nans_only(field_id); @@ -212,17 +203,13 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { ROWS_MIGHT_NOT_MATCH } - fn not_nan( - &mut self, - reference: &BoundReference, - _predicate: &BoundPredicate, - ) -> crate::Result { + fn not_nan(&mut self, reference: &BoundReference, _predicate: &BoundPredicate) -> Result { let field_id = reference.field().id; - if let Some(&nan_count) = self.nan_count(field_id) { - if nan_count == 0 { - return ROWS_MUST_MATCH; - } + if let Some(&nan_count) = self.nan_count(field_id) + && nan_count == 0 + { + return ROWS_MUST_MATCH; } if self.contains_nulls_only(field_id) { @@ -237,7 +224,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { self.visit_inequality(reference, datum, PartialOrd::lt, false) } @@ -246,7 +233,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { self.visit_inequality(reference, datum, PartialOrd::le, false) } @@ -255,13 +242,13 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field_id = reference.field().id; - if let Some(lower) = self.lower_bound(field_id) { - if lower.is_nan() { - return ROWS_MIGHT_NOT_MATCH; - } + if let Some(lower) = self.lower_bound(field_id) + && lower.is_nan() + { + return ROWS_MIGHT_NOT_MATCH; } self.visit_inequality(reference, datum, PartialOrd::gt, true) @@ -272,7 +259,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { self.visit_inequality(reference, datum, PartialOrd::ge, true) } @@ -281,7 +268,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field_id = reference.field().id; if self.may_contain_null(field_id) || self.may_contain_nan(field_id) { @@ -307,7 +294,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { reference: &BoundReference, datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field_id = reference.field().id; if self.contains_nulls_only(field_id) || self.contains_nans_only(field_id) { @@ -340,7 +327,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { _reference: &BoundReference, _datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { ROWS_MIGHT_NOT_MATCH } @@ -349,7 +336,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { _reference: &BoundReference, _datum: &Datum, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { ROWS_MIGHT_NOT_MATCH } @@ -358,7 +345,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { reference: &BoundReference, literals: &FnvHashSet, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field_id = reference.field().id; if self.may_contain_null(field_id) || self.may_contain_nan(field_id) { @@ -382,7 +369,7 @@ impl BoundPredicateVisitor for StrictMetricsEvaluator<'_> { reference: &BoundReference, literals: &FnvHashSet, _predicate: &BoundPredicate, - ) -> crate::Result { + ) -> Result { let field_id = reference.field().id; if self.contains_nulls_only(field_id) || self.contains_nans_only(field_id) { @@ -578,7 +565,7 @@ mod test { ]), column_sizes: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -604,7 +591,7 @@ mod test { lower_bounds: Default::default(), upper_bounds: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -630,7 +617,7 @@ mod test { upper_bounds: HashMap::from([(1, Datum::int(42))]), column_sizes: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -657,7 +644,7 @@ mod test { upper_bounds: HashMap::from([(3, Datum::string("dC"))]), column_sizes: Default::default(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -1045,8 +1032,7 @@ mod test { // For zero-record files, strict eval returns MUST_MATCH. assert!( result, - "Strict eval: Should read zero-record file for expression {:?}", - expr + "Strict eval: Should read zero-record file for expression {expr:?}" ); } } @@ -1142,11 +1128,7 @@ mod test { let result = StrictMetricsEvaluator::eval(&less_than_int("id", INT_MIN_VALUE), &file).unwrap(); - assert!( - !result, - "Strict eval: id < {} should be false", - INT_MIN_VALUE - ); + assert!(!result, "Strict eval: id < {INT_MIN_VALUE} should be false"); let result = StrictMetricsEvaluator::eval(&less_than_int("id", INT_MIN_VALUE + 1), &file).unwrap(); @@ -1158,11 +1140,7 @@ mod test { let result = StrictMetricsEvaluator::eval(&less_than_int("id", INT_MAX_VALUE), &file).unwrap(); - assert!( - !result, - "Strict eval: id < {} should be false", - INT_MAX_VALUE - ); + assert!(!result, "Strict eval: id < {INT_MAX_VALUE} should be false"); let result = StrictMetricsEvaluator::eval(&less_than_int("id", INT_MAX_VALUE + 1), &file).unwrap(); @@ -1200,18 +1178,13 @@ mod test { .unwrap(); assert!( !result, - "Strict eval: id <= {} should be false", - INT_MIN_VALUE + "Strict eval: id <= {INT_MIN_VALUE} should be false" ); let result = StrictMetricsEvaluator::eval(&less_than_or_equal_int("id", INT_MAX_VALUE), &file) .unwrap(); - assert!( - result, - "Strict eval: id <= {} should be true", - INT_MAX_VALUE - ); + assert!(result, "Strict eval: id <= {INT_MAX_VALUE} should be true"); let result = StrictMetricsEvaluator::eval(&less_than_or_equal_int("id", INT_MAX_VALUE + 1), &file) @@ -1238,19 +1211,11 @@ mod test { let result = StrictMetricsEvaluator::eval(&greater_than_int("id", INT_MAX_VALUE), &file).unwrap(); - assert!( - !result, - "Strict eval: id > {} should be false", - INT_MAX_VALUE - ); + assert!(!result, "Strict eval: id > {INT_MAX_VALUE} should be false"); let result = StrictMetricsEvaluator::eval(&greater_than_int("id", INT_MIN_VALUE), &file).unwrap(); - assert!( - !result, - "Strict eval: id > {} should be false", - INT_MIN_VALUE - ); + assert!(!result, "Strict eval: id > {INT_MIN_VALUE} should be false"); let result = StrictMetricsEvaluator::eval(&greater_than_int("id", INT_MIN_VALUE - 1), &file) @@ -1302,18 +1267,13 @@ mod test { .unwrap(); assert!( !result, - "Strict eval: id >= {} should be false", - INT_MAX_VALUE + "Strict eval: id >= {INT_MAX_VALUE} should be false" ); let result = StrictMetricsEvaluator::eval(&greater_than_or_equal_int("id", INT_MIN_VALUE), &file) .unwrap(); - assert!( - result, - "Strict eval: id >= {} should be true", - INT_MIN_VALUE - ); + assert!(result, "Strict eval: id >= {INT_MIN_VALUE} should be true"); let result = StrictMetricsEvaluator::eval( &greater_than_or_equal_int("id", INT_MIN_VALUE - 1), diff --git a/crates/iceberg/src/expr/visitors/strict_projection.rs b/crates/iceberg/src/expr/visitors/strict_projection.rs index ebc6212c76..2188e6df08 100644 --- a/crates/iceberg/src/expr/visitors/strict_projection.rs +++ b/crates/iceberg/src/expr/visitors/strict_projection.rs @@ -2761,7 +2761,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 10000)".to_string() + "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 100.00)".to_string() ); // test less or eq @@ -2780,7 +2780,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 10000)".to_string() + "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 100.00)".to_string() ); // test greater @@ -2799,7 +2799,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 > 100) AND (pcol2 > 100)) AND (pcol3 > 10000)".to_string() + "((pcol1 > 100) AND (pcol2 > 100)) AND (pcol3 > 100.00)".to_string() ); // test greater or eq @@ -2818,7 +2818,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 9990)".to_string() + "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 99.90)".to_string() ); // test not eq @@ -2837,7 +2837,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 != 100) AND (pcol2 != 100)) AND (pcol3 != 10000)".to_string() + "((pcol1 != 100) AND (pcol2 != 100)) AND (pcol3 != 100.00)".to_string() ); // test not in @@ -2880,7 +2880,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 NOT IN (100, 90)) AND (pcol2 NOT IN (100, 90))) AND (pcol3 NOT IN (10000, 10100, 9900))" + "((pcol1 NOT IN (100, 90)) AND (pcol2 NOT IN (100, 90))) AND (pcol3 NOT IN (100.00, 99.00, 101.00))" .to_string() ); @@ -2984,7 +2984,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 < 90) AND (pcol2 < 90)) AND (pcol3 < 9990)".to_string() + "((pcol1 < 90) AND (pcol2 < 90)) AND (pcol3 < 99.90)".to_string() ); // test less or eq @@ -3003,7 +3003,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 10000)".to_string() + "((pcol1 < 100) AND (pcol2 < 100)) AND (pcol3 < 100.00)".to_string() ); // test greater @@ -3022,7 +3022,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 9990)".to_string() + "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 99.90)".to_string() ); // test greater or eq @@ -3041,7 +3041,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 9990)".to_string() + "((pcol1 > 90) AND (pcol2 > 90)) AND (pcol3 > 99.90)".to_string() ); // test not eq @@ -3060,7 +3060,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 != 90) AND (pcol2 != 90)) AND (pcol3 != 9990)".to_string() + "((pcol1 != 90) AND (pcol2 != 90)) AND (pcol3 != 99.90)".to_string() ); // test not in @@ -3103,7 +3103,7 @@ mod tests { let result = strict_projection.strict_project(&predicate).unwrap(); assert_eq!( result.to_string(), - "((pcol1 NOT IN (100, 90)) AND (pcol2 NOT IN (100, 90))) AND (pcol3 NOT IN (9890, 9990, 10090))" + "((pcol1 NOT IN (100, 90)) AND (pcol2 NOT IN (100, 90))) AND (pcol3 NOT IN (98.90, 100.90, 99.90))" .to_string() ); diff --git a/crates/iceberg/src/inspect/history.rs b/crates/iceberg/src/inspect/history.rs new file mode 100644 index 0000000000..8fec7d5776 --- /dev/null +++ b/crates/iceberg/src/inspect/history.rs @@ -0,0 +1,539 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! History table: the chronological log of every snapshot that was ever the +//! table's current snapshot. +//! +//! Each row is one entry of the metadata snapshot-log and answers two questions +//! about the table's lineage: *when* a snapshot became current, and *whether* +//! it is still an ancestor of the current state. Snapshots that were rolled back +//! remain visible in the log yet are flagged `is_current_ancestor = false`, so +//! the live lineage stays distinguishable from abandoned history. +//! +//! References: +//! - + +use std::collections::HashSet; +use std::sync::Arc; + +use arrow_array::RecordBatch; +use arrow_array::builder::{BooleanBuilder, PrimitiveBuilder}; +use arrow_array::types::{Int64Type, TimestampMicrosecondType}; +use futures::{StreamExt, stream}; + +use crate::Result; +use crate::arrow::{UTC_TIME_ZONE, schema_to_arrow_schema}; +use crate::scan::ArrowRecordBatchStream; +use crate::spec::{NestedField, PrimitiveType, Type}; +use crate::table::Table; +use crate::util::snapshot::ancestors_of; + +/// History table. +pub struct HistoryTable<'a> { + table: &'a Table, +} + +impl<'a> HistoryTable<'a> { + /// Create a new History table instance. + pub fn new(table: &'a Table) -> Self { + Self { table } + } + + /// Returns the iceberg schema of the history table. + pub fn schema(&self) -> crate::spec::Schema { + let fields = vec![ + NestedField::required( + 1, + "made_current_at", + Type::Primitive(PrimitiveType::Timestamptz), + ), + NestedField::required(2, "snapshot_id", Type::Primitive(PrimitiveType::Long)), + NestedField::optional(3, "parent_id", Type::Primitive(PrimitiveType::Long)), + NestedField::required( + 4, + "is_current_ancestor", + Type::Primitive(PrimitiveType::Boolean), + ), + ]; + crate::spec::Schema::builder() + .with_fields(fields.into_iter().map(|f| f.into())) + .build() + .unwrap() + } + + /// Scans the history table. + pub async fn scan(&self) -> Result { + let schema = schema_to_arrow_schema(&self.schema())?; + let metadata = self.table.metadata(); + + // Walk the current snapshot's parent chain once: membership in this set + // is what tells the live lineage apart from rolled-back history entries. + let ancestors: HashSet = metadata + .current_snapshot_id() + .map(|id| { + ancestors_of(&self.table.metadata_ref(), id) + .map(|snapshot| snapshot.snapshot_id()) + .collect() + }) + .unwrap_or_default(); + + let mut made_current_at = + PrimitiveBuilder::::new().with_timezone(UTC_TIME_ZONE); + let mut snapshot_id = PrimitiveBuilder::::new(); + let mut parent_id = PrimitiveBuilder::::new(); + let mut is_current_ancestor = BooleanBuilder::new(); + + for entry in metadata.history() { + made_current_at.append_value(entry.timestamp_ms.saturating_mul(1000)); // ms -> µs + snapshot_id.append_value(entry.snapshot_id); + parent_id.append_option( + metadata + .snapshot_by_id(entry.snapshot_id) + .and_then(|snapshot| snapshot.parent_snapshot_id()), + ); + is_current_ancestor.append_value(ancestors.contains(&entry.snapshot_id)); + } + + let batch = RecordBatch::try_new(Arc::new(schema), vec![ + Arc::new(made_current_at.finish()), + Arc::new(snapshot_id.finish()), + Arc::new(parent_id.finish()), + Arc::new(is_current_ancestor.finish()), + ])?; + + Ok(stream::iter(vec![Ok(batch)]).boxed()) + } +} + +#[cfg(test)] +mod tests { + use expect_test::expect; + use futures::TryStreamExt; + + use crate::TableIdent; + use crate::io::FileIO; + use crate::spec::TableMetadata; + use crate::table::Table; + use crate::test_utils::scan::TableTestFixture; + use crate::test_utils::{check_record_batches, test_runtime}; + + fn table_from_metadata_json(metadata_json: &str) -> Table { + let metadata = serde_json::from_str::(metadata_json).unwrap(); + Table::builder() + .metadata(metadata) + .identifier(TableIdent::from_strs(["db", "history"]).unwrap()) + .file_io(FileIO::new_with_fs()) + .metadata_location("s3://bucket/test/location/metadata/v3.json") + .runtime(test_runtime()) + .build() + .unwrap() + } + + #[tokio::test] + async fn test_history_table() { + let table = TableTestFixture::new().table; + + let batch_stream = table.inspect().history().scan().await.unwrap(); + + check_record_batches( + batch_stream.try_collect::>().await.unwrap(), + expect![[r#" + Field { "made_current_at": Timestamp(µs, "+00:00"), metadata: {"PARQUET:field_id": "1"} }, + Field { "snapshot_id": Int64, metadata: {"PARQUET:field_id": "2"} }, + Field { "parent_id": nullable Int64, metadata: {"PARQUET:field_id": "3"} }, + Field { "is_current_ancestor": Boolean, metadata: {"PARQUET:field_id": "4"} }"#]], + expect![[r#" + made_current_at: PrimitiveArray + [ + 2018-01-04T21:22:35.770+00:00, + 2019-04-12T20:29:15.770+00:00, + ], + snapshot_id: PrimitiveArray + [ + 3051729675574597004, + 3055729675574597004, + ], + parent_id: PrimitiveArray + [ + null, + 3051729675574597004, + ], + is_current_ancestor: BooleanArray + [ + true, + true, + ]"#]], + &[], + Some("made_current_at"), + ); + } + + /// A rolled-back snapshot (S2) stays in the log but drops out of the current + /// lineage: the current snapshot (S3) descends from S1, so only S1 and S3 are + /// current ancestors while S2 is flagged `false`. The log also keeps an entry + /// for an expired snapshot (99, absent from `snapshots`) which renders with a + /// null parent and `is_current_ancestor = false`, and a second entry for S1 + /// (a roll-forward makes a snapshot current again) which yields one row per + /// log entry. + #[tokio::test] + async fn test_history_table_with_rolled_back_snapshot() { + let metadata_json = r#"{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 3, + "last-updated-ms": 1600000000000, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": true, "type": "long"}]} + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "current-snapshot-id": 3, + "snapshots": [ + { + "snapshot-id": 1, + "timestamp-ms": 1515100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-1.avro", + "schema-id": 0 + }, + { + "snapshot-id": 2, + "parent-snapshot-id": 1, + "timestamp-ms": 1555100955770, + "sequence-number": 2, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-2.avro", + "schema-id": 0 + }, + { + "snapshot-id": 3, + "parent-snapshot-id": 1, + "timestamp-ms": 1600000000000, + "sequence-number": 3, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-3.avro", + "schema-id": 0 + } + ], + "snapshot-log": [ + {"snapshot-id": 99, "timestamp-ms": 1500000000000}, + {"snapshot-id": 1, "timestamp-ms": 1515100955770}, + {"snapshot-id": 2, "timestamp-ms": 1555100955770}, + {"snapshot-id": 1, "timestamp-ms": 1580000000000}, + {"snapshot-id": 3, "timestamp-ms": 1600000000000} + ], + "metadata-log": [], + "refs": {"main": {"snapshot-id": 3, "type": "branch"}} + }"#; + + let table = table_from_metadata_json(metadata_json); + + let batch_stream = table.inspect().history().scan().await.unwrap(); + + check_record_batches( + batch_stream.try_collect::>().await.unwrap(), + expect![[r#" + Field { "made_current_at": Timestamp(µs, "+00:00"), metadata: {"PARQUET:field_id": "1"} }, + Field { "snapshot_id": Int64, metadata: {"PARQUET:field_id": "2"} }, + Field { "parent_id": nullable Int64, metadata: {"PARQUET:field_id": "3"} }, + Field { "is_current_ancestor": Boolean, metadata: {"PARQUET:field_id": "4"} }"#]], + expect![[r#" + made_current_at: PrimitiveArray + [ + 2017-07-14T02:40:00+00:00, + 2018-01-04T21:22:35.770+00:00, + 2019-04-12T20:29:15.770+00:00, + 2020-01-26T00:53:20+00:00, + 2020-09-13T12:26:40+00:00, + ], + snapshot_id: PrimitiveArray + [ + 99, + 1, + 2, + 1, + 3, + ], + parent_id: PrimitiveArray + [ + null, + null, + 1, + null, + 1, + ], + is_current_ancestor: BooleanArray + [ + false, + true, + false, + true, + true, + ]"#]], + &[], + Some("made_current_at"), + ); + } + + /// Corrupt metadata whose parent pointers form a cycle (S1 and S2 claim each + /// other as parent) must not hang the scan: the ancestor traversal visits each + /// snapshot once, so both log entries still render, flagged as ancestors. + #[tokio::test] + async fn test_history_table_with_parent_cycle() { + let metadata_json = r#"{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 2, + "last-updated-ms": 1555100955770, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": true, "type": "long"}]} + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "current-snapshot-id": 2, + "snapshots": [ + { + "snapshot-id": 1, + "parent-snapshot-id": 2, + "timestamp-ms": 1515100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-1.avro", + "schema-id": 0 + }, + { + "snapshot-id": 2, + "parent-snapshot-id": 1, + "timestamp-ms": 1555100955770, + "sequence-number": 2, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-2.avro", + "schema-id": 0 + } + ], + "snapshot-log": [ + {"snapshot-id": 1, "timestamp-ms": 1515100955770}, + {"snapshot-id": 2, "timestamp-ms": 1555100955770} + ], + "metadata-log": [], + "refs": {"main": {"snapshot-id": 2, "type": "branch"}} + }"#; + + let table = table_from_metadata_json(metadata_json); + + let batch_stream = table.inspect().history().scan().await.unwrap(); + + check_record_batches( + batch_stream.try_collect::>().await.unwrap(), + expect![[r#" + Field { "made_current_at": Timestamp(µs, "+00:00"), metadata: {"PARQUET:field_id": "1"} }, + Field { "snapshot_id": Int64, metadata: {"PARQUET:field_id": "2"} }, + Field { "parent_id": nullable Int64, metadata: {"PARQUET:field_id": "3"} }, + Field { "is_current_ancestor": Boolean, metadata: {"PARQUET:field_id": "4"} }"#]], + expect![[r#" + made_current_at: PrimitiveArray + [ + 2018-01-04T21:22:35.770+00:00, + 2019-04-12T20:29:15.770+00:00, + ], + snapshot_id: PrimitiveArray + [ + 1, + 2, + ], + parent_id: PrimitiveArray + [ + 2, + 1, + ], + is_current_ancestor: BooleanArray + [ + true, + true, + ]"#]], + &[], + Some("made_current_at"), + ); + } + + /// An empty snapshot-log must yield a zero-row batch that still carries the + /// full history schema. + #[tokio::test] + async fn test_history_table_with_empty_snapshot_log() { + let metadata_json = r#"{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 1, + "last-updated-ms": 1515100955770, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": true, "type": "long"}]} + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "current-snapshot-id": 1, + "snapshots": [ + { + "snapshot-id": 1, + "timestamp-ms": 1515100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-1.avro", + "schema-id": 0 + } + ], + "snapshot-log": [], + "metadata-log": [], + "refs": {"main": {"snapshot-id": 1, "type": "branch"}} + }"#; + + let table = table_from_metadata_json(metadata_json); + + let batch_stream = table.inspect().history().scan().await.unwrap(); + + check_record_batches( + batch_stream.try_collect::>().await.unwrap(), + expect![[r#" + Field { "made_current_at": Timestamp(µs, "+00:00"), metadata: {"PARQUET:field_id": "1"} }, + Field { "snapshot_id": Int64, metadata: {"PARQUET:field_id": "2"} }, + Field { "parent_id": nullable Int64, metadata: {"PARQUET:field_id": "3"} }, + Field { "is_current_ancestor": Boolean, metadata: {"PARQUET:field_id": "4"} }"#]], + expect![[r#" + made_current_at: PrimitiveArray + [ + ], + snapshot_id: PrimitiveArray + [ + ], + parent_id: PrimitiveArray + [ + ], + is_current_ancestor: BooleanArray + [ + ]"#]], + &[], + Some("made_current_at"), + ); + } + + /// Without a current snapshot (`current-snapshot-id: -1`) there is no live + /// lineage, so every log entry renders with `is_current_ancestor = false`. + #[tokio::test] + async fn test_history_table_without_current_snapshot() { + let metadata_json = r#"{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 2, + "last-updated-ms": 1555100955770, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [ + {"type": "struct", "schema-id": 0, "fields": [{"id": 1, "name": "x", "required": true, "type": "long"}]} + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [ + { + "snapshot-id": 1, + "timestamp-ms": 1515100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-1.avro", + "schema-id": 0 + }, + { + "snapshot-id": 2, + "parent-snapshot-id": 1, + "timestamp-ms": 1555100955770, + "sequence-number": 2, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-2.avro", + "schema-id": 0 + } + ], + "snapshot-log": [ + {"snapshot-id": 1, "timestamp-ms": 1515100955770}, + {"snapshot-id": 2, "timestamp-ms": 1555100955770} + ], + "metadata-log": [], + "refs": {} + }"#; + + let table = table_from_metadata_json(metadata_json); + + let batch_stream = table.inspect().history().scan().await.unwrap(); + + check_record_batches( + batch_stream.try_collect::>().await.unwrap(), + expect![[r#" + Field { "made_current_at": Timestamp(µs, "+00:00"), metadata: {"PARQUET:field_id": "1"} }, + Field { "snapshot_id": Int64, metadata: {"PARQUET:field_id": "2"} }, + Field { "parent_id": nullable Int64, metadata: {"PARQUET:field_id": "3"} }, + Field { "is_current_ancestor": Boolean, metadata: {"PARQUET:field_id": "4"} }"#]], + expect![[r#" + made_current_at: PrimitiveArray + [ + 2018-01-04T21:22:35.770+00:00, + 2019-04-12T20:29:15.770+00:00, + ], + snapshot_id: PrimitiveArray + [ + 1, + 2, + ], + parent_id: PrimitiveArray + [ + null, + 1, + ], + is_current_ancestor: BooleanArray + [ + false, + false, + ]"#]], + &[], + Some("made_current_at"), + ); + } +} diff --git a/crates/iceberg/src/inspect/manifests.rs b/crates/iceberg/src/inspect/manifests.rs index 60854b8bae..c9f4c3f403 100644 --- a/crates/iceberg/src/inspect/manifests.rs +++ b/crates/iceberg/src/inspect/manifests.rs @@ -28,6 +28,7 @@ use futures::{StreamExt, stream}; use crate::Result; use crate::arrow::schema_to_arrow_schema; +use crate::error::invalid_data; use crate::scan::ArrowRecordBatchStream; use crate::spec::{Datum, FieldSummary, ListType, NestedField, PrimitiveType, StructType, Type}; use crate::table::Table; @@ -161,9 +162,7 @@ impl<'a> ManifestsTable<'a> { let mut partition_summaries = self.partition_summary_builder()?; if let Some(snapshot) = self.table.metadata().current_snapshot() { - let manifest_list = snapshot - .load_manifest_list(self.table.file_io(), &self.table.metadata_ref()) - .await?; + let manifest_list = self.table.manifest_list_reader(snapshot).load().await?; for manifest in manifest_list.entries() { content.append_value(manifest.content as i32); path.append_value(manifest.manifest_path.clone()); @@ -186,13 +185,17 @@ impl<'a> ManifestsTable<'a> { .table .metadata() .partition_spec_by_id(manifest.partition_spec_id) - .unwrap(); - let spec_struct = spec - .partition_type(self.table.metadata().current_schema()) - .unwrap(); + .ok_or_else(|| { + invalid_data!( + "Partition spec {} for manifest {} is not in table metadata", + manifest.partition_spec_id, + manifest.manifest_path + ) + })?; + let spec_struct = spec.partition_type(self.table.metadata().current_schema())?; self.append_partition_summaries( &mut partition_summaries, - &manifest.partitions.clone().unwrap_or_else(Vec::new), + manifest.partitions.as_deref().unwrap_or(&[]), spec_struct, ); } @@ -256,22 +259,22 @@ impl<'a> ManifestsTable<'a> { .unwrap() .append_option(summary.contains_nan); - partition_summaries_builder - .field_builder::(2) - .unwrap() - .append_option(summary.lower_bound.as_ref().map(|v| { - Datum::try_from_bytes(v, field.field_type.as_primitive_type().unwrap().clone()) - .unwrap() - .to_string() - })); - partition_summaries_builder - .field_builder::(3) - .unwrap() - .append_option(summary.upper_bound.as_ref().map(|v| { - Datum::try_from_bytes(v, field.field_type.as_primitive_type().unwrap().clone()) - .unwrap() - .to_string() - })); + let field_type = field.field_type.as_primitive_type().unwrap(); + for (index, bound) in [(2, &summary.lower_bound), (3, &summary.upper_bound)] { + // Bounds cannot be decoded when the source column's type is unknown. + let bound = bound + .as_ref() + .filter(|_| *field_type != PrimitiveType::Unknown) + .map(|v| { + Datum::try_from_bytes(v, field_type.clone()) + .unwrap() + .to_string() + }); + partition_summaries_builder + .field_builder::(index) + .unwrap() + .append_option(bound); + } partition_summaries_builder.append(true); } builder.append(true); @@ -280,11 +283,15 @@ impl<'a> ManifestsTable<'a> { #[cfg(test)] mod tests { + use std::sync::Arc; + + use arrow_array::{Array, ListArray, StructArray}; use expect_test::expect; use futures::TryStreamExt; - use crate::scan::tests::TableTestFixture; + use crate::spec::TableMetadata; use crate::test_utils::check_record_batches; + use crate::test_utils::scan::TableTestFixture; #[tokio::test] async fn test_manifests_table() { @@ -296,18 +303,18 @@ mod tests { check_record_batches( record_batch.try_collect::>().await.unwrap(), expect![[r#" - Field { name: "content", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "14"} }, - Field { name: "path", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "1"} }, - Field { name: "length", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "2"} }, - Field { name: "partition_spec_id", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "3"} }, - Field { name: "added_snapshot_id", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "4"} }, - Field { name: "added_data_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "5"} }, - Field { name: "existing_data_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "6"} }, - Field { name: "deleted_data_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "7"} }, - Field { name: "added_delete_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "15"} }, - Field { name: "existing_delete_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "16"} }, - Field { name: "deleted_delete_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "17"} }, - Field { name: "partition_summaries", data_type: List(Field { name: "item", data_type: Struct([Field { name: "contains_null", data_type: Boolean, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "10"} }, Field { name: "contains_nan", data_type: Boolean, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "11"} }, Field { name: "lower_bound", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "12"} }, Field { name: "upper_bound", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "13"} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "9"} }), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "8"} }"#]], + Field { "content": Int32, metadata: {"PARQUET:field_id": "14"} }, + Field { "path": Utf8, metadata: {"PARQUET:field_id": "1"} }, + Field { "length": Int64, metadata: {"PARQUET:field_id": "2"} }, + Field { "partition_spec_id": Int32, metadata: {"PARQUET:field_id": "3"} }, + Field { "added_snapshot_id": Int64, metadata: {"PARQUET:field_id": "4"} }, + Field { "added_data_files_count": Int32, metadata: {"PARQUET:field_id": "5"} }, + Field { "existing_data_files_count": Int32, metadata: {"PARQUET:field_id": "6"} }, + Field { "deleted_data_files_count": Int32, metadata: {"PARQUET:field_id": "7"} }, + Field { "added_delete_files_count": Int32, metadata: {"PARQUET:field_id": "15"} }, + Field { "existing_delete_files_count": Int32, metadata: {"PARQUET:field_id": "16"} }, + Field { "deleted_delete_files_count": Int32, metadata: {"PARQUET:field_id": "17"} }, + Field { "partition_summaries": List(non-null Struct("contains_null": non-null Boolean, metadata: {"PARQUET:field_id": "10"}, "contains_nan": Boolean, metadata: {"PARQUET:field_id": "11"}, "lower_bound": Utf8, metadata: {"PARQUET:field_id": "12"}, "upper_bound": Utf8, metadata: {"PARQUET:field_id": "13"}), metadata: {"PARQUET:field_id": "9"}), metadata: {"PARQUET:field_id": "8"} }"#]], expect![[r#" content: PrimitiveArray [ @@ -381,4 +388,59 @@ mod tests { Some("path"), ); } + + #[tokio::test] + async fn test_manifests_table_with_dropped_partition_source_column() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Evolve the table so that the manifests reference a historical spec whose source + // column is no longer in the current schema: add an unpartitioned default spec, then + // drop the source column of the original spec. + let mut metadata = serde_json::to_value(fixture.table.metadata()).unwrap(); + let current_schema_id = metadata["current-schema-id"].clone(); + let schemas = metadata["schemas"].as_array_mut().unwrap(); + for schema in schemas { + if schema["schema-id"] == current_schema_id { + let fields = schema["fields"].as_array_mut().unwrap(); + fields.retain(|field| field["id"] != 1); + let identifier_ids = schema["identifier-field-ids"].as_array_mut().unwrap(); + identifier_ids.retain(|id| *id != 1); + } + } + + metadata["partition-specs"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({"spec-id": 1, "fields": []})); + metadata["default-spec-id"] = serde_json::json!(1); + + let metadata: TableMetadata = serde_json::from_value(metadata).unwrap(); + let table = fixture.table.clone().with_metadata(Arc::new(metadata)); + + let batches: Vec<_> = table + .inspect() + .manifests() + .scan() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 1); + let summaries = batches[0] + .column_by_name("partition_summaries") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let summary = summaries.value(0); + let summary = summary.as_any().downcast_ref::().unwrap(); + assert_eq!(summary.len(), 1); + assert!(summary.column_by_name("contains_null").unwrap().is_valid(0)); + assert!(summary.column_by_name("contains_nan").unwrap().is_valid(0)); + assert!(summary.column_by_name("lower_bound").unwrap().is_null(0)); + assert!(summary.column_by_name("upper_bound").unwrap().is_null(0)); + } } diff --git a/crates/iceberg/src/inspect/metadata_table.rs b/crates/iceberg/src/inspect/metadata_table.rs index 92571db181..0e83c20aed 100644 --- a/crates/iceberg/src/inspect/metadata_table.rs +++ b/crates/iceberg/src/inspect/metadata_table.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use super::{ManifestsTable, SnapshotsTable}; +use super::{HistoryTable, ManifestsTable, SnapshotsTable}; use crate::table::Table; /// Metadata table is used to inspect a table's history, snapshots, and other metadata as a table. @@ -34,6 +34,8 @@ pub enum MetadataTableType { Snapshots, /// [`ManifestsTable`] Manifests, + /// [`HistoryTable`] + History, } impl MetadataTableType { @@ -42,6 +44,7 @@ impl MetadataTableType { match self { MetadataTableType::Snapshots => "snapshots", MetadataTableType::Manifests => "manifests", + MetadataTableType::History => "history", } } @@ -55,10 +58,11 @@ impl MetadataTableType { impl TryFrom<&str> for MetadataTableType { type Error = String; - fn try_from(value: &str) -> std::result::Result { + fn try_from(value: &str) -> Result { match value { "snapshots" => Ok(Self::Snapshots), "manifests" => Ok(Self::Manifests), + "history" => Ok(Self::History), _ => Err(format!("invalid metadata table type: {value}")), } } @@ -71,12 +75,17 @@ impl<'a> MetadataTable<'a> { } /// Get the snapshots table. - pub fn snapshots(&self) -> SnapshotsTable { + pub fn snapshots(&self) -> SnapshotsTable<'_> { SnapshotsTable::new(self.0) } /// Get the manifests table. - pub fn manifests(&self) -> ManifestsTable { + pub fn manifests(&self) -> ManifestsTable<'_> { ManifestsTable::new(self.0) } + + /// Get the history table. + pub fn history(&self) -> HistoryTable<'_> { + HistoryTable::new(self.0) + } } diff --git a/crates/iceberg/src/inspect/mod.rs b/crates/iceberg/src/inspect/mod.rs index b64420ea11..540715c097 100644 --- a/crates/iceberg/src/inspect/mod.rs +++ b/crates/iceberg/src/inspect/mod.rs @@ -17,10 +17,12 @@ //! Metadata table APIs. +mod history; mod manifests; mod metadata_table; mod snapshots; +pub use history::HistoryTable; pub use manifests::ManifestsTable; pub use metadata_table::*; pub use snapshots::SnapshotsTable; diff --git a/crates/iceberg/src/inspect/snapshots.rs b/crates/iceberg/src/inspect/snapshots.rs index 6081ec165b..22046daba7 100644 --- a/crates/iceberg/src/inspect/snapshots.rs +++ b/crates/iceberg/src/inspect/snapshots.rs @@ -139,8 +139,8 @@ mod tests { use expect_test::expect; use futures::TryStreamExt; - use crate::scan::tests::TableTestFixture; use crate::test_utils::check_record_batches; + use crate::test_utils::scan::TableTestFixture; #[tokio::test] async fn test_snapshots_table() { @@ -151,14 +151,14 @@ mod tests { check_record_batches( batch_stream.try_collect::>().await.unwrap(), expect![[r#" - Field { name: "committed_at", data_type: Timestamp(Microsecond, Some("+00:00")), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "1"} }, - Field { name: "snapshot_id", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "2"} }, - Field { name: "parent_id", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "3"} }, - Field { name: "operation", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "4"} }, - Field { name: "manifest_list", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "5"} }, - Field { name: "summary", data_type: Map(Field { name: "key_value", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "7"} }, Field { name: "value", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "8"} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "6"} }"#]], + Field { "committed_at": Timestamp(µs, "+00:00"), metadata: {"PARQUET:field_id": "1"} }, + Field { "snapshot_id": Int64, metadata: {"PARQUET:field_id": "2"} }, + Field { "parent_id": nullable Int64, metadata: {"PARQUET:field_id": "3"} }, + Field { "operation": nullable Utf8, metadata: {"PARQUET:field_id": "4"} }, + Field { "manifest_list": nullable Utf8, metadata: {"PARQUET:field_id": "5"} }, + Field { "summary": nullable Map("key_value": non-null Struct("key": non-null Utf8, metadata: {"PARQUET:field_id": "7"}, "value": Utf8, metadata: {"PARQUET:field_id": "8"}), unsorted), metadata: {"PARQUET:field_id": "6"} }"#]], expect![[r#" - committed_at: PrimitiveArray + committed_at: PrimitiveArray [ 2018-01-04T21:22:35.770+00:00, 2019-04-12T20:29:15.770+00:00, diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 6e2d152ed7..42eec4db56 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -15,93 +15,171 @@ // specific language governing permissions and limitations // under the License. -use std::any::{Any, TypeId}; -use std::collections::HashMap; use std::ops::Range; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use bytes::Bytes; -use opendal::Operator; -use url::Url; +use futures::{Stream, StreamExt}; -use super::storage::Storage; -use crate::{Error, ErrorKind, Result}; +use super::storage::{ + LocalFsStorageFactory, MemoryStorageFactory, Storage, StorageConfig, StorageFactory, +}; +use crate::Result; /// FileIO implementation, used to manipulate files in underlying storage. /// +/// FileIO wraps a `dyn Storage` with lazy initialization via `StorageFactory`. +/// The storage is created on first use and cached for subsequent operations. +/// /// # Note /// -/// All path passed to `FileIO` must be absolute path starting with scheme string used to construct `FileIO`. -/// For example, if you construct `FileIO` with `s3a` scheme, then all path passed to `FileIO` must start with `s3a://`. +/// All paths passed to `FileIO` must be absolute paths starting with the scheme string +/// appropriate for the storage backend being used. +/// +/// This crate provides native support for local filesystem (`file://`) and +/// memory (`memory://`) storage. For extensive storage backend support (S3, GCS, +/// OSS, Azure, etc.), use the +/// [`iceberg-storage-opendal`](https://crates.io/crates/iceberg-storage-opendal) crate. +/// +/// # Example +/// +/// ```rust,ignore +/// use iceberg::io::{FileIO, FileIOBuilder}; +/// use iceberg::io::{LocalFsStorageFactory, MemoryStorageFactory}; +/// use std::sync::Arc; /// -/// Supported storages: +/// // Create FileIO with memory storage for testing +/// let file_io = FileIO::new_with_memory(); /// -/// | Storage | Feature Flag | Expected Path Format | Schemes | -/// |--------------------|-------------------|----------------------------------| ------------------------------| -/// | Local file system | `storage-fs` | `file` | `file://path/to/file` | -/// | Memory | `storage-memory` | `memory` | `memory://path/to/file` | -/// | S3 | `storage-s3` | `s3`, `s3a` | `s3:///path/to/file` | -/// | GCS | `storage-gcs` | `gs`, `gcs` | `gs:///path/to/file` | -/// | OSS | `storage-oss` | `oss` | `oss:///path/to/file` | -/// | Azure Datalake | `storage-azdls` | `abfs`, `abfss`, `wasb`, `wasbs` | `abfs://@.dfs.core.windows.net/path/to/file` or `wasb://@.blob.core.windows.net/path/to/file` | +/// // Create FileIO with local filesystem storage +/// let file_io = FileIO::new_with_fs(); +/// +/// // Create FileIO with custom factory +/// let file_io = FileIOBuilder::new(Arc::new(LocalFsStorageFactory)) +/// .with_prop("key", "value") +/// .build(); +/// ``` #[derive(Clone, Debug)] pub struct FileIO { - builder: FileIOBuilder, + /// Storage configuration containing properties + config: StorageConfig, + /// Factory for creating storage instances + factory: Arc, + /// Cached storage instance (lazily initialized) + storage: Arc>>, +} + +mod _serde { + use std::sync::Arc; - inner: Arc, + use serde::{Deserialize, Serialize}; + + use super::{StorageConfig, StorageFactory}; + + #[derive(Serialize)] + pub(super) struct SerializableFileIO<'a> { + pub(super) config: &'a StorageConfig, + pub(super) factory: &'a Arc, + } + + #[derive(Deserialize)] + pub(super) struct DeserializedFileIO { + pub(super) config: StorageConfig, + pub(super) factory: Arc, + } } impl FileIO { - /// Convert FileIO into [`FileIOBuilder`] which used to build this FileIO. + /// Create a new FileIO backed by in-memory storage. + /// + /// This is useful for testing scenarios where persistent storage is not needed. + pub fn new_with_memory() -> Self { + Self { + config: StorageConfig::new(), + factory: Arc::new(MemoryStorageFactory), + storage: Arc::new(OnceLock::new()), + } + } + + /// Create a new FileIO backed by local filesystem storage. /// - /// This function is useful when you want serialize and deserialize FileIO across - /// distributed systems. - pub fn into_builder(self) -> FileIOBuilder { - self.builder + /// This is useful for local development and testing with real files. + pub fn new_with_fs() -> Self { + Self { + config: StorageConfig::new(), + factory: Arc::new(LocalFsStorageFactory), + storage: Arc::new(OnceLock::new()), + } } - /// Try to infer file io scheme from path. See [`FileIO`] for supported schemes. + /// Serializes all portable state of this `FileIO` into a byte vector. /// - /// - If it's a valid url, for example `s3://bucket/a`, url scheme will be used, and the rest of the url will be ignored. - /// - If it's not a valid url, will try to detect if it's a file path. + /// This includes the storage configuration and factory, but not the cached storage instance. + /// The storage cache is rebuilt lazily on first use after calling [`FileIO::deserialize_all`]. /// - /// Otherwise will return parsing error. - pub fn from_path(path: impl AsRef) -> crate::Result { - let url = Url::parse(path.as_ref()) - .map_err(Error::from) - .or_else(|e| { - Url::from_file_path(path.as_ref()).map_err(|_| { - Error::new( - ErrorKind::DataInvalid, - "Input is neither a valid url nor path", - ) - .with_context("input", path.as_ref().to_string()) - .with_source(e) - }) - })?; - - Ok(FileIOBuilder::new(url.scheme())) + /// The serialized representation is not a stable format and may change between crate versions. + /// Applications should not rely on it for long-term storage or exchange it between incompatible + /// versions of this crate. + /// + /// All storage configuration properties are included in the serialized representation. These + /// properties may contain credentials or other sensitive values, so the returned bytes must be + /// protected in transit and at rest by the application embedding this crate. + /// + /// Storage factories are serialized through [`typetag`](https://docs.rs/typetag). Third-party + /// factories must use `#[typetag::serde]` on their [`StorageFactory`] implementation. + pub fn serialize_all(&self) -> Result> { + Ok(serde_json::to_vec(&_serde::SerializableFileIO { + config: &self.config, + factory: &self.factory, + })?) } - /// Deletes file. + /// Deserializes a `FileIO` previously produced by [`FileIO::serialize_all`]. /// - /// # Arguments + /// The receiving binary must use a compatible crate version and link the concrete factory + /// implementation so it is registered with `typetag`. Backend-specific requirements are + /// documented by each storage factory implementation. + pub fn deserialize_all(bytes: &[u8]) -> Result { + let _serde::DeserializedFileIO { config, factory } = serde_json::from_slice(bytes)?; + Ok(Self { + config, + factory, + storage: Arc::new(OnceLock::new()), + }) + } + + /// Get the storage configuration. + pub fn config(&self) -> &StorageConfig { + &self.config + } + + /// Get or create the storage instance. /// - /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. - pub async fn delete(&self, path: impl AsRef) -> Result<()> { - let (op, relative_path) = self.inner.create_operator(&path)?; - Ok(op.delete(relative_path).await?) + /// The factory is invoked on first access and the result is cached + /// for all subsequent operations. + fn get_storage(&self) -> Result> { + // Check if already initialized + if let Some(storage) = self.storage.get() { + return Ok(storage.clone()); + } + + // Build the storage + let storage = self.factory.build(&self.config)?; + + // Try to set it (another thread might have set it first) + let _ = self.storage.set(storage.clone()); + + // Return whatever is in the cell (either ours or another thread's) + Ok(self.storage.get().unwrap().clone()) } - /// Remove the path and all nested dirs and files recursively. + /// Deletes file. /// /// # Arguments /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. - #[deprecated(note = "use remove_dir_all instead", since = "0.4.0")] - pub async fn remove_all(&self, path: impl AsRef) -> Result<()> { - let (op, relative_path) = self.inner.create_operator(&path)?; - Ok(op.remove_all(relative_path).await?) + pub async fn delete(&self, path: impl AsRef) -> Result<()> { + self.get_storage()?.delete(path.as_ref()).await } /// Remove the path and all nested dirs and files recursively. @@ -115,14 +193,20 @@ impl FileIO { /// - If the path is a file or not exist, this function will be no-op. /// - If the path is a empty directory, this function will remove the directory itself. /// - If the path is a non-empty directory, this function will remove the directory and all nested files and directories. - pub async fn remove_dir_all(&self, path: impl AsRef) -> Result<()> { - let (op, relative_path) = self.inner.create_operator(&path)?; - let path = if relative_path.ends_with('/') { - relative_path.to_string() - } else { - format!("{relative_path}/") - }; - Ok(op.remove_all(&path).await?) + pub async fn delete_prefix(&self, path: impl AsRef) -> Result<()> { + self.get_storage()?.delete_prefix(path.as_ref()).await + } + + /// Delete multiple files from a stream of paths. + /// + /// # Arguments + /// + /// * paths: A stream of absolute paths starting with the scheme string used to construct [`FileIO`]. + pub async fn delete_stream( + &self, + paths: impl Stream + Send + 'static, + ) -> Result<()> { + self.get_storage()?.delete_stream(paths.boxed()).await } /// Check file exists. @@ -131,8 +215,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub async fn exists(&self, path: impl AsRef) -> Result { - let (op, relative_path) = self.inner.create_operator(&path)?; - Ok(op.exists(relative_path).await?) + self.get_storage()?.exists(path.as_ref()).await } /// Creates input file. @@ -141,14 +224,7 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub fn new_input(&self, path: impl AsRef) -> Result { - let (op, relative_path) = self.inner.create_operator(&path)?; - let path = path.as_ref().to_string(); - let relative_path_pos = path.len() - relative_path.len(); - Ok(InputFile { - op, - path, - relative_path_pos, - }) + self.get_storage()?.new_input(path.as_ref()) } /// Creates output file. @@ -157,127 +233,60 @@ impl FileIO { /// /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. pub fn new_output(&self, path: impl AsRef) -> Result { - let (op, relative_path) = self.inner.create_operator(&path)?; - let path = path.as_ref().to_string(); - let relative_path_pos = path.len() - relative_path.len(); - Ok(OutputFile { - op, - path, - relative_path_pos, - }) - } -} - -/// Container for storing type-safe extensions used to configure underlying FileIO behavior. -#[derive(Clone, Debug, Default)] -pub struct Extensions(HashMap>); - -impl Extensions { - /// Add an extension. - pub fn add(&mut self, ext: T) { - self.0.insert(TypeId::of::(), Arc::new(ext)); - } - - /// Extends the current set of extensions with another set of extensions. - pub fn extend(&mut self, extensions: Extensions) { - self.0.extend(extensions.0); - } - - /// Fetch an extension. - pub fn get(&self) -> Option> - where T: 'static + Send + Sync + Clone { - let type_id = TypeId::of::(); - self.0 - .get(&type_id) - .and_then(|arc_any| Arc::clone(arc_any).downcast::().ok()) + self.get_storage()?.new_output(path.as_ref()) } } /// Builder for [`FileIO`]. +/// +/// The builder accepts an explicit `StorageFactory` and configuration properties. +/// Storage is lazily initialized on first use. #[derive(Clone, Debug)] pub struct FileIOBuilder { - /// This is used to infer scheme of operator. - /// - /// If this is `None`, then [`FileIOBuilder::build`](FileIOBuilder::build) will build a local file io. - scheme_str: Option, - /// Arguments for operator. - props: HashMap, - /// Optional extensions to configure the underlying FileIO behavior. - extensions: Extensions, + /// Factory for creating storage instances + factory: Arc, + /// Storage configuration + config: StorageConfig, } impl FileIOBuilder { - /// Creates a new builder with scheme. - /// See [`FileIO`] for supported schemes. - pub fn new(scheme_str: impl ToString) -> Self { - Self { - scheme_str: Some(scheme_str.to_string()), - props: HashMap::default(), - extensions: Extensions::default(), - } - } - - /// Creates a new builder for local file io. - pub fn new_fs_io() -> Self { + /// Creates a new builder with the given storage factory. + pub fn new(factory: Arc) -> Self { Self { - scheme_str: None, - props: HashMap::default(), - extensions: Extensions::default(), + factory, + config: StorageConfig::new(), } } - /// Fetch the scheme string. - /// - /// The scheme_str will be empty if it's None. - pub fn into_parts(self) -> (String, HashMap, Extensions) { - ( - self.scheme_str.unwrap_or_default(), - self.props, - self.extensions, - ) - } - - /// Add argument for operator. + /// Add a configuration property. pub fn with_prop(mut self, key: impl ToString, value: impl ToString) -> Self { - self.props.insert(key.to_string(), value.to_string()); + self.config = self.config.with_prop(key.to_string(), value.to_string()); self } - /// Add argument for operator. + /// Add multiple configuration properties. pub fn with_props( mut self, args: impl IntoIterator, ) -> Self { - self.props - .extend(args.into_iter().map(|e| (e.0.to_string(), e.1.to_string()))); - self - } - - /// Add an extension to the file IO builder. - pub fn with_extension(mut self, ext: T) -> Self { - self.extensions.add(ext); + self.config = self + .config + .with_props(args.into_iter().map(|e| (e.0.to_string(), e.1.to_string()))); self } - /// Adds multiple extensions to the file IO builder. - pub fn with_extensions(mut self, extensions: Extensions) -> Self { - self.extensions.extend(extensions); - self - } - - /// Fetch an extension from the file IO builder. - pub fn extension(&self) -> Option> - where T: 'static + Send + Sync + Clone { - self.extensions.get::() + /// Get the storage configuration. + pub fn config(&self) -> &StorageConfig { + &self.config } /// Builds [`FileIO`]. - pub fn build(self) -> Result { - let storage = Storage::build(self.clone())?; - Ok(FileIO { - builder: self, - inner: Arc::new(storage), - }) + pub fn build(self) -> FileIO { + FileIO { + config: self.config, + factory: self.factory, + storage: Arc::new(OnceLock::new()), + } } } @@ -292,7 +301,6 @@ pub struct FileMetadata { /// Trait for reading file. /// /// # TODO -/// /// It's possible for us to remove the async_trait, but we need to figure /// out how to handle the object safety. #[async_trait::async_trait] @@ -300,62 +308,57 @@ pub trait FileRead: Send + Sync + Unpin + 'static { /// Read file content with given range. /// /// TODO: we can support reading non-contiguous bytes in the future. - async fn read(&self, range: Range) -> crate::Result; + async fn read(&self, range: Range) -> Result; } #[async_trait::async_trait] -impl FileRead for opendal::Reader { - async fn read(&self, range: Range) -> crate::Result { - Ok(opendal::Reader::read(self, range).await?.to_bytes()) +impl + Send + Sync + Unpin + 'static> FileRead for T { + async fn read(&self, range: Range) -> Result { + self.as_ref().read(range).await } } /// Input file is used for reading from files. #[derive(Debug)] pub struct InputFile { - op: Operator, - // Absolution path of file. + storage: Arc, + // Absolute path of file. path: String, - // Relative path of file to uri, starts at [`relative_path_pos`] - relative_path_pos: usize, } impl InputFile { + /// Creates a new input file. + pub fn new(storage: Arc, path: String) -> Self { + Self { storage, path } + } + /// Absolute path to root uri. pub fn location(&self) -> &str { &self.path } /// Check if file exists. - pub async fn exists(&self) -> crate::Result { - Ok(self.op.exists(&self.path[self.relative_path_pos..]).await?) + pub async fn exists(&self) -> Result { + self.storage.exists(&self.path).await } /// Fetch and returns metadata of file. - pub async fn metadata(&self) -> crate::Result { - let meta = self.op.stat(&self.path[self.relative_path_pos..]).await?; - - Ok(FileMetadata { - size: meta.content_length(), - }) + pub async fn metadata(&self) -> Result { + self.storage.metadata(&self.path).await } /// Read and returns whole content of file. /// /// For continuous reading, use [`Self::reader`] instead. - pub async fn read(&self) -> crate::Result { - Ok(self - .op - .read(&self.path[self.relative_path_pos..]) - .await? - .to_bytes()) + pub async fn read(&self) -> Result { + self.storage.read(&self.path).await } /// Creates [`FileRead`] for continuous reading. /// /// For one-time reading, use [`Self::read`] instead. - pub async fn reader(&self) -> crate::Result> { - Ok(self.op.reader(&self.path[self.relative_path_pos..]).await?) + pub async fn reader(&self) -> Result> { + self.storage.reader(&self.path).await } } @@ -370,48 +373,28 @@ pub trait FileWrite: Send + Unpin + 'static { /// Write bytes to file. /// /// TODO: we can support writing non-contiguous bytes in the future. - async fn write(&mut self, bs: Bytes) -> crate::Result<()>; + async fn write(&mut self, bs: Bytes) -> Result<()>; - /// Close file. + /// Close the file and return its stored size, including encryption overhead for encrypted files. /// /// Calling close on closed file will generate an error. - async fn close(&mut self) -> crate::Result<()>; -} - -#[async_trait::async_trait] -impl FileWrite for opendal::Writer { - async fn write(&mut self, bs: Bytes) -> crate::Result<()> { - Ok(opendal::Writer::write(self, bs).await?) - } - - async fn close(&mut self) -> crate::Result<()> { - let _ = opendal::Writer::close(self).await?; - Ok(()) - } -} - -#[async_trait::async_trait] -impl FileWrite for Box { - async fn write(&mut self, bs: Bytes) -> crate::Result<()> { - self.as_mut().write(bs).await - } - - async fn close(&mut self) -> crate::Result<()> { - self.as_mut().close().await - } + async fn close(&mut self) -> Result; } /// Output file is used for writing to files.. #[derive(Debug)] pub struct OutputFile { - op: Operator, - // Absolution path of file. + storage: Arc, + // Absolute path of file. path: String, - // Relative path of file to uri, starts at [`relative_path_pos`] - relative_path_pos: usize, } impl OutputFile { + /// Creates a new output file. + pub fn new(storage: Arc, path: String) -> Self { + Self { storage, path } + } + /// Relative path to root uri. pub fn location(&self) -> &str { &self.path @@ -419,22 +402,21 @@ impl OutputFile { /// Checks if file exists. pub async fn exists(&self) -> Result { - Ok(self.op.exists(&self.path[self.relative_path_pos..]).await?) + self.storage.exists(&self.path).await } /// Deletes file. /// /// If the file does not exist, it will not return error. pub async fn delete(&self) -> Result<()> { - Ok(self.op.delete(&self.path[self.relative_path_pos..]).await?) + self.storage.delete(&self.path).await } /// Converts into [`InputFile`]. pub fn to_input_file(self) -> InputFile { InputFile { - op: self.op, + storage: self.storage, path: self.path, - relative_path_pos: self.relative_path_pos, } } @@ -444,10 +426,8 @@ impl OutputFile { /// /// Calling `write` will overwrite the file if it exists. /// For continuous writing, use [`Self::writer`]. - pub async fn write(&self, bs: Bytes) -> crate::Result<()> { - let mut writer = self.writer().await?; - writer.write(bs).await?; - writer.close().await + pub async fn write(&self, bs: Bytes) -> Result<()> { + self.storage.write(&self.path, bs).await } /// Creates output file for continuous writing. @@ -455,10 +435,8 @@ impl OutputFile { /// # Notes /// /// For one-time writing, use [`Self::write`] instead. - pub async fn writer(&self) -> crate::Result> { - Ok(Box::new( - self.op.writer(&self.path[self.relative_path_pos..]).await?, - )) + pub async fn writer(&self) -> Result> { + self.storage.writer(&self.path).await } } @@ -467,6 +445,7 @@ mod tests { use std::fs::{File, create_dir_all}; use std::io::Write; use std::path::Path; + use std::sync::Arc; use bytes::Bytes; use futures::AsyncReadExt; @@ -474,9 +453,10 @@ mod tests { use tempfile::TempDir; use super::{FileIO, FileIOBuilder}; + use crate::io::{LocalFsStorageFactory, MemoryStorageFactory}; fn create_local_file_io() -> FileIO { - FileIOBuilder::new_fs_io().build().unwrap() + FileIO::new_with_fs() } fn write_to_file>(s: &str, path: P) { @@ -506,7 +486,6 @@ mod tests { let input_file = file_io.new_input(&full_path).unwrap(); assert!(input_file.exists().await.unwrap()); - // Remove heading slash assert_eq!(&full_path, input_file.location()); let read_content = read_from_file(full_path).await; @@ -529,14 +508,14 @@ mod tests { assert!(file_io.exists(&a_path).await.unwrap()); // Remove a file should be no-op. - file_io.remove_dir_all(&a_path).await.unwrap(); + file_io.delete_prefix(&a_path).await.unwrap(); assert!(file_io.exists(&a_path).await.unwrap()); // Remove a not exist dir should be no-op. - file_io.remove_dir_all("not_exists/").await.unwrap(); + file_io.delete_prefix("not_exists/").await.unwrap(); // Remove a dir should remove all files in it. - file_io.remove_dir_all(&sub_dir_path).await.unwrap(); + file_io.delete_prefix(&sub_dir_path).await.unwrap(); assert!(!file_io.exists(&b_path).await.unwrap()); assert!(!file_io.exists(&c_path).await.unwrap()); assert!(file_io.exists(&a_path).await.unwrap()); @@ -555,7 +534,7 @@ mod tests { let file_io = create_local_file_io(); assert!(!file_io.exists(&full_path).await.unwrap()); assert!(file_io.delete(&full_path).await.is_ok()); - assert!(file_io.remove_dir_all(&full_path).await.is_ok()); + assert!(file_io.delete_prefix(&full_path).await.is_ok()); } #[tokio::test] @@ -582,27 +561,9 @@ mod tests { assert_eq!(content, &read_content); } - #[test] - fn test_create_file_from_path() { - let io = FileIO::from_path("/tmp/a").unwrap(); - assert_eq!("file", io.scheme_str.unwrap().as_str()); - - let io = FileIO::from_path("file:/tmp/b").unwrap(); - assert_eq!("file", io.scheme_str.unwrap().as_str()); - - let io = FileIO::from_path("file:///tmp/c").unwrap(); - assert_eq!("file", io.scheme_str.unwrap().as_str()); - - let io = FileIO::from_path("s3://bucket/a").unwrap(); - assert_eq!("s3", io.scheme_str.unwrap().as_str()); - - let io = FileIO::from_path("tmp/||c"); - assert!(io.is_err()); - } - #[tokio::test] async fn test_memory_io() { - let io = FileIOBuilder::new("memory").build().unwrap(); + let io = FileIO::new_with_memory(); let path = format!("{}/1.txt", TempDir::new().unwrap().path().to_str().unwrap()); @@ -617,4 +578,103 @@ mod tests { io.delete(&path).await.unwrap(); assert!(!io.exists(&path).await.unwrap()); } + + #[tokio::test] + async fn test_file_io_builder_with_props() { + let factory = Arc::new(MemoryStorageFactory); + let file_io = FileIOBuilder::new(factory) + .with_prop("key1", "value1") + .with_prop("key2", "value2") + .build(); + + assert_eq!(file_io.config().get("key1"), Some(&"value1".to_string())); + assert_eq!(file_io.config().get("key2"), Some(&"value2".to_string())); + } + + #[tokio::test] + async fn test_file_io_builder_with_multiple_props() { + let factory = Arc::new(LocalFsStorageFactory); + let props = vec![("key1", "value1"), ("key2", "value2")]; + let file_io = FileIOBuilder::new(factory).with_props(props).build(); + + assert_eq!(file_io.config().get("key1"), Some(&"value1".to_string())); + assert_eq!(file_io.config().get("key2"), Some(&"value2".to_string())); + } + + #[tokio::test] + async fn test_memory_file_io_serialization_roundtrip() { + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("test-property", "test-value") + .with_prop("s3.session-token", "test-token") + .build(); + + file_io + .new_output("memory://test/file.txt") + .unwrap() + .write("test".into()) + .await + .unwrap(); + assert!(file_io.storage.get().is_some()); + + let serialized = file_io.serialize_all().unwrap(); + let deserialized = FileIO::deserialize_all(&serialized).unwrap(); + assert!(deserialized.storage.get().is_none()); + assert_eq!( + deserialized.config().get("test-property"), + Some(&"test-value".to_string()) + ); + assert_eq!( + deserialized.config().get("s3.session-token"), + Some(&"test-token".to_string()) + ); + + deserialized + .new_output("memory://test/roundtrip.txt") + .unwrap() + .write("roundtrip".into()) + .await + .unwrap(); + assert_eq!( + deserialized + .new_input("memory://test/roundtrip.txt") + .unwrap() + .read() + .await + .unwrap(), + Bytes::from("roundtrip") + ); + assert!(deserialized.storage.get().is_some()); + } + + #[tokio::test] + async fn test_local_fs_file_io_serialization_roundtrip() { + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().join("roundtrip.txt"); + let path = path.to_str().unwrap(); + let file_io = FileIOBuilder::new(Arc::new(LocalFsStorageFactory)) + .with_prop("test-property", "test-value") + .build(); + + file_io + .new_output(path) + .unwrap() + .write("roundtrip".into()) + .await + .unwrap(); + assert!(file_io.storage.get().is_some()); + + let serialized = file_io.serialize_all().unwrap(); + let deserialized = FileIO::deserialize_all(&serialized).unwrap(); + assert!(deserialized.storage.get().is_none()); + assert_eq!( + deserialized.config().get("test-property"), + Some(&"test-value".to_string()) + ); + assert!(deserialized.exists(path).await.unwrap()); + assert_eq!( + deserialized.new_input(path).unwrap().read().await.unwrap(), + Bytes::from("roundtrip") + ); + assert!(deserialized.storage.get().is_some()); + } } diff --git a/crates/iceberg/src/io/mod.rs b/crates/iceberg/src/io/mod.rs index 5eb5964345..59a21c4b66 100644 --- a/crates/iceberg/src/io/mod.rs +++ b/crates/iceberg/src/io/mod.rs @@ -19,42 +19,24 @@ //! //! # How to build `FileIO` //! -//! We provided a `FileIOBuilder` to build `FileIO` from scratch. For example: +//! `FileIO` uses explicit `StorageFactory` injection for storage creation. +//! We provide convenience constructors for common use cases: //! -//! ```rust -//! use iceberg::Result; -//! use iceberg::io::{FileIOBuilder, S3_REGION}; +//! ```rust,ignore +//! use iceberg::io::{FileIO, FileIOBuilder}; +//! use iceberg::io::{LocalFsStorageFactory, MemoryStorageFactory}; +//! use std::sync::Arc; //! -//! # fn test() -> Result<()> { -//! // Build a memory file io. -//! let file_io = FileIOBuilder::new("memory").build()?; -//! // Build an fs file io. -//! let file_io = FileIOBuilder::new("fs").build()?; -//! // Build an s3 file io. -//! let file_io = FileIOBuilder::new("s3") -//! .with_prop(S3_REGION, "us-east-1") -//! .build()?; -//! # Ok(()) -//! # } -//! ``` -//! -//! Or you can pass a path to ask `FileIO` to infer schema for you: +//! // Build a memory file io for testing +//! let file_io = FileIO::new_with_memory(); //! -//! ```rust -//! use iceberg::Result; -//! use iceberg::io::{FileIO, S3_REGION}; +//! // Build a local filesystem file io +//! let file_io = FileIO::new_with_fs(); //! -//! # fn test() -> Result<()> { -//! // Build a memory file io. -//! let file_io = FileIO::from_path("memory:///")?.build()?; -//! // Build an fs file io. -//! let file_io = FileIO::from_path("fs:///tmp")?.build()?; -//! // Build an s3 file io. -//! let file_io = FileIO::from_path("s3://bucket/a")? -//! .with_prop(S3_REGION, "us-east-1") -//! .build()?; -//! # Ok(()) -//! # } +//! // Build with explicit factory and configuration +//! let file_io = FileIOBuilder::new(Arc::new(LocalFsStorageFactory)) +//! .with_prop("key", "value") +//! .build(); //! ``` //! //! # How to use `FileIO` @@ -62,6 +44,7 @@ //! Currently `FileIO` provides simple methods for file operations: //! //! - `delete`: Delete file. +//! - `delete_prefix`: Delete all files with a given prefix. //! - `exists`: Check if file exists. //! - `new_input`: Create input file for reading. //! - `new_output`: Create output file for writing. @@ -70,33 +53,9 @@ mod file_io; mod storage; pub use file_io::*; -pub(crate) mod object_cache; +pub use storage::*; -#[cfg(feature = "storage-azdls")] -mod storage_azdls; -#[cfg(feature = "storage-fs")] -mod storage_fs; -#[cfg(feature = "storage-gcs")] -mod storage_gcs; -#[cfg(feature = "storage-memory")] -mod storage_memory; -#[cfg(feature = "storage-oss")] -mod storage_oss; -#[cfg(feature = "storage-s3")] -mod storage_s3; - -#[cfg(feature = "storage-azdls")] -pub use storage_azdls::*; -#[cfg(feature = "storage-fs")] -use storage_fs::*; -#[cfg(feature = "storage-gcs")] -pub use storage_gcs::*; -#[cfg(feature = "storage-memory")] -use storage_memory::*; -#[cfg(feature = "storage-oss")] -pub use storage_oss::*; -#[cfg(feature = "storage-s3")] -pub use storage_s3::*; +pub(crate) mod object_cache; pub(crate) fn is_truthy(value: &str) -> bool { ["true", "t", "1", "on"].contains(&value.to_lowercase().as_str()) diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index a23ff36b32..37ded38946 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -16,11 +16,13 @@ // under the License. use std::mem::size_of_val; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; +use crate::encryption::EncryptionManager; use crate::io::FileIO; use crate::spec::{ - FormatVersion, Manifest, ManifestFile, ManifestList, SchemaId, SnapshotRef, TableMetadataRef, + Manifest, ManifestFile, ManifestList, ManifestListReader, ManifestReader, SnapshotRef, + TableMetadataRef, }; use crate::{Error, ErrorKind, Result}; @@ -34,69 +36,109 @@ pub(crate) enum CachedItem { #[derive(Clone, Debug, Hash, Eq, PartialEq)] pub(crate) enum CachedObjectKey { - ManifestList((String, FormatVersion, SchemaId)), - Manifest(String), + // Keyed by location alone. A location holds one set of immutable bytes, and + // a cache is bound to one table, so the first access parses those bytes + // under the table's then-current format version and every later access + // shares that entry. The version can still shift under a live cache across + // an in-place upgrade (`Table::with_metadata`), which is why the key omits it. + ManifestList(String), + // Keyed by the path plus every manifest-list value that `ManifestReader` + // inherits onto the parsed entries. The same physical manifest can be + // referenced under different values: `first_row_id` differs across + // snapshots and branches, and `sequence_number` differs when a transaction + // retry reuses a manifest it wrote under an earlier attempt's manifest + // list. A key missing any of these would serve one reference's inherited + // values for another. + Manifest(ManifestKey), +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +pub(crate) struct ManifestKey { + path: String, + added_snapshot_id: i64, + sequence_number: i64, + first_row_id: Option, +} + +impl ManifestKey { + fn new(manifest_file: &ManifestFile) -> Self { + Self { + path: manifest_file.manifest_path.clone(), + added_snapshot_id: manifest_file.added_snapshot_id, + sequence_number: manifest_file.sequence_number, + first_row_id: manifest_file.first_row_id, + } + } } /// Caches metadata objects deserialized from immutable files #[derive(Clone, Debug)] pub struct ObjectCache { - cache: moka::future::Cache, + cache: Arc>>, file_io: FileIO, - cache_disabled: bool, + cache_size_bytes: u64, + encryption_manager: Option>, } impl ObjectCache { /// Creates a new [`ObjectCache`] /// with the default cache size - pub(crate) fn new(file_io: FileIO) -> Self { - Self::new_with_capacity(file_io, DEFAULT_CACHE_SIZE_BYTES) + pub(crate) fn new(file_io: FileIO, encryption_manager: Option>) -> Self { + Self::new_with_capacity(file_io, DEFAULT_CACHE_SIZE_BYTES, encryption_manager) } /// Creates a new [`ObjectCache`] /// with a specific cache size - pub(crate) fn new_with_capacity(file_io: FileIO, cache_size_bytes: u64) -> Self { - if cache_size_bytes == 0 { - Self::with_disabled_cache(file_io) - } else { - Self { - cache: moka::future::Cache::builder() - .weigher(|_, val: &CachedItem| match val { - CachedItem::ManifestList(item) => size_of_val(item.as_ref()), - CachedItem::Manifest(item) => size_of_val(item.as_ref()), - } as u32) - .max_capacity(cache_size_bytes) - .build(), - file_io, - cache_disabled: false, - } + pub(crate) fn new_with_capacity( + file_io: FileIO, + cache_size_bytes: u64, + encryption_manager: Option>, + ) -> Self { + Self { + cache: Arc::new(OnceLock::new()), + file_io, + cache_size_bytes, + encryption_manager, } } /// Creates a new [`ObjectCache`] /// with caching disabled - pub(crate) fn with_disabled_cache(file_io: FileIO) -> Self { - Self { - cache: moka::future::Cache::new(0), - file_io, - cache_disabled: true, + pub(crate) fn with_disabled_cache( + file_io: FileIO, + encryption_manager: Option>, + ) -> Self { + Self::new_with_capacity(file_io, 0, encryption_manager) + } + + fn cache(&self) -> Option<&moka::future::Cache> { + if self.cache_size_bytes == 0 { + return None; } + Some(self.cache.get_or_init(|| { + moka::future::Cache::builder() + .weigher(|_, val: &CachedItem| match val { + CachedItem::ManifestList(item) => size_of_val(item.as_ref()), + CachedItem::Manifest(item) => size_of_val(item.as_ref()), + } as u32) + .max_capacity(self.cache_size_bytes) + .build() + })) } /// Retrieves an Arc [`Manifest`] from the cache /// or retrieves one from FileIO and parses it if not present pub(crate) async fn get_manifest(&self, manifest_file: &ManifestFile) -> Result> { - if self.cache_disabled { - return manifest_file - .load_manifest(&self.file_io) + let Some(cache) = self.cache() else { + return ManifestReader::new(self.file_io.clone()) + .read(manifest_file) .await .map(Arc::new); - } + }; - let key = CachedObjectKey::Manifest(manifest_file.manifest_path.clone()); + let key = CachedObjectKey::Manifest(ManifestKey::new(manifest_file)); - let cache_entry = self - .cache + let cache_entry = cache .entry_by_ref(&key) .or_try_insert_with(self.fetch_and_parse_manifest(manifest_file)) .await @@ -113,7 +155,7 @@ impl ObjectCache { CachedItem::Manifest(arc_manifest) => Ok(arc_manifest), _ => Err(Error::new( ErrorKind::Unexpected, - format!("cached object for key '{:?}' is not a Manifest", key), + format!("cached object for key '{key:?}' is not a Manifest"), )), } } @@ -125,20 +167,20 @@ impl ObjectCache { snapshot: &SnapshotRef, table_metadata: &TableMetadataRef, ) -> Result> { - if self.cache_disabled { - return snapshot - .load_manifest_list(&self.file_io, table_metadata) - .await - .map(Arc::new); - } + let Some(cache) = self.cache() else { + return ManifestListReader::new( + snapshot.clone(), + self.file_io.clone(), + table_metadata.clone(), + self.encryption_manager.clone(), + ) + .load() + .await + .map(Arc::new); + }; - let key = CachedObjectKey::ManifestList(( - snapshot.manifest_list().to_string(), - table_metadata.format_version, - snapshot.schema_id().unwrap(), - )); - let cache_entry = self - .cache + let key = CachedObjectKey::ManifestList(snapshot.manifest_list().to_string()); + let cache_entry = cache .entry_by_ref(&key) .or_try_insert_with(self.fetch_and_parse_manifest_list(snapshot, table_metadata)) .await @@ -157,13 +199,15 @@ impl ObjectCache { CachedItem::ManifestList(arc_manifest_list) => Ok(arc_manifest_list), _ => Err(Error::new( ErrorKind::Unexpected, - format!("cached object for path '{:?}' is not a manifest list", key), + format!("cached object for path '{key:?}' is not a manifest list"), )), } } async fn fetch_and_parse_manifest(&self, manifest_file: &ManifestFile) -> Result { - let manifest = manifest_file.load_manifest(&self.file_io).await?; + let manifest = ManifestReader::new(self.file_io.clone()) + .read(manifest_file) + .await?; Ok(CachedItem::Manifest(Arc::new(manifest))) } @@ -173,9 +217,14 @@ impl ObjectCache { snapshot: &SnapshotRef, table_metadata: &TableMetadataRef, ) -> Result { - let manifest_list = snapshot - .load_manifest_list(&self.file_io, table_metadata) - .await?; + let manifest_list = ManifestListReader::new( + snapshot.clone(), + self.file_io.clone(), + table_metadata.clone(), + self.encryption_manager.clone(), + ) + .load() + .await?; Ok(CachedItem::ManifestList(Arc::new(manifest_list))) } @@ -183,20 +232,30 @@ impl ObjectCache { #[cfg(test)] mod tests { + use std::collections::HashMap; use std::fs; + use minijinja::value::Value; + use minijinja::{AutoEscape, Environment, context}; use tempfile::TempDir; - use tera::{Context, Tera}; use uuid::Uuid; use super::*; use crate::TableIdent; use crate::io::{FileIO, OutputFile}; use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Literal, ManifestEntry, - ManifestListWriter, ManifestStatus, ManifestWriterBuilder, Struct, TableMetadata, + DataContentType, DataFileBuilder, DataFileFormat, FormatVersion, Literal, + ManifestContentType, ManifestEntry, ManifestListWriter, ManifestStatus, + ManifestWriterBuilder, Operation, Snapshot, Struct, Summary, TableMetadata, }; use crate::table::Table; + use crate::test_utils::test_runtime; + + fn render_template(template: &str, ctx: Value) -> String { + let mut env = Environment::new(); + env.set_auto_escape_callback(|_| AutoEscape::None); + env.render_str(template, ctx).unwrap() + } struct TableTestFixture { table_location: String, @@ -211,10 +270,7 @@ mod tests { let manifest_list2_location = table_location.join("metadata/manifests_list_2.avro"); let table_metadata1_location = table_location.join("metadata/v1.json"); - let file_io = FileIO::from_path(table_location.as_os_str().to_str().unwrap()) - .unwrap() - .build() - .unwrap(); + let file_io = FileIO::new_with_fs(); let table_metadata = { let template_json_str = fs::read_to_string(format!( @@ -222,13 +278,12 @@ mod tests { env!("CARGO_MANIFEST_DIR") )) .unwrap(); - let mut context = Context::new(); - context.insert("table_location", &table_location); - context.insert("manifest_list_1_location", &manifest_list1_location); - context.insert("manifest_list_2_location", &manifest_list2_location); - context.insert("table_metadata_1_location", &table_metadata1_location); - - let metadata_json = Tera::one_off(&template_json_str, &context, false).unwrap(); + let metadata_json = render_template(&template_json_str, context! { + table_location => &table_location, + manifest_list_1_location => &manifest_list1_location, + manifest_list_2_location => &manifest_list2_location, + table_metadata_1_location => &table_metadata1_location, + }); serde_json::from_str::(&metadata_json).unwrap() }; @@ -237,6 +292,7 @@ mod tests { .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) .file_io(file_io.clone()) .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) + .runtime(test_runtime()) .build() .unwrap(); @@ -266,7 +322,6 @@ mod tests { let mut writer = ManifestWriterBuilder::new( self.next_manifest_file(), Some(current_snapshot.snapshot_id()), - None, current_schema.clone(), current_partition_spec.as_ref().clone(), ) @@ -293,11 +348,16 @@ mod tests { let data_file_manifest = writer.write_manifest_file().await.unwrap(); // Write to manifest list + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); let mut manifest_list_write = ManifestListWriter::v2( - self.table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap(), + manifest_list_writer, current_snapshot.snapshot_id(), current_snapshot.parent_snapshot_id(), current_snapshot.sequence_number(), @@ -307,6 +367,75 @@ mod tests { .unwrap(); manifest_list_write.close().await.unwrap(); } + + async fn setup_v1_manifest_files(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // Write data files + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v1(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v1( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + } + + #[test] + fn cache_initializes_once_on_first_use() { + let original = ObjectCache::new_with_capacity(FileIO::new_with_memory(), 1024, None); + let cloned = original.clone(); + let another_clone = original.clone(); + assert!(original.cache.get().is_none()); + drop(original); + assert!(cloned.cache.get().is_none()); + assert!(another_clone.cache.get().is_none()); + + let cache = cloned.cache().unwrap(); + assert!(std::ptr::eq(cache, another_clone.cache().unwrap())); + assert_eq!(cache.policy().max_capacity(), Some(1024)); } #[tokio::test] @@ -314,7 +443,7 @@ mod tests { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - let object_cache = ObjectCache::with_disabled_cache(fixture.table.file_io().clone()); + let object_cache = ObjectCache::with_disabled_cache(fixture.table.file_io().clone(), None); let result_manifest_list = object_cache .get_manifest_list( @@ -340,6 +469,7 @@ mod tests { .unwrap(), "1.parquet" ); + assert!(object_cache.cache.get().is_none()); } #[tokio::test] @@ -347,7 +477,9 @@ mod tests { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - let object_cache = ObjectCache::new(fixture.table.file_io().clone()); + let object_cache = ObjectCache::new(fixture.table.file_io().clone(), None); + let cloned = object_cache.clone(); + assert!(object_cache.cache.get().is_none()); // not in cache let result_manifest_list = object_cache @@ -361,7 +493,8 @@ mod tests { assert_eq!(result_manifest_list.entries().len(), 1); // retrieve cached version - let result_manifest_list = object_cache + let first_manifest_list = result_manifest_list; + let result_manifest_list = cloned .get_manifest_list( fixture.table.metadata().current_snapshot().unwrap(), &fixture.table.metadata_ref(), @@ -369,7 +502,7 @@ mod tests { .await .unwrap(); - assert_eq!(result_manifest_list.entries().len(), 1); + assert!(Arc::ptr_eq(&first_manifest_list, &result_manifest_list)); let manifest_file = result_manifest_list.entries().first().unwrap(); @@ -389,18 +522,238 @@ mod tests { ); // retrieve cached version - let result_manifest = object_cache.get_manifest(manifest_file).await.unwrap(); + let cached_manifest = cloned.get_manifest(manifest_file).await.unwrap(); + assert!(Arc::ptr_eq(&result_manifest, &cached_manifest)); + } + #[tokio::test] + async fn test_get_manifest_list_with_no_schema_id() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + let current_snapshot = fixture.table.metadata().current_snapshot().unwrap(); + + // The spec marks `schema-id` optional in every version (v1-v3), so a + // snapshot may omit it; fetching its manifest list must not depend on the + // schema-id being present. + let snapshot_without_schema_id: SnapshotRef = Snapshot::builder() + .with_snapshot_id(current_snapshot.snapshot_id()) + .with_sequence_number(current_snapshot.sequence_number()) + .with_timestamp_ms(current_snapshot.timestamp_ms()) + .with_manifest_list(current_snapshot.manifest_list()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .build() + .into(); + assert!(snapshot_without_schema_id.schema_id().is_none()); + assert!(current_snapshot.schema_id().is_some()); + + let object_cache = ObjectCache::new(fixture.table.file_io().clone(), None); + + // Cold miss: the schema-id-less snapshot populates the cache. + let inserted = object_cache + .get_manifest_list(&snapshot_without_schema_id, &fixture.table.metadata_ref()) + .await + .unwrap(); + assert_eq!(inserted.entries().len(), 1); + + // Warm hit: the original snapshot carries a schema-id but points at the same + // manifest-list location, so it returns the same cached entry. + let cached = object_cache + .get_manifest_list(current_snapshot, &fixture.table.metadata_ref()) + .await + .unwrap(); + assert!( + Arc::ptr_eq(&inserted, &cached), + "snapshots with and without schema-id at one location must share a cache entry" + ); + } + + #[tokio::test] + async fn test_get_manifest_list_serves_cached_v1_parse_after_upgrade() { + let mut fixture = TableTestFixture::new(); + fixture.setup_v1_manifest_files().await; + + let current_snapshot = fixture.table.metadata().current_snapshot().unwrap(); + + // The manifest list on disk is v1. The fixture table is v2, so clone its + // metadata as v1 to model the state before an in-place upgrade; the real + // metadata_ref models the state after it. Both drive lookups at the same + // location through one cache. + let metadata_v2 = fixture.table.metadata_ref(); + let mut metadata_v1 = fixture.table.metadata().clone(); + metadata_v1.format_version = FormatVersion::V1; + let metadata_v1: TableMetadataRef = Arc::new(metadata_v1); + assert_ne!( + metadata_v2.format_version, metadata_v1.format_version, + "setup invariant: the two metadata refs must differ in format version" + ); + + let object_cache = ObjectCache::new(fixture.table.file_io().clone(), None); + + // Cold miss under v1 parses the v1 bytes as v1: content defaults to Data and + // the sequence number to 0. + let cached_under_v1 = object_cache + .get_manifest_list(current_snapshot, &metadata_v1) + .await + .unwrap(); + let entry = cached_under_v1.entries().first().unwrap(); + assert_eq!(entry.content, ManifestContentType::Data); + assert_eq!(entry.sequence_number, 0); + + // Warm hit under v2 at the same location returns the cached v1 parse rather + // than reparsing the v1 bytes as v2, so the upgraded caller sees the same + // correct content. + let cached_under_v2 = object_cache + .get_manifest_list(current_snapshot, &metadata_v2) + .await + .unwrap(); + assert!( + Arc::ptr_eq(&cached_under_v1, &cached_under_v2), + "lookups at one location must share a cache entry across format versions" + ); + let entry = cached_under_v2.entries().first().unwrap(); + assert_eq!(entry.content, ManifestContentType::Data); + assert_eq!(entry.sequence_number, 0); + } + + #[tokio::test] + async fn test_get_manifest_keys_on_first_row_id() { + use crate::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SchemaRef, Type}; + + let io = FileIO::new_with_memory(); + let path = "memory:///metadata/first_row_id_manifest.avro"; + + let schema: SchemaRef = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap() + .into(); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .build() + .unwrap(); + + let mut writer = ManifestWriterBuilder::new( + io.new_output(path).unwrap(), + Some(1), + schema, + partition_spec, + ) + .build_v3_data(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let manifest_file = writer.write_manifest_file().await.unwrap(); + + // Two manifest-list references to the same physical manifest carrying + // different offsets, as time travel or two branches would produce. + let mut manifest_file_a = manifest_file.clone(); + manifest_file_a.first_row_id = Some(1000); + let mut manifest_file_b = manifest_file; + manifest_file_b.first_row_id = Some(2000); + + let object_cache = ObjectCache::new(io, None); + + let manifest_a = object_cache.get_manifest(&manifest_file_a).await.unwrap(); + let manifest_b = object_cache.get_manifest(&manifest_file_b).await.unwrap(); + + // A cache keyed on path alone would serve manifest_a's inherited ids for + // the second read; keying on first_row_id keeps them distinct. assert_eq!( - result_manifest - .entries() - .first() - .unwrap() - .file_path() - .split("/") - .last() - .unwrap(), - "1.parquet" + manifest_a.entries()[0].data_file().first_row_id(), + Some(1000) + ); + assert_eq!( + manifest_b.entries()[0].data_file().first_row_id(), + Some(2000) ); } + + /// A transaction retry reuses the manifest it wrote during an earlier + /// attempt, but the retry's manifest list assigns it a newer sequence + /// number. Entries inherit that number on read, so a cache that already + /// holds the earlier attempt's parse must not serve it for the retry. + #[tokio::test] + async fn test_get_manifest_keys_on_sequence_number() { + use crate::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SchemaRef, Type}; + + let io = FileIO::new_with_memory(); + let path = "memory:///metadata/retried_manifest.avro"; + + let schema: SchemaRef = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap() + .into(); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .build() + .unwrap(); + + // A v2 manifest: entries carry no sequence number of their own and + // `first_row_id` stays `None`, so only the sequence number distinguishes + // the two references below. + let mut writer = ManifestWriterBuilder::new( + io.new_output(path).unwrap(), + Some(1), + schema, + partition_spec, + ) + .build_v2_data(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let manifest_file = writer.write_manifest_file().await.unwrap(); + assert_eq!(manifest_file.first_row_id, None); + + // First attempt's manifest list assigns 35; after a rebase the retry's + // manifest list assigns 36 to the same physical manifest. + let mut first_attempt = manifest_file.clone(); + first_attempt.sequence_number = 35; + let mut retry = manifest_file; + retry.sequence_number = 36; + + let object_cache = ObjectCache::new(io, None); + + let first = object_cache.get_manifest(&first_attempt).await.unwrap(); + let second = object_cache.get_manifest(&retry).await.unwrap(); + + assert_eq!(first.entries()[0].sequence_number(), Some(35)); + assert_eq!(second.entries()[0].sequence_number(), Some(36)); + } } diff --git a/crates/iceberg/src/io/storage.rs b/crates/iceberg/src/io/storage.rs deleted file mode 100644 index 3de4f10dbf..0000000000 --- a/crates/iceberg/src/io/storage.rs +++ /dev/null @@ -1,234 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::sync::Arc; - -use opendal::layers::RetryLayer; -#[cfg(feature = "storage-azdls")] -use opendal::services::AzdlsConfig; -#[cfg(feature = "storage-gcs")] -use opendal::services::GcsConfig; -#[cfg(feature = "storage-oss")] -use opendal::services::OssConfig; -#[cfg(feature = "storage-s3")] -use opendal::services::S3Config; -use opendal::{Operator, Scheme}; - -#[cfg(feature = "storage-azdls")] -use super::AzureStorageScheme; -use super::FileIOBuilder; -#[cfg(feature = "storage-s3")] -use crate::io::CustomAwsCredentialLoader; -use crate::{Error, ErrorKind}; - -/// The storage carries all supported storage services in iceberg -#[derive(Debug)] -pub(crate) enum Storage { - #[cfg(feature = "storage-memory")] - Memory(Operator), - #[cfg(feature = "storage-fs")] - LocalFs, - /// Expects paths of the form `s3[a]:///`. - #[cfg(feature = "storage-s3")] - S3 { - /// s3 storage could have `s3://` and `s3a://`. - /// Storing the scheme string here to return the correct path. - configured_scheme: String, - config: Arc, - customized_credential_load: Option, - }, - #[cfg(feature = "storage-gcs")] - Gcs { config: Arc }, - #[cfg(feature = "storage-oss")] - Oss { config: Arc }, - /// Expects paths of the form - /// `abfs[s]://@.dfs./` or - /// `wasb[s]://@.blob./`. - #[cfg(feature = "storage-azdls")] - Azdls { - /// Because Azdls accepts multiple possible schemes, we store the full - /// passed scheme here to later validate schemes passed via paths. - configured_scheme: AzureStorageScheme, - config: Arc, - }, -} - -impl Storage { - /// Convert iceberg config to opendal config. - pub(crate) fn build(file_io_builder: FileIOBuilder) -> crate::Result { - let (scheme_str, props, extensions) = file_io_builder.into_parts(); - let scheme = Self::parse_scheme(&scheme_str)?; - - match scheme { - #[cfg(feature = "storage-memory")] - Scheme::Memory => Ok(Self::Memory(super::memory_config_build()?)), - #[cfg(feature = "storage-fs")] - Scheme::Fs => Ok(Self::LocalFs), - #[cfg(feature = "storage-s3")] - Scheme::S3 => Ok(Self::S3 { - configured_scheme: scheme_str, - config: super::s3_config_parse(props)?.into(), - customized_credential_load: extensions - .get::() - .map(Arc::unwrap_or_clone), - }), - #[cfg(feature = "storage-gcs")] - Scheme::Gcs => Ok(Self::Gcs { - config: super::gcs_config_parse(props)?.into(), - }), - #[cfg(feature = "storage-oss")] - Scheme::Oss => Ok(Self::Oss { - config: super::oss_config_parse(props)?.into(), - }), - #[cfg(feature = "storage-azdls")] - Scheme::Azdls => { - let scheme = scheme_str.parse::()?; - Ok(Self::Azdls { - config: super::azdls_config_parse(props)?.into(), - configured_scheme: scheme, - }) - } - // Update doc on [`FileIO`] when adding new schemes. - _ => Err(Error::new( - ErrorKind::FeatureUnsupported, - format!("Constructing file io from scheme: {scheme} not supported now",), - )), - } - } - - /// Creates operator from path. - /// - /// # Arguments - /// - /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`]. - /// - /// # Returns - /// - /// The return value consists of two parts: - /// - /// * An [`opendal::Operator`] instance used to operate on file. - /// * Relative path to the root uri of [`opendal::Operator`]. - pub(crate) fn create_operator<'a>( - &self, - path: &'a impl AsRef, - ) -> crate::Result<(Operator, &'a str)> { - let path = path.as_ref(); - let (operator, relative_path): (Operator, &str) = match self { - #[cfg(feature = "storage-memory")] - Storage::Memory(op) => { - if let Some(stripped) = path.strip_prefix("memory:/") { - Ok::<_, crate::Error>((op.clone(), stripped)) - } else { - Ok::<_, crate::Error>((op.clone(), &path[1..])) - } - } - #[cfg(feature = "storage-fs")] - Storage::LocalFs => { - let op = super::fs_config_build()?; - - if let Some(stripped) = path.strip_prefix("file:/") { - Ok::<_, crate::Error>((op, stripped)) - } else { - Ok::<_, crate::Error>((op, &path[1..])) - } - } - #[cfg(feature = "storage-s3")] - Storage::S3 { - configured_scheme, - config, - customized_credential_load, - } => { - let op = super::s3_config_build(config, customized_credential_load, path)?; - let op_info = op.info(); - - // Check prefix of s3 path. - let prefix = format!("{}://{}/", configured_scheme, op_info.name()); - if path.starts_with(&prefix) { - Ok((op, &path[prefix.len()..])) - } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid s3 url: {}, should start with {}", path, prefix), - )) - } - } - #[cfg(feature = "storage-gcs")] - Storage::Gcs { config } => { - let operator = super::gcs_config_build(config, path)?; - let prefix = format!("gs://{}/", operator.info().name()); - if path.starts_with(&prefix) { - Ok((operator, &path[prefix.len()..])) - } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid gcs url: {}, should start with {}", path, prefix), - )) - } - } - #[cfg(feature = "storage-oss")] - Storage::Oss { config } => { - let op = super::oss_config_build(config, path)?; - - // Check prefix of oss path. - let prefix = format!("oss://{}/", op.info().name()); - if path.starts_with(&prefix) { - Ok((op, &path[prefix.len()..])) - } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid oss url: {}, should start with {}", path, prefix), - )) - } - } - #[cfg(feature = "storage-azdls")] - Storage::Azdls { - configured_scheme, - config, - } => super::azdls_create_operator(path, config, configured_scheme), - #[cfg(all( - not(feature = "storage-s3"), - not(feature = "storage-fs"), - not(feature = "storage-gcs"), - not(feature = "storage-oss"), - not(feature = "storage-azdls"), - ))] - _ => Err(Error::new( - ErrorKind::FeatureUnsupported, - "No storage service has been enabled", - )), - }?; - - // Transient errors are common for object stores; however there's no - // harm in retrying temporary failures for other storage backends as well. - let operator = operator.layer(RetryLayer::new()); - - Ok((operator, relative_path)) - } - - /// Parse scheme. - fn parse_scheme(scheme: &str) -> crate::Result { - match scheme { - "memory" => Ok(Scheme::Memory), - "file" | "" => Ok(Scheme::Fs), - "s3" | "s3a" => Ok(Scheme::S3), - "gs" | "gcs" => Ok(Scheme::Gcs), - "oss" => Ok(Scheme::Oss), - "abfss" | "abfs" | "wasbs" | "wasb" => Ok(Scheme::Azdls), - s => Ok(s.parse::()?), - } - } -} diff --git a/crates/iceberg/src/io/storage/config/azdls.rs b/crates/iceberg/src/io/storage/config/azdls.rs new file mode 100644 index 0000000000..a9541e7791 --- /dev/null +++ b/crates/iceberg/src/io/storage/config/azdls.rs @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Azure Data Lake Storage configuration. +//! +//! This module provides configuration constants and types for Azure Data Lake Storage. + +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use super::StorageConfig; +use crate::Result; + +/// A connection string. +/// +/// Note, this string is parsed first, and any other passed adls.* properties +/// will override values from the connection string. +pub const ADLS_CONNECTION_STRING: &str = "adls.connection-string"; +/// The account that you want to connect to. +pub const ADLS_ACCOUNT_NAME: &str = "adls.account-name"; +/// The key to authentication against the account. +pub const ADLS_ACCOUNT_KEY: &str = "adls.account-key"; +/// The shared access signature. +pub const ADLS_SAS_TOKEN: &str = "adls.sas-token"; +/// The tenant-id. +pub const ADLS_TENANT_ID: &str = "adls.tenant-id"; +/// The client-id. +pub const ADLS_CLIENT_ID: &str = "adls.client-id"; +/// The client-secret. +pub const ADLS_CLIENT_SECRET: &str = "adls.client-secret"; +/// The authority host of the service principal. +/// - required for client_credentials authentication +/// - default value: `https://login.microsoftonline.com` +pub const ADLS_AUTHORITY_HOST: &str = "adls.authority-host"; + +/// Azure Data Lake Storage configuration. +/// +/// This struct contains all the configuration options for connecting to Azure Data Lake Storage. +/// Use the builder pattern via `AzdlsConfig::builder()` to construct instances. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +pub struct AzdlsConfig { + /// Connection string. + #[builder(default, setter(strip_option, into))] + pub connection_string: Option, + /// Account name. + #[builder(default, setter(strip_option, into))] + pub account_name: Option, + /// Account key. + #[builder(default, setter(strip_option, into))] + pub account_key: Option, + /// SAS token. + #[builder(default, setter(strip_option, into))] + pub sas_token: Option, + /// Tenant ID. + #[builder(default, setter(strip_option, into))] + pub tenant_id: Option, + /// Client ID. + #[builder(default, setter(strip_option, into))] + pub client_id: Option, + /// Client secret. + #[builder(default, setter(strip_option, into))] + pub client_secret: Option, + /// Authority host. + #[builder(default, setter(strip_option, into))] + pub authority_host: Option, + /// Endpoint URL. + #[builder(default, setter(strip_option, into))] + pub endpoint: Option, + /// Filesystem name. + #[builder(default, setter(into))] + pub filesystem: String, +} + +impl TryFrom<&StorageConfig> for AzdlsConfig { + type Error = crate::Error; + + fn try_from(config: &StorageConfig) -> Result { + let props = config.props(); + + let mut cfg = AzdlsConfig::default(); + + if let Some(connection_string) = props.get(ADLS_CONNECTION_STRING) { + cfg.connection_string = Some(connection_string.clone()); + } + if let Some(account_name) = props.get(ADLS_ACCOUNT_NAME) { + cfg.account_name = Some(account_name.clone()); + } + if let Some(account_key) = props.get(ADLS_ACCOUNT_KEY) { + cfg.account_key = Some(account_key.clone()); + } + if let Some(sas_token) = props.get(ADLS_SAS_TOKEN) { + cfg.sas_token = Some(sas_token.clone()); + } + if let Some(tenant_id) = props.get(ADLS_TENANT_ID) { + cfg.tenant_id = Some(tenant_id.clone()); + } + if let Some(client_id) = props.get(ADLS_CLIENT_ID) { + cfg.client_id = Some(client_id.clone()); + } + if let Some(client_secret) = props.get(ADLS_CLIENT_SECRET) { + cfg.client_secret = Some(client_secret.clone()); + } + if let Some(authority_host) = props.get(ADLS_AUTHORITY_HOST) { + cfg.authority_host = Some(authority_host.clone()); + } + + Ok(cfg) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_azdls_config_builder() { + let config = AzdlsConfig::builder() + .account_name("myaccount") + .account_key("my-account-key") + .build(); + + assert_eq!(config.account_name.as_deref(), Some("myaccount")); + assert_eq!(config.account_key.as_deref(), Some("my-account-key")); + } + + #[test] + fn test_azdls_config_from_storage_config() { + let storage_config = StorageConfig::new() + .with_prop(ADLS_ACCOUNT_NAME, "myaccount") + .with_prop(ADLS_ACCOUNT_KEY, "my-account-key"); + + let azdls_config = AzdlsConfig::try_from(&storage_config).unwrap(); + + assert_eq!(azdls_config.account_name.as_deref(), Some("myaccount")); + assert_eq!(azdls_config.account_key.as_deref(), Some("my-account-key")); + } + + #[test] + fn test_azdls_config_with_sas_token() { + let storage_config = StorageConfig::new() + .with_prop(ADLS_ACCOUNT_NAME, "myaccount") + .with_prop(ADLS_SAS_TOKEN, "my-sas-token"); + + let azdls_config = AzdlsConfig::try_from(&storage_config).unwrap(); + + assert_eq!(azdls_config.account_name.as_deref(), Some("myaccount")); + assert_eq!(azdls_config.sas_token.as_deref(), Some("my-sas-token")); + } + + #[test] + fn test_azdls_config_with_client_credentials() { + let storage_config = StorageConfig::new() + .with_prop(ADLS_ACCOUNT_NAME, "myaccount") + .with_prop(ADLS_TENANT_ID, "my-tenant") + .with_prop(ADLS_CLIENT_ID, "my-client") + .with_prop(ADLS_CLIENT_SECRET, "my-secret"); + + let azdls_config = AzdlsConfig::try_from(&storage_config).unwrap(); + + assert_eq!(azdls_config.account_name.as_deref(), Some("myaccount")); + assert_eq!(azdls_config.tenant_id.as_deref(), Some("my-tenant")); + assert_eq!(azdls_config.client_id.as_deref(), Some("my-client")); + assert_eq!(azdls_config.client_secret.as_deref(), Some("my-secret")); + } +} diff --git a/crates/iceberg/src/io/storage/config/gcs.rs b/crates/iceberg/src/io/storage/config/gcs.rs new file mode 100644 index 0000000000..99867e5dfc --- /dev/null +++ b/crates/iceberg/src/io/storage/config/gcs.rs @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Google Cloud Storage configuration. +//! +//! This module provides configuration constants and types for Google Cloud Storage. +//! Reference: + +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use super::StorageConfig; +use crate::Result; +use crate::io::is_truthy; + +/// Google Cloud Project ID. +pub const GCS_PROJECT_ID: &str = "gcs.project-id"; +/// Google Cloud Storage endpoint. +pub const GCS_SERVICE_HOST: &str = "gcs.service.host"; +/// Google Cloud user project. +pub const GCS_USER_PROJECT: &str = "gcs.user-project"; +/// Allow unauthenticated requests. +pub const GCS_NO_AUTH: &str = "gcs.no-auth"; +/// Google Cloud Storage credentials JSON string, base64 encoded. +/// +/// E.g. base64::prelude::BASE64_STANDARD.encode(serde_json::to_string(credential).as_bytes()) +pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json"; +/// Google Cloud Storage token. +pub const GCS_TOKEN: &str = "gcs.oauth2.token"; +/// Option to skip signing requests (e.g. for public buckets/folders). +pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous"; +/// Option to skip loading the credential from GCE metadata server. +pub const GCS_DISABLE_VM_METADATA: &str = "gcs.disable-vm-metadata"; +/// Option to skip loading configuration from config file and the env. +pub const GCS_DISABLE_CONFIG_LOAD: &str = "gcs.disable-config-load"; + +/// Google Cloud Storage configuration. +/// +/// This struct contains all the configuration options for connecting to Google Cloud Storage. +/// Use the builder pattern via `GcsConfig::builder()` to construct instances. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +pub struct GcsConfig { + /// Google Cloud Project ID. + #[builder(default, setter(strip_option, into))] + pub project_id: Option, + /// GCS service endpoint. + #[builder(default, setter(strip_option, into))] + pub endpoint: Option, + /// User project for requester pays buckets. + #[builder(default, setter(strip_option, into))] + pub user_project: Option, + /// Credentials JSON (base64 encoded). + #[builder(default, setter(strip_option, into))] + pub credential: Option, + /// OAuth2 token. + #[builder(default, setter(strip_option, into))] + pub token: Option, + /// Allow anonymous access. + #[builder(default)] + pub allow_anonymous: bool, + /// Disable VM metadata. + #[builder(default)] + pub disable_vm_metadata: bool, + /// Disable config load. + #[builder(default)] + pub disable_config_load: bool, +} + +impl TryFrom<&StorageConfig> for GcsConfig { + type Error = crate::Error; + + fn try_from(config: &StorageConfig) -> Result { + let props = config.props(); + + let mut cfg = GcsConfig::default(); + + if let Some(project_id) = props.get(GCS_PROJECT_ID) { + cfg.project_id = Some(project_id.clone()); + } + if let Some(endpoint) = props.get(GCS_SERVICE_HOST) { + cfg.endpoint = Some(endpoint.clone()); + } + if let Some(user_project) = props.get(GCS_USER_PROJECT) { + cfg.user_project = Some(user_project.clone()); + } + if let Some(credential) = props.get(GCS_CREDENTIALS_JSON) { + cfg.credential = Some(credential.clone()); + } + if let Some(token) = props.get(GCS_TOKEN) { + cfg.token = Some(token.clone()); + } + + // GCS_NO_AUTH enables all anonymous/no-auth options + if props.get(GCS_NO_AUTH).is_some() { + cfg.allow_anonymous = true; + cfg.disable_vm_metadata = true; + cfg.disable_config_load = true; + } + + if let Some(allow_anonymous) = props.get(GCS_ALLOW_ANONYMOUS) + && is_truthy(allow_anonymous.to_lowercase().as_str()) + { + cfg.allow_anonymous = true; + } + if let Some(disable_vm_metadata) = props.get(GCS_DISABLE_VM_METADATA) + && is_truthy(disable_vm_metadata.to_lowercase().as_str()) + { + cfg.disable_vm_metadata = true; + } + if let Some(disable_config_load) = props.get(GCS_DISABLE_CONFIG_LOAD) + && is_truthy(disable_config_load.to_lowercase().as_str()) + { + cfg.disable_config_load = true; + } + + Ok(cfg) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gcs_config_builder() { + let config = GcsConfig::builder() + .project_id("my-project") + .credential("base64-creds") + .endpoint("http://localhost:4443") + .build(); + + assert_eq!(config.project_id.as_deref(), Some("my-project")); + assert_eq!(config.credential.as_deref(), Some("base64-creds")); + assert_eq!(config.endpoint.as_deref(), Some("http://localhost:4443")); + } + + #[test] + fn test_gcs_config_from_storage_config() { + let storage_config = StorageConfig::new() + .with_prop(GCS_PROJECT_ID, "my-project") + .with_prop(GCS_CREDENTIALS_JSON, "base64-creds") + .with_prop(GCS_SERVICE_HOST, "http://localhost:4443"); + + let gcs_config = GcsConfig::try_from(&storage_config).unwrap(); + + assert_eq!(gcs_config.project_id.as_deref(), Some("my-project")); + assert_eq!(gcs_config.credential.as_deref(), Some("base64-creds")); + assert_eq!( + gcs_config.endpoint.as_deref(), + Some("http://localhost:4443") + ); + } + + #[test] + fn test_gcs_config_no_auth() { + let storage_config = StorageConfig::new().with_prop(GCS_NO_AUTH, "true"); + + let gcs_config = GcsConfig::try_from(&storage_config).unwrap(); + + assert!(gcs_config.allow_anonymous); + assert!(gcs_config.disable_vm_metadata); + assert!(gcs_config.disable_config_load); + } + + #[test] + fn test_gcs_config_allow_anonymous() { + let storage_config = StorageConfig::new().with_prop(GCS_ALLOW_ANONYMOUS, "true"); + + let gcs_config = GcsConfig::try_from(&storage_config).unwrap(); + + assert!(gcs_config.allow_anonymous); + assert!(!gcs_config.disable_vm_metadata); + } +} diff --git a/crates/iceberg/src/io/storage/config/hdfs_native.rs b/crates/iceberg/src/io/storage/config/hdfs_native.rs new file mode 100644 index 0000000000..ef9e16bf65 --- /dev/null +++ b/crates/iceberg/src/io/storage/config/hdfs_native.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HDFS storage configuration. + +/// HDFS NameNode RPC endpoint(s) as `host:port`, e.g. `hdfs://namenode:8020`; +/// a comma-separated list enables HA failover. Serves authority-less paths +/// (`hdfs:///path`) only; a path authority with a port is used as is. A +/// portless authority (`hdfs://ns1/path`) must be a nameservice declared in +/// properties: `hdfs.name-node.` takes the same list and is +/// sugar for Hadoop's `dfs.ha.namenodes.` and +/// `dfs.namenode.rpc-address..`, which are honored as well +/// when passed through [`HDFS_HADOOP_CONF_PREFIX`]. Unlike Hadoop, an +/// undeclared one is an error: it is neither resolved from +/// `$HADOOP_CONF_DIR` nor dialed on the default port 8020. +pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; +/// NameNode host for authority-less paths, as in PyIceberg; paths that carry +/// an authority ignore it, as they do there. Combined with [`HDFS_PORT`] into +/// Hadoop's `fs.defaultFS`. PyIceberg's `hdfs.user` and `hdfs.kerberos_ticket` +/// have no equivalent: the client reads `HADOOP_USER_NAME` and the default +/// Kerberos credential cache. +pub const HDFS_HOST: &str = "hdfs.host"; +/// NameNode port for [`HDFS_HOST`]; defaults to `8020` and has no effect +/// without the host. +pub const HDFS_PORT: &str = "hdfs.port"; +/// Prefix for properties forwarded to the HDFS client configuration with the +/// prefix stripped: `hadoop.dfs.client.failover.random.order` is forwarded as +/// `dfs.client.failover.random.order`. Forwarded values override those loaded +/// from `$HADOOP_CONF_DIR`; `hadoop.fs.defaultFS` also serves authority-less +/// paths. +pub const HDFS_HADOOP_CONF_PREFIX: &str = "hadoop."; diff --git a/crates/iceberg/src/io/storage/config/hf.rs b/crates/iceberg/src/io/storage/config/hf.rs new file mode 100644 index 0000000000..fdb79a5af2 --- /dev/null +++ b/crates/iceberg/src/io/storage/config/hf.rs @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HuggingFace Hub storage configuration. + +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use super::StorageConfig; +use crate::Result; + +/// HuggingFace Hub authentication token. +pub const HF_TOKEN: &str = "hf.token"; +/// HuggingFace Hub endpoint URL. Defaults to `https://huggingface.co`. +pub const HF_ENDPOINT: &str = "hf.endpoint"; +/// Default git revision/branch for all paths that don't specify one. Defaults to `main`. +pub const HF_REVISION: &str = "hf.revision"; + +/// HuggingFace Hub storage configuration. +/// +/// Repo type, repo ID, and revision are normally encoded in the file path URI +/// (`hf:////[@]/`, where `` +/// is one of `models`, `datasets`, `spaces`, or `buckets`). +/// The fields here provide credentials and a default revision fallback. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +pub struct HfConfig { + /// HuggingFace Hub API token (required for private repos and write access). + #[builder(default, setter(strip_option, into))] + pub token: Option, + /// HuggingFace Hub endpoint. Defaults to `https://huggingface.co`. + #[builder(default, setter(strip_option, into))] + pub endpoint: Option, + /// Default revision to use when a path URI does not specify one. + #[builder(default, setter(strip_option, into))] + pub revision: Option, +} + +impl TryFrom<&StorageConfig> for HfConfig { + type Error = crate::Error; + + fn try_from(config: &StorageConfig) -> Result { + let props = config.props(); + let mut cfg = HfConfig::default(); + if let Some(token) = props.get(HF_TOKEN) { + cfg.token = Some(token.clone()); + } + if let Some(endpoint) = props.get(HF_ENDPOINT) { + cfg.endpoint = Some(endpoint.clone()); + } + if let Some(revision) = props.get(HF_REVISION) { + cfg.revision = Some(revision.clone()); + } + Ok(cfg) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hf_config_builder() { + let cfg = HfConfig::builder() + .token("hf_mytoken") + .endpoint("https://huggingface.co") + .revision("dev") + .build(); + assert_eq!(cfg.token.as_deref(), Some("hf_mytoken")); + assert_eq!(cfg.endpoint.as_deref(), Some("https://huggingface.co")); + assert_eq!(cfg.revision.as_deref(), Some("dev")); + } + + #[test] + fn test_hf_config_from_storage_config() { + let storage_config = StorageConfig::new() + .with_prop(HF_TOKEN, "hf_abc123") + .with_prop(HF_ENDPOINT, "https://huggingface.co"); + + let cfg = HfConfig::try_from(&storage_config).unwrap(); + assert_eq!(cfg.token.as_deref(), Some("hf_abc123")); + assert_eq!(cfg.endpoint.as_deref(), Some("https://huggingface.co")); + } + + #[test] + fn test_hf_config_empty() { + let cfg = HfConfig::try_from(&StorageConfig::new()).unwrap(); + assert_eq!(cfg.token, None); + assert_eq!(cfg.endpoint, None); + } +} diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs new file mode 100644 index 0000000000..8ea4af080e --- /dev/null +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// TODO Add specific configs +//! Storage configuration for storage backends. +//! +//! This module provides configuration types for various storage backends. +//! The configuration types are designed to be used with the `StorageFactory` +//! trait to create storage instances. +//! +//! # Available Configurations +//! +//! - [`StorageConfig`]: Base configuration containing properties for storage backends +//! - [`S3Config`]: Amazon S3 specific configuration +//! - [`GcsConfig`]: Google Cloud Storage specific configuration +//! - [`OssConfig`]: Alibaba Cloud OSS specific configuration +//! - [`AzdlsConfig`]: Azure Data Lake Storage specific configuration + +mod azdls; +mod gcs; +mod hdfs_native; +mod hf; +mod oss; +mod s3; + +use std::collections::HashMap; + +pub use azdls::*; +pub use gcs::*; +pub use hdfs_native::*; +pub use hf::*; +pub use oss::*; +pub use s3::*; +use serde::{Deserialize, Serialize}; + +/// Configuration properties for storage backends. +/// +/// This struct contains only configuration properties without specifying +/// which storage backend to use. The storage type is determined by the +/// explicit factory selection. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct StorageConfig { + /// Configuration properties for the storage backend + props: HashMap, +} + +impl StorageConfig { + /// Create a new empty StorageConfig. + pub fn new() -> Self { + Self { + props: HashMap::new(), + } + } + + /// Create a StorageConfig from existing properties. + /// + /// # Arguments + /// + /// * `props` - Configuration properties for the storage backend + pub fn from_props(props: HashMap) -> Self { + Self { props } + } + + /// Get all configuration properties. + pub fn props(&self) -> &HashMap { + &self.props + } + + /// Get a specific configuration property by key. + /// + /// # Arguments + /// + /// * `key` - The property key to look up + /// + /// # Returns + /// + /// An `Option` containing a reference to the property value if it exists. + pub fn get(&self, key: &str) -> Option<&String> { + self.props.get(key) + } + + /// Add a configuration property. + /// + /// This is a builder-style method that returns `self` for chaining. + /// + /// # Arguments + /// + /// * `key` - The property key + /// * `value` - The property value + pub fn with_prop(mut self, key: impl Into, value: impl Into) -> Self { + self.props.insert(key.into(), value.into()); + self + } + + /// Add multiple configuration properties. + /// + /// This is a builder-style method that returns `self` for chaining. + /// + /// # Arguments + /// + /// * `props` - An iterator of key-value pairs to add + pub fn with_props( + mut self, + props: impl IntoIterator, impl Into)>, + ) -> Self { + self.props + .extend(props.into_iter().map(|(k, v)| (k.into(), v.into()))); + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_storage_config_new() { + let config = StorageConfig::new(); + + assert!(config.props().is_empty()); + } + + #[test] + fn test_storage_config_from_props() { + let props = HashMap::from([ + ("region".to_string(), "us-east-1".to_string()), + ("bucket".to_string(), "my-bucket".to_string()), + ]); + let config = StorageConfig::from_props(props.clone()); + + assert_eq!(config.props(), &props); + } + + #[test] + fn test_storage_config_default() { + let config = StorageConfig::default(); + + assert!(config.props().is_empty()); + } + + #[test] + fn test_storage_config_get() { + let config = StorageConfig::new().with_prop("region", "us-east-1"); + + assert_eq!(config.get("region"), Some(&"us-east-1".to_string())); + assert_eq!(config.get("nonexistent"), None); + } + + #[test] + fn test_storage_config_with_prop() { + let config = StorageConfig::new() + .with_prop("region", "us-east-1") + .with_prop("bucket", "my-bucket"); + + assert_eq!(config.get("region"), Some(&"us-east-1".to_string())); + assert_eq!(config.get("bucket"), Some(&"my-bucket".to_string())); + } + + #[test] + fn test_storage_config_with_props() { + let additional_props = vec![("key1", "value1"), ("key2", "value2")]; + let config = StorageConfig::new().with_props(additional_props); + + assert_eq!(config.get("key1"), Some(&"value1".to_string())); + assert_eq!(config.get("key2"), Some(&"value2".to_string())); + } + + #[test] + fn test_storage_config_clone() { + let config = StorageConfig::new().with_prop("region", "us-east-1"); + let cloned = config.clone(); + + assert_eq!(config, cloned); + assert_eq!(cloned.get("region"), Some(&"us-east-1".to_string())); + } + + #[test] + fn test_storage_config_serialization_roundtrip() { + let config = StorageConfig::new() + .with_prop("region", "us-east-1") + .with_prop("bucket", "my-bucket"); + + let serialized = serde_json::to_string(&config).unwrap(); + let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(config, deserialized); + } + + #[test] + fn test_storage_config_clone_independence() { + let original = StorageConfig::new().with_prop("region", "us-east-1"); + let mut cloned = original.clone(); + + // Modify the clone + cloned = cloned.with_prop("region", "eu-west-1"); + cloned = cloned.with_prop("new_key", "new_value"); + + // Original should be unchanged + assert_eq!(original.get("region"), Some(&"us-east-1".to_string())); + assert_eq!(original.get("new_key"), None); + + // Clone should have the new values + assert_eq!(cloned.get("region"), Some(&"eu-west-1".to_string())); + assert_eq!(cloned.get("new_key"), Some(&"new_value".to_string())); + } + + #[test] + fn test_storage_config_from_props_empty() { + let config = StorageConfig::from_props(HashMap::new()); + + assert!(config.props().is_empty()); + } + + #[test] + fn test_storage_config_serialization_empty() { + let config = StorageConfig::new(); + + let serialized = serde_json::to_string(&config).unwrap(); + let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(config, deserialized); + assert!(deserialized.props().is_empty()); + } +} diff --git a/crates/iceberg/src/io/storage/config/oss.rs b/crates/iceberg/src/io/storage/config/oss.rs new file mode 100644 index 0000000000..0ac45c8da3 --- /dev/null +++ b/crates/iceberg/src/io/storage/config/oss.rs @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Alibaba Cloud OSS storage configuration. +//! +//! This module provides configuration constants and types for Alibaba Cloud OSS storage. + +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use super::StorageConfig; +use crate::Result; + +/// Aliyun OSS endpoint. +pub const OSS_ENDPOINT: &str = "oss.endpoint"; +/// Aliyun OSS access key ID. +pub const OSS_ACCESS_KEY_ID: &str = "oss.access-key-id"; +/// Aliyun OSS access key secret. +pub const OSS_ACCESS_KEY_SECRET: &str = "oss.access-key-secret"; + +/// Alibaba Cloud OSS storage configuration. +/// +/// This struct contains all the configuration options for connecting to Alibaba Cloud OSS. +/// Use the builder pattern via `OssConfig::builder()` to construct instances. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +pub struct OssConfig { + /// OSS endpoint URL. + #[builder(default, setter(strip_option, into))] + pub endpoint: Option, + /// OSS access key ID. + #[builder(default, setter(strip_option, into))] + pub access_key_id: Option, + /// OSS access key secret. + #[builder(default, setter(strip_option, into))] + pub access_key_secret: Option, +} + +impl TryFrom<&StorageConfig> for OssConfig { + type Error = crate::Error; + + fn try_from(config: &StorageConfig) -> Result { + let props = config.props(); + + let mut cfg = OssConfig::default(); + if let Some(endpoint) = props.get(OSS_ENDPOINT) { + cfg.endpoint = Some(endpoint.clone()); + } + if let Some(access_key_id) = props.get(OSS_ACCESS_KEY_ID) { + cfg.access_key_id = Some(access_key_id.clone()); + } + if let Some(access_key_secret) = props.get(OSS_ACCESS_KEY_SECRET) { + cfg.access_key_secret = Some(access_key_secret.clone()); + } + + Ok(cfg) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_oss_config_builder() { + let config = OssConfig::builder() + .endpoint("https://oss-cn-hangzhou.aliyuncs.com") + .access_key_id("my-access-key") + .access_key_secret("my-secret-key") + .build(); + + assert_eq!( + config.endpoint.as_deref(), + Some("https://oss-cn-hangzhou.aliyuncs.com") + ); + assert_eq!(config.access_key_id.as_deref(), Some("my-access-key")); + assert_eq!(config.access_key_secret.as_deref(), Some("my-secret-key")); + } + + #[test] + fn test_oss_config_from_storage_config() { + let storage_config = StorageConfig::new() + .with_prop(OSS_ENDPOINT, "https://oss-cn-hangzhou.aliyuncs.com") + .with_prop(OSS_ACCESS_KEY_ID, "my-access-key") + .with_prop(OSS_ACCESS_KEY_SECRET, "my-secret-key"); + + let oss_config = OssConfig::try_from(&storage_config).unwrap(); + + assert_eq!( + oss_config.endpoint.as_deref(), + Some("https://oss-cn-hangzhou.aliyuncs.com") + ); + assert_eq!(oss_config.access_key_id.as_deref(), Some("my-access-key")); + assert_eq!( + oss_config.access_key_secret.as_deref(), + Some("my-secret-key") + ); + } + + #[test] + fn test_oss_config_empty() { + let storage_config = StorageConfig::new(); + + let oss_config = OssConfig::try_from(&storage_config).unwrap(); + + assert_eq!(oss_config.endpoint, None); + assert_eq!(oss_config.access_key_id, None); + assert_eq!(oss_config.access_key_secret, None); + } +} diff --git a/crates/iceberg/src/io/storage/config/s3.rs b/crates/iceberg/src/io/storage/config/s3.rs new file mode 100644 index 0000000000..fcee3916e8 --- /dev/null +++ b/crates/iceberg/src/io/storage/config/s3.rs @@ -0,0 +1,343 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Amazon S3 storage configuration. +//! +//! This module provides configuration constants and types for Amazon S3 storage. +//! These are based on the [Iceberg S3 FileIO configuration](https://py.iceberg.apache.org/configuration/#s3). + +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use super::StorageConfig; +use crate::error::invalid_data; +use crate::io::is_truthy; +use crate::{Error, Result}; + +/// S3 endpoint URL. +pub const S3_ENDPOINT: &str = "s3.endpoint"; +/// S3 access key ID. +pub const S3_ACCESS_KEY_ID: &str = "s3.access-key-id"; +/// S3 secret access key. +pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret-access-key"; +/// S3 session token (required when using temporary credentials). +pub const S3_SESSION_TOKEN: &str = "s3.session-token"; +/// S3 region. +pub const S3_REGION: &str = "s3.region"; +/// Region to use for the S3 client (takes precedence over [`S3_REGION`]). +pub const CLIENT_REGION: &str = "client.region"; +/// S3 Path Style Access. +pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access"; +/// S3 Server Side Encryption Type. +pub const S3_SSE_TYPE: &str = "s3.sse.type"; +/// S3 Server Side Encryption Key. +/// If S3 encryption type is kms, input is a KMS Key ID. +/// In case this property is not set, default key "aws/s3" is used. +/// If encryption type is custom, input is a custom base-64 AES256 symmetric key. +pub const S3_SSE_KEY: &str = "s3.sse.key"; +/// S3 Server Side Encryption MD5. +pub const S3_SSE_MD5: &str = "s3.sse.md5"; +/// If set, all AWS clients will assume a role of the given ARN, instead of using the default +/// credential chain. +pub const S3_ASSUME_ROLE_ARN: &str = "client.assume-role.arn"; +/// Optional external ID used to assume an IAM role. +pub const S3_ASSUME_ROLE_EXTERNAL_ID: &str = "client.assume-role.external-id"; +/// Optional session name used to assume an IAM role. +pub const S3_ASSUME_ROLE_SESSION_NAME: &str = "client.assume-role.session-name"; +/// Option to skip signing requests (e.g. for public buckets/folders). +pub const S3_ALLOW_ANONYMOUS: &str = "s3.allow-anonymous"; +/// Option to skip loading the credential from EC2 metadata (typically used in conjunction with +/// `S3_ALLOW_ANONYMOUS`). +pub const S3_DISABLE_EC2_METADATA: &str = "s3.disable-ec2-metadata"; +/// Option to skip loading configuration from config file and the env. +pub const S3_DISABLE_CONFIG_LOAD: &str = "s3.disable-config-load"; +/// Size in bytes of each part of a multipart upload. Defaults to 32 MiB, matching Java +/// `S3FileIOProperties.MULTIPART_SIZE_DEFAULT`. Must be at least 5 MiB, the S3 minimum, +/// and at most `i32::MAX`, since Java reads the property as an `int`. +/// +/// Each part travels as one request, so the part size bounds how much data has to +/// transfer within the per-operation IO timeout. Lower it on a slow connection; if a +/// 5 MiB part still cannot finish in time, raise `opendal.io-timeout-ms` instead. +/// +/// Only `iceberg-storage-opendal` reads this property, so [`S3Config`] has no matching field. +pub const S3_MULTIPART_PART_SIZE_BYTES: &str = "s3.multipart.part-size-bytes"; +/// Number of times to retry an S3 operation, `0` to disable retrying. Retries back off exponentially, +/// with jitter, between [`S3_RETRY_MIN_WAIT_MS`] and [`S3_RETRY_MAX_WAIT_MS`]. +pub const S3_RETRY_NUM_RETRIES: &str = "s3.retry.num-retries"; +/// Minimum wait in milliseconds before retrying an S3 operation. +pub const S3_RETRY_MIN_WAIT_MS: &str = "s3.retry.min-wait-ms"; +/// Maximum wait in milliseconds between two retries of an S3 operation. +pub const S3_RETRY_MAX_WAIT_MS: &str = "s3.retry.max-wait-ms"; + +/// Amazon S3 storage configuration. +/// +/// This struct contains all the configuration options for connecting to Amazon S3. +/// Use the builder pattern via `S3Config::builder()` to construct instances. +/// +/// Defaults follow the Iceberg `S3FileIOProperties` spec (see +/// [`PATH_STYLE_ACCESS_DEFAULT = false`](https://github.com/apache/iceberg/blob/main/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java)), +/// i.e. virtual-host-style addressing is enabled unless +/// `s3.path-style-access=true` is explicitly set. This matches what +/// Java clients do out of the box and is required for a number of +/// S3-compatible stores that do not support path-style URLs. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +pub struct S3Config { + /// S3 endpoint URL. + #[builder(default, setter(strip_option, into))] + pub endpoint: Option, + /// S3 access key ID. + #[builder(default, setter(strip_option, into))] + pub access_key_id: Option, + /// S3 secret access key. + #[builder(default, setter(strip_option, into))] + pub secret_access_key: Option, + /// S3 session token. + #[builder(default, setter(strip_option, into))] + pub session_token: Option, + /// S3 region. + #[builder(default, setter(strip_option, into))] + pub region: Option, + /// Enable virtual host style (opposite of path style access). + /// + /// Defaults to `true` to match Iceberg `S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT = false`. + #[builder(default = true)] + pub enable_virtual_host_style: bool, + /// Server side encryption type. + #[builder(default, setter(strip_option, into))] + pub server_side_encryption: Option, + /// Server side encryption AWS KMS key ID. + #[builder(default, setter(strip_option, into))] + pub server_side_encryption_aws_kms_key_id: Option, + /// Server side encryption customer algorithm. + #[builder(default, setter(strip_option, into))] + pub server_side_encryption_customer_algorithm: Option, + /// Server side encryption customer key. + #[builder(default, setter(strip_option, into))] + pub server_side_encryption_customer_key: Option, + /// Server side encryption customer key MD5. + #[builder(default, setter(strip_option, into))] + pub server_side_encryption_customer_key_md5: Option, + /// Role ARN for assuming a role. + #[builder(default, setter(strip_option, into))] + pub role_arn: Option, + /// External ID for assuming a role. + #[builder(default, setter(strip_option, into))] + pub external_id: Option, + /// Session name for assuming a role. + #[builder(default, setter(strip_option, into))] + pub role_session_name: Option, + /// Allow anonymous access. + #[builder(default)] + pub allow_anonymous: bool, + /// Disable EC2 metadata. + #[builder(default)] + pub disable_ec2_metadata: bool, + /// Disable config load. + #[builder(default)] + pub disable_config_load: bool, +} + +impl Default for S3Config { + fn default() -> Self { + Self::builder().build() + } +} + +impl TryFrom<&StorageConfig> for S3Config { + type Error = Error; + + fn try_from(config: &StorageConfig) -> Result { + let props = config.props(); + + let mut cfg = S3Config::default(); + + if let Some(endpoint) = props.get(S3_ENDPOINT) { + cfg.endpoint = Some(endpoint.clone()); + } + if let Some(access_key_id) = props.get(S3_ACCESS_KEY_ID) { + cfg.access_key_id = Some(access_key_id.clone()); + } + if let Some(secret_access_key) = props.get(S3_SECRET_ACCESS_KEY) { + cfg.secret_access_key = Some(secret_access_key.clone()); + } + if let Some(session_token) = props.get(S3_SESSION_TOKEN) { + cfg.session_token = Some(session_token.clone()); + } + if let Some(region) = props.get(S3_REGION) { + cfg.region = Some(region.clone()); + } + // CLIENT_REGION takes precedence over S3_REGION + if let Some(region) = props.get(CLIENT_REGION) { + cfg.region = Some(region.clone()); + } + if let Some(path_style_access) = props.get(S3_PATH_STYLE_ACCESS) { + cfg.enable_virtual_host_style = !is_truthy(path_style_access.to_lowercase().as_str()); + } + if let Some(arn) = props.get(S3_ASSUME_ROLE_ARN) { + cfg.role_arn = Some(arn.clone()); + } + if let Some(external_id) = props.get(S3_ASSUME_ROLE_EXTERNAL_ID) { + cfg.external_id = Some(external_id.clone()); + } + if let Some(session_name) = props.get(S3_ASSUME_ROLE_SESSION_NAME) { + cfg.role_session_name = Some(session_name.clone()); + } + + // Handle SSE configuration + let s3_sse_key = props.get(S3_SSE_KEY).cloned(); + if let Some(sse_type) = props.get(S3_SSE_TYPE) { + match sse_type.to_lowercase().as_str() { + // No Server Side Encryption + "none" => {} + // S3 SSE-S3 encryption (S3 managed keys). https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html + "s3" => { + cfg.server_side_encryption = Some("AES256".to_string()); + } + // S3 SSE KMS, either using default or custom KMS key. https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + "kms" => { + cfg.server_side_encryption = Some("aws:kms".to_string()); + cfg.server_side_encryption_aws_kms_key_id = s3_sse_key; + } + // S3 SSE-C, using customer managed keys. https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html + "custom" => { + cfg.server_side_encryption_customer_algorithm = Some("AES256".to_string()); + cfg.server_side_encryption_customer_key = s3_sse_key; + cfg.server_side_encryption_customer_key_md5 = props.get(S3_SSE_MD5).cloned(); + } + _ => { + return Err(invalid_data!( + "Invalid {S3_SSE_TYPE}: {sse_type}. Expected one of (custom, kms, s3, none)" + )); + } + } + } + + if let Some(allow_anonymous) = props.get(S3_ALLOW_ANONYMOUS) + && is_truthy(allow_anonymous.to_lowercase().as_str()) + { + cfg.allow_anonymous = true; + } + if let Some(disable_ec2_metadata) = props.get(S3_DISABLE_EC2_METADATA) + && is_truthy(disable_ec2_metadata.to_lowercase().as_str()) + { + cfg.disable_ec2_metadata = true; + } + if let Some(disable_config_load) = props.get(S3_DISABLE_CONFIG_LOAD) + && is_truthy(disable_config_load.to_lowercase().as_str()) + { + cfg.disable_config_load = true; + } + + Ok(cfg) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_s3_config_builder() { + let config = S3Config::builder() + .region("us-east-1") + .access_key_id("my-access-key") + .secret_access_key("my-secret-key") + .endpoint("http://localhost:9000") + .build(); + + assert_eq!(config.region.as_deref(), Some("us-east-1")); + assert_eq!(config.access_key_id.as_deref(), Some("my-access-key")); + assert_eq!(config.secret_access_key.as_deref(), Some("my-secret-key")); + assert_eq!(config.endpoint.as_deref(), Some("http://localhost:9000")); + } + + #[test] + fn test_s3_config_from_storage_config() { + let storage_config = StorageConfig::new() + .with_prop(S3_REGION, "us-east-1") + .with_prop(S3_ACCESS_KEY_ID, "my-access-key") + .with_prop(S3_SECRET_ACCESS_KEY, "my-secret-key") + .with_prop(S3_ENDPOINT, "http://localhost:9000"); + + let s3_config = S3Config::try_from(&storage_config).unwrap(); + + assert_eq!(s3_config.region.as_deref(), Some("us-east-1")); + assert_eq!(s3_config.access_key_id.as_deref(), Some("my-access-key")); + assert_eq!( + s3_config.secret_access_key.as_deref(), + Some("my-secret-key") + ); + assert_eq!(s3_config.endpoint.as_deref(), Some("http://localhost:9000")); + } + + #[test] + fn test_s3_config_client_region_precedence() { + let storage_config = StorageConfig::new() + .with_prop(S3_REGION, "us-east-1") + .with_prop(CLIENT_REGION, "eu-west-1"); + + let s3_config = S3Config::try_from(&storage_config).unwrap(); + + // CLIENT_REGION should take precedence + assert_eq!(s3_config.region.as_deref(), Some("eu-west-1")); + } + + #[test] + fn test_s3_config_default_is_virtual_host_style() { + // Matches Iceberg S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT = false. + assert!(S3Config::default().enable_virtual_host_style); + assert!( + S3Config::try_from(&StorageConfig::new()) + .unwrap() + .enable_virtual_host_style + ); + } + + #[test] + fn test_s3_config_path_style_access() { + let storage_config = StorageConfig::new().with_prop(S3_PATH_STYLE_ACCESS, "true"); + + let s3_config = S3Config::try_from(&storage_config).unwrap(); + + // path style access = true means virtual host style = false + assert!(!s3_config.enable_virtual_host_style); + } + + #[test] + fn test_s3_config_sse_kms() { + let storage_config = StorageConfig::new() + .with_prop(S3_SSE_TYPE, "kms") + .with_prop(S3_SSE_KEY, "my-kms-key-id"); + + let s3_config = S3Config::try_from(&storage_config).unwrap(); + + assert_eq!(s3_config.server_side_encryption.as_deref(), Some("aws:kms")); + assert_eq!( + s3_config.server_side_encryption_aws_kms_key_id.as_deref(), + Some("my-kms-key-id") + ); + } + + #[test] + fn test_s3_config_allow_anonymous() { + let storage_config = StorageConfig::new().with_prop(S3_ALLOW_ANONYMOUS, "true"); + + let s3_config = S3Config::try_from(&storage_config).unwrap(); + + assert!(s3_config.allow_anonymous); + } +} diff --git a/crates/iceberg/src/io/storage/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs new file mode 100644 index 0000000000..cb780e86e7 --- /dev/null +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -0,0 +1,591 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Local filesystem storage implementation for testing. +//! +//! This module provides a `LocalFsStorage` implementation that uses standard +//! Rust filesystem operations. It is primarily intended for unit testing +//! scenarios where tests need to read/write files on the local filesystem. + +use std::fs; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::ops::Range; +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::StreamExt; +use futures::stream::BoxStream; +use serde::{Deserialize, Serialize}; + +use crate::error::invalid_data; +use crate::io::{ + FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, + StorageFactory, +}; +use crate::{Error, ErrorKind, Result}; + +/// Local filesystem storage implementation. +/// +/// This storage implementation uses standard Rust filesystem operations, +/// making it suitable for unit tests that need to read/write files on disk. +/// +/// # Path Normalization +/// +/// The storage normalizes paths to handle various formats: +/// - `file:///path/to/file` -> `/path/to/file` +/// - `file:/path/to/file` -> `/path/to/file` +/// - `/path/to/file` -> `/path/to/file` +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct LocalFsStorage; + +impl LocalFsStorage { + /// Create a new `LocalFsStorage` instance. + pub fn new() -> Self { + Self + } + + /// Normalize a path by removing scheme prefixes. + /// + /// This handles the following formats: + /// - `file:///path` -> `/path` + /// - `file://path` -> `/path` (treats as absolute) + /// - `file:/path` -> `/path` + /// - `/path` -> `/path` + pub(crate) fn normalize_path(path: &str) -> PathBuf { + let path = if let Some(stripped) = path.strip_prefix("file://") { + // file:///path -> /path or file://path -> /path + if stripped.starts_with('/') { + stripped.to_string() + } else { + format!("/{stripped}") + } + } else if let Some(stripped) = path.strip_prefix("file:") { + // file:/path -> /path + if stripped.starts_with('/') { + stripped.to_string() + } else { + format!("/{stripped}") + } + } else { + path.to_string() + }; + PathBuf::from(path) + } +} + +#[async_trait] +#[typetag::serde] +impl Storage for LocalFsStorage { + async fn exists(&self, path: &str) -> Result { + let path = Self::normalize_path(path); + Ok(path.exists()) + } + + async fn metadata(&self, path: &str) -> Result { + let path = Self::normalize_path(path); + let metadata = fs::metadata(&path) + .map_err(|e| invalid_data!("Failed to get metadata for {}: {}", path.display(), e))?; + Ok(FileMetadata { + size: metadata.len(), + }) + } + + async fn read(&self, path: &str) -> Result { + let path = Self::normalize_path(path); + let content = fs::read(&path) + .map_err(|e| invalid_data!("Failed to read file {}: {}", path.display(), e))?; + Ok(Bytes::from(content)) + } + + async fn reader(&self, path: &str) -> Result> { + let path = Self::normalize_path(path); + let file = fs::File::open(&path) + .map_err(|e| invalid_data!("Failed to open file {}: {}", path.display(), e))?; + Ok(Box::new(LocalFsFileRead::new(file))) + } + + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { + let path = Self::normalize_path(path); + + // Create parent directories if they don't exist + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to create directory {}: {}", parent.display(), e), + ) + })?; + } + + fs::write(&path, &bs).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to write file {}: {}", path.display(), e), + ) + })?; + Ok(()) + } + + async fn writer(&self, path: &str) -> Result> { + let path = Self::normalize_path(path); + + // Create parent directories if they don't exist + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to create directory {}: {}", parent.display(), e), + ) + })?; + } + + let file = fs::File::create(&path).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to create file {}: {}", path.display(), e), + ) + })?; + Ok(Box::new(LocalFsFileWrite::new(file))) + } + + async fn delete(&self, path: &str) -> Result<()> { + let path = Self::normalize_path(path); + if path.exists() { + fs::remove_file(&path).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to delete file {}: {}", path.display(), e), + ) + })?; + } + Ok(()) + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + let path = Self::normalize_path(path); + if path.is_dir() { + fs::remove_dir_all(&path).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to delete directory {}: {}", path.display(), e), + ) + })?; + } + Ok(()) + } + + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + while let Some(path) = paths.next().await { + self.delete(&path).await?; + } + Ok(()) + } + + fn new_input(&self, path: &str) -> Result { + Ok(InputFile::new(Arc::new(self.clone()), path.to_string())) + } + + fn new_output(&self, path: &str) -> Result { + Ok(OutputFile::new(Arc::new(self.clone()), path.to_string())) + } +} + +/// File reader for local filesystem storage. +#[derive(Debug)] +pub struct LocalFsFileRead { + file: std::sync::Mutex, +} + +impl LocalFsFileRead { + /// Create a new `LocalFsFileRead` with the given file. + pub fn new(file: fs::File) -> Self { + Self { + file: std::sync::Mutex::new(file), + } + } +} + +#[async_trait] +impl FileRead for LocalFsFileRead { + async fn read(&self, range: Range) -> Result { + let mut file = self.file.lock().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to acquire file lock: {e}"), + ) + })?; + + file.seek(SeekFrom::Start(range.start)) + .map_err(|e| invalid_data!("Failed to seek to position {}: {}", range.start, e))?; + + let len = (range.end - range.start) as usize; + let mut buffer = vec![0u8; len]; + file.read_exact(&mut buffer) + .map_err(|e| invalid_data!("Failed to read {len} bytes: {e}"))?; + + Ok(Bytes::from(buffer)) + } +} + +/// File writer for local filesystem storage. +/// +/// This struct implements `FileWrite` for writing to local files. +#[derive(Debug)] +pub struct LocalFsFileWrite { + file: Option, + bytes_written: u64, +} + +impl LocalFsFileWrite { + /// Create a new `LocalFsFileWrite` for the given file. + pub fn new(file: fs::File) -> Self { + Self { + file: Some(file), + bytes_written: 0, + } + } +} + +#[async_trait] +impl FileWrite for LocalFsFileWrite { + async fn write(&mut self, bs: Bytes) -> Result<()> { + let file = self + .file + .as_mut() + .ok_or_else(|| invalid_data!("Cannot write to closed file"))?; + + file.write_all(&bs).map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to write to file: {e}"), + ) + })?; + self.bytes_written += bs.len() as u64; + + Ok(()) + } + + async fn close(&mut self) -> Result { + let file = self + .file + .take() + .ok_or_else(|| invalid_data!("File already closed"))?; + + file.sync_all() + .map_err(|e| Error::new(ErrorKind::Unexpected, format!("Failed to sync file: {e}")))?; + + Ok(FileMetadata { + size: self.bytes_written, + }) + } +} + +/// Factory for creating `LocalFsStorage` instances. +/// +/// This factory implements `StorageFactory` and creates `LocalFsStorage` +/// instances for the "file" scheme. +/// +/// # Example +/// +/// ```rust,ignore +/// use iceberg::io::{StorageConfig, StorageFactory, LocalFsStorageFactory}; +/// +/// let factory = LocalFsStorageFactory; +/// let config = StorageConfig::new(); +/// let storage = factory.build(&config)?; +/// ``` +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct LocalFsStorageFactory; + +#[typetag::serde] +impl StorageFactory for LocalFsStorageFactory { + fn build(&self, _config: &StorageConfig) -> Result> { + Ok(Arc::new(LocalFsStorage::new())) + } +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + + #[test] + fn test_normalize_path() { + // Test file:/// prefix + assert_eq!( + LocalFsStorage::normalize_path("file:///path/to/file"), + PathBuf::from("/path/to/file") + ); + + // Test file:// prefix (without leading slash in path) + assert_eq!( + LocalFsStorage::normalize_path("file://path/to/file"), + PathBuf::from("/path/to/file") + ); + + // Test file:/ prefix + assert_eq!( + LocalFsStorage::normalize_path("file:/path/to/file"), + PathBuf::from("/path/to/file") + ); + + // Test bare path + assert_eq!( + LocalFsStorage::normalize_path("/path/to/file"), + PathBuf::from("/path/to/file") + ); + } + + #[tokio::test] + async fn test_local_fs_storage_write_read() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let path = tmp_dir.path().join("test.txt"); + let path_str = path.to_str().unwrap(); + let content = Bytes::from("Hello, World!"); + + // Write + storage.write(path_str, content.clone()).await.unwrap(); + + // Read + let read_content = storage.read(path_str).await.unwrap(); + assert_eq!(read_content, content); + } + + #[tokio::test] + async fn test_local_fs_storage_exists() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let path = tmp_dir.path().join("test.txt"); + let path_str = path.to_str().unwrap(); + + // File doesn't exist initially + assert!(!storage.exists(path_str).await.unwrap()); + + // Write file + storage.write(path_str, Bytes::from("test")).await.unwrap(); + + // File exists now + assert!(storage.exists(path_str).await.unwrap()); + } + + #[tokio::test] + async fn test_local_fs_storage_metadata() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let path = tmp_dir.path().join("test.txt"); + let path_str = path.to_str().unwrap(); + let content = Bytes::from("Hello, World!"); + + storage.write(path_str, content.clone()).await.unwrap(); + + let metadata = storage.metadata(path_str).await.unwrap(); + assert_eq!(metadata.size, content.len() as u64); + } + + #[tokio::test] + async fn test_local_fs_storage_delete() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let path = tmp_dir.path().join("test.txt"); + let path_str = path.to_str().unwrap(); + + storage.write(path_str, Bytes::from("test")).await.unwrap(); + assert!(storage.exists(path_str).await.unwrap()); + + storage.delete(path_str).await.unwrap(); + assert!(!storage.exists(path_str).await.unwrap()); + } + + #[tokio::test] + async fn test_local_fs_storage_delete_prefix() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let dir_path = tmp_dir.path().join("subdir"); + let file1 = dir_path.join("file1.txt"); + let file2 = dir_path.join("file2.txt"); + + // Create files in subdirectory + storage + .write(file1.to_str().unwrap(), Bytes::from("1")) + .await + .unwrap(); + storage + .write(file2.to_str().unwrap(), Bytes::from("2")) + .await + .unwrap(); + + // Delete prefix (directory) + storage + .delete_prefix(dir_path.to_str().unwrap()) + .await + .unwrap(); + + // Directory should be deleted + assert!(!dir_path.exists()); + } + + #[tokio::test] + async fn test_local_fs_storage_reader() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let path = tmp_dir.path().join("test.txt"); + let path_str = path.to_str().unwrap(); + let content = Bytes::from("Hello, World!"); + + storage.write(path_str, content.clone()).await.unwrap(); + + let reader = storage.reader(path_str).await.unwrap(); + let read_content = reader.read(0..content.len() as u64).await.unwrap(); + assert_eq!(read_content, content); + + // Test partial read + let partial = reader.read(0..5).await.unwrap(); + assert_eq!(partial, Bytes::from("Hello")); + } + + #[tokio::test] + async fn test_local_fs_storage_writer() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let path = tmp_dir.path().join("test.txt"); + let path_str = path.to_str().unwrap(); + + let mut writer = storage.writer(path_str).await.unwrap(); + writer.write(Bytes::from("Hello, ")).await.unwrap(); + writer.write(Bytes::from("World!")).await.unwrap(); + let metadata = writer.close().await.unwrap(); + + let content = storage.read(path_str).await.unwrap(); + assert_eq!(content, Bytes::from("Hello, World!")); + assert_eq!(metadata.size, content.len() as u64); + } + + #[tokio::test] + async fn test_local_fs_file_write_double_close() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let path = tmp_dir.path().join("test.txt"); + let path_str = path.to_str().unwrap(); + + let mut writer = storage.writer(path_str).await.unwrap(); + writer.write(Bytes::from("test")).await.unwrap(); + writer.close().await.unwrap(); + + // Second close should fail + let result = writer.close().await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_local_fs_file_write_after_close() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let path = tmp_dir.path().join("test.txt"); + let path_str = path.to_str().unwrap(); + + let mut writer = storage.writer(path_str).await.unwrap(); + assert_eq!(writer.close().await.unwrap().size, 0); + + // Write after close should fail + let result = writer.write(Bytes::from("test")).await; + assert!(result.is_err()); + } + + #[test] + fn test_local_fs_storage_factory() { + let factory = LocalFsStorageFactory; + let config = StorageConfig::new(); + let storage = factory.build(&config).unwrap(); + + // Verify we got a valid storage instance + assert!(format!("{storage:?}").contains("LocalFsStorage")); + } + + #[tokio::test] + async fn test_local_fs_creates_parent_directories() { + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + let path = tmp_dir.path().join("a/b/c/test.txt"); + let path_str = path.to_str().unwrap(); + + // Write should create parent directories + storage.write(path_str, Bytes::from("test")).await.unwrap(); + + assert!(path.exists()); + } + + #[tokio::test] + async fn test_local_fs_storage_delete_stream() { + use futures::stream; + + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + + // Create multiple files + let file1 = tmp_dir.path().join("file1.txt"); + let file2 = tmp_dir.path().join("file2.txt"); + let file3 = tmp_dir.path().join("file3.txt"); + + storage + .write(file1.to_str().unwrap(), Bytes::from("1")) + .await + .unwrap(); + storage + .write(file2.to_str().unwrap(), Bytes::from("2")) + .await + .unwrap(); + storage + .write(file3.to_str().unwrap(), Bytes::from("3")) + .await + .unwrap(); + + // Verify files exist + assert!(storage.exists(file1.to_str().unwrap()).await.unwrap()); + assert!(storage.exists(file2.to_str().unwrap()).await.unwrap()); + assert!(storage.exists(file3.to_str().unwrap()).await.unwrap()); + + // Delete multiple files using stream + let paths = vec![ + file1.to_str().unwrap().to_string(), + file2.to_str().unwrap().to_string(), + ]; + let path_stream = stream::iter(paths).boxed(); + storage.delete_stream(path_stream).await.unwrap(); + + // Verify deleted files no longer exist + assert!(!storage.exists(file1.to_str().unwrap()).await.unwrap()); + assert!(!storage.exists(file2.to_str().unwrap()).await.unwrap()); + + // Verify file3 still exists + assert!(storage.exists(file3.to_str().unwrap()).await.unwrap()); + } + + #[tokio::test] + async fn test_local_fs_storage_delete_stream_empty() { + use futures::stream; + + let storage = LocalFsStorage::new(); + + // Delete with empty stream should succeed + let path_stream = stream::iter(Vec::::new()).boxed(); + storage.delete_stream(path_stream).await.unwrap(); + } +} diff --git a/crates/iceberg/src/io/storage/memory.rs b/crates/iceberg/src/io/storage/memory.rs new file mode 100644 index 0000000000..fefa7973e8 --- /dev/null +++ b/crates/iceberg/src/io/storage/memory.rs @@ -0,0 +1,645 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Pure Rust in-memory storage implementation for testing. +//! +//! This module provides a `MemoryStorage` implementation that stores data +//! in a thread-safe `HashMap`, without any external dependencies. +//! It is primarily intended for unit testing and scenarios where persistent +//! storage is not needed. + +use std::collections::HashMap; +use std::ops::Range; +use std::sync::{Arc, RwLock}; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::StreamExt; +use futures::stream::BoxStream; +use serde::{Deserialize, Serialize}; + +use crate::error::invalid_data; +use crate::io::{ + FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, + StorageFactory, +}; +use crate::{Error, ErrorKind, Result}; + +/// In-memory storage implementation. +/// +/// This storage implementation stores all data in a thread-safe `HashMap`, +/// making it suitable for unit tests and scenarios where persistent storage +/// is not needed. +/// +/// # Path Normalization +/// +/// The storage normalizes paths to handle various formats: +/// - `memory://path/to/file` -> `path/to/file` +/// - `memory:/path/to/file` -> `path/to/file` +/// - `/path/to/file` -> `path/to/file` +/// - `path/to/file` -> `path/to/file` +/// +/// # Serialization +/// +/// When serialized, `MemoryStorage` serializes to an empty state. When +/// deserialized, it creates a new empty instance. This is intentional +/// because in-memory data cannot be meaningfully serialized across +/// process boundaries. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct MemoryStorage { + #[serde(skip, default = "default_memory_data")] + data: Arc>>, +} + +fn default_memory_data() -> Arc>> { + Arc::new(RwLock::new(HashMap::new())) +} + +impl MemoryStorage { + /// Create a new empty `MemoryStorage` instance. + pub fn new() -> Self { + Self { + data: Arc::new(RwLock::new(HashMap::new())), + } + } + + /// Normalize a path by removing scheme prefixes and leading slashes. + /// + /// This handles the following formats: + /// - `memory://path` -> `path` + /// - `memory:/path` -> `path` + /// - `/path` -> `path` + /// - `path` -> `path` + pub(crate) fn normalize_path(path: &str) -> String { + // Handle memory:// prefix (with double slash) + let path = path.strip_prefix("memory://").unwrap_or(path); + // Handle memory:/ prefix (with single slash) + let path = path.strip_prefix("memory:/").unwrap_or(path); + // Remove any leading slashes + path.trim_start_matches('/').to_string() + } +} + +#[async_trait] +#[typetag::serde] +impl Storage for MemoryStorage { + async fn exists(&self, path: &str) -> Result { + let normalized = Self::normalize_path(path); + let data = self.data.read().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to acquire read lock: {e}"), + ) + })?; + Ok(data.contains_key(&normalized)) + } + + async fn metadata(&self, path: &str) -> Result { + let normalized = Self::normalize_path(path); + let data = self.data.read().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to acquire read lock: {e}"), + ) + })?; + match data.get(&normalized) { + Some(bytes) => Ok(FileMetadata { + size: bytes.len() as u64, + }), + None => Err(invalid_data!("File not found: {path}")), + } + } + + async fn read(&self, path: &str) -> Result { + let normalized = Self::normalize_path(path); + let data = self.data.read().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to acquire read lock: {e}"), + ) + })?; + match data.get(&normalized) { + Some(bytes) => Ok(bytes.clone()), + None => Err(invalid_data!("File not found: {path}")), + } + } + + async fn reader(&self, path: &str) -> Result> { + let normalized = Self::normalize_path(path); + let data = self.data.read().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to acquire read lock: {e}"), + ) + })?; + match data.get(&normalized) { + Some(bytes) => Ok(Box::new(MemoryFileRead::new(bytes.clone()))), + None => Err(invalid_data!("File not found: {path}")), + } + } + + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { + let normalized = Self::normalize_path(path); + let mut data = self.data.write().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to acquire write lock: {e}"), + ) + })?; + data.insert(normalized, bs); + Ok(()) + } + + async fn writer(&self, path: &str) -> Result> { + let normalized = Self::normalize_path(path); + Ok(Box::new(MemoryFileWrite::new( + self.data.clone(), + normalized, + ))) + } + + async fn delete(&self, path: &str) -> Result<()> { + let normalized = Self::normalize_path(path); + let mut data = self.data.write().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to acquire write lock: {e}"), + ) + })?; + data.remove(&normalized); + Ok(()) + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + let normalized = Self::normalize_path(path); + let prefix = if normalized.ends_with('/') { + normalized + } else { + format!("{normalized}/") + }; + + let mut data = self.data.write().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to acquire write lock: {e}"), + ) + })?; + + // Collect keys to remove (can't modify while iterating) + let keys_to_remove: Vec = data + .keys() + .filter(|k| k.starts_with(&prefix)) + .cloned() + .collect(); + + for key in keys_to_remove { + data.remove(&key); + } + + Ok(()) + } + + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + while let Some(path) = paths.next().await { + self.delete(&path).await?; + } + Ok(()) + } + + fn new_input(&self, path: &str) -> Result { + Ok(InputFile::new(Arc::new(self.clone()), path.to_string())) + } + + fn new_output(&self, path: &str) -> Result { + Ok(OutputFile::new(Arc::new(self.clone()), path.to_string())) + } +} + +/// Factory for creating `MemoryStorage` instances. +/// +/// This factory implements `StorageFactory` and creates `MemoryStorage` +/// instances. Since the factory is explicitly chosen, no scheme validation +/// is performed - the storage will validate paths during operations. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct MemoryStorageFactory; + +#[typetag::serde] +impl StorageFactory for MemoryStorageFactory { + fn build(&self, _config: &StorageConfig) -> Result> { + Ok(Arc::new(MemoryStorage::new())) + } +} + +/// File reader for in-memory storage. +#[derive(Debug)] +pub struct MemoryFileRead { + data: Bytes, +} + +impl MemoryFileRead { + /// Create a new `MemoryFileRead` with the given data. + pub fn new(data: Bytes) -> Self { + Self { data } + } +} + +#[async_trait] +impl FileRead for MemoryFileRead { + async fn read(&self, range: Range) -> Result { + let start = range.start as usize; + let end = range.end as usize; + + if start > self.data.len() || end > self.data.len() { + return Err(invalid_data!( + "Range {}..{} is out of bounds for data of length {}", + start, + end, + self.data.len() + )); + } + + Ok(self.data.slice(start..end)) + } +} + +/// File writer for in-memory storage. +/// +/// This struct implements `FileWrite` for writing to in-memory storage. +/// Data is buffered until `close()` is called, at which point it is +/// flushed to the storage. +#[derive(Debug)] +pub struct MemoryFileWrite { + data: Arc>>, + path: String, + buffer: Vec, + closed: bool, +} + +impl MemoryFileWrite { + /// Create a new `MemoryFileWrite` for the given path. + pub fn new(data: Arc>>, path: String) -> Self { + Self { + data, + path, + buffer: Vec::new(), + closed: false, + } + } +} + +#[async_trait] +impl FileWrite for MemoryFileWrite { + async fn write(&mut self, bs: Bytes) -> Result<()> { + if self.closed { + return Err(invalid_data!("Cannot write to closed file")); + } + self.buffer.extend_from_slice(&bs); + Ok(()) + } + + async fn close(&mut self) -> Result { + if self.closed { + return Err(invalid_data!("File already closed")); + } + + let mut data = self.data.write().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to acquire write lock: {e}"), + ) + })?; + + let size = self.buffer.len() as u64; + data.insert( + self.path.clone(), + Bytes::from(std::mem::take(&mut self.buffer)), + ); + self.closed = true; + Ok(FileMetadata { size }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_path() { + // Test memory:// prefix + assert_eq!( + MemoryStorage::normalize_path("memory://path/to/file"), + "path/to/file" + ); + + // Test memory:/ prefix + assert_eq!( + MemoryStorage::normalize_path("memory:/path/to/file"), + "path/to/file" + ); + + // Test leading slash + assert_eq!( + MemoryStorage::normalize_path("/path/to/file"), + "path/to/file" + ); + + // Test bare path + assert_eq!( + MemoryStorage::normalize_path("path/to/file"), + "path/to/file" + ); + + // Test multiple leading slashes + assert_eq!( + MemoryStorage::normalize_path("///path/to/file"), + "path/to/file" + ); + + // Test memory:// with leading slash in path + assert_eq!( + MemoryStorage::normalize_path("memory:///path/to/file"), + "path/to/file" + ); + } + + #[tokio::test] + async fn test_memory_storage_write_read() { + let storage = MemoryStorage::new(); + let path = "memory://test/file.txt"; + let content = Bytes::from("Hello, World!"); + + // Write + storage.write(path, content.clone()).await.unwrap(); + + // Read + let read_content = storage.read(path).await.unwrap(); + assert_eq!(read_content, content); + } + + #[tokio::test] + async fn test_memory_storage_exists() { + let storage = MemoryStorage::new(); + let path = "memory://test/file.txt"; + + // File doesn't exist initially + assert!(!storage.exists(path).await.unwrap()); + + // Write file + storage.write(path, Bytes::from("test")).await.unwrap(); + + // File exists now + assert!(storage.exists(path).await.unwrap()); + } + + #[tokio::test] + async fn test_memory_storage_metadata() { + let storage = MemoryStorage::new(); + let path = "memory://test/file.txt"; + let content = Bytes::from("Hello, World!"); + + storage.write(path, content.clone()).await.unwrap(); + + let metadata = storage.metadata(path).await.unwrap(); + assert_eq!(metadata.size, content.len() as u64); + } + + #[tokio::test] + async fn test_memory_storage_delete() { + let storage = MemoryStorage::new(); + let path = "memory://test/file.txt"; + + storage.write(path, Bytes::from("test")).await.unwrap(); + assert!(storage.exists(path).await.unwrap()); + + storage.delete(path).await.unwrap(); + assert!(!storage.exists(path).await.unwrap()); + } + + #[tokio::test] + async fn test_memory_storage_delete_prefix() { + let storage = MemoryStorage::new(); + + // Create multiple files + storage + .write("memory://dir/file1.txt", Bytes::from("1")) + .await + .unwrap(); + storage + .write("memory://dir/file2.txt", Bytes::from("2")) + .await + .unwrap(); + storage + .write("memory://other/file.txt", Bytes::from("3")) + .await + .unwrap(); + + // Delete prefix + storage.delete_prefix("memory://dir").await.unwrap(); + + // Files in dir should be deleted + assert!(!storage.exists("memory://dir/file1.txt").await.unwrap()); + assert!(!storage.exists("memory://dir/file2.txt").await.unwrap()); + + // File in other dir should still exist + assert!(storage.exists("memory://other/file.txt").await.unwrap()); + } + + #[tokio::test] + async fn test_memory_storage_reader() { + let storage = MemoryStorage::new(); + let path = "memory://test/file.txt"; + let content = Bytes::from("Hello, World!"); + + storage.write(path, content.clone()).await.unwrap(); + + let reader = storage.reader(path).await.unwrap(); + let read_content = reader.read(0..content.len() as u64).await.unwrap(); + assert_eq!(read_content, content); + + // Test partial read + let partial = reader.read(0..5).await.unwrap(); + assert_eq!(partial, Bytes::from("Hello")); + } + + #[tokio::test] + async fn test_memory_storage_writer() { + let storage = MemoryStorage::new(); + let path = "memory://test/file.txt"; + + let mut writer = storage.writer(path).await.unwrap(); + writer.write(Bytes::from("Hello, ")).await.unwrap(); + writer.write(Bytes::from("World!")).await.unwrap(); + let metadata = writer.close().await.unwrap(); + + let content = storage.read(path).await.unwrap(); + assert_eq!(content, Bytes::from("Hello, World!")); + assert_eq!(metadata.size, content.len() as u64); + } + + #[tokio::test] + async fn test_memory_file_write_double_close() { + let storage = MemoryStorage::new(); + let path = "memory://test/file.txt"; + + let mut writer = storage.writer(path).await.unwrap(); + writer.write(Bytes::from("test")).await.unwrap(); + writer.close().await.unwrap(); + + // Second close should fail + let result = writer.close().await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_memory_file_write_after_close() { + let storage = MemoryStorage::new(); + let path = "memory://test/file.txt"; + + let mut writer = storage.writer(path).await.unwrap(); + assert_eq!(writer.close().await.unwrap().size, 0); + + // Write after close should fail + let result = writer.write(Bytes::from("test")).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_memory_file_read_out_of_bounds() { + let storage = MemoryStorage::new(); + let path = "memory://test/file.txt"; + let content = Bytes::from("Hello"); + + storage.write(path, content).await.unwrap(); + + let reader = storage.reader(path).await.unwrap(); + let result = reader.read(0..100).await; + assert!(result.is_err()); + } + + #[test] + fn test_memory_storage_serialization() { + let storage = MemoryStorage::new(); + + // Serialize + let serialized = serde_json::to_string(&storage).unwrap(); + + // Deserialize + let deserialized: MemoryStorage = serde_json::from_str(&serialized).unwrap(); + + // Deserialized storage should be empty (new instance) + assert!(deserialized.data.read().unwrap().is_empty()); + } + + #[test] + fn test_memory_storage_factory() { + let factory = MemoryStorageFactory; + let config = StorageConfig::new(); + let storage = factory.build(&config).unwrap(); + + // Verify we got a valid storage instance + assert!(format!("{storage:?}").contains("MemoryStorage")); + } + + #[test] + fn test_memory_storage_factory_serialization() { + let factory = MemoryStorageFactory; + + // Serialize + let serialized = serde_json::to_string(&factory).unwrap(); + + // Deserialize + let deserialized: MemoryStorageFactory = serde_json::from_str(&serialized).unwrap(); + + // Verify the deserialized factory works + let config = StorageConfig::new(); + let storage = deserialized.build(&config).unwrap(); + assert!(format!("{storage:?}").contains("MemoryStorage")); + } + + #[tokio::test] + async fn test_path_normalization_consistency() { + let storage = MemoryStorage::new(); + let content = Bytes::from("test content"); + + // Write with one format + storage + .write("memory://path/to/file", content.clone()) + .await + .unwrap(); + + // Read with different formats - all should work + assert_eq!( + storage.read("memory://path/to/file").await.unwrap(), + content + ); + assert_eq!(storage.read("memory:/path/to/file").await.unwrap(), content); + assert_eq!(storage.read("/path/to/file").await.unwrap(), content); + assert_eq!(storage.read("path/to/file").await.unwrap(), content); + } + + #[tokio::test] + async fn test_memory_storage_delete_stream() { + use futures::stream; + + let storage = MemoryStorage::new(); + + // Create multiple files + storage + .write("memory://file1.txt", Bytes::from("1")) + .await + .unwrap(); + storage + .write("memory://file2.txt", Bytes::from("2")) + .await + .unwrap(); + storage + .write("memory://file3.txt", Bytes::from("3")) + .await + .unwrap(); + + // Verify files exist + assert!(storage.exists("memory://file1.txt").await.unwrap()); + assert!(storage.exists("memory://file2.txt").await.unwrap()); + assert!(storage.exists("memory://file3.txt").await.unwrap()); + + // Delete multiple files using stream + let paths = vec![ + "memory://file1.txt".to_string(), + "memory://file2.txt".to_string(), + ]; + let path_stream = stream::iter(paths).boxed(); + storage.delete_stream(path_stream).await.unwrap(); + + // Verify deleted files no longer exist + assert!(!storage.exists("memory://file1.txt").await.unwrap()); + assert!(!storage.exists("memory://file2.txt").await.unwrap()); + + // Verify file3 still exists + assert!(storage.exists("memory://file3.txt").await.unwrap()); + } + + #[tokio::test] + async fn test_memory_storage_delete_stream_empty() { + use futures::stream; + + let storage = MemoryStorage::new(); + + // Delete with empty stream should succeed + let path_stream = stream::iter(Vec::::new()).boxed(); + storage.delete_stream(path_stream).await.unwrap(); + } +} diff --git a/crates/iceberg/src/io/storage/mod.rs b/crates/iceberg/src/io/storage/mod.rs new file mode 100644 index 0000000000..5276c7771f --- /dev/null +++ b/crates/iceberg/src/io/storage/mod.rs @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Storage interfaces for Iceberg. + +mod config; +mod local_fs; +mod memory; + +use std::fmt::Debug; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +pub use config::*; +use futures::stream::BoxStream; +pub use local_fs::{LocalFsStorage, LocalFsStorageFactory}; +pub use memory::{MemoryStorage, MemoryStorageFactory}; + +use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile}; +use crate::Result; + +/// Trait for storage operations in Iceberg. +/// +/// The trait supports serialization via `typetag`, allowing storage instances to be +/// serialized and deserialized across process boundaries. +/// +/// Third-party implementations can implement this trait to provide custom storage backends. +/// +/// # Implementing Custom Storage +/// +/// To implement a custom storage backend: +/// +/// 1. Create a struct that implements this trait +/// 2. Add `#[typetag::serde]` attribute for serialization support +/// 3. Implement all required methods +/// +/// # Example +/// +/// ```rust,ignore +/// #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +/// struct MyStorage { +/// // custom fields +/// } +/// +/// #[async_trait] +/// #[typetag::serde] +/// impl Storage for MyStorage { +/// async fn exists(&self, path: &str) -> Result { +/// // implementation +/// todo!() +/// } +/// // ... implement other methods +/// } +/// ``` +#[async_trait] +#[typetag::serde(tag = "type")] +pub trait Storage: Debug + Send + Sync { + /// Check if a file exists at the given path + async fn exists(&self, path: &str) -> Result; + + /// Get metadata from an input path + async fn metadata(&self, path: &str) -> Result; + + /// Read bytes from a path + async fn read(&self, path: &str) -> Result; + + /// Get FileRead from a path + async fn reader(&self, path: &str) -> Result>; + + /// Write bytes to an output path + async fn write(&self, path: &str, bs: Bytes) -> Result<()>; + + /// Get FileWrite from a path + async fn writer(&self, path: &str) -> Result>; + + /// Delete a file at the given path + async fn delete(&self, path: &str) -> Result<()>; + + /// Delete all files with the given prefix + async fn delete_prefix(&self, path: &str) -> Result<()>; + + /// Delete multiple files from a stream of paths. + async fn delete_stream(&self, paths: BoxStream<'static, String>) -> Result<()>; + + /// Create a new input file for reading + fn new_input(&self, path: &str) -> Result; + + /// Create a new output file for writing + fn new_output(&self, path: &str) -> Result; +} + +/// Factory for creating Storage instances from configuration. +/// +/// Implement this trait to provide custom storage backends. The factory pattern +/// allows for lazy initialization of storage instances and enables users to +/// inject custom storage implementations into catalogs. +/// +/// # Example +/// +/// ```rust,ignore +/// #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +/// struct MyCustomStorageFactory { +/// // custom configuration +/// } +/// +/// #[typetag::serde] +/// impl StorageFactory for MyCustomStorageFactory { +/// fn build(&self, config: &StorageConfig) -> Result> { +/// // Create and return custom storage implementation +/// todo!() +/// } +/// } +/// ``` +#[typetag::serde(tag = "type")] +pub trait StorageFactory: Debug + Send + Sync { + /// Build a new Storage instance from the given configuration. + /// + /// # Arguments + /// + /// * `config` - The storage configuration containing scheme and properties + /// + /// # Returns + /// + /// A `Result` containing an `Arc` on success, or an error + /// if the storage could not be created. + fn build(&self, config: &StorageConfig) -> Result>; +} diff --git a/crates/iceberg/src/io/storage_s3.rs b/crates/iceberg/src/io/storage_s3.rs deleted file mode 100644 index f2408331c5..0000000000 --- a/crates/iceberg/src/io/storage_s3.rs +++ /dev/null @@ -1,216 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::sync::Arc; - -use async_trait::async_trait; -use opendal::services::S3Config; -use opendal::{Configurator, Operator}; -pub use reqsign::{AwsCredential, AwsCredentialLoad}; -use reqwest::Client; -use url::Url; - -use crate::io::is_truthy; -use crate::{Error, ErrorKind, Result}; - -/// Following are arguments for [s3 file io](https://py.iceberg.apache.org/configuration/#s3). -/// S3 endpoint. -pub const S3_ENDPOINT: &str = "s3.endpoint"; -/// S3 access key id. -pub const S3_ACCESS_KEY_ID: &str = "s3.access-key-id"; -/// S3 secret access key. -pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret-access-key"; -/// S3 session token. -/// This is required when using temporary credentials. -pub const S3_SESSION_TOKEN: &str = "s3.session-token"; -/// S3 region. -pub const S3_REGION: &str = "s3.region"; -/// Region to use for the S3 client. -/// -/// This takes precedence over [`S3_REGION`]. -pub const CLIENT_REGION: &str = "client.region"; -/// S3 Path Style Access. -pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access"; -/// S3 Server Side Encryption Type. -pub const S3_SSE_TYPE: &str = "s3.sse.type"; -/// S3 Server Side Encryption Key. -/// If S3 encryption type is kms, input is a KMS Key ID. -/// In case this property is not set, default key "aws/s3" is used. -/// If encryption type is custom, input is a custom base-64 AES256 symmetric key. -pub const S3_SSE_KEY: &str = "s3.sse.key"; -/// S3 Server Side Encryption MD5. -pub const S3_SSE_MD5: &str = "s3.sse.md5"; -/// If set, all AWS clients will assume a role of the given ARN, instead of using the default -/// credential chain. -pub const S3_ASSUME_ROLE_ARN: &str = "client.assume-role.arn"; -/// Optional external ID used to assume an IAM role. -pub const S3_ASSUME_ROLE_EXTERNAL_ID: &str = "client.assume-role.external-id"; -/// Optional session name used to assume an IAM role. -pub const S3_ASSUME_ROLE_SESSION_NAME: &str = "client.assume-role.session-name"; -/// Option to skip signing requests (e.g. for public buckets/folders). -pub const S3_ALLOW_ANONYMOUS: &str = "s3.allow-anonymous"; -/// Option to skip loading the credential from EC2 metadata (typically used in conjunction with -/// `S3_ALLOW_ANONYMOUS`). -pub const S3_DISABLE_EC2_METADATA: &str = "s3.disable-ec2-metadata"; -/// Option to skip loading configuration from config file and the env. -pub const S3_DISABLE_CONFIG_LOAD: &str = "s3.disable-config-load"; - -/// Parse iceberg props to s3 config. -pub(crate) fn s3_config_parse(mut m: HashMap) -> Result { - let mut cfg = S3Config::default(); - if let Some(endpoint) = m.remove(S3_ENDPOINT) { - cfg.endpoint = Some(endpoint); - }; - if let Some(access_key_id) = m.remove(S3_ACCESS_KEY_ID) { - cfg.access_key_id = Some(access_key_id); - }; - if let Some(secret_access_key) = m.remove(S3_SECRET_ACCESS_KEY) { - cfg.secret_access_key = Some(secret_access_key); - }; - if let Some(session_token) = m.remove(S3_SESSION_TOKEN) { - cfg.session_token = Some(session_token); - }; - if let Some(region) = m.remove(S3_REGION) { - cfg.region = Some(region); - }; - if let Some(region) = m.remove(CLIENT_REGION) { - cfg.region = Some(region); - }; - if let Some(path_style_access) = m.remove(S3_PATH_STYLE_ACCESS) { - cfg.enable_virtual_host_style = !is_truthy(path_style_access.to_lowercase().as_str()); - }; - if let Some(arn) = m.remove(S3_ASSUME_ROLE_ARN) { - cfg.role_arn = Some(arn); - } - if let Some(external_id) = m.remove(S3_ASSUME_ROLE_EXTERNAL_ID) { - cfg.external_id = Some(external_id); - }; - if let Some(session_name) = m.remove(S3_ASSUME_ROLE_SESSION_NAME) { - cfg.role_session_name = Some(session_name); - }; - let s3_sse_key = m.remove(S3_SSE_KEY); - if let Some(sse_type) = m.remove(S3_SSE_TYPE) { - match sse_type.to_lowercase().as_str() { - // No Server Side Encryption - "none" => {} - // S3 SSE-S3 encryption (S3 managed keys). https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html - "s3" => { - cfg.server_side_encryption = Some("AES256".to_string()); - } - // S3 SSE KMS, either using default or custom KMS key. https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html - "kms" => { - cfg.server_side_encryption = Some("aws:kms".to_string()); - cfg.server_side_encryption_aws_kms_key_id = s3_sse_key; - } - // S3 SSE-C, using customer managed keys. https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html - "custom" => { - cfg.server_side_encryption_customer_algorithm = Some("AES256".to_string()); - cfg.server_side_encryption_customer_key = s3_sse_key; - cfg.server_side_encryption_customer_key_md5 = m.remove(S3_SSE_MD5); - } - _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid {}: {}. Expected one of (custom, kms, s3, none)", - S3_SSE_TYPE, sse_type - ), - )); - } - } - }; - - if let Some(allow_anonymous) = m.remove(S3_ALLOW_ANONYMOUS) { - if is_truthy(allow_anonymous.to_lowercase().as_str()) { - cfg.allow_anonymous = true; - } - } - if let Some(disable_ec2_metadata) = m.remove(S3_DISABLE_EC2_METADATA) { - if is_truthy(disable_ec2_metadata.to_lowercase().as_str()) { - cfg.disable_ec2_metadata = true; - } - }; - if let Some(disable_config_load) = m.remove(S3_DISABLE_CONFIG_LOAD) { - if is_truthy(disable_config_load.to_lowercase().as_str()) { - cfg.disable_config_load = true; - } - }; - - Ok(cfg) -} - -/// Build new opendal operator from give path. -pub(crate) fn s3_config_build( - cfg: &S3Config, - customized_credential_load: &Option, - path: &str, -) -> Result { - let url = Url::parse(path)?; - let bucket = url.host_str().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid s3 url: {}, missing bucket", path), - ) - })?; - - let mut builder = cfg - .clone() - .into_builder() - // Set bucket name. - .bucket(bucket); - - if let Some(customized_credential_load) = customized_credential_load { - builder = builder - .customized_credential_load(customized_credential_load.clone().into_opendal_loader()); - } - - Ok(Operator::new(builder)?.finish()) -} - -/// Custom AWS credential loader. -/// This can be used to load credentials from a custom source, such as the AWS SDK. -/// -/// This should be set as an extension on `FileIOBuilder`. -#[derive(Clone)] -pub struct CustomAwsCredentialLoader(Arc); - -impl std::fmt::Debug for CustomAwsCredentialLoader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CustomAwsCredentialLoader") - .finish_non_exhaustive() - } -} - -impl CustomAwsCredentialLoader { - /// Create a new custom AWS credential loader. - pub fn new(loader: Arc) -> Self { - Self(loader) - } - - /// Convert this loader into an opendal compatible loader for customized AWS credentials. - pub fn into_opendal_loader(self) -> Box { - Box::new(self) - } -} - -#[async_trait] -impl AwsCredentialLoad for CustomAwsCredentialLoader { - async fn load_credential(&self, client: Client) -> anyhow::Result> { - self.0.load_credential(client).await - } -} diff --git a/crates/iceberg/src/lib.rs b/crates/iceberg/src/lib.rs index aae8efed74..2ca71fb35c 100644 --- a/crates/iceberg/src/lib.rs +++ b/crates/iceberg/src/lib.rs @@ -51,7 +51,7 @@ //! .scan() //! .select(["name", "id"]) //! .build()? -//! .to_arrow() +//! .to_arrow(Default::default()) //! .await?; //! //! // Consume this stream like arrow record batch stream. @@ -65,18 +65,25 @@ #[macro_use] extern crate derive_builder; extern crate core; +// Required so `#[derive(Properties)]` output can name `::iceberg::Error`. +extern crate self as iceberg; mod error; pub use error::{Error, ErrorKind, Result}; +pub mod sensitive; + mod catalog; +pub use catalog::utils::{delete_removed_metadata_files, drop_table_data}; pub use catalog::*; pub mod table; mod avro; pub mod cache; +pub mod compression; +pub mod cow_rewrite; pub mod io; pub mod spec; @@ -88,12 +95,18 @@ pub mod transaction; pub mod transform; mod runtime; +pub use runtime::{JoinHandle, Runtime, RuntimeHandle}; pub mod arrow; pub(crate) mod delete_file_index; -pub mod test_utils; -mod utils; +pub mod encryption; +#[cfg(test)] +mod test_utils; pub mod writer; -mod delete_vector; +pub mod delete_vector; +pub mod metadata_columns; +pub mod partitioning; pub mod puffin; +/// Utility functions and modules. +pub mod util; diff --git a/crates/iceberg/src/metadata_columns.rs b/crates/iceberg/src/metadata_columns.rs new file mode 100644 index 0000000000..aed4183250 --- /dev/null +++ b/crates/iceberg/src/metadata_columns.rs @@ -0,0 +1,688 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Metadata columns (virtual/reserved fields) for Iceberg tables. +//! +//! This module defines metadata columns that can be requested in projections +//! but are not stored in data files. Instead, they are computed on-the-fly +//! during reading. Examples include the _file column (file path) and future +//! columns like partition values or row numbers. + +use std::sync::Arc; + +use once_cell::sync::Lazy; + +use crate::spec::{NestedField, NestedFieldRef, PrimitiveType, Type}; +use crate::{Error, ErrorKind, Result}; + +/// Reserved field ID for the file path (_file) column per Iceberg spec +pub const RESERVED_FIELD_ID_FILE: i32 = i32::MAX - 1; + +/// Reserved field ID for the position (_pos) column per Iceberg spec +pub const RESERVED_FIELD_ID_POS: i32 = i32::MAX - 2; + +/// Reserved field ID for the deleted (_deleted) column per Iceberg spec +pub const RESERVED_FIELD_ID_DELETED: i32 = i32::MAX - 3; + +/// Reserved field ID for the spec ID (_spec_id) column per Iceberg spec +pub const RESERVED_FIELD_ID_SPEC_ID: i32 = i32::MAX - 4; + +/// Reserved field ID for the partition (_partition) column per Iceberg spec +pub const RESERVED_FIELD_ID_PARTITION: i32 = i32::MAX - 5; + +/// Reserved field ID for the file path in position delete files +pub const RESERVED_FIELD_ID_DELETE_FILE_PATH: i32 = i32::MAX - 101; + +/// Reserved field ID for the position in position delete files +pub const RESERVED_FIELD_ID_DELETE_FILE_POS: i32 = i32::MAX - 102; + +/// Reserved field ID for the change type (_change_type) column per Iceberg spec +pub const RESERVED_FIELD_ID_CHANGE_TYPE: i32 = i32::MAX - 104; + +/// Reserved field ID for the change ordinal (_change_ordinal) column per Iceberg spec +pub const RESERVED_FIELD_ID_CHANGE_ORDINAL: i32 = i32::MAX - 105; + +/// Reserved field ID for the commit snapshot ID (_commit_snapshot_id) column per Iceberg spec +pub const RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID: i32 = i32::MAX - 106; + +/// Reserved field ID for the row ID (_row_id) column per Iceberg spec +pub const RESERVED_FIELD_ID_ROW_ID: i32 = i32::MAX - 107; + +/// Reserved field ID for the last updated sequence number (_last_updated_sequence_number) column per Iceberg spec +pub const RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER: i32 = i32::MAX - 108; + +/// Reserved column name for the file path metadata column +pub const RESERVED_COL_NAME_FILE: &str = "_file"; + +/// Reserved column name for the position metadata column +pub const RESERVED_COL_NAME_POS: &str = "_pos"; + +/// Reserved column name for the deleted metadata column +pub const RESERVED_COL_NAME_DELETED: &str = "_deleted"; + +/// Reserved column name for the spec ID metadata column +pub const RESERVED_COL_NAME_SPEC_ID: &str = "_spec_id"; + +/// Reserved column name for the partition metadata column +pub const RESERVED_COL_NAME_PARTITION: &str = "_partition"; + +/// Reserved column name for the file path in position delete files +pub const RESERVED_COL_NAME_DELETE_FILE_PATH: &str = "file_path"; + +/// Reserved column name for the position in position delete files +pub const RESERVED_COL_NAME_DELETE_FILE_POS: &str = "pos"; + +/// Reserved column name for the change type metadata column +pub const RESERVED_COL_NAME_CHANGE_TYPE: &str = "_change_type"; + +/// Reserved column name for the change ordinal metadata column +pub const RESERVED_COL_NAME_CHANGE_ORDINAL: &str = "_change_ordinal"; + +/// Reserved column name for the commit snapshot ID metadata column +pub const RESERVED_COL_NAME_COMMIT_SNAPSHOT_ID: &str = "_commit_snapshot_id"; + +/// Reserved column name for the row ID metadata column +pub const RESERVED_COL_NAME_ROW_ID: &str = "_row_id"; + +/// Reserved column name for the last updated sequence number metadata column +pub const RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER: &str = "_last_updated_sequence_number"; + +/// Lazy-initialized Iceberg field definition for the _file metadata column. +/// This field represents the file path as a required string field. +static FILE_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_FILE, + RESERVED_COL_NAME_FILE, + Type::Primitive(PrimitiveType::String), + ) + .with_doc("Path of the file in which a row is stored"), + ) +}); + +/// Lazy-initialized Iceberg field definition for the _pos metadata column. +/// This field represents the ordinal position of a row in the source data file. +static POS_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_POS, + RESERVED_COL_NAME_POS, + Type::Primitive(PrimitiveType::Long), + ) + .with_doc("Ordinal position of a row in the source data file"), + ) +}); + +/// Lazy-initialized Iceberg field definition for the _deleted metadata column. +/// This field indicates whether a row has been deleted. +static DELETED_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_DELETED, + RESERVED_COL_NAME_DELETED, + Type::Primitive(PrimitiveType::Boolean), + ) + .with_doc("Whether the row has been deleted"), + ) +}); + +/// Lazy-initialized Iceberg field definition for the _spec_id metadata column. +/// This field represents the spec ID used to track the file containing a row. +static SPEC_ID_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_SPEC_ID, + RESERVED_COL_NAME_SPEC_ID, + Type::Primitive(PrimitiveType::Int), + ) + .with_doc("Spec ID used to track the file containing a row"), + ) +}); + +/// Lazy-initialized Iceberg field definition for the file_path column in position delete files. +/// This field represents the path of a file in position-based delete files. +static DELETE_FILE_PATH_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_DELETE_FILE_PATH, + RESERVED_COL_NAME_DELETE_FILE_PATH, + Type::Primitive(PrimitiveType::String), + ) + .with_doc("Path of a file, used in position-based delete files"), + ) +}); + +/// Lazy-initialized Iceberg field definition for the pos column in position delete files. +/// This field represents the ordinal position of a row in position-based delete files. +static DELETE_FILE_POS_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_DELETE_FILE_POS, + RESERVED_COL_NAME_DELETE_FILE_POS, + Type::Primitive(PrimitiveType::Long), + ) + .with_doc("Ordinal position of a row, used in position-based delete files"), + ) +}); + +/// Lazy-initialized Iceberg field definition for the _change_type metadata column. +/// This field represents the record type in the changelog. +static CHANGE_TYPE_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_CHANGE_TYPE, + RESERVED_COL_NAME_CHANGE_TYPE, + Type::Primitive(PrimitiveType::String), + ) + .with_doc( + "The record type in the changelog (INSERT, DELETE, UPDATE_BEFORE, or UPDATE_AFTER)", + ), + ) +}); + +/// Lazy-initialized Iceberg field definition for the _change_ordinal metadata column. +/// This field represents the order of the change. +static CHANGE_ORDINAL_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_CHANGE_ORDINAL, + RESERVED_COL_NAME_CHANGE_ORDINAL, + Type::Primitive(PrimitiveType::Int), + ) + .with_doc("The order of the change"), + ) +}); + +/// Lazy-initialized Iceberg field definition for the _commit_snapshot_id metadata column. +/// This field represents the snapshot ID in which the change occurred. +static COMMIT_SNAPSHOT_ID_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID, + RESERVED_COL_NAME_COMMIT_SNAPSHOT_ID, + Type::Primitive(PrimitiveType::Long), + ) + .with_doc("The snapshot ID in which the change occurred"), + ) +}); + +/// Lazy-initialized Iceberg field definition for the _row_id metadata column. +/// This field represents a unique long assigned for row lineage. +static ROW_ID_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::optional( + RESERVED_FIELD_ID_ROW_ID, + RESERVED_COL_NAME_ROW_ID, + Type::Primitive(PrimitiveType::Long), + ) + .with_doc("A unique long assigned for row lineage"), + ) +}); + +/// Lazy-initialized Iceberg field definition for the _last_updated_sequence_number metadata column. +/// This field represents the sequence number which last updated this row. +static LAST_UPDATED_SEQUENCE_NUMBER_FIELD: Lazy = Lazy::new(|| { + Arc::new( + NestedField::optional( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + Type::Primitive(PrimitiveType::Long), + ) + .with_doc("The sequence number which last updated this row"), + ) +}); + +/// Returns the Iceberg field definition for the _file metadata column. +/// +/// # Returns +/// A reference to the _file field definition as an Iceberg NestedField +pub fn file_field() -> &'static NestedFieldRef { + &FILE_FIELD +} + +/// Returns the Iceberg field definition for the _pos metadata column. +/// +/// # Returns +/// A reference to the _pos field definition as an Iceberg NestedField +pub fn pos_field() -> &'static NestedFieldRef { + &POS_FIELD +} + +/// Returns the Iceberg field definition for the _deleted metadata column. +/// +/// # Returns +/// A reference to the _deleted field definition as an Iceberg NestedField +pub fn deleted_field() -> &'static NestedFieldRef { + &DELETED_FIELD +} + +/// Returns the Iceberg field definition for the _spec_id metadata column. +/// +/// # Returns +/// A reference to the _spec_id field definition as an Iceberg NestedField +pub fn spec_id_field() -> &'static NestedFieldRef { + &SPEC_ID_FIELD +} + +/// Returns the Iceberg field definition for the file_path column in position delete files. +/// +/// # Returns +/// A reference to the file_path field definition as an Iceberg NestedField +pub fn delete_file_path_field() -> &'static NestedFieldRef { + &DELETE_FILE_PATH_FIELD +} + +/// Returns the Iceberg field definition for the pos column in position delete files. +/// +/// # Returns +/// A reference to the pos field definition as an Iceberg NestedField +pub fn delete_file_pos_field() -> &'static NestedFieldRef { + &DELETE_FILE_POS_FIELD +} + +/// Returns the Iceberg field definition for the _change_type metadata column. +/// +/// # Returns +/// A reference to the _change_type field definition as an Iceberg NestedField +pub fn change_type_field() -> &'static NestedFieldRef { + &CHANGE_TYPE_FIELD +} + +/// Returns the Iceberg field definition for the _change_ordinal metadata column. +/// +/// # Returns +/// A reference to the _change_ordinal field definition as an Iceberg NestedField +pub fn change_ordinal_field() -> &'static NestedFieldRef { + &CHANGE_ORDINAL_FIELD +} + +/// Returns the Iceberg field definition for the _commit_snapshot_id metadata column. +/// +/// # Returns +/// A reference to the _commit_snapshot_id field definition as an Iceberg NestedField +pub fn commit_snapshot_id_field() -> &'static NestedFieldRef { + &COMMIT_SNAPSHOT_ID_FIELD +} + +/// Returns the Iceberg field definition for the _row_id metadata column. +/// +/// # Returns +/// A reference to the _row_id field definition as an Iceberg NestedField +pub fn row_id_field() -> &'static NestedFieldRef { + &ROW_ID_FIELD +} + +/// Returns the Iceberg field definition for the _last_updated_sequence_number metadata column. +/// +/// # Returns +/// A reference to the _last_updated_sequence_number field definition as an Iceberg NestedField +pub fn last_updated_sequence_number_field() -> &'static NestedFieldRef { + &LAST_UPDATED_SEQUENCE_NUMBER_FIELD +} + +/// Creates the Iceberg field definition for the _partition metadata column. +/// +/// The _partition field is a struct whose fields depend on the partition spec. +/// This function creates the field dynamically with the provided partition fields. +/// +/// # Arguments +/// * `partition_fields` - The fields that make up the partition struct +/// +/// # Returns +/// A new _partition field definition as an Iceberg NestedField +/// +/// # Example +/// ``` +/// use std::sync::Arc; +/// +/// use iceberg::metadata_columns::partition_field; +/// use iceberg::spec::{NestedField, PrimitiveType, Type}; +/// +/// let fields = vec![ +/// Arc::new(NestedField::required( +/// 1, +/// "year", +/// Type::Primitive(PrimitiveType::Int), +/// )), +/// Arc::new(NestedField::required( +/// 2, +/// "month", +/// Type::Primitive(PrimitiveType::Int), +/// )), +/// ]; +/// let partition_field = partition_field(fields); +/// ``` +pub fn partition_field(partition_fields: Vec) -> NestedFieldRef { + use crate::spec::StructType; + + Arc::new( + NestedField::required( + RESERVED_FIELD_ID_PARTITION, + RESERVED_COL_NAME_PARTITION, + Type::Struct(StructType::new(partition_fields)), + ) + .with_doc("Partition to which a row belongs"), + ) +} + +/// Returns the Iceberg field definition for a metadata field ID. +/// +/// Note: This function does not support `_partition` (field ID `i32::MAX - 5`) because +/// it's a struct field that requires dynamic partition fields. Use `partition_field()` +/// instead to create the `_partition` field with the appropriate partition fields. +/// +/// # Arguments +/// * `field_id` - The metadata field ID +/// +/// # Returns +/// The Iceberg field definition for the metadata column, or an error if not a metadata field +pub fn get_metadata_field(field_id: i32) -> Result<&'static NestedFieldRef> { + match field_id { + RESERVED_FIELD_ID_FILE => Ok(file_field()), + RESERVED_FIELD_ID_POS => Ok(pos_field()), + RESERVED_FIELD_ID_DELETED => Ok(deleted_field()), + RESERVED_FIELD_ID_SPEC_ID => Ok(spec_id_field()), + RESERVED_FIELD_ID_PARTITION => Err(Error::new( + ErrorKind::Unexpected, + "The _partition field must be created using partition_field() with appropriate partition fields", + )), + RESERVED_FIELD_ID_DELETE_FILE_PATH => Ok(delete_file_path_field()), + RESERVED_FIELD_ID_DELETE_FILE_POS => Ok(delete_file_pos_field()), + RESERVED_FIELD_ID_CHANGE_TYPE => Ok(change_type_field()), + RESERVED_FIELD_ID_CHANGE_ORDINAL => Ok(change_ordinal_field()), + RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID => Ok(commit_snapshot_id_field()), + RESERVED_FIELD_ID_ROW_ID => Ok(row_id_field()), + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER => Ok(last_updated_sequence_number_field()), + _ if is_metadata_field(field_id) => { + // Future metadata fields can be added here + Err(Error::new( + ErrorKind::Unexpected, + format!( + "Metadata field ID {field_id} recognized but field definition not implemented" + ), + )) + } + _ => Err(Error::new( + ErrorKind::Unexpected, + format!("Field ID {field_id} is not a metadata field"), + )), + } +} + +/// Returns the field ID for a metadata column name. +/// +/// This maps every reserved name to its field id, including the position-delete-file +/// internal columns `pos` and `file_path`, which are not `_`-prefixed. It is therefore +/// not a membership test for data-table metadata columns: `get_metadata_field_id("pos")` +/// returns `Ok` even though `pos` is a valid user data column name. To decide whether a +/// name is a projectable data-table metadata column, use [`is_metadata_column_name`]. +/// +/// # Arguments +/// * `column_name` - The metadata column name +/// +/// # Returns +/// The field ID of the metadata column, or an error if the column name is not recognized +pub fn get_metadata_field_id(column_name: &str) -> Result { + match column_name { + RESERVED_COL_NAME_FILE => Ok(RESERVED_FIELD_ID_FILE), + RESERVED_COL_NAME_POS => Ok(RESERVED_FIELD_ID_POS), + RESERVED_COL_NAME_DELETED => Ok(RESERVED_FIELD_ID_DELETED), + RESERVED_COL_NAME_SPEC_ID => Ok(RESERVED_FIELD_ID_SPEC_ID), + RESERVED_COL_NAME_PARTITION => Ok(RESERVED_FIELD_ID_PARTITION), + RESERVED_COL_NAME_DELETE_FILE_PATH => Ok(RESERVED_FIELD_ID_DELETE_FILE_PATH), + RESERVED_COL_NAME_DELETE_FILE_POS => Ok(RESERVED_FIELD_ID_DELETE_FILE_POS), + RESERVED_COL_NAME_CHANGE_TYPE => Ok(RESERVED_FIELD_ID_CHANGE_TYPE), + RESERVED_COL_NAME_CHANGE_ORDINAL => Ok(RESERVED_FIELD_ID_CHANGE_ORDINAL), + RESERVED_COL_NAME_COMMIT_SNAPSHOT_ID => Ok(RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID), + RESERVED_COL_NAME_ROW_ID => Ok(RESERVED_FIELD_ID_ROW_ID), + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER => { + Ok(RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER) + } + _ => Err(Error::new( + ErrorKind::Unexpected, + format!("Unknown/unsupported metadata column name: {column_name}"), + )), + } +} + +/// Checks if a field ID is a data-table metadata column. +/// +/// Mirrors Java `MetadataColumns.isMetadataColumn(int)` (backed by `META_IDS`): only the +/// field ids of columns projectable in a data-table scan return `true`. The +/// position-delete-file internal columns (`file_path`/`pos`, ids `i32::MAX - 101/-102`) +/// and the changelog columns (`_change_type`/`_change_ordinal`/`_commit_snapshot_id`) are +/// deliberately excluded, so their ids return `false` here even though +/// [`get_metadata_field`] can still resolve them. +/// +/// # Arguments +/// * `field_id` - The field ID to check +/// +/// # Returns +/// `true` if the field ID is a data-table metadata column, `false` otherwise +pub fn is_metadata_field(field_id: i32) -> bool { + matches!( + field_id, + RESERVED_FIELD_ID_FILE + | RESERVED_FIELD_ID_POS + | RESERVED_FIELD_ID_DELETED + | RESERVED_FIELD_ID_SPEC_ID + | RESERVED_FIELD_ID_PARTITION + | RESERVED_FIELD_ID_ROW_ID + | RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER + ) +} + +/// Checks if a column name is a data-table metadata column. +/// +/// Mirrors Java `MetadataColumns.isMetadataColumn(String)` (backed by its `META_COLUMNS` +/// allowlist plus the `_partition` special case): a column name is a data-table metadata +/// column only if it is one of the reserved names that a scan can project. +/// +/// This deliberately excludes two groups of reserved names that Java also keeps out of +/// `META_COLUMNS`: +/// - the position-delete-file internal columns `pos`, `file_path`, and `row`, so a user +/// data column of the same name is not shadowed during a scan (issue #2837); +/// - the changelog columns `_change_type`, `_change_ordinal`, and `_commit_snapshot_id`. +/// +/// [`get_metadata_field_id`] still maps the reserved names it knows (including `pos` and +/// `file_path`) to their field ids; only this membership check excludes them. (`row` has +/// no reserved id in this crate, so `get_metadata_field_id("row")` is an error.) +/// +/// # Arguments +/// * `column_name` - The column name to check +/// +/// # Returns +/// `true` if the column name is a data-table metadata column, `false` otherwise +pub fn is_metadata_column_name(column_name: &str) -> bool { + matches!( + column_name, + RESERVED_COL_NAME_FILE + | RESERVED_COL_NAME_POS + | RESERVED_COL_NAME_DELETED + | RESERVED_COL_NAME_SPEC_ID + | RESERVED_COL_NAME_PARTITION + | RESERVED_COL_NAME_ROW_ID + | RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spec::PrimitiveType; + + #[test] + fn test_partition_field_creation() { + // Create partition fields for a hypothetical year/month partition + let partition_fields = vec![ + Arc::new(NestedField::required( + 1000, + "year", + Type::Primitive(PrimitiveType::Int), + )), + Arc::new(NestedField::required( + 1001, + "month", + Type::Primitive(PrimitiveType::Int), + )), + ]; + + // Create the _partition metadata field + let partition = partition_field(partition_fields); + + // Verify field properties + assert_eq!(partition.id, RESERVED_FIELD_ID_PARTITION); + assert_eq!(partition.name, RESERVED_COL_NAME_PARTITION); + assert!(partition.required); + + // Verify it's a struct type with correct fields + if let Type::Struct(struct_type) = partition.field_type.as_ref() { + assert_eq!(struct_type.fields().len(), 2); + assert_eq!(struct_type.fields()[0].name, "year"); + assert_eq!(struct_type.fields()[1].name, "month"); + } else { + panic!("Expected struct type for _partition field"); + } + } + + #[test] + fn test_partition_field_id_recognized() { + assert!(is_metadata_field(RESERVED_FIELD_ID_PARTITION)); + } + + #[test] + fn test_partition_field_name_recognized() { + assert_eq!( + get_metadata_field_id(RESERVED_COL_NAME_PARTITION).unwrap(), + RESERVED_FIELD_ID_PARTITION + ); + } + + #[test] + fn test_get_metadata_field_returns_error_for_partition() { + // partition field requires dynamic creation, so get_metadata_field should return an error + let result = get_metadata_field(RESERVED_FIELD_ID_PARTITION); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("partition_field()") + ); + } + + #[test] + fn test_all_metadata_field_ids() { + // Test that all non-partition metadata fields can be retrieved + assert!(get_metadata_field(RESERVED_FIELD_ID_FILE).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_POS).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_DELETED).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_SPEC_ID).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_DELETE_FILE_PATH).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_DELETE_FILE_POS).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_CHANGE_TYPE).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_CHANGE_ORDINAL).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_ROW_ID).is_ok()); + assert!(get_metadata_field(RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER).is_ok()); + } + + #[test] + fn test_row_lineage_metadata_fields_are_optional() { + // The spec requires readers to produce null for these columns in + // legitimate cases (e.g. a data file with a null first_row_id), so the + // fields must be optional rather than required. + assert!(!row_id_field().required); + assert!(!last_updated_sequence_number_field().required); + } + + #[test] + fn test_data_table_metadata_columns() { + // Both sides of the data-table metadata check (name and field id) must accept + // these seven columns. On the name side Java keeps six in `META_COLUMNS` and + // recognizes `_partition` via the `isMetadataColumn(String)` special case; on the + // id side all seven are in `META_IDS`. + assert!(is_metadata_column_name(RESERVED_COL_NAME_FILE)); + assert!(is_metadata_column_name(RESERVED_COL_NAME_POS)); + assert!(is_metadata_column_name(RESERVED_COL_NAME_DELETED)); + assert!(is_metadata_column_name(RESERVED_COL_NAME_SPEC_ID)); + assert!(is_metadata_column_name(RESERVED_COL_NAME_PARTITION)); + assert!(is_metadata_column_name(RESERVED_COL_NAME_ROW_ID)); + assert!(is_metadata_column_name( + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER + )); + + assert!(is_metadata_field(RESERVED_FIELD_ID_FILE)); + assert!(is_metadata_field(RESERVED_FIELD_ID_POS)); + assert!(is_metadata_field(RESERVED_FIELD_ID_DELETED)); + assert!(is_metadata_field(RESERVED_FIELD_ID_SPEC_ID)); + assert!(is_metadata_field(RESERVED_FIELD_ID_PARTITION)); + assert!(is_metadata_field(RESERVED_FIELD_ID_ROW_ID)); + assert!(is_metadata_field( + RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER + )); + } + + #[test] + fn test_changelog_columns_are_not_data_table_metadata_columns() { + // Changelog columns are not in Java's `META_COLUMNS`, so they are not projectable + // data-table metadata columns (`isMetadataColumn` is false for them). + assert!(!is_metadata_column_name(RESERVED_COL_NAME_CHANGE_TYPE)); + assert!(!is_metadata_column_name(RESERVED_COL_NAME_CHANGE_ORDINAL)); + assert!(!is_metadata_column_name( + RESERVED_COL_NAME_COMMIT_SNAPSHOT_ID + )); + assert!(!is_metadata_field(RESERVED_FIELD_ID_CHANGE_TYPE)); + assert!(!is_metadata_field(RESERVED_FIELD_ID_CHANGE_ORDINAL)); + assert!(!is_metadata_field(RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID)); + } + + #[test] + fn test_delete_file_columns_are_not_data_table_metadata_columns() { + // `pos`, `file_path`, and `row` are the internal columns of a position-delete file, + // not projectable metadata columns of a data table. They must not shadow real data + // columns of the same name during a scan (issue #2837). Java returns false for both + // their names and their field ids; mirror that on both sides here. + assert!(!is_metadata_column_name("pos")); + assert!(!is_metadata_column_name("file_path")); + assert!(!is_metadata_column_name("row")); + assert!(!is_metadata_field(RESERVED_FIELD_ID_DELETE_FILE_POS)); + assert!(!is_metadata_field(RESERVED_FIELD_ID_DELETE_FILE_PATH)); + // `row` (Java's DELETE_FILE_ROW_FIELD_ID, i32::MAX - 103) has no reserved id in + // this crate, so it is not a metadata field either. + assert!(!is_metadata_field(i32::MAX - 103)); + + // `get_metadata_field_id` intentionally still maps the two bare names it knows to + // their reserved field ids; only the data-table membership check excludes them. + assert_eq!( + get_metadata_field_id(RESERVED_COL_NAME_DELETE_FILE_POS).unwrap(), + RESERVED_FIELD_ID_DELETE_FILE_POS + ); + assert_eq!( + get_metadata_field_id(RESERVED_COL_NAME_DELETE_FILE_PATH).unwrap(), + RESERVED_FIELD_ID_DELETE_FILE_PATH + ); + } + + #[test] + fn test_ordinary_data_column_names_are_not_metadata_columns() { + assert!(!is_metadata_column_name("id")); + assert!(!is_metadata_column_name("name")); + // A user column that merely starts with an underscore but is not reserved. + assert!(!is_metadata_column_name("_custom")); + } +} diff --git a/crates/iceberg/src/partitioning.rs b/crates/iceberg/src/partitioning.rs new file mode 100644 index 0000000000..6ac62c68c4 --- /dev/null +++ b/crates/iceberg/src/partitioning.rs @@ -0,0 +1,469 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Partition type utilities for Iceberg tables. + +use std::cmp::Reverse; +use std::collections::{HashMap, HashSet}; + +use crate::error::invalid_data; +use crate::spec::{ + NestedField, NestedFieldRef, PartitionField, PartitionSpec, Schema, StructType, Transform, Type, +}; +use crate::{Error, ErrorKind, Result}; + +/// Computes the unified partition type across all partition specs in the table. +/// +/// This is equivalent to Java's `Partitioning.partitionType(table)`. The result is a +/// StructType containing all partition fields ever used across all specs, enabling correct +/// representation of the `_partition` metadata column when partition evolution has occurred. +/// +/// Matches Java's `buildPartitionProjectionType` behavior: +/// - Specs are sorted by spec_id in descending order (newer specs first), so newer field +/// names take precedence when deduplicating by field_id. +/// - Unknown transforms cause an error. +/// - Fields whose source column was dropped from the schema are skipped. +/// - Two specs defining the same field_id must be compatible (same source, compatible +/// transforms); V1 tables do not guarantee field ids are unique across specs. +/// - When a newer spec marks a field as Void (dropped) but an older spec has it with a +/// real transform, the older spec's type is preserved while the newer spec's name is kept. +/// - Fields are deduplicated by field_id; each unique field_id appears exactly once. +/// - Output fields are sorted by field_id ascending. +/// +/// # Arguments +/// * `partition_specs` - Iterator over all partition specs in the table +/// * `schema` - The current table schema (needed to determine result types of transforms) +pub fn compute_unified_partition_type<'a>( + partition_specs: impl Iterator, + schema: &Schema, +) -> Result { + let mut specs: Vec<&PartitionSpec> = partition_specs.collect(); + specs.sort_by_key(|s| Reverse(s.spec_id())); + + let active_field_ids = all_active_field_ids(specs.iter().copied(), schema); + + let mut field_map: HashMap = HashMap::new(); + let mut type_map: HashMap = HashMap::new(); + let mut name_map: HashMap = HashMap::new(); + + for spec in &specs { + for field in spec.fields() { + let field_id = field.field_id; + + // Reject unknown transforms up front: we cannot determine their result type, + // so we cannot build a partition column for them. This check must precede the + // active_field_ids filter below, otherwise an unknown transform could be + // silently skipped. + if matches!(field.transform, Transform::Unknown) { + return Err(invalid_data!( + "Partition field '{}' uses an unknown transform whose result type \ + cannot be determined", + field.name + )); + } + + if !active_field_ids.contains(&field_id) { + continue; + } + + let source_field = match schema.field_by_id(field.source_id) { + Some(f) => f, + None => continue, + }; + + match field_map.get(&field_id) { + None => { + let res_type = field.transform.result_type(&source_field.field_type)?; + field_map.insert(field_id, field); + type_map.insert(field_id, res_type); + name_map.insert(field_id, field.name.clone()); + } + Some(existing) => { + // V1 tables do not guarantee field ids are unique across specs, so two + // specs may define the same field id. They must be compatible. + if !equivalent_ignoring_names(field, existing) { + return Err(invalid_data!( + "Conflicting partition fields for field id {field_id}: \ + '{}' and '{}'", + field.name, + existing.name + )); + } + + // Use the correct type for dropped partitions in v1 tables: if the + // newer spec voided the field but an older spec has a real transform, + // keep the older spec's type. + if is_void_transform(existing) && !is_void_transform(field) { + let res_type = field.transform.result_type(&source_field.field_type)?; + field_map.insert(field_id, field); + type_map.insert(field_id, res_type); + } + } + } + } + } + + let mut field_ids: Vec = field_map.keys().copied().collect(); + field_ids.sort(); + + let struct_fields = field_ids + .into_iter() + .map(|fid| -> Result { + let name = name_map.get(&fid).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!("Missing name for partition field {fid}"), + ) + })?; + let ty = type_map.remove(&fid).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!("Missing type for partition field {fid}"), + ) + })?; + Ok(NestedField::optional(fid, name, ty).into()) + }) + .collect::>>()?; + + Ok(StructType::new(struct_fields)) +} + +fn is_void_transform(field: &PartitionField) -> bool { + matches!(field.transform, Transform::Void) +} + +/// Two partition fields with the same field id are compatible if they share the same +/// source id and have compatible transforms. Matches Java's +/// `Partitioning.equivalentIgnoringNames`. +fn equivalent_ignoring_names(field: &PartitionField, other: &PartitionField) -> bool { + field.field_id == other.field_id + && field.source_id == other.source_id + && compatible_transforms(&field.transform, &other.transform) +} + +/// Transforms are compatible if they are equal, or if either is Void (a dropped field). +/// Matches Java's `Partitioning.compatibleTransforms`. +fn compatible_transforms(t1: &Transform, t2: &Transform) -> bool { + t1 == t2 || matches!(t1, Transform::Void) || matches!(t2, Transform::Void) +} + +fn all_active_field_ids<'a>( + partition_specs: impl Iterator, + schema: &Schema, +) -> HashSet { + partition_specs + .flat_map(|spec| spec.fields().iter()) + .filter(|field| schema.field_by_id(field.source_id).is_some()) + .map(|field| field.field_id) + .collect() +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + use crate::spec::{ + NestedField, PrimitiveType, Transform, Type, UnboundPartitionField, UnboundPartitionSpec, + }; + + fn test_schema() -> Schema { + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "ts", Type::Primitive(PrimitiveType::Timestamp)).into(), + NestedField::required(4, "category", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap() + } + + fn build_spec( + schema: &Schema, + spec_id: i32, + fields: Vec<(i32, &str, Transform)>, + ) -> PartitionSpec { + let mut builder = UnboundPartitionSpec::builder().with_spec_id(spec_id); + for (source_id, name, transform) in fields { + builder = builder + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![source_id]) + .name(name) + .transform(transform) + .build() + .unwrap(), + ) + .unwrap(); + } + builder.build().bind(schema.clone()).unwrap() + } + + #[test] + fn test_single_spec_identity() { + let schema = test_schema(); + let spec = build_spec(&schema, 0, vec![(4, "category", Transform::Identity)]); + + let result = compute_unified_partition_type([&spec].into_iter(), &schema).unwrap(); + assert_eq!(result.fields().len(), 1); + assert_eq!(result.fields()[0].name, "category"); + assert_eq!( + *result.fields()[0].field_type, + Type::Primitive(PrimitiveType::String) + ); + } + + #[test] + fn test_single_spec_with_year_transform() { + let schema = test_schema(); + let spec = build_spec(&schema, 0, vec![(3, "ts_year", Transform::Year)]); + + let result = compute_unified_partition_type([&spec].into_iter(), &schema).unwrap(); + assert_eq!(result.fields().len(), 1); + assert_eq!(result.fields()[0].name, "ts_year"); + assert_eq!( + *result.fields()[0].field_type, + Type::Primitive(PrimitiveType::Int) + ); + } + + #[test] + fn test_unpartitioned() { + let schema = test_schema(); + let spec = PartitionSpec::unpartition_spec(); + let result = compute_unified_partition_type([&spec].into_iter(), &schema).unwrap(); + assert!(result.fields().is_empty()); + } + + #[test] + fn test_multiple_fields_sorted_by_id() { + let schema = test_schema(); + let spec = build_spec(&schema, 0, vec![ + (3, "ts_year", Transform::Year), + (4, "category", Transform::Identity), + ]); + + let result = compute_unified_partition_type([&spec].into_iter(), &schema).unwrap(); + assert_eq!(result.fields().len(), 2); + assert!(result.fields()[0].id < result.fields()[1].id); + } + + #[test] + fn test_newer_name_takes_precedence() { + let schema = test_schema(); + + // Spec 0: old name + let spec_v0 = PartitionSpec::builder(Arc::new(schema.clone())) + .with_spec_id(0) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("cat_old".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build() + .unwrap(); + + // Spec 1: newer name, same field_id + let spec_v1 = PartitionSpec::builder(Arc::new(schema.clone())) + .with_spec_id(1) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("cat_new".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build() + .unwrap(); + + let result = + compute_unified_partition_type([&spec_v0, &spec_v1].into_iter(), &schema).unwrap(); + assert_eq!(result.fields().len(), 1); + assert_eq!(result.fields()[0].name, "cat_new"); + } + + #[test] + fn test_void_replaced_by_older_non_void() { + let schema = test_schema(); + + // Spec 0 (older): category partitioned by identity + let spec_v0 = PartitionSpec::builder(Arc::new(schema.clone())) + .with_spec_id(0) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("category".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build() + .unwrap(); + + // Spec 1 (newer): same field_id voided (partition dropped) + let spec_v1 = PartitionSpec::builder(Arc::new(schema.clone())) + .with_spec_id(1) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("category_v2".to_string()) + .transform(Transform::Void) + .build() + .unwrap(), + ) + .unwrap() + .build() + .unwrap(); + + let result = + compute_unified_partition_type([&spec_v0, &spec_v1].into_iter(), &schema).unwrap(); + + assert_eq!(result.fields().len(), 1); + // Name from newer spec + assert_eq!(result.fields()[0].name, "category_v2"); + // Type from older non-void spec + assert_eq!( + *result.fields()[0].field_type, + Type::Primitive(PrimitiveType::String) + ); + } + + #[test] + fn test_dropped_source_column_skipped() { + // Schema without field 4 (category was dropped) + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + // Spec references source_id=4 which no longer exists in the schema. + // Deserialize directly since the builder rejects unknown source columns. + let spec = serde_json::from_value::(serde_json::json!({ + "spec-id": 0, + "fields": [{ + "source-id": 4, + "field-id": 1000, + "name": "category", + "transform": "identity" + }] + })) + .unwrap(); + + let result = compute_unified_partition_type([&spec].into_iter(), &schema).unwrap(); + assert!(result.fields().is_empty()); + } + + #[test] + fn test_evolution_adds_new_field() { + let schema = test_schema(); + + // Spec 0: partition by category + let spec_v0 = build_spec(&schema, 0, vec![(4, "category", Transform::Identity)]); + + // Spec 1: partition by category + ts_year + let spec_v1 = PartitionSpec::builder(Arc::new(schema.clone())) + .with_spec_id(1) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(spec_v0.fields()[0].field_id) + .name("category".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .add_partition_field("ts", "ts_year", Transform::Year) + .unwrap() + .build() + .unwrap(); + + let result = + compute_unified_partition_type([&spec_v0, &spec_v1].into_iter(), &schema).unwrap(); + assert_eq!(result.fields().len(), 2); + } + + #[test] + fn test_unknown_transform_errors() { + let schema = test_schema(); + + // A spec using an unknown transform. Deserialize directly since the builder + // validates transforms. + let spec = serde_json::from_value::(serde_json::json!({ + "spec-id": 0, + "fields": [{ + "source-id": 4, + "field-id": 1000, + "name": "category", + "transform": "unknown" + }] + })) + .unwrap(); + + let err = compute_unified_partition_type([&spec].into_iter(), &schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_conflicting_partition_fields_error() { + let schema = test_schema(); + + // Spec 0: field id 1000 -> source 4 (category), identity + let spec_v0 = serde_json::from_value::(serde_json::json!({ + "spec-id": 0, + "fields": [{ + "source-id": 4, + "field-id": 1000, + "name": "category", + "transform": "identity" + }] + })) + .unwrap(); + + // Spec 1: field id 1000 reused for a different source (ts) and transform (year). + // This conflicts with spec 0 and must be rejected. + let spec_v1 = serde_json::from_value::(serde_json::json!({ + "spec-id": 1, + "fields": [{ + "source-id": 3, + "field-id": 1000, + "name": "ts_year", + "transform": "year" + }] + })) + .unwrap(); + + let err = + compute_unified_partition_type([&spec_v0, &spec_v1].into_iter(), &schema).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } +} diff --git a/crates/iceberg/src/puffin/blob.rs b/crates/iceberg/src/puffin/blob.rs index 7e4316ddd0..c5a7a341cb 100644 --- a/crates/iceberg/src/puffin/blob.rs +++ b/crates/iceberg/src/puffin/blob.rs @@ -37,7 +37,7 @@ pub struct Blob { impl Blob { #[inline] - /// See blob types: https://iceberg.apache.org/puffin-spec/#blob-types + /// See blob types: pub fn blob_type(&self) -> &str { &self.r#type } diff --git a/crates/iceberg/src/puffin/compression.rs b/crates/iceberg/src/puffin/compression.rs deleted file mode 100644 index a9a56ef12c..0000000000 --- a/crates/iceberg/src/puffin/compression.rs +++ /dev/null @@ -1,123 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use serde::{Deserialize, Serialize}; - -use crate::{Error, ErrorKind, Result}; - -/// Data compression formats -#[derive(Debug, PartialEq, Eq, Clone, Copy, Default, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum CompressionCodec { - #[default] - /// No compression - None, - /// LZ4 single compression frame with content size present - Lz4, - /// Zstandard single compression frame with content size present - Zstd, -} - -impl CompressionCodec { - pub(crate) fn decompress(&self, bytes: Vec) -> Result> { - match self { - CompressionCodec::None => Ok(bytes), - CompressionCodec::Lz4 => Err(Error::new( - ErrorKind::FeatureUnsupported, - "LZ4 decompression is not supported currently", - )), - CompressionCodec::Zstd => { - let decompressed = zstd::stream::decode_all(&bytes[..])?; - Ok(decompressed) - } - } - } - - pub(crate) fn compress(&self, bytes: Vec) -> Result> { - match self { - CompressionCodec::None => Ok(bytes), - CompressionCodec::Lz4 => Err(Error::new( - ErrorKind::FeatureUnsupported, - "LZ4 compression is not supported currently", - )), - CompressionCodec::Zstd => { - let writer = Vec::::new(); - let mut encoder = zstd::stream::Encoder::new(writer, 3)?; - encoder.include_checksum(true)?; - encoder.set_pledged_src_size(Some(bytes.len().try_into()?))?; - std::io::copy(&mut &bytes[..], &mut encoder)?; - let compressed = encoder.finish()?; - Ok(compressed) - } - } - } - - pub(crate) fn is_none(&self) -> bool { - matches!(self, CompressionCodec::None) - } -} - -#[cfg(test)] -mod tests { - use crate::puffin::compression::CompressionCodec; - - #[tokio::test] - async fn test_compression_codec_none() { - let compression_codec = CompressionCodec::None; - let bytes_vec = [0_u8; 100].to_vec(); - - let compressed = compression_codec.compress(bytes_vec.clone()).unwrap(); - assert_eq!(bytes_vec, compressed); - - let decompressed = compression_codec.decompress(compressed.clone()).unwrap(); - assert_eq!(compressed, decompressed) - } - - #[tokio::test] - async fn test_compression_codec_lz4() { - let compression_codec = CompressionCodec::Lz4; - let bytes_vec = [0_u8; 100].to_vec(); - - assert_eq!( - compression_codec - .compress(bytes_vec.clone()) - .unwrap_err() - .to_string(), - "FeatureUnsupported => LZ4 compression is not supported currently", - ); - - assert_eq!( - compression_codec - .decompress(bytes_vec.clone()) - .unwrap_err() - .to_string(), - "FeatureUnsupported => LZ4 decompression is not supported currently", - ) - } - - #[tokio::test] - async fn test_compression_codec_zstd() { - let compression_codec = CompressionCodec::Zstd; - let bytes_vec = [0_u8; 100].to_vec(); - - let compressed = compression_codec.compress(bytes_vec.clone()).unwrap(); - assert!(compressed.len() < bytes_vec.len()); - - let decompressed = compression_codec.decompress(compressed.clone()).unwrap(); - assert_eq!(decompressed, bytes_vec) - } -} diff --git a/crates/iceberg/src/puffin/metadata.rs b/crates/iceberg/src/puffin/metadata.rs index 6039c7f820..e237edf516 100644 --- a/crates/iceberg/src/puffin/metadata.rs +++ b/crates/iceberg/src/puffin/metadata.rs @@ -20,16 +20,17 @@ use std::collections::{HashMap, HashSet}; use bytes::Bytes; use serde::{Deserialize, Serialize}; -use crate::io::{FileRead, InputFile}; -use crate::puffin::compression::CompressionCodec; -use crate::{Error, ErrorKind, Result}; +use crate::Result; +use crate::compression::CompressionCodec; +use crate::error::invalid_data; +use crate::io::FileRead; /// Human-readable identification of the application writing the file, along with its version. /// Example: "Trino version 381" pub const CREATED_BY_PROPERTY: &str = "created-by"; /// Metadata about a blob. -/// For more information, see: https://iceberg.apache.org/puffin-spec/#blobmetadata +/// For more information, see: #[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] #[serde(rename_all = "kebab-case")] pub struct BlobMetadata { @@ -49,7 +50,7 @@ pub struct BlobMetadata { impl BlobMetadata { #[inline] - /// See blob types: https://iceberg.apache.org/puffin-spec/#blob-types + /// See blob types: pub fn blob_type(&self) -> &str { &self.r#type } @@ -119,12 +120,8 @@ impl Flag { if Flag::FooterPayloadCompressed.matches(byte_idx, bit_idx) { Ok(Flag::FooterPayloadCompressed) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Unknown flag byte {} and bit {} combination", - byte_idx, bit_idx - ), + Err(invalid_data!( + "Unknown flag byte {byte_idx} and bit {bit_idx} combination" )) } } @@ -132,7 +129,7 @@ impl Flag { /// Metadata about a puffin file. /// -/// For more information, see: https://iceberg.apache.org/puffin-spec/#filemetadata +/// For more information, see: #[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] pub struct FileMetadata { pub(crate) blobs: Vec, @@ -177,6 +174,11 @@ impl FileMetadata { pub(crate) const FOOTER_STRUCT_LENGTH: u8 = FileMetadata::FOOTER_STRUCT_MAGIC_OFFSET + FileMetadata::MAGIC_LENGTH; + /// Smallest possible Puffin file: the file header magic, followed by a footer + /// holding an empty payload (footer magic + FOOTER_STRUCT). + const MIN_FILE_LENGTH: u64 = + (FileMetadata::MAGIC_LENGTH as u64) * 2 + (FileMetadata::FOOTER_STRUCT_LENGTH as u64); + /// Constructs new puffin `FileMetadata` pub fn new(blobs: Vec, properties: HashMap) -> Self { Self { blobs, properties } @@ -186,13 +188,10 @@ impl FileMetadata { if bytes == FileMetadata::MAGIC { Ok(()) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Bad magic value: {:?} should be {:?}", - bytes, - FileMetadata::MAGIC - ), + Err(invalid_data!( + "Bad magic value: {:?} should be {:?}", + bytes, + FileMetadata::MAGIC )) } } @@ -218,7 +217,13 @@ impl FileMetadata { let footer_length = footer_payload_length as u64 + FileMetadata::FOOTER_STRUCT_LENGTH as u64 + FileMetadata::MAGIC_LENGTH as u64; - let start = input_file_length - footer_length; + let start = input_file_length + .checked_sub(footer_length) + .ok_or_else(|| { + invalid_data!( + "Footer length {footer_length} exceeds file length {input_file_length}" + ) + })?; let end = input_file_length; file_read.read(start..end).await } @@ -232,9 +237,9 @@ impl FileMetadata { - FileMetadata::FOOTER_STRUCT_FLAGS_LENGTH as usize + byte_idx as usize; - let flag_byte = *footer_bytes.get(byte_offset).ok_or_else(|| { - Error::new(ErrorKind::DataInvalid, "Index range is out of bounds.") - })?; + let flag_byte = *footer_bytes + .get(byte_offset) + .ok_or_else(|| invalid_data!("Index range is out of bounds."))?; for bit_idx in 0..8 { if ((flag_byte >> bit_idx) & 1) != 0 { @@ -263,35 +268,36 @@ impl FileMetadata { FileMetadata::MAGIC_LENGTH as usize + usize::try_from(footer_payload_length)?; let footer_payload_bytes = footer_bytes .get(start_offset..end_offset) - .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "Index range is out of bounds."))?; + .ok_or_else(|| invalid_data!("Index range is out of bounds."))?; let decompressed_footer_payload_bytes = footer_compression_codec.decompress(footer_payload_bytes.into())?; - String::from_utf8(decompressed_footer_payload_bytes).map_err(|src| { - Error::new(ErrorKind::DataInvalid, "Footer is not a valid UTF-8 string") - .with_source(src) - }) + String::from_utf8(decompressed_footer_payload_bytes) + .map_err(|src| invalid_data!("Footer is not a valid UTF-8 string").with_source(src)) } fn from_json_str(string: &str) -> Result { - serde_json::from_str::(string).map_err(|src| { - Error::new(ErrorKind::DataInvalid, "Given string is not valid JSON").with_source(src) - }) + serde_json::from_str::(string) + .map_err(|src| invalid_data!("Given string is not valid JSON").with_source(src)) } /// Returns the file metadata about a Puffin file - pub(crate) async fn read(input_file: &InputFile) -> Result { - let file_read = input_file.reader().await?; + pub(crate) async fn read(file_read: &dyn FileRead, file_length: u64) -> Result { + if file_length < FileMetadata::MIN_FILE_LENGTH { + return Err(invalid_data!( + "File length {} is too short to be a Puffin file, expected at least {} bytes", + file_length, + FileMetadata::MIN_FILE_LENGTH + )); + } let first_four_bytes = file_read.read(0..FileMetadata::MAGIC_LENGTH.into()).await?; FileMetadata::check_magic(&first_four_bytes)?; - let input_file_length = input_file.metadata().await?.size; let footer_payload_length = - FileMetadata::read_footer_payload_length(&file_read, input_file_length).await?; + FileMetadata::read_footer_payload_length(file_read, file_length).await?; let footer_bytes = - FileMetadata::read_footer_bytes(&file_read, input_file_length, footer_payload_length) - .await?; + FileMetadata::read_footer_bytes(file_read, file_length, footer_payload_length).await?; let magic_length = FileMetadata::MAGIC_LENGTH as usize; // check first four bytes of footer @@ -312,21 +318,23 @@ impl FileMetadata { /// read option. #[allow(dead_code)] pub(crate) async fn read_with_prefetch( - input_file: &InputFile, + file_read: &dyn FileRead, + file_length: u64, prefetch_hint: u8, ) -> Result { if prefetch_hint > 16 { - let input_file_length = input_file.metadata().await?.size; - let file_read = input_file.reader().await?; - // Hint cannot be larger than input file - if prefetch_hint as u64 > input_file_length { - return FileMetadata::read(input_file).await; + if prefetch_hint as u64 > file_length { + return FileMetadata::read(file_read, file_length).await; } - // Read footer based on prefetchi hint - let start = input_file_length - prefetch_hint as u64; - let end = input_file_length; + // Validate file header magic + let first_four_bytes = file_read.read(0..FileMetadata::MAGIC_LENGTH.into()).await?; + FileMetadata::check_magic(&first_four_bytes)?; + + // Read footer based on prefetch hint + let start = file_length - prefetch_hint as u64; + let end = file_length; let footer_bytes = file_read.read(start..end).await?; let payload_length_start = @@ -346,7 +354,7 @@ impl FileMetadata { + FileMetadata::FOOTER_STRUCT_LENGTH as usize + FileMetadata::MAGIC_LENGTH as usize; if footer_length > prefetch_hint as usize { - return FileMetadata::read(input_file).await; + return FileMetadata::read(file_read, file_length).await; } // Read footer bytes @@ -365,7 +373,7 @@ impl FileMetadata { return FileMetadata::from_json_str(&footer_payload_str); } - FileMetadata::read(input_file).await + FileMetadata::read(file_read, file_length).await } #[inline] @@ -388,19 +396,21 @@ mod tests { use bytes::Bytes; use tempfile::TempDir; - use crate::io::{FileIOBuilder, InputFile}; + use crate::ErrorKind; + use crate::io::{FileIO, InputFile}; use crate::puffin::metadata::{BlobMetadata, CompressionCodec, FileMetadata}; use crate::puffin::test_utils::{ empty_footer_payload, empty_footer_payload_bytes, empty_footer_payload_bytes_length_bytes, java_empty_uncompressed_input_file, java_uncompressed_metric_input_file, - java_zstd_compressed_metric_input_file, uncompressed_metric_file_metadata, + java_zstd_compressed_metric_input_file, read_file_metadata, + read_file_metadata_with_prefetch, uncompressed_metric_file_metadata, zstd_compressed_metric_file_metadata, }; const INVALID_MAGIC_VALUE: [u8; 4] = [80, 70, 65, 0]; async fn input_file_with_bytes(temp_dir: &TempDir, slice: &[u8]) -> InputFile { - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let path_buf = temp_dir.path().join("abc.puffin"); let temp_path = path_buf.to_str().unwrap(); @@ -443,7 +453,7 @@ mod tests { let input_file = input_file_with_bytes(&temp_dir, &bytes).await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -466,7 +476,7 @@ mod tests { let input_file = input_file_with_bytes(&temp_dir, &bytes).await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -489,7 +499,7 @@ mod tests { let input_file = input_file_with_bytes(&temp_dir, &bytes).await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -514,7 +524,7 @@ mod tests { let input_file = input_file_with_bytes(&temp_dir, &bytes).await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -539,7 +549,7 @@ mod tests { let input_file = input_file_with_bytes(&temp_dir, &bytes).await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -562,7 +572,7 @@ mod tests { let input_file = input_file_with_bytes(&temp_dir, &bytes).await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -585,7 +595,7 @@ mod tests { let input_file = input_file_with_bytes(&temp_dir, &bytes).await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -611,7 +621,7 @@ mod tests { let input_file = input_file_with_bytes(&temp_dir, &bytes).await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -619,6 +629,44 @@ mod tests { ) } + #[tokio::test] + async fn test_file_shorter_than_minimum_length_returns_error() { + let temp_dir = TempDir::new().unwrap(); + + // Only the file header magic, nothing else. + let input_file = input_file_with_bytes(&temp_dir, &FileMetadata::MAGIC).await; + + let err = read_file_metadata(&input_file).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains("too short to be a Puffin file"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_footer_payload_length_larger_than_file_returns_error() { + let temp_dir = TempDir::new().unwrap(); + + let mut bytes = vec![]; + bytes.extend(FileMetadata::MAGIC.to_vec()); + bytes.extend(FileMetadata::MAGIC.to_vec()); + bytes.extend(empty_footer_payload_bytes()); + // Declared footer payload length is far larger than the file itself. + bytes.extend(u32::to_le_bytes(u32::MAX)); + bytes.extend(vec![0, 0, 0, 0]); + bytes.extend(FileMetadata::MAGIC); + + let input_file = input_file_with_bytes(&temp_dir, &bytes).await; + + let err = read_file_metadata(&input_file).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains("exceeds file length"), + "unexpected error: {err}" + ); + } + #[tokio::test] async fn test_minimal_valid_file_returns_file_metadata() { let temp_dir = TempDir::new().unwrap(); @@ -634,7 +682,7 @@ mod tests { let input_file = input_file_with_bytes(&temp_dir, &bytes).await; assert_eq!( - FileMetadata::read(&input_file).await.unwrap(), + read_file_metadata(&input_file).await.unwrap(), FileMetadata { blobs: vec![], properties: HashMap::new(), @@ -658,7 +706,7 @@ mod tests { .await; assert_eq!( - FileMetadata::read(&input_file).await.unwrap(), + read_file_metadata(&input_file).await.unwrap(), FileMetadata { blobs: vec![], properties: { @@ -687,7 +735,7 @@ mod tests { .await; assert_eq!( - FileMetadata::read(&input_file).await.unwrap(), + read_file_metadata(&input_file).await.unwrap(), FileMetadata { blobs: vec![], properties: { @@ -713,7 +761,7 @@ mod tests { .await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -736,7 +784,7 @@ mod tests { .await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -776,7 +824,7 @@ mod tests { .await; assert_eq!( - FileMetadata::read(&input_file).await.unwrap(), + read_file_metadata(&input_file).await.unwrap(), FileMetadata { blobs: vec![ BlobMetadata { @@ -830,7 +878,7 @@ mod tests { .await; assert_eq!( - FileMetadata::read(&input_file).await.unwrap(), + read_file_metadata(&input_file).await.unwrap(), FileMetadata { blobs: vec![BlobMetadata { r#type: "type-a".to_string(), @@ -864,27 +912,25 @@ mod tests { "blobs" : [ {{ "type" : "type-a", - "fields" : [ {} ], + "fields" : [ {out_of_i32_range_number} ], "snapshot-id" : 14, "sequence-number" : 3, "offset" : 4, "length" : 16 }} ] - }}"#, - out_of_i32_range_number + }}"# ), ) .await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), format!( - "DataInvalid => Given string is not valid JSON, source: invalid value: integer `{}`, expected i32 at line 5 column 51", - out_of_i32_range_number + "DataInvalid => Given string is not valid JSON, source: invalid value: integer `{out_of_i32_range_number}`, expected i32 at line 5 column 51" ), ) } @@ -896,7 +942,7 @@ mod tests { let input_file = input_file_with_payload(&temp_dir, r#""blobs" = []"#).await; assert_eq!( - FileMetadata::read(&input_file) + read_file_metadata(&input_file) .await .unwrap_err() .to_string(), @@ -908,7 +954,7 @@ mod tests { async fn test_read_file_metadata_of_uncompressed_empty_file() { let input_file = java_empty_uncompressed_input_file(); - let file_metadata = FileMetadata::read(&input_file).await.unwrap(); + let file_metadata = read_file_metadata(&input_file).await.unwrap(); assert_eq!(file_metadata, empty_footer_payload()) } @@ -916,7 +962,7 @@ mod tests { async fn test_read_file_metadata_of_uncompressed_metric_data() { let input_file = java_uncompressed_metric_input_file(); - let file_metadata = FileMetadata::read(&input_file).await.unwrap(); + let file_metadata = read_file_metadata(&input_file).await.unwrap(); assert_eq!(file_metadata, uncompressed_metric_file_metadata()) } @@ -924,7 +970,7 @@ mod tests { async fn test_read_file_metadata_of_zstd_compressed_metric_data() { let input_file = java_zstd_compressed_metric_input_file(); - let file_metadata = FileMetadata::read_with_prefetch(&input_file, 64) + let file_metadata = read_file_metadata_with_prefetch(&input_file, 64) .await .unwrap(); assert_eq!(file_metadata, zstd_compressed_metric_file_metadata()) @@ -933,7 +979,7 @@ mod tests { #[tokio::test] async fn test_read_file_metadata_of_empty_file_with_prefetching() { let input_file = java_empty_uncompressed_input_file(); - let file_metadata = FileMetadata::read_with_prefetch(&input_file, 64) + let file_metadata = read_file_metadata_with_prefetch(&input_file, 64) .await .unwrap(); @@ -943,7 +989,7 @@ mod tests { #[tokio::test] async fn test_read_file_metadata_of_uncompressed_metric_data_with_prefetching() { let input_file = java_uncompressed_metric_input_file(); - let file_metadata = FileMetadata::read_with_prefetch(&input_file, 64) + let file_metadata = read_file_metadata_with_prefetch(&input_file, 64) .await .unwrap(); @@ -953,10 +999,75 @@ mod tests { #[tokio::test] async fn test_read_file_metadata_of_zstd_compressed_metric_data_with_prefetching() { let input_file = java_zstd_compressed_metric_input_file(); - let file_metadata = FileMetadata::read_with_prefetch(&input_file, 64) + let file_metadata = read_file_metadata_with_prefetch(&input_file, 64) .await .unwrap(); assert_eq!(file_metadata, zstd_compressed_metric_file_metadata()); } + + #[tokio::test] + async fn test_read_with_incorrect_header_magic() { + let temp_dir = TempDir::new().unwrap(); + + let prefetch_hint: u8 = 64; + let mut bytes = vec![]; + // Invalid header magic + bytes.extend([0x00, 0x00, 0x00, 0x00]); + // Intentionally keep file size larger than prefetch_hint. + bytes.extend(vec![0u8; prefetch_hint as usize]); + // Valid footer: magic + payload + footer struct + bytes.extend(FileMetadata::MAGIC); + bytes.extend(empty_footer_payload_bytes()); + bytes.extend(empty_footer_payload_bytes_length_bytes()); + bytes.extend(vec![0, 0, 0, 0]); // flags + bytes.extend(FileMetadata::MAGIC); + + let input_file = input_file_with_bytes(&temp_dir, &bytes).await; + + assert_eq!( + read_file_metadata(&input_file).await.unwrap_err().kind(), + ErrorKind::DataInvalid, + ); + assert_eq!( + read_file_metadata_with_prefetch(&input_file, prefetch_hint) + .await + .unwrap_err() + .kind(), + ErrorKind::DataInvalid, + ); + } + + #[tokio::test] + async fn test_gzip_compression_allowed_in_metadata() { + let temp_dir = TempDir::new().unwrap(); + + // Create a JSON payload with Gzip compression codec + // Metadata should be readable, but accessing the blob will fail + let payload = r#"{ + "blobs": [ + { + "type": "test-type", + "fields": [1], + "snapshot-id": 1, + "sequence-number": 1, + "offset": 4, + "length": 10, + "compression-codec": "gzip" + } + ] + }"#; + + let input_file = input_file_with_payload(&temp_dir, payload).await; + + // Reading metadata should succeed (lazy validation) + let result = read_file_metadata(&input_file).await; + assert!(result.is_ok()); + let metadata = result.unwrap(); + assert_eq!(metadata.blobs.len(), 1); + assert_eq!( + metadata.blobs[0].compression_codec, + CompressionCodec::gzip_default() + ); + } } diff --git a/crates/iceberg/src/puffin/mod.rs b/crates/iceberg/src/puffin/mod.rs index 0a03781655..998520cb8d 100644 --- a/crates/iceberg/src/puffin/mod.rs +++ b/crates/iceberg/src/puffin/mod.rs @@ -19,11 +19,28 @@ #![deny(missing_docs)] +use crate::Result; +use crate::error::invalid_data; + mod blob; pub use blob::{APACHE_DATASKETCHES_THETA_V1, Blob, DELETION_VECTOR_V1}; -mod compression; -pub use compression::CompressionCodec; +pub use crate::compression::CompressionCodec; + +/// Validates that the compression codec is supported for Puffin files. +/// Returns an error if the codec is not supported. +fn validate_puffin_compression(codec: CompressionCodec) -> Result<()> { + match codec { + CompressionCodec::None | CompressionCodec::Lz4 | CompressionCodec::Zstd(_) => Ok(()), + other => Err(invalid_data!( + "Compression codec {} is not supported for Puffin files. Only {}, {}, and {} are supported.", + other.name(), + CompressionCodec::None.name(), + CompressionCodec::Lz4.name(), + CompressionCodec::zstd_default().name() + )), + } +} mod metadata; pub use metadata::{BlobMetadata, CREATED_BY_PROPERTY, FileMetadata}; @@ -36,3 +53,20 @@ pub use writer::PuffinWriter; #[cfg(test)] mod test_utils; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_puffin_codec_validation() { + // Supported codecs + assert!(validate_puffin_compression(CompressionCodec::None).is_ok()); + assert!(validate_puffin_compression(CompressionCodec::Lz4).is_ok()); + assert!(validate_puffin_compression(CompressionCodec::zstd_default()).is_ok()); + assert!(validate_puffin_compression(CompressionCodec::Zstd(5)).is_ok()); + + // Unsupported codecs + assert!(validate_puffin_compression(CompressionCodec::gzip_default()).is_err()); + } +} diff --git a/crates/iceberg/src/puffin/reader.rs b/crates/iceberg/src/puffin/reader.rs index dce53d93f0..a7b178ac84 100644 --- a/crates/iceberg/src/puffin/reader.rs +++ b/crates/iceberg/src/puffin/reader.rs @@ -17,22 +17,43 @@ use tokio::sync::OnceCell; +use super::validate_puffin_compression; use crate::Result; +use crate::encryption::EncryptedInputFile; use crate::io::{FileRead, InputFile}; use crate::puffin::blob::Blob; use crate::puffin::metadata::{BlobMetadata, FileMetadata}; /// Puffin reader pub struct PuffinReader { - input_file: InputFile, + file_read: Box, + file_length: u64, file_metadata: OnceCell, } impl PuffinReader { - /// Returns a new Puffin reader - pub fn new(input_file: InputFile) -> Self { + /// Returns a new Puffin reader for an unencrypted file. + pub async fn new(input_file: InputFile) -> Result { + let file_length = input_file.metadata().await?.size; + let file_read = input_file.reader().await?; + Ok(Self::from_parts(file_read, file_length)) + } + + /// Returns a new Puffin reader from an [`EncryptedInputFile`]. + /// + /// Use this when reading Puffin files with transparent decryption. The + /// reader operates over plaintext offsets and length, so all blob and + /// footer positions match those written to the unencrypted file. + pub async fn new_from_encrypted(encrypted_input: EncryptedInputFile) -> Result { + let file_length = encrypted_input.metadata()?.size; + let file_read = encrypted_input.reader().await?; + Ok(Self::from_parts(file_read, file_length)) + } + + fn from_parts(file_read: Box, file_length: u64) -> Self { Self { - input_file, + file_read, + file_length, file_metadata: OnceCell::new(), } } @@ -40,17 +61,18 @@ impl PuffinReader { /// Returns file metadata pub async fn file_metadata(&self) -> Result<&FileMetadata> { self.file_metadata - .get_or_try_init(|| FileMetadata::read(&self.input_file)) + .get_or_try_init(|| FileMetadata::read(self.file_read.as_ref(), self.file_length)) .await } /// Returns blob pub async fn blob(&self, blob_metadata: &BlobMetadata) -> Result { - let file_read = self.input_file.reader().await?; + validate_puffin_compression(blob_metadata.compression_codec)?; + let start = blob_metadata.offset; let end = start + blob_metadata.length; - let bytes = file_read.read(start..end).await?.to_vec(); - let data = blob_metadata.compression_codec.decompress(bytes)?; + let bytes = self.file_read.read(start..end).await?; + let data = blob_metadata.compression_codec.decompress(bytes.to_vec())?; Ok(Blob { r#type: blob_metadata.r#type.clone(), @@ -65,7 +87,11 @@ impl PuffinReader { #[cfg(test)] mod tests { + use std::collections::HashMap; + use crate::ErrorKind; + use crate::compression::CompressionCodec; + use crate::puffin::metadata::BlobMetadata; use crate::puffin::reader::PuffinReader; use crate::puffin::test_utils::{ blob_0, blob_1, java_uncompressed_metric_input_file, @@ -76,7 +102,7 @@ mod tests { #[tokio::test] async fn test_puffin_reader_uncompressed_metric_data() { let input_file = java_uncompressed_metric_input_file(); - let puffin_reader = PuffinReader::new(input_file); + let puffin_reader = PuffinReader::new(input_file).await.unwrap(); let file_metadata = puffin_reader.file_metadata().await.unwrap().clone(); assert_eq!(file_metadata, uncompressed_metric_file_metadata()); @@ -101,7 +127,7 @@ mod tests { #[tokio::test] async fn test_puffin_reader_zstd_compressed_metric_data() { let input_file = java_zstd_compressed_metric_input_file(); - let puffin_reader = PuffinReader::new(input_file); + let puffin_reader = PuffinReader::new(input_file).await.unwrap(); let file_metadata = puffin_reader.file_metadata().await.unwrap().clone(); assert_eq!(file_metadata, zstd_compressed_metric_file_metadata()); @@ -122,4 +148,34 @@ mod tests { blob_1(), ) } + + #[tokio::test] + async fn test_gzip_compression_rejected_on_blob_access() { + // Use a real puffin file + let input_file = java_uncompressed_metric_input_file(); + let reader = PuffinReader::new(input_file).await.unwrap(); + + // Create a BlobMetadata with Gzip compression + let gzip_blob_metadata = BlobMetadata { + r#type: "test-type".to_string(), + fields: vec![1], + snapshot_id: 1, + sequence_number: 1, + offset: 4, + length: 10, + compression_codec: CompressionCodec::gzip_default(), + properties: HashMap::new(), + }; + + // Attempting to access the blob should fail + let result = reader.blob(&gzip_blob_metadata).await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("gzip")); + assert!( + err.to_string() + .contains("is not supported for Puffin files") + ); + } } diff --git a/crates/iceberg/src/puffin/test_utils.rs b/crates/iceberg/src/puffin/test_utils.rs index ca91fa217b..26a6834a65 100644 --- a/crates/iceberg/src/puffin/test_utils.rs +++ b/crates/iceberg/src/puffin/test_utils.rs @@ -18,8 +18,9 @@ use std::collections::HashMap; use super::blob::Blob; -use crate::io::{FileIOBuilder, InputFile}; -use crate::puffin::compression::CompressionCodec; +use crate::Result; +use crate::compression::CompressionCodec; +use crate::io::{FileIO, InputFile}; use crate::puffin::metadata::{BlobMetadata, CREATED_BY_PROPERTY, FileMetadata}; const JAVA_TESTDATA: &str = "testdata/puffin/java-generated"; @@ -28,13 +29,28 @@ const METRIC_UNCOMPRESSED: &str = "sample-metric-data-uncompressed.bin"; const METRIC_ZSTD_COMPRESSED: &str = "sample-metric-data-compressed-zstd.bin"; fn input_file_for_test_data(path: &str) -> InputFile { - FileIOBuilder::new_fs_io() - .build() - .unwrap() + FileIO::new_with_fs() .new_input(env!("CARGO_MANIFEST_DIR").to_owned() + "/" + path) .unwrap() } +/// Reads file metadata from an [`InputFile`], resolving its reader and length. +pub(crate) async fn read_file_metadata(input_file: &InputFile) -> Result { + let file_read = input_file.reader().await?; + let file_length = input_file.metadata().await?.size; + FileMetadata::read(file_read.as_ref(), file_length).await +} + +/// Reads file metadata with a prefetch hint from an [`InputFile`]. +pub(crate) async fn read_file_metadata_with_prefetch( + input_file: &InputFile, + prefetch_hint: u8, +) -> Result { + let file_read = input_file.reader().await?; + let file_length = input_file.metadata().await?.size; + FileMetadata::read_with_prefetch(file_read.as_ref(), file_length, prefetch_hint).await +} + pub(crate) fn java_empty_uncompressed_input_file() -> InputFile { input_file_for_test_data(&[JAVA_TESTDATA, EMPTY_UNCOMPRESSED].join("/")) } @@ -79,7 +95,7 @@ pub(crate) fn zstd_compressed_metric_blob_0_metadata() -> BlobMetadata { sequence_number: METRIC_BLOB_0_SEQUENCE_NUMBER, offset: 4, length: 22, - compression_codec: CompressionCodec::Zstd, + compression_codec: CompressionCodec::zstd_default(), properties: HashMap::new(), } } @@ -136,7 +152,7 @@ pub(crate) fn zstd_compressed_metric_blob_1_metadata() -> BlobMetadata { sequence_number: METRIC_BLOB_1_SEQUENCE_NUMBER, offset: 26, length: 77, - compression_codec: CompressionCodec::Zstd, + compression_codec: CompressionCodec::zstd_default(), properties: HashMap::new(), } } diff --git a/crates/iceberg/src/puffin/writer.rs b/crates/iceberg/src/puffin/writer.rs index f68efda889..e6882a4844 100644 --- a/crates/iceberg/src/puffin/writer.rs +++ b/crates/iceberg/src/puffin/writer.rs @@ -19,10 +19,12 @@ use std::collections::{HashMap, HashSet}; use bytes::Bytes; +use super::validate_puffin_compression; use crate::Result; -use crate::io::{FileWrite, OutputFile}; +use crate::compression::CompressionCodec; +use crate::encryption::EncryptedOutputFile; +use crate::io::{FileMetadata as IoFileMetadata, FileWrite, OutputFile}; use crate::puffin::blob::Blob; -use crate::puffin::compression::CompressionCodec; use crate::puffin::metadata::{BlobMetadata, FileMetadata, Flag}; /// Puffin writer @@ -37,12 +39,41 @@ pub struct PuffinWriter { } impl PuffinWriter { - /// Returns a new Puffin writer + /// Returns a new Puffin writer for an unencrypted file. pub async fn new( output_file: &OutputFile, properties: HashMap, compress_footer: bool, ) -> Result { + Ok(Self::from_writer( + output_file.writer().await?, + properties, + compress_footer, + )) + } + + /// Returns a new Puffin writer from an [`EncryptedOutputFile`]. + /// + /// Use this when writing Puffin files with transparent encryption. Blob + /// and footer offsets are recorded as plaintext positions, matching what + /// an unencrypted writer would produce. + pub async fn new_from_encrypted( + encrypted_output: &EncryptedOutputFile, + properties: HashMap, + compress_footer: bool, + ) -> Result { + Ok(Self::from_writer( + encrypted_output.writer().await?, + properties, + compress_footer, + )) + } + + fn from_writer( + writer: Box, + properties: HashMap, + compress_footer: bool, + ) -> Self { let mut flags = HashSet::::new(); let footer_compression_codec = if compress_footer { flags.insert(Flag::FooterPayloadCompressed); @@ -51,19 +82,21 @@ impl PuffinWriter { CompressionCodec::None }; - Ok(Self { - writer: output_file.writer().await?, + Self { + writer, is_header_written: false, num_bytes_written: 0, written_blobs_metadata: Vec::new(), properties, footer_compression_codec, flags, - }) + } } /// Adds blob to Puffin file pub async fn add(&mut self, blob: Blob, compression_codec: CompressionCodec) -> Result<()> { + validate_puffin_compression(compression_codec)?; + self.write_header_once().await?; let offset = self.num_bytes_written; @@ -84,12 +117,11 @@ impl PuffinWriter { Ok(()) } - /// Finalizes the Puffin file - pub async fn close(mut self) -> Result<()> { + /// Finalizes the Puffin file and returns its stored size. + pub async fn close(mut self) -> Result { self.write_header_once().await?; self.write_footer().await?; - self.writer.close().await?; - Ok(()) + self.writer.close().await } async fn write(&mut self, bytes: Bytes) -> Result<()> { @@ -114,8 +146,7 @@ impl PuffinWriter { properties: self.properties.clone(), }; let json = serde_json::to_string::(&file_metadata)?; - let bytes = json.as_bytes(); - self.footer_compression_codec.compress(bytes.to_vec()) + self.footer_compression_codec.compress(json.into_bytes()) } fn flags_bytes(&self) -> [u8; FileMetadata::FOOTER_STRUCT_FLAGS_LENGTH as usize] { @@ -148,26 +179,26 @@ mod tests { use tempfile::TempDir; - use crate::Result; - use crate::io::{FileIOBuilder, InputFile, OutputFile}; + use crate::compression::CompressionCodec; + use crate::io::{FileIO, InputFile, OutputFile}; use crate::puffin::blob::Blob; - use crate::puffin::compression::CompressionCodec; use crate::puffin::metadata::FileMetadata; use crate::puffin::reader::PuffinReader; use crate::puffin::test_utils::{ blob_0, blob_1, empty_footer_payload, empty_footer_payload_bytes, file_properties, java_empty_uncompressed_input_file, java_uncompressed_metric_input_file, - java_zstd_compressed_metric_input_file, uncompressed_metric_file_metadata, - zstd_compressed_metric_file_metadata, + java_zstd_compressed_metric_input_file, read_file_metadata, + uncompressed_metric_file_metadata, zstd_compressed_metric_file_metadata, }; use crate::puffin::writer::PuffinWriter; + use crate::{ErrorKind, Result}; async fn write_puffin_file( temp_dir: &TempDir, blobs: Vec<(Blob, CompressionCodec)>, properties: HashMap, ) -> Result { - let file_io = FileIOBuilder::new_fs_io().build()?; + let file_io = FileIO::new_with_fs(); let path_buf = temp_dir.path().join("temp_puffin.bin"); let temp_path = path_buf.to_str().unwrap(); @@ -183,7 +214,7 @@ mod tests { } async fn read_all_blobs_from_puffin_file(input_file: InputFile) -> Vec { - let puffin_reader = PuffinReader::new(input_file); + let puffin_reader = PuffinReader::new(input_file).await.unwrap(); let mut blobs = Vec::new(); let blobs_metadata = puffin_reader.file_metadata().await.unwrap().clone().blobs; for blob_metadata in blobs_metadata { @@ -202,7 +233,7 @@ mod tests { .to_input_file(); assert_eq!( - FileMetadata::read(&input_file).await.unwrap(), + read_file_metadata(&input_file).await.unwrap(), empty_footer_payload() ); @@ -238,7 +269,7 @@ mod tests { .to_input_file(); assert_eq!( - FileMetadata::read(&input_file).await.unwrap(), + read_file_metadata(&input_file).await.unwrap(), uncompressed_metric_file_metadata() ); @@ -249,7 +280,8 @@ mod tests { async fn test_write_zstd_compressed_metric_data() { let temp_dir = TempDir::new().unwrap(); let blobs = vec![blob_0(), blob_1()]; - let blobs_with_compression = blobs_with_compression(blobs.clone(), CompressionCodec::Zstd); + let blobs_with_compression = + blobs_with_compression(blobs.clone(), CompressionCodec::zstd_default()); let input_file = write_puffin_file(&temp_dir, blobs_with_compression, file_properties()) .await @@ -257,7 +289,7 @@ mod tests { .to_input_file(); assert_eq!( - FileMetadata::read(&input_file).await.unwrap(), + read_file_metadata(&input_file).await.unwrap(), zstd_compressed_metric_file_metadata() ); @@ -321,7 +353,8 @@ mod tests { async fn test_zstd_compressed_metric_data_is_bit_identical_to_java_generated_file() { let temp_dir = TempDir::new().unwrap(); let blobs = vec![blob_0(), blob_1()]; - let blobs_with_compression = blobs_with_compression(blobs, CompressionCodec::Zstd); + let blobs_with_compression = + blobs_with_compression(blobs, CompressionCodec::zstd_default()); assert_files_are_bit_identical( write_puffin_file(&temp_dir, blobs_with_compression, file_properties()) @@ -331,4 +364,75 @@ mod tests { ) .await } + + #[tokio::test] + async fn test_gzip_compression_rejected() { + let temp_dir = TempDir::new().unwrap(); + let blobs = vec![blob_0()]; + let blobs_with_compression = + blobs_with_compression(blobs, CompressionCodec::gzip_default()); + + let result = write_puffin_file(&temp_dir, blobs_with_compression, file_properties()).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("gzip")); + assert!( + err.to_string() + .contains("is not supported for Puffin files") + ); + } + + #[tokio::test] + async fn test_encrypted_write_read_roundtrip() { + use crate::encryption::{EncryptedInputFile, EncryptedOutputFile, StandardKeyMetadata}; + + let key_metadata = || { + StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!") + }; + + let file_io = FileIO::new_with_memory(); + let path = "memory:///test/encrypted.puffin"; + let blobs = vec![blob_0(), blob_1()]; + + // Write through the encrypting writer. + let encrypted_output = + EncryptedOutputFile::new(file_io.new_output(path).unwrap(), key_metadata()); + let mut writer = + PuffinWriter::new_from_encrypted(&encrypted_output, file_properties(), false) + .await + .unwrap(); + for blob in blobs.clone() { + writer.add(blob, CompressionCodec::None).await.unwrap(); + } + let file_metadata = writer.close().await.unwrap(); + + // The ciphertext on disk must not equal a plaintext puffin file. + let raw = file_io.new_input(path).unwrap().read().await.unwrap(); + assert_ne!( + &raw[..FileMetadata::MAGIC_LENGTH as usize], + FileMetadata::MAGIC + ); + + // Read back through the decrypting reader over plaintext offsets. + let key_metadata = encrypted_output.key_metadata_with_saved_file_metadata(&file_metadata); + assert_eq!(key_metadata.file_length(), Some(raw.len() as u64)); + let encrypted_input = + EncryptedInputFile::new(file_io.new_input(path).unwrap(), key_metadata); + let reader = PuffinReader::new_from_encrypted(encrypted_input) + .await + .unwrap(); + + let file_metadata = reader.file_metadata().await.unwrap().clone(); + assert_eq!(file_metadata, uncompressed_metric_file_metadata()); + + let mut read_blobs = Vec::new(); + for blob_metadata in &file_metadata.blobs { + read_blobs.push(reader.blob(blob_metadata).await.unwrap()); + } + assert_eq!(read_blobs, blobs); + } } diff --git a/crates/iceberg/src/runtime/mod.rs b/crates/iceberg/src/runtime/mod.rs index d0a3ce6602..8ee4a65326 100644 --- a/crates/iceberg/src/runtime/mod.rs +++ b/crates/iceberg/src/runtime/mod.rs @@ -17,97 +17,324 @@ // This module contains the async runtime abstraction for iceberg. +use std::fmt; use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; -pub enum JoinHandle { - #[cfg(feature = "tokio")] - Tokio(tokio::task::JoinHandle), - #[cfg(all(feature = "smol", not(feature = "tokio")))] - Smol(smol::Task), - #[cfg(all(not(feature = "smol"), not(feature = "tokio")))] - Unimplemented(Box), +use tokio::task::JoinHandle as TokioJoinHandle; + +use crate::{Error, ErrorKind, Result}; + +/// Wrapper around tokio's [`TokioJoinHandle`] that converts task failures into +/// [`crate::Error`]. +/// +/// Tokio's `JoinHandle` resolves to `Result`, where a +/// [`tokio::task::JoinError`] means the task either panicked or was cancelled +/// (typically from runtime shutdown or `abort`). +/// Both are surfaced here as `ErrorKind::Unexpected` with the original +/// [`tokio::task::JoinError`] preserved as the source. +pub struct JoinHandle(TokioJoinHandle); + +impl JoinHandle { + /// Ties the task's lifetime to this handle: see [`AbortOnDropHandle`]. + pub(crate) fn abort_on_drop(self) -> AbortOnDropHandle { + AbortOnDropHandle(self) + } } +impl Unpin for JoinHandle {} + impl Future for JoinHandle { - type Output = T; + type Output = Result; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - match self.get_mut() { - #[cfg(feature = "tokio")] - JoinHandle::Tokio(handle) => Pin::new(handle) - .poll(cx) - .map(|h| h.expect("tokio spawned task failed")), - #[cfg(all(feature = "smol", not(feature = "tokio")))] - JoinHandle::Smol(handle) => Pin::new(handle).poll(cx), - #[cfg(all(not(feature = "smol"), not(feature = "tokio")))] - JoinHandle::Unimplemented(_) => unimplemented!("no runtime has been enabled"), - } + Pin::new(&mut self.get_mut().0).poll(cx).map(|r| { + r.map_err(|e| Error::new(ErrorKind::Unexpected, "spawned task failed").with_source(e)) + }) } } -#[allow(dead_code)] -pub fn spawn(f: F) -> JoinHandle -where - F: Future + Send + 'static, - F::Output: Send + 'static, -{ - #[cfg(feature = "tokio")] - return JoinHandle::Tokio(tokio::task::spawn(f)); +/// A [`JoinHandle`] that aborts its task when dropped. +/// +/// Dropping a plain `JoinHandle` leaves the task running with nobody waiting for +/// its result. Use this instead when only the caller wants that result, so that +/// dropping the caller stops the work. +pub(crate) struct AbortOnDropHandle(JoinHandle); - #[cfg(all(feature = "smol", not(feature = "tokio")))] - return JoinHandle::Smol(smol::spawn(f)); +impl Drop for AbortOnDropHandle { + fn drop(&mut self) { + self.0.0.abort(); + } +} + +impl Future for AbortOnDropHandle { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.get_mut().0).poll(cx) + } +} + +/// Handle to a single tokio runtime. +/// +/// Wraps a [`tokio::runtime::Handle`], which is cheap to clone. The caller is +/// responsible for keeping the underlying runtime alive while this handle is +/// in use; spawning on a shut-down runtime will surface as a [`tokio::task::JoinError`] +/// via the returned [`JoinHandle`]. +#[derive(Clone)] +pub struct RuntimeHandle { + handle: tokio::runtime::Handle, +} - #[cfg(all(not(feature = "smol"), not(feature = "tokio")))] - unimplemented!("no runtime has been enabled") +impl fmt::Debug for RuntimeHandle { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RuntimeHandle").finish() + } } -#[allow(dead_code)] -pub fn spawn_blocking(f: F) -> JoinHandle -where - F: FnOnce() -> T + Send + 'static, - T: Send + 'static, -{ - #[cfg(feature = "tokio")] - return JoinHandle::Tokio(tokio::task::spawn_blocking(f)); +impl RuntimeHandle { + fn from_tokio_handle(handle: tokio::runtime::Handle) -> Self { + Self { handle } + } + + /// Spawn an async task. + pub fn spawn(&self, future: F) -> JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + JoinHandle(self.handle.spawn(future)) + } - #[cfg(all(feature = "smol", not(feature = "tokio")))] - return JoinHandle::Smol(smol::unblock(f)); + /// Spawn a blocking task. + pub fn spawn_blocking(&self, f: F) -> JoinHandle + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, + { + JoinHandle(self.handle.spawn_blocking(f)) + } +} - #[cfg(all(not(feature = "smol"), not(feature = "tokio")))] - unimplemented!("no runtime has been enabled") +/// Iceberg's runtime abstraction. +/// +/// Contains separate handles for IO-bound and CPU-bound work. When constructed +/// with a single tokio runtime, both `io()` and `cpu()` route to the same one. +/// Use [`Runtime::new_with_split`] to provide dedicated runtimes for each +/// category. +/// +/// # Lifetime +/// +/// A `Runtime` stores only `tokio::runtime::Handle`s (weak references). +/// The caller owns the tokio runtime's lifetime. If the underlying runtime is +/// dropped while iceberg is still using it, subsequent spawns will surface as +/// task cancellation errors via the returned `JoinHandle`. +/// +/// Cloning is cheap. +#[derive(Clone)] +pub struct Runtime { + io: RuntimeHandle, + cpu: RuntimeHandle, +} + +impl fmt::Debug for Runtime { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Runtime").finish() + } +} + +impl Runtime { + /// Create a Runtime backed by a single tokio runtime for all work. + pub fn new(runtime: &tokio::runtime::Runtime) -> Self { + let handle = RuntimeHandle::from_tokio_handle(runtime.handle().clone()); + Self { + io: handle.clone(), + cpu: handle, + } + } + + /// Create a Runtime with separate tokio runtimes for IO and CPU work. + pub fn new_with_split( + io_runtime: &tokio::runtime::Runtime, + cpu_runtime: &tokio::runtime::Runtime, + ) -> Self { + Self { + io: RuntimeHandle::from_tokio_handle(io_runtime.handle().clone()), + cpu: RuntimeHandle::from_tokio_handle(cpu_runtime.handle().clone()), + } + } + + /// Borrows the tokio runtime the caller is currently running in. + /// + /// Panics if called outside a tokio runtime context. Use + /// [`Runtime::try_current`] for a fallible version. + /// + /// Iceberg never implicitly spawns its own runtime; callers outside a + /// tokio context must construct one explicitly via [`Runtime::new`] or + /// [`Runtime::new_with_split`]. + pub fn current() -> Self { + Self::try_current().expect( + "Runtime::current() called outside a tokio runtime context. \ + Call it from within #[tokio::main] / #[tokio::test], or construct \ + a Runtime explicitly via Runtime::new / Runtime::new_with_split.", + ) + } + + /// Fallible variant of [`Runtime::current`]. Returns an error if no tokio + /// runtime is available in the current context. + pub fn try_current() -> Result { + let handle = tokio::runtime::Handle::try_current().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + "no tokio runtime in context; call Runtime::try_current() \ + from within a tokio runtime, or construct a Runtime explicitly \ + via Runtime::new / Runtime::new_with_split", + ) + .with_source(e) + })?; + let rh = RuntimeHandle::from_tokio_handle(handle); + Ok(Self { + io: rh.clone(), + cpu: rh, + }) + } + + /// Handle for IO-bound work (network fetches, file reads). + pub fn io(&self) -> &RuntimeHandle { + &self.io + } + + /// Handle for CPU-bound work (decoding, predicate eval, projection). + pub fn cpu(&self) -> &RuntimeHandle { + &self.cpu + } } #[cfg(test)] mod tests { use super::*; - #[cfg(feature = "tokio")] - #[tokio::test] - async fn test_tokio_spawn() { - let handle = spawn(async { 1 + 1 }); - assert_eq!(handle.await, 2); + /// A test harness that owns a tokio runtime and exposes a `Runtime` handle + /// plus a `block_on` helper for sync test bodies. + struct TestRuntime { + tokio: tokio::runtime::Runtime, + rt: Runtime, } - #[cfg(feature = "tokio")] - #[tokio::test] - async fn test_tokio_spawn_blocking() { - let handle = spawn_blocking(|| 1 + 1); - assert_eq!(handle.await, 2); + impl TestRuntime { + fn new() -> Self { + let tokio = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("Failed to build tokio runtime"); + let rt = Runtime::new(&tokio); + Self { tokio, rt } + } + + fn block_on(&self, f: F) -> F::Output { + self.tokio.block_on(f) + } } - #[cfg(all(feature = "smol", not(feature = "tokio")))] - #[smol::test] - async fn test_smol_spawn() { - let handle = spawn(async { 1 + 1 }); - assert_eq!(handle.await, 2); + #[test] + fn test_runtime_spawn_io() { + let h = TestRuntime::new(); + let handle = h.rt.io().spawn(async { 1 + 1 }); + assert_eq!(h.block_on(handle).unwrap(), 2); } - #[cfg(all(feature = "smol", not(feature = "tokio")))] - #[smo::test] - async fn test_smol_spawn_blocking() { - let handle = spawn_blocking(|| 1 + 1); - assert_eq!(handle.await, 2); + #[test] + fn test_runtime_spawn_cpu() { + let h = TestRuntime::new(); + let handle = h.rt.cpu().spawn(async { 3 + 4 }); + assert_eq!(h.block_on(handle).unwrap(), 7); + } + + #[test] + fn test_runtime_spawn_blocking() { + let h = TestRuntime::new(); + let handle = h.rt.cpu().spawn_blocking(|| 1 + 1); + assert_eq!(h.block_on(handle).unwrap(), 2); + } + + #[test] + fn test_runtime_new_with_custom_runtime() { + let h = TestRuntime::new(); + let handle = h.rt.io().spawn(async { 42 }); + assert_eq!(h.block_on(handle).unwrap(), 42); + } + + #[test] + fn test_runtime_split_uses_separate_handles() { + let io_rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + let cpu_rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + let rt = Runtime::new_with_split(&io_rt, &cpu_rt); + // Spawn on each and confirm both are distinct live runtimes. We use + // `io_rt`/`cpu_rt` directly to `block_on` since our `Runtime` doesn't + // expose one. + let io_result = io_rt.block_on(async { rt.io().spawn(async { "io" }).await.unwrap() }); + let cpu_result = cpu_rt.block_on(async { rt.cpu().spawn(async { "cpu" }).await.unwrap() }); + assert_eq!(io_result, "io"); + assert_eq!(cpu_result, "cpu"); + } + + #[test] + fn test_runtime_clone() { + let h = TestRuntime::new(); + let rt2 = h.rt.clone(); + let handle = rt2.io().spawn(async { 5 }); + assert_eq!(h.block_on(handle).unwrap(), 5); + } + + #[test] + fn test_runtime_debug() { + let h = TestRuntime::new(); + let debug_str = format!("{:?}", h.rt); + assert!(debug_str.contains("Runtime")); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_try_current_in_runtime() { + let rt = Runtime::try_current().expect("should find current runtime"); + let result = rt.io().spawn(async { 7 }).await.unwrap(); + assert_eq!(result, 7); + } + + #[test] + fn test_try_current_outside_runtime() { + let err = Runtime::try_current().expect_err("must fail outside runtime"); + assert_eq!(err.kind(), ErrorKind::Unexpected); + } + + /// Verifies that when the caller drops the underlying tokio runtime, a + /// subsequent spawn surfaces as a `JoinError` via our `JoinHandle` rather + /// than hanging or misbehaving. + #[test] + fn test_spawn_after_runtime_drop_errors() { + let driver = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let owned = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + let rt = Runtime::new(&owned); + // Drop the caller's owned runtime. Our `Runtime` only holds a `Handle`, + // so this shuts down the underlying tokio runtime. + drop(owned); + + // Spawning after shutdown returns a handle that resolves to a cancelled + // JoinError, which our wrapper maps to iceberg::Error. + let handle = rt.io().spawn(async { 1 }); + let result = driver.block_on(handle); + assert!(result.is_err(), "expected error after runtime shutdown"); } } diff --git a/crates/iceberg/src/scan/cache.rs b/crates/iceberg/src/scan/cache.rs index fc7e559e61..b87786e6cf 100644 --- a/crates/iceberg/src/scan/cache.rs +++ b/crates/iceberg/src/scan/cache.rs @@ -66,43 +66,68 @@ impl PartitionFilterCache { .partition_spec_by_id(spec_id) .ok_or(Error::new( ErrorKind::Unexpected, - format!("Could not find partition spec for id {}", spec_id), + format!("Could not find partition spec for id {spec_id}"), ))?; - let partition_type = partition_spec.partition_type(schema)?; - let partition_fields = partition_type.fields().to_owned(); - let partition_schema = Arc::new( - Schema::builder() - .with_schema_id(partition_spec.spec_id()) - .with_fields(partition_fields) - .build()?, - ); + // A historical spec may reference a source column that was later dropped from the + // schema, which is a legitimate v2+ state. Such a spec cannot be resolved to a + // partition type, so it falls back to an always-true filter: files under the spec + // are not partition-pruned but still receive the row filter. Any other resolution + // failure is unexpected and propagates. The fallback is cached by spec id like any + // other filter; this is safe only because the cache lives per-scan in `PlanContext` + // with a fixed schema and predicate. Hoisting it to table or catalog scope would + // pin a spec to always-true even for a later scan whose schema could resolve it. + // TODO(https://github.com/apache/iceberg-rust/issues/2844): derive partition types from + // transforms where possible to restore pruning on a historical spec's still-live fields, + // which also makes the fallback per-term (dropped fields only) instead of per-spec. + let has_dropped_source_column = partition_spec + .fields() + .iter() + .any(|field| schema.field_by_id(field.source_id).is_none()); - let mut inclusive_projection = InclusiveProjection::new(partition_spec.clone()); + let partition_filter = if has_dropped_source_column { + BoundPredicate::AlwaysTrue + } else { + let partition_type = partition_spec.partition_type(schema)?; + let partition_fields = partition_type.fields().to_owned(); + let partition_schema = Arc::new( + Schema::builder() + .with_schema_id(partition_spec.spec_id()) + .with_fields(partition_fields) + .build()?, + ); - let partition_filter = inclusive_projection - .project(&filter)? - .rewrite_not() - .bind(partition_schema.clone(), case_sensitive)?; + let mut inclusive_projection = InclusiveProjection::new(partition_spec.clone()); - self.0 - .write() - .map_err(|_| { - Error::new( - ErrorKind::Unexpected, - "PartitionFilterCache RwLock was poisoned", - ) - })? - .insert(spec_id, Arc::new(partition_filter)); + inclusive_projection + .project(&filter)? + .rewrite_not() + .bind(partition_schema.clone(), case_sensitive)? + }; - let read = self.0.read().map_err(|_| { + let mut write = self.0.write().map_err(|_| { Error::new( ErrorKind::Unexpected, "PartitionFilterCache RwLock was poisoned", ) })?; - Ok(read.get(&spec_id).unwrap().clone()) + // Another thread may have populated the entry while the filter was computed without a + // lock held. Only warn and insert when this call is the one that populates it, so the + // always-true fallback is logged once per spec rather than once per racing thread. + let partition_filter = write.entry(spec_id).or_insert_with(|| { + if has_dropped_source_column { + tracing::warn!( + spec_id, + "Partition spec references a source column not in the scan schema; \ + skipping partition pruning for its manifests" + ); + } + + Arc::new(partition_filter) + }); + + Ok(partition_filter.clone()) } } @@ -235,3 +260,88 @@ impl ExpressionEvaluatorCache { Ok(read.get(&spec_id).unwrap().clone()) } } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use super::PartitionFilterCache; + use crate::expr::{Bind, BoundPredicate, Reference}; + use crate::spec::{ + Datum, FormatVersion, NestedField, PrimitiveType, Schema, SortOrder, TableMetadataBuilder, + Transform, Type, UnboundPartitionField, UnboundPartitionSpec, + }; + + /// A historical spec whose source column was dropped from the current schema resolves to an + /// always-true filter, and the fallback is cached under the spec id. + #[test] + fn dropped_partition_source_column_falls_back_to_always_true() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "part", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + // Spec 0 partitions on `part` (id 2). + let spec = UnboundPartitionSpec::builder() + .with_spec_id(0) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("part") + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build(); + + // Evolve the schema so that `part` is no longer present, leaving spec 0 unresolvable. + let evolved_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + // Make an unpartitioned spec the default before dropping `part`, so spec 0 survives + // only as a historical spec that can no longer be resolved against the schema. + let table_metadata = Arc::new( + TableMetadataBuilder::new( + schema, + spec, + SortOrder::unsorted_order(), + "s3://bucket/test".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .add_default_partition_spec(UnboundPartitionSpec::builder().build()) + .unwrap() + .add_current_schema(evolved_schema.clone()) + .unwrap() + .build() + .unwrap() + .metadata, + ); + + let filter = Reference::new("id") + .greater_than_or_equal_to(Datum::long(5)) + .bind(Arc::new(evolved_schema.clone()), true) + .unwrap(); + + let cache = PartitionFilterCache::new(); + let partition_filter = cache + .get(0, &table_metadata, &evolved_schema, true, filter.clone()) + .unwrap(); + assert!(matches!(*partition_filter, BoundPredicate::AlwaysTrue)); + + // The fallback is cached, so a second lookup returns the same instance. + let cached = cache + .get(0, &table_metadata, &evolved_schema, true, filter) + .unwrap(); + assert!(Arc::ptr_eq(&partition_filter, &cached)); + } +} diff --git a/crates/iceberg/src/scan/context.rs b/crates/iceberg/src/scan/context.rs index 3f7c29dbf4..8188445123 100644 --- a/crates/iceberg/src/scan/context.rs +++ b/crates/iceberg/src/scan/context.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::sync::Arc; use futures::channel::mpsc::Sender; @@ -24,11 +25,12 @@ use crate::delete_file_index::DeleteFileIndex; use crate::expr::{Bind, BoundPredicate, Predicate}; use crate::io::object_cache::ObjectCache; use crate::scan::{ - BoundPredicates, ExpressionEvaluatorCache, FileScanTask, ManifestEvaluatorCache, - PartitionFilterCache, + BoundPredicates, ColumnStatsColumns, ExpressionEvaluatorCache, FileScanTask, + FileScanTaskColumnStats, ManifestEvaluatorCache, PartitionFilterCache, }; use crate::spec::{ - ManifestContentType, ManifestEntryRef, ManifestFile, ManifestList, SchemaRef, SnapshotRef, + ManifestContentType, ManifestEntryRef, ManifestFile, ManifestList, NameMapping, + PartitionSpecRef, PrimitiveType, SchemaRef, SnapshotRef, SortOrderRef, StructType, TableMetadataRef, }; use crate::{Error, ErrorKind, Result}; @@ -46,6 +48,12 @@ pub(crate) struct ManifestFileContext { snapshot_schema: SchemaRef, expression_evaluator_cache: Arc, delete_file_index: DeleteFileIndex, + name_mapping: Option>, + case_sensitive: bool, + partition_spec: Option, + unified_partition_type: Option>, + sort_orders: Arc>, + column_stats_columns: Option, } /// Wraps a [`ManifestEntryRef`] alongside the objects that are needed @@ -59,11 +67,39 @@ pub(crate) struct ManifestEntryContext { pub partition_spec_id: i32, pub snapshot_schema: SchemaRef, pub delete_file_index: DeleteFileIndex, + pub name_mapping: Option>, + pub case_sensitive: bool, + pub partition_spec: Option, + pub unified_partition_type: Option>, + pub sort_order_id: Option, + pub sort_order: Option, + pub column_stats_columns: Option, +} + +/// Returns whether any `int` or `float` field of `written` is widened in `current`, to +/// `long`/`double` or to `unknown` once its source column is dropped, i.e. partition +/// values written with `written` need normalizing before they compare against predicates +/// bound to the current schema or against delete-file partitions. +fn partition_types_differ(current: &StructType, written: &StructType) -> bool { + current.fields().iter().zip(written.fields()).any(|(c, w)| { + matches!( + ( + w.field_type.as_primitive_type(), + c.field_type.as_primitive_type() + ), + ( + Some(PrimitiveType::Int), + Some(PrimitiveType::Long | PrimitiveType::Unknown) + ) | ( + Some(PrimitiveType::Float), + Some(PrimitiveType::Double | PrimitiveType::Unknown) + ) + ) + }) } impl ManifestFileContext { - /// Consumes this [`ManifestFileContext`], fetching its Manifest from FileIO and then - /// streaming its constituent [`ManifestEntries`] to the channel provided in the context + /// Fetches the manifest and streams its live entries to the context's channel. pub(crate) async fn fetch_manifest_and_stream_manifest_entries(self) -> Result<()> { let ManifestFileContext { object_cache, @@ -74,21 +110,73 @@ impl ManifestFileContext { mut sender, expression_evaluator_cache, delete_file_index, - .. + name_mapping, + case_sensitive, + partition_spec, + unified_partition_type, + sort_orders, + column_stats_columns, } = self; let manifest = object_cache.get_manifest(&manifest_file).await?; + // A manifest written under an older schema stores partition values in that + // schema's (narrower) types. When the current partition type widens the one the + // manifest was written with (e.g. an int source column promoted to long), widen + // each entry's partition tuple here so the pruning evaluator and the delete-file + // index compare like-typed values. The gate compares the partition types + // themselves, not schema ids, since v1 may omit them (defaulting to 0). Column + // bounds are widened lazily where the metrics evaluator compares them. + let promotion_target: Option<(StructType, StructType)> = match partition_spec.as_ref() { + Some(spec) => { + let current = spec.partition_type(&snapshot_schema)?; + let written = spec.partition_type(manifest.metadata().schema())?; + partition_types_differ(¤t, &written).then_some((current, written)) + } + None => None, + }; + for manifest_entry in manifest.entries() { + if !manifest_entry.is_alive() { + continue; + } + + let manifest_entry = match promotion_target.as_ref() { + Some((current, written)) => { + let mut entry = manifest_entry.as_ref().clone(); + entry.data_file.partition = + entry.data_file.promoted_partition(current, written)?; + Arc::new(entry) + } + None => manifest_entry.clone(), + }; + + // Carry the raw id through unresolved, then resolve it to an order for the + // reader. The resolved order is `None` for the unsorted order (empty fields) so + // that `Some` always means genuinely sorted, matching Java's `isSorted()` gate; + // the raw id above keeps the "physically sorted but definition dropped" case + // recoverable. + let sort_order_id = manifest_entry.data_file().sort_order_id(); + let sort_order = sort_order_id + .and_then(|id| sort_orders.get(&(id as i64))) + .filter(|order| !order.is_unsorted()) + .cloned(); + let manifest_entry_context = ManifestEntryContext { - // TODO: refactor to avoid the expensive ManifestEntry clone - manifest_entry: manifest_entry.clone(), + manifest_entry, expression_evaluator_cache: expression_evaluator_cache.clone(), field_ids: field_ids.clone(), partition_spec_id: manifest_file.partition_spec_id, bound_predicates: bound_predicates.clone(), snapshot_schema: snapshot_schema.clone(), delete_file_index: delete_file_index.clone(), + name_mapping: name_mapping.clone(), + case_sensitive, + partition_spec: partition_spec.clone(), + unified_partition_type: unified_partition_type.clone(), + sort_order_id, + sort_order, + column_stats_columns: column_stats_columns.clone(), }; sender @@ -111,23 +199,52 @@ impl ManifestEntryContext { self.manifest_entry.data_file(), self.manifest_entry.sequence_number(), ) - .await; - - Ok(FileScanTask { - start: 0, - length: self.manifest_entry.file_size_in_bytes(), - record_count: Some(self.manifest_entry.record_count()), - - data_file_path: self.manifest_entry.file_path().to_string(), - data_file_format: self.manifest_entry.file_format(), - - schema: self.snapshot_schema, - project_field_ids: self.field_ids.to_vec(), - predicate: self - .bound_predicates - .map(|x| x.as_ref().snapshot_bound_predicate.clone()), + .await?; + + FileScanTask::builder() + .with_file_size_in_bytes(self.manifest_entry.file_size_in_bytes()) + .with_start(0) + .with_length(self.manifest_entry.file_size_in_bytes()) + .with_record_count(Some(self.manifest_entry.record_count())) + .with_first_row_id(self.manifest_entry.data_file().first_row_id()) + .with_data_sequence_number(self.manifest_entry.sequence_number()) + .with_data_file_path(self.manifest_entry.file_path().to_string()) + .with_data_file_format(self.manifest_entry.file_format()) + .with_schema(self.snapshot_schema) + .with_project_field_ids(self.field_ids.to_vec()) + .with_predicate( + self.bound_predicates + .map(|x| x.as_ref().snapshot_bound_predicate.clone()), + ) + .with_deletes(deletes) + .with_partition(Some(self.manifest_entry.data_file.partition.clone())) + .with_partition_spec(self.partition_spec.clone()) + .with_name_mapping(self.name_mapping) + .with_unified_partition_type(self.unified_partition_type.clone()) + .with_case_sensitive(self.case_sensitive) + .with_key_metadata(self.manifest_entry.data_file.key_metadata().map(Box::from)) + .with_sort_order_id(self.sort_order_id) + .with_sort_order(self.sort_order) + .with_column_stats(self.column_stats_columns.as_deref().map(|columns| { + FileScanTaskColumnStats::from_data_file(self.manifest_entry.data_file(), columns) + })) + .build() + } - deletes, + /// Consume this `ManifestEntryContext`, returning a COW rewrite file candidate. + /// + /// The scan task keeps everything planned for a normal read (projection, + /// delete files, partition metadata, ...) except the row predicate: the + /// candidate was already selected by that predicate during planning, and + /// the rewrite must read the whole file. + pub(crate) async fn into_cow_rewrite_file(self) -> Result { + let old_data_file = self.manifest_entry.data_file().clone(); + let mut scan_task = self.into_file_scan_task().await?; + scan_task.clear_predicate(); + + Ok(crate::cow_rewrite::CowRewriteFile { + old_data_file, + scan_task, }) } } @@ -139,16 +256,30 @@ pub(crate) struct PlanContext { pub snapshot: SnapshotRef, pub table_metadata: TableMetadataRef, + /// The schema the scan projects. For a scan of the current state, this is + /// the table's current schema, not the snapshot's; for time travel, it is + /// the schema the snapshot was written with. pub snapshot_schema: SchemaRef, pub case_sensitive: bool, pub predicate: Option>, pub snapshot_bound_predicate: Option>, pub object_cache: Arc, pub field_ids: Arc>, + pub name_mapping: Option>, pub partition_filter_cache: Arc, pub manifest_evaluator_cache: Arc, pub expression_evaluator_cache: Arc, + + pub unified_partition_type: Option>, + + /// The table's sort orders keyed by id, precomputed once so each + /// [`ManifestFileContext`] carries only this narrow map rather than the full table + /// metadata. Mirrors how `unified_partition_type` carries a precomputed value. + pub sort_orders: Arc>, + + /// Field ids whose column statistics each [`FileScanTask`] retains, or `None` to retain none. + pub column_stats_columns: Option, } impl PlanContext { @@ -159,6 +290,8 @@ impl PlanContext { .await } + /// Returns the partition filter for a manifest. See [`PartitionFilterCache::get`] for the + /// always-true fallback when the manifest's spec cannot be resolved against the scan schema. fn get_partition_filter(&self, manifest_file: &ManifestFile) -> Result> { let partition_spec_id = manifest_file.partition_spec_id; @@ -187,7 +320,17 @@ impl PlanContext { delete_file_idx: DeleteFileIndex, delete_file_tx: Sender, ) -> Result> + 'static>> { - let manifest_files = manifest_list.entries().iter(); + let mut manifest_files = manifest_list.entries().iter().collect::>(); + // Sort manifest files to process delete manifests first. + // This avoids a deadlock where the producer blocks on sending data manifest entries + // (because the data channel is full) while the delete manifest consumer is waiting + // for delete manifest entries (which haven't been produced yet). + // By processing delete manifests first, we ensure the delete consumer can finish, + // which then allows the data consumer to start draining the data channel. + manifest_files.sort_by_key(|m| match m.content { + ManifestContentType::Deletes => 0, + ManifestContentType::Data => 1, + }); // TODO: Ideally we could ditch this intermediate Vec as we return an iterator. let mut filtered_mfcs = vec![]; @@ -260,6 +403,15 @@ impl PlanContext { field_ids: self.field_ids.clone(), expression_evaluator_cache: self.expression_evaluator_cache.clone(), delete_file_index, + name_mapping: self.name_mapping.clone(), + case_sensitive: self.case_sensitive, + partition_spec: self + .table_metadata + .partition_spec_by_id(manifest_file.partition_spec_id) + .cloned(), + unified_partition_type: self.unified_partition_type.clone(), + sort_orders: self.sort_orders.clone(), + column_stats_columns: self.column_stats_columns.clone(), } } } diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 3d14b3cce4..1f488796c9 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -23,6 +23,8 @@ mod context; use context::*; mod task; +use std::collections::HashMap; +use std::future::Future; use std::sync::Arc; use arrow_array::RecordBatch; @@ -31,34 +33,157 @@ use futures::stream::BoxStream; use futures::{SinkExt, StreamExt, TryStreamExt}; pub use task::*; -use crate::arrow::ArrowReaderBuilder; +use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions}; +pub use crate::arrow::{ScanMetrics, ScanResult}; use crate::delete_file_index::DeleteFileIndex; +use crate::error::invalid_data; use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator; use crate::expr::{Bind, BoundPredicate, Predicate}; use crate::io::FileIO; -use crate::runtime::spawn; -use crate::spec::{DataContentType, SnapshotRef}; +use crate::metadata_columns::{ + RESERVED_FIELD_ID_PARTITION, get_metadata_field_id, is_metadata_column_name, +}; +use crate::runtime::Runtime; +use crate::spec::{DataContentType, Schema, SchemaRef, SnapshotRef, SortOrderRef, StructType}; use crate::table::Table; -use crate::utils::available_parallelism; +use crate::util::available_parallelism; +use crate::util::snapshot::snapshot_id_as_of_time; use crate::{Error, ErrorKind, Result}; /// A stream of arrow [`RecordBatch`]es. pub type ArrowRecordBatchStream = BoxStream<'static, Result>; +/// Resolves a column name to its field ID, honouring the scan's case sensitivity. +fn resolve_field_id( + schema: &Schema, + column_name: &str, + case_sensitive: bool, +) -> Result> { + if case_sensitive { + Ok(schema.field_id_by_name(column_name)) + } else { + let field = schema.field_by_name_case_insensitive_checked(column_name)?; + Ok(field.map(|field| field.id)) + } +} + +/// Resolves a column name to its field ID, failing if no field matches. +fn require_field_id(schema: &Schema, column_name: &str, case_sensitive: bool) -> Result { + resolve_field_id(schema, column_name, case_sensitive)? + .ok_or_else(|| invalid_data!("Column {column_name} not found in table. Schema: {schema}")) +} + +fn collect_scan_field_ids( + schema: &Schema, + column_names: Option<&[String]>, + case_sensitive: bool, +) -> Result> { + let Some(column_names) = column_names else { + return Ok(schema.as_struct().fields().iter().map(|f| f.id).collect()); + }; + + column_names + .iter() + .map(|column_name| { + if is_metadata_column_name(column_name) { + return get_metadata_field_id(column_name); + } + + let field_id = require_field_id(schema, column_name, case_sensitive)?; + + schema + .as_struct() + .field_by_id(field_id) + .ok_or_else(|| { + Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Column {column_name} is not a direct child of schema but a nested field, which is not supported now. Schema: {schema}" + ), + ) + })?; + + Ok(field_id) + }) + .collect() +} + +/// Resolves the columns to keep statistics for into their field ids and types. +fn resolve_column_stats_columns( + schema: &Schema, + column_names: Option<&[String]>, + case_sensitive: bool, +) -> Result> { + column_names + .map(|column_names| { + column_names + .iter() + .map(|column_name| { + if is_metadata_column_name(column_name) { + return Err(invalid_data!( + "Metadata column {column_name} has no column statistics" + )); + } + let field_id = require_field_id(schema, column_name, case_sensitive)?; + let column_type = schema + .field_by_id(field_id) + .and_then(|field| field.field_type.as_primitive_type()) + .ok_or_else(|| { + invalid_data!( + "Column {column_name} is not a primitive field and has no column statistics" + ) + })?; + Ok((field_id, column_type.clone())) + }) + .collect() + }) + .transpose() +} + +fn bind_scan_predicate( + schema: &SchemaRef, + predicate: Option<&Predicate>, + case_sensitive: bool, +) -> Result>> { + predicate + .map(|predicate| predicate.bind(schema.clone(), case_sensitive)) + .transpose() + .map(|predicate| predicate.map(Arc::new)) +} + +fn projected_partition_type( + table: &Table, + schema: &Schema, + field_ids: &[i32], +) -> Result>> { + if !field_ids.contains(&RESERVED_FIELD_ID_PARTITION) { + return Ok(None); + } + + table + .metadata() + .unified_partition_type(schema) + .map(Arc::new) + .map(Some) +} + +/// Selects a snapshot by ID or timestamp. +enum SnapshotSelection { + SnapshotId(i64), + AsOfTime(i64), +} + /// Builder to create table scan. pub struct TableScanBuilder<'a> { table: &'a Table, // Defaults to none which means select all columns column_names: Option>, - snapshot_id: Option, - batch_size: Option, + column_stats_names: Option>, + snapshot_selection: Option, case_sensitive: bool, filter: Option, - concurrency_limit_data_files: usize, concurrency_limit_manifest_entries: usize, concurrency_limit_manifest_files: usize, - row_group_filtering_enabled: bool, - row_selection_enabled: bool, } impl<'a> TableScanBuilder<'a> { @@ -68,25 +193,15 @@ impl<'a> TableScanBuilder<'a> { Self { table, column_names: None, - snapshot_id: None, - batch_size: None, + column_stats_names: None, + snapshot_selection: None, case_sensitive: true, filter: None, - concurrency_limit_data_files: num_cpus, concurrency_limit_manifest_entries: num_cpus, concurrency_limit_manifest_files: num_cpus, - row_group_filtering_enabled: true, - row_selection_enabled: false, } } - /// Sets the desired size of batches in the response - /// to something other than the default - pub fn with_batch_size(mut self, batch_size: Option) -> Self { - self.batch_size = batch_size; - self - } - /// Sets the scan's case sensitivity pub fn with_case_sensitive(mut self, case_sensitive: bool) -> Self { self.case_sensitive = case_sensitive; @@ -124,156 +239,167 @@ impl<'a> TableScanBuilder<'a> { self } - /// Set the snapshot to scan. When not set, it uses current snapshot. - pub fn snapshot_id(mut self, snapshot_id: i64) -> Self { - self.snapshot_id = Some(snapshot_id); + /// Retains the data file's column statistics for the named columns on each planned + /// [`FileScanTask`], exposed as [`FileScanTask::column_stats`]. + /// + /// Names resolve against the scan's schema with the scan's case sensitivity and must name + /// primitive fields, such as `s.f` for a field of struct `s`. By default, tasks carry no + /// statistics. + /// + /// [`build`](Self::build) fails if a name matches no field, matches more than one field when + /// the scan is case-insensitive, or names a struct, list, map or metadata column. + /// + /// ``` + /// # use futures::TryStreamExt; + /// # use iceberg::table::Table; + /// # async fn example(table: &Table) -> iceberg::Result<()> { + /// let scan = table.scan().with_column_stats(["x"]).build()?; + /// let mut tasks = scan.plan_files().await?; + /// while let Some(task) = tasks.try_next().await? { + /// let x = task.schema().field_id_by_name("x").unwrap(); + /// let lower_bound = task + /// .column_stats() + /// .and_then(|stats| stats.lower_bounds().get(&x)); + /// } + /// # Ok(()) + /// # } + /// ``` + pub fn with_column_stats( + mut self, + column_names: impl IntoIterator, + ) -> Self { + self.column_stats_names = Some( + column_names + .into_iter() + .map(|item| item.to_string()) + .collect(), + ); self } - /// Sets the concurrency limit for both manifest files and manifest - /// entries for this scan - pub fn with_concurrency_limit(mut self, limit: usize) -> Self { - self.concurrency_limit_manifest_files = limit; - self.concurrency_limit_manifest_entries = limit; - self.concurrency_limit_data_files = limit; + /// Set the snapshot to scan. The scan projects the schema that snapshot was + /// written with. + /// + /// The last call to this or [`Self::as_of_time`] takes precedence. + /// If neither is set, the scan reads the current snapshot with the table's + /// current schema. + pub fn snapshot_id(mut self, snapshot_id: i64) -> Self { + self.snapshot_selection = Some(SnapshotSelection::SnapshotId(snapshot_id)); self } - /// Sets the data file concurrency limit for this scan - pub fn with_data_file_concurrency_limit(mut self, limit: usize) -> Self { - self.concurrency_limit_data_files = limit; + /// Select a snapshot at or before a timestamp in milliseconds since the Unix epoch. + /// + /// Uses the table's main history, and projects the schema the selected snapshot + /// was written with. The last call to this or [`Self::snapshot_id`] takes precedence. + /// + /// ``` + /// # use iceberg::TableIdent; + /// # use iceberg::io::FileIO; + /// # use iceberg::table::StaticTable; + /// # #[tokio::main] + /// # async fn main() -> iceberg::Result<()> { + /// # let location = concat!(env!("CARGO_MANIFEST_DIR"), "/testdata/example_table_metadata_v2.json"); + /// # let ident = TableIdent::from_strs(["ns", "t"])?; + /// # let table = StaticTable::from_metadata_file(location, ident, FileIO::new_with_fs()).await?; + /// // The snapshot log has entries at 1515100955770 and 1555100955770. + /// let scan = table.scan().as_of_time(1_555_100_955_769).build()?; + /// assert_eq!(scan.snapshot().unwrap().snapshot_id(), 3051729675574597004); + /// # Ok(()) + /// # } + /// ``` + pub fn as_of_time(mut self, timestamp_ms: i64) -> Self { + self.snapshot_selection = Some(SnapshotSelection::AsOfTime(timestamp_ms)); self } - /// Sets the manifest entry concurrency limit for this scan - pub fn with_manifest_entry_concurrency_limit(mut self, limit: usize) -> Self { + /// Sets manifest file and manifest entry concurrency for file planning. + pub fn with_concurrency_limit(mut self, limit: usize) -> Self { + self.concurrency_limit_manifest_files = limit; self.concurrency_limit_manifest_entries = limit; self } - /// Determines whether to enable row group filtering. - /// When enabled, if a read is performed with a filter predicate, - /// then the metadata for each row group in the parquet file is - /// evaluated against the filter predicate and row groups - /// that cant contain matching rows will be skipped entirely. - /// - /// Defaults to enabled, as it generally improves performance or - /// keeps it the same, with performance degradation unlikely. - pub fn with_row_group_filtering_enabled(mut self, row_group_filtering_enabled: bool) -> Self { - self.row_group_filtering_enabled = row_group_filtering_enabled; - self - } - - /// Determines whether to enable row selection. - /// When enabled, if a read is performed with a filter predicate, - /// then (for row groups that have not been skipped) the page index - /// for each row group in a parquet file is parsed and evaluated - /// against the filter predicate to determine if ranges of rows - /// within a row group can be skipped, based upon the page-level - /// statistics for each column. - /// - /// Defaults to being disabled. Enabling requires parsing the parquet page - /// index, which can be slow enough that parsing the page index outweighs any - /// gains from the reduced number of rows that need scanning. - /// It is recommended to experiment with partitioning, sorting, row group size, - /// page size, and page row limit Iceberg settings on the table being scanned in - /// order to get the best performance from using row selection. - pub fn with_row_selection_enabled(mut self, row_selection_enabled: bool) -> Self { - self.row_selection_enabled = row_selection_enabled; + /// Sets the manifest entry concurrency limit for this scan + pub fn with_manifest_entry_concurrency_limit(mut self, limit: usize) -> Self { + self.concurrency_limit_manifest_entries = limit; self } /// Build the table scan. pub fn build(self) -> Result { - let snapshot = match self.snapshot_id { - Some(snapshot_id) => self + let selected_snapshot = match self.snapshot_selection { + Some(SnapshotSelection::SnapshotId(id)) => { + Some((id, format!("Snapshot with id {id} not found"))) + } + Some(SnapshotSelection::AsOfTime(timestamp_ms)) => { + let id = snapshot_id_as_of_time(&self.table.metadata_ref(), timestamp_ms)?; + // History outlives expired snapshots; name the timestamp that resolved to `id`. + Some(( + id, + format!( + "Snapshot with id {id} resolved from timestamp {timestamp_ms} ms not found; it may have expired" + ), + )) + } + None => None, + }; + let is_time_travel = selected_snapshot.is_some(); + let snapshot = match selected_snapshot { + Some((snapshot_id, not_found_message)) => self .table .metadata() .snapshot_by_id(snapshot_id) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Snapshot with id {} not found", snapshot_id), - ) - })? + .ok_or_else(|| invalid_data!(not_found_message))? .clone(), None => { let Some(current_snapshot_id) = self.table.metadata().current_snapshot() else { return Ok(TableScan { - batch_size: self.batch_size, column_names: self.column_names, file_io: self.table.file_io().clone(), plan_context: None, - concurrency_limit_data_files: self.concurrency_limit_data_files, concurrency_limit_manifest_entries: self.concurrency_limit_manifest_entries, concurrency_limit_manifest_files: self.concurrency_limit_manifest_files, - row_group_filtering_enabled: self.row_group_filtering_enabled, - row_selection_enabled: self.row_selection_enabled, + runtime: self.table.runtime().clone(), }); }; current_snapshot_id.clone() } }; - let schema = snapshot.schema(self.table.metadata())?; - - // Check that all column names exist in the schema. - if let Some(column_names) = self.column_names.as_ref() { - for column_name in column_names { - if schema.field_by_name(column_name).is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Column {} not found in table. Schema: {}", - column_name, schema - ), - )); - } - } - } - - let mut field_ids = vec![]; - let column_names = self.column_names.clone().unwrap_or_else(|| { - schema - .as_struct() - .fields() - .iter() - .map(|f| f.name.clone()) - .collect() - }); - - for column_name in column_names.iter() { - let field_id = schema.field_id_by_name(column_name).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Column {} not found in table. Schema: {}", - column_name, schema - ), - ) - })?; - - schema - .as_struct() - .field_by_id(field_id) - .ok_or_else(|| { - Error::new( - ErrorKind::FeatureUnsupported, - format!( - "Column {} is not a direct child of schema but a nested field, which is not supported now. Schema: {}", - column_name, schema - ), - ) - })?; - - field_ids.push(field_id); - } - - let snapshot_bound_predicate = if let Some(ref predicates) = self.filter { - Some(predicates.bind(schema.clone(), true)?) + // A scan of the current state uses the table's current schema; time travel uses the + // schema its snapshot was written with. + let schema = if is_time_travel { + snapshot.schema(self.table.metadata())? } else { - None + self.table.metadata().current_schema().clone() }; + let field_ids = + collect_scan_field_ids(&schema, self.column_names.as_deref(), self.case_sensitive)?; + let column_stats_columns = resolve_column_stats_columns( + &schema, + self.column_stats_names.as_deref(), + self.case_sensitive, + )?; + let snapshot_bound_predicate = + bind_scan_predicate(&schema, self.filter.as_ref(), self.case_sensitive)?; + let name_mapping = self + .table + .metadata() + .table_properties() + .default_name_mapping()? + .map(Arc::new); + let unified_partition_type = projected_partition_type(self.table, &schema, &field_ids)?; + + // Precompute the table's sort orders once, keyed by id, so each manifest-file + // context carries only this narrow map instead of the full table metadata. + let sort_orders = Arc::new( + self.table + .metadata() + .sort_orders_iter() + .map(|order| (order.order_id, order.clone())) + .collect::>(), + ); let plan_context = PlanContext { snapshot, @@ -281,24 +407,25 @@ impl<'a> TableScanBuilder<'a> { snapshot_schema: schema, case_sensitive: self.case_sensitive, predicate: self.filter.map(Arc::new), - snapshot_bound_predicate: snapshot_bound_predicate.map(Arc::new), + snapshot_bound_predicate, object_cache: self.table.object_cache(), field_ids: Arc::new(field_ids), + name_mapping, partition_filter_cache: Arc::new(PartitionFilterCache::new()), manifest_evaluator_cache: Arc::new(ManifestEvaluatorCache::new()), expression_evaluator_cache: Arc::new(ExpressionEvaluatorCache::new()), + unified_partition_type, + sort_orders, + column_stats_columns, }; Ok(TableScan { - batch_size: self.batch_size, column_names: self.column_names, file_io: self.table.file_io().clone(), plan_context: Some(plan_context), - concurrency_limit_data_files: self.concurrency_limit_data_files, concurrency_limit_manifest_entries: self.concurrency_limit_manifest_entries, concurrency_limit_manifest_files: self.concurrency_limit_manifest_files, - row_group_filtering_enabled: self.row_group_filtering_enabled, - row_selection_enabled: self.row_selection_enabled, + runtime: self.table.runtime().clone(), }) } } @@ -310,7 +437,6 @@ pub struct TableScan { /// /// If this is None, then the scan contains no rows. plan_context: Option, - batch_size: Option, file_io: FileIO, column_names: Option>, /// The maximum number of manifest files that will be @@ -321,17 +447,31 @@ pub struct TableScan { /// be processed in parallel concurrency_limit_manifest_entries: usize, - /// The maximum number of [`ManifestEntry`]s that will - /// be processed in parallel - concurrency_limit_data_files: usize, - - row_group_filtering_enabled: bool, - row_selection_enabled: bool, + runtime: Runtime, } impl TableScan { /// Returns a stream of [`FileScanTask`]s. pub async fn plan_files(&self) -> Result { + self.plan_data_files(|ctx| ctx.into_file_scan_task()).await + } + + pub(crate) async fn plan_cow_rewrite_files( + &self, + ) -> Result>> { + self.plan_data_files(|ctx| ctx.into_cow_rewrite_file()) + .await + } + + async fn plan_data_files( + &self, + build_result: F, + ) -> Result>> + where + T: Send + 'static, + F: Fn(ManifestEntryContext) -> Fut + Copy + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { let Some(plan_context) = self.plan_context.as_ref() else { return Ok(Box::pin(futures::stream::empty())); }; @@ -345,10 +485,10 @@ impl TableScan { let (manifest_entry_delete_ctx_tx, manifest_entry_delete_ctx_rx) = channel(concurrency_limit_manifest_files); - // used to stream the results back to the caller - let (file_scan_task_tx, file_scan_task_rx) = channel(concurrency_limit_manifest_entries); + // used to stream the planned data file results back to the caller + let (planned_file_tx, planned_file_rx) = channel(concurrency_limit_manifest_entries); - let (delete_file_idx, delete_file_tx) = DeleteFileIndex::new(); + let (delete_file_idx, delete_file_tx) = DeleteFileIndex::new(self.runtime.clone()); let manifest_list = plan_context.get_manifest_list().await?; @@ -362,10 +502,14 @@ impl TableScan { manifest_entry_delete_ctx_tx, )?; - let mut channel_for_manifest_error = file_scan_task_tx.clone(); + let mut channel_for_manifest_error = planned_file_tx.clone(); + let mut channel_for_data_manifest_entry_error = planned_file_tx.clone(); + let mut channel_for_delete_manifest_entry_error = planned_file_tx.clone(); + + let rt = self.runtime.clone(); // Concurrently load all [`Manifest`]s and stream their [`ManifestEntry`]s - spawn(async move { + rt.io().spawn(async move { let result = futures::stream::iter(manifest_file_contexts) .try_for_each_concurrent(concurrency_limit_manifest_files, |ctx| async move { ctx.fetch_manifest_and_stream_manifest_entries().await @@ -377,67 +521,118 @@ impl TableScan { } }); - let mut channel_for_data_manifest_entry_error = file_scan_task_tx.clone(); - let mut channel_for_delete_manifest_entry_error = file_scan_task_tx.clone(); - // Process the delete file [`ManifestEntry`] stream in parallel - spawn(async move { - let result = manifest_entry_delete_ctx_rx - .map(|me_ctx| Ok((me_ctx, delete_file_tx.clone()))) - .try_for_each_concurrent( - concurrency_limit_manifest_entries, - |(manifest_entry_context, tx)| async move { - spawn(async move { - Self::process_delete_manifest_entry(manifest_entry_context, tx).await - }) - .await - }, - ) - .await; - - if let Err(error) = result { - let _ = channel_for_delete_manifest_entry_error - .send(Err(error)) + { + let rt = rt.clone(); + let rt_inner = rt.clone(); + rt.cpu().spawn(async move { + let result = manifest_entry_delete_ctx_rx + .map(|me_ctx| Ok((me_ctx, delete_file_tx.clone()))) + .try_for_each_concurrent( + concurrency_limit_manifest_entries, + |(manifest_entry_context, tx)| { + let rt_inner = rt_inner.clone(); + async move { + rt_inner + .cpu() + .spawn(async move { + Self::process_delete_manifest_entry( + manifest_entry_context, + tx, + ) + .await + }) + .await? + } + }, + ) .await; - } - }) - .await; + + if let Err(error) = result { + let _ = channel_for_delete_manifest_entry_error + .send(Err(error)) + .await; + } + }); + } // Process the data file [`ManifestEntry`] stream in parallel - spawn(async move { - let result = manifest_entry_data_ctx_rx - .map(|me_ctx| Ok((me_ctx, file_scan_task_tx.clone()))) - .try_for_each_concurrent( - concurrency_limit_manifest_entries, - |(manifest_entry_context, tx)| async move { - spawn(async move { - Self::process_data_manifest_entry(manifest_entry_context, tx).await - }) - .await - }, - ) - .await; + { + let rt_inner = rt.clone(); + rt.cpu().spawn(async move { + let result = manifest_entry_data_ctx_rx + .map(|me_ctx| Ok((me_ctx, planned_file_tx.clone()))) + .try_for_each_concurrent( + concurrency_limit_manifest_entries, + |(manifest_entry_context, tx)| { + let rt_inner = rt_inner.clone(); + async move { + rt_inner + .cpu() + .spawn(async move { + Self::process_data_manifest_entry( + manifest_entry_context, + tx, + build_result, + ) + .await + }) + .await? + } + }, + ) + .await; - if let Err(error) = result { - let _ = channel_for_data_manifest_entry_error.send(Err(error)).await; - } - }); + if let Err(error) = result { + let _ = channel_for_data_manifest_entry_error.send(Err(error)).await; + } + }); + } - Ok(file_scan_task_rx.boxed()) + Ok(planned_file_rx.boxed()) } - /// Returns an [`ArrowRecordBatchStream`]. - pub async fn to_arrow(&self) -> Result { - let mut arrow_reader_builder = ArrowReaderBuilder::new(self.file_io.clone()) - .with_data_file_concurrency_limit(self.concurrency_limit_data_files) - .with_row_group_filtering_enabled(self.row_group_filtering_enabled) - .with_row_selection_enabled(self.row_selection_enabled); - - if let Some(batch_size) = self.batch_size { - arrow_reader_builder = arrow_reader_builder.with_batch_size(batch_size); - } - - arrow_reader_builder.build().read(self.plan_files().await?) + /// Returns an [`ArrowRecordBatchStream`] using the supplied reader settings. + /// + /// Reader settings are independent of file planning. Unset data-file concurrency + /// uses execution-host parallelism; zero is treated as one. + /// [`TableScanBuilder::with_concurrency_limit`] only controls manifest planning. + /// + /// ```no_run + /// # use iceberg::{Result, arrow::{ArrowReaderOptions, ParquetReadOptions}, table::Table}; + /// # async fn read(table: &Table) -> Result<()> { + /// let options = ArrowReaderOptions::builder() + /// .with_batch_size(1024) + /// .with_row_selection_enabled(true) + /// .with_parquet_options( + /// ParquetReadOptions::builder() + /// .with_metadata_size_hint(512 * 1024) + /// .build(), + /// ) + /// .build(); + /// let stream = table.scan().build()?.to_arrow(options).await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// When upgrading, move reader-setting calls from `TableScanBuilder` to + /// `ArrowReaderOptions::builder()` and pass the result to `to_arrow(options)`. + /// Use `to_arrow(Default::default())` for default settings. To preserve a + /// data-file concurrency bound previously set through the scan, set + /// `with_data_file_concurrency_limit(limit)` on the reader options explicitly. + /// + /// Locally constructed readers accept the same options through + /// [`ArrowReaderBuilder::with_options`], including Parquet I/O tuning. + /// This replaces the complete configuration. Configure individual settings + /// through the options builders; per-field setters on `ArrowReaderBuilder` + /// have been removed. [`ArrowReaderBuilder::with_parquet_options`] replaces + /// only the nested Parquet configuration. + pub async fn to_arrow(&self, options: ArrowReaderOptions) -> Result { + ArrowReaderBuilder::new(self.file_io.clone(), self.runtime.clone()) + .with_options(options) + .build() + .read(self.plan_files().await?) + .map(|result| result.stream()) } /// Returns a reference to the column names of the table scan. @@ -450,15 +645,15 @@ impl TableScan { self.plan_context.as_ref().map(|x| &x.snapshot) } - async fn process_data_manifest_entry( + async fn process_data_manifest_entry( manifest_entry_context: ManifestEntryContext, - mut file_scan_task_tx: Sender>, - ) -> Result<()> { - // skip processing this manifest entry if it has been marked as deleted - if !manifest_entry_context.manifest_entry.is_alive() { - return Ok(()); - } - + mut planned_file_tx: Sender>, + build_result: F, + ) -> Result<()> + where + F: Fn(ManifestEntryContext) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { // abort the plan if we encounter a manifest entry for a delete file if manifest_entry_context.manifest_entry.content_type() != DataContentType::Data { return Err(Error::new( @@ -498,10 +693,10 @@ impl TableScan { } // congratulations! the manifest entry has made its way through the - // entire plan without getting filtered out. Create a corresponding - // FileScanTask and push it to the result stream - file_scan_task_tx - .send(Ok(manifest_entry_context.into_file_scan_task().await?)) + // entire plan without getting filtered out. Build the planned file before + // sending so delete-file lookup preserves the original scan timing. + planned_file_tx + .send(Ok(build_result(manifest_entry_context).await?)) .await?; Ok(()) @@ -511,11 +706,6 @@ impl TableScan { manifest_entry_context: ManifestEntryContext, mut delete_file_ctx_tx: Sender, ) -> Result<()> { - // skip processing this manifest entry if it has been marked as deleted - if !manifest_entry_context.manifest_entry.is_alive() { - return Ok(()); - } - // abort the plan if we encounter a manifest entry that is not for a delete file if manifest_entry_context.manifest_entry.content_type() == DataContentType::Data { return Err(Error::new( @@ -557,647 +747,277 @@ pub(crate) struct BoundPredicates { } #[cfg(test)] -pub mod tests { +mod tests { //! shared tests for the table scan API - #![allow(missing_docs)] use std::collections::HashMap; use std::fs; use std::fs::File; use std::sync::Arc; + use arrow_array::cast::AsArray; + use arrow_array::types::Int32Type; use arrow_array::{ - ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray, + Array, ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RecordBatch, RunArray, + StringArray, }; use futures::{TryStreamExt, stream}; - use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; - use parquet::basic::Compression; - use parquet::file::properties::WriterProperties; - use tempfile::TempDir; - use tera::{Context, Tera}; + use parquet::arrow::ArrowWriter; use uuid::Uuid; - use crate::TableIdent; - use crate::arrow::ArrowReaderBuilder; + use super::{resolve_column_stats_columns, resolve_field_id}; + use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions, ParquetReadOptions}; use crate::expr::{BoundPredicate, Reference}; - use crate::io::{FileIO, OutputFile}; - use crate::scan::FileScanTask; + use crate::io::FileIO; + use crate::metadata_columns::{ + RESERVED_COL_NAME_FILE, RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, + RESERVED_COL_NAME_POS, RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_POS, + }; + use crate::scan::{FileScanTask, FileScanTaskColumnStats, FileScanTaskDeleteFile, TableScan}; use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Datum, Literal, ManifestEntry, - ManifestListWriter, ManifestStatus, ManifestWriterBuilder, NestedField, PartitionSpec, - PrimitiveType, Schema, Struct, StructType, TableMetadata, Type, + DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MAIN_BRANCH, + ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, MappedField, + NameMapping, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema, + Snapshot, SortDirection, SortField, SortOrder, Struct, StructType, Summary, + TableMetadataBuilder, TableProperties, Transform, Type, UnboundPartitionSpec, }; use crate::table::Table; + use crate::test_utils::scan::{ + ROWS_PER_FIXTURE_FILE, TableTestFixture, assert_last_updated_seq_all, + }; + use crate::test_utils::test_runtime; + use crate::util::snapshot::snapshot_id_as_of_time; + use crate::{ErrorKind, Runtime, TableIdent}; - pub struct TableTestFixture { - pub table_location: String, - pub table: Table, - } - - impl TableTestFixture { - #[allow(clippy::new_without_default)] - pub fn new() -> Self { - let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().join("table1"); - let manifest_list1_location = table_location.join("metadata/manifests_list_1.avro"); - let manifest_list2_location = table_location.join("metadata/manifests_list_2.avro"); - let table_metadata1_location = table_location.join("metadata/v1.json"); - - let file_io = FileIO::from_path(table_location.as_os_str().to_str().unwrap()) - .unwrap() - .build() - .unwrap(); + #[tokio::test] + async fn test_table_scan_columns() { + let table = TableTestFixture::new().table; - let table_metadata = { - let template_json_str = fs::read_to_string(format!( - "{}/testdata/example_table_metadata_v2.json", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(); - let mut context = Context::new(); - context.insert("table_location", &table_location); - context.insert("manifest_list_1_location", &manifest_list1_location); - context.insert("manifest_list_2_location", &manifest_list2_location); - context.insert("table_metadata_1_location", &table_metadata1_location); - - let metadata_json = Tera::one_off(&template_json_str, &context, false).unwrap(); - serde_json::from_str::(&metadata_json).unwrap() - }; + let table_scan = table.scan().select(["x", "y"]).build().unwrap(); + assert_eq!( + Some(vec!["x".to_string(), "y".to_string()]), + table_scan.column_names + ); - let table = Table::builder() - .metadata(table_metadata) - .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) - .file_io(file_io.clone()) - .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) - .build() - .unwrap(); + let table_scan = table + .scan() + .select(["x", "y"]) + .select(["z"]) + .build() + .unwrap(); + assert_eq!(Some(vec!["z".to_string()]), table_scan.column_names); + } - Self { - table_location: table_location.to_str().unwrap().to_string(), - table, - } - } + #[tokio::test] + async fn test_select_all() { + let table = TableTestFixture::new().table; - #[allow(clippy::new_without_default)] - pub fn new_empty() -> Self { - let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().join("table1"); - let table_metadata1_location = table_location.join("metadata/v1.json"); + let table_scan = table.scan().select_all().build().unwrap(); + assert!(table_scan.column_names.is_none()); + } - let file_io = FileIO::from_path(table_location.as_os_str().to_str().unwrap()) - .unwrap() - .build() - .unwrap(); + #[test] + fn test_select_no_exist_column() { + let table = TableTestFixture::new().table; - let table_metadata = { - let template_json_str = fs::read_to_string(format!( - "{}/testdata/example_empty_table_metadata_v2.json", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(); - let mut context = Context::new(); - context.insert("table_location", &table_location); - context.insert("table_metadata_1_location", &table_metadata1_location); + let table_scan = table.scan().select(["x", "y", "z", "a", "b"]).build(); + assert!(table_scan.is_err()); + } - let metadata_json = Tera::one_off(&template_json_str, &context, false).unwrap(); - serde_json::from_str::(&metadata_json).unwrap() - }; + #[test] + fn test_case_sensitive_scan_rejects_mismatched_column_case() { + let table = TableTestFixture::new().table; - let table = Table::builder() - .metadata(table_metadata) - .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) - .file_io(file_io.clone()) - .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) + // Case sensitivity defaults to true, so "X" must not resolve to "x". + assert!(table.scan().select(["X"]).build().is_err()); + assert!( + table + .scan() + .with_filter(Reference::new("X").greater_than(Datum::long(1))) .build() - .unwrap(); - - Self { - table_location: table_location.to_str().unwrap().to_string(), - table, - } - } + .is_err() + ); + } - pub fn new_unpartitioned() -> Self { - let tmp_dir = TempDir::new().unwrap(); - let table_location = tmp_dir.path().join("table1"); - let manifest_list1_location = table_location.join("metadata/manifests_list_1.avro"); - let manifest_list2_location = table_location.join("metadata/manifests_list_2.avro"); - let table_metadata1_location = table_location.join("metadata/v1.json"); + /// A schema with a struct `s`, a list `l` and a long `n`, whose field ids are 1 to 5. + fn nested_schema() -> Schema { + Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "s", + Type::Struct(StructType::new(vec![ + NestedField::optional(2, "f", PrimitiveType::Long).into(), + ])), + ) + .into(), + NestedField::required( + 3, + "l", + Type::List(ListType::new( + NestedField::list_element(4, PrimitiveType::Long, true).into(), + )), + ) + .into(), + NestedField::required(5, "n", PrimitiveType::Long).into(), + ]) + .build() + .unwrap() + } - let file_io = FileIO::from_path(table_location.to_str().unwrap()) - .unwrap() - .build() - .unwrap(); + fn column_names(names: &[&str]) -> Vec { + names.iter().map(|name| name.to_string()).collect() + } - let mut table_metadata = { - let template_json_str = fs::read_to_string(format!( - "{}/testdata/example_table_metadata_v2.json", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(); - let mut context = Context::new(); - context.insert("table_location", &table_location); - context.insert("manifest_list_1_location", &manifest_list1_location); - context.insert("manifest_list_2_location", &manifest_list2_location); - context.insert("table_metadata_1_location", &table_metadata1_location); - - let metadata_json = Tera::one_off(&template_json_str, &context, false).unwrap(); - serde_json::from_str::(&metadata_json).unwrap() - }; + #[test] + fn test_resolve_column_stats_columns_resolves_primitive_names() { + let schema = nested_schema(); + let long = PrimitiveType::Long; - table_metadata.default_spec = Arc::new(PartitionSpec::unpartition_spec()); - table_metadata.partition_specs.clear(); - table_metadata.default_partition_type = StructType::new(vec![]); - table_metadata - .partition_specs - .insert(0, table_metadata.default_spec.clone()); - - let table = Table::builder() - .metadata(table_metadata) - .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) - .file_io(file_io.clone()) - .metadata_location(table_metadata1_location.to_str().unwrap()) - .build() - .unwrap(); + assert_eq!( + resolve_column_stats_columns(&schema, Some(&column_names(&["s.f", "n"])), true) + .unwrap(), + Some(Arc::<[_]>::from([(2, long.clone()), (5, long.clone())])) + ); + assert_eq!( + resolve_column_stats_columns(&schema, Some(&column_names(&["N"])), false).unwrap(), + Some(Arc::<[_]>::from([(5, long)])) + ); + assert_eq!( + resolve_column_stats_columns(&schema, None, true).unwrap(), + None + ); + } - Self { - table_location: table_location.to_str().unwrap().to_string(), - table, - } + #[test] + fn test_resolve_column_stats_columns_rejects_names_without_statistics() { + let schema = nested_schema(); + let cases = [ + ( + "s", + "Column s is not a primitive field and has no column statistics", + ), + ( + "l", + "Column l is not a primitive field and has no column statistics", + ), + ( + RESERVED_COL_NAME_FILE, + "Metadata column _file has no column statistics", + ), + ("missing", "Column missing not found in table"), + ("N", "Column N not found in table"), + ]; + + for (name, expected) in cases { + let err = resolve_column_stats_columns(&schema, Some(&column_names(&[name])), true) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid, "{name}"); + assert!(err.message().starts_with(expected), "{name}: {err}"); } + } - fn next_manifest_file(&self) -> OutputFile { - self.table - .file_io() - .new_output(format!( - "{}/metadata/manifest_{}.avro", - self.table_location, - Uuid::new_v4() - )) - .unwrap() - } + #[test] + fn test_resolve_field_id_case_insensitive_rejects_ambiguous_name() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", PrimitiveType::Int).into(), + NestedField::optional(2, "ID", PrimitiveType::Int).into(), + NestedField::optional(3, "data", PrimitiveType::Int).into(), + ]) + .build() + .unwrap(); - pub async fn setup_manifest_files(&mut self) { - let current_snapshot = self.table.metadata().current_snapshot().unwrap(); - let parent_snapshot = current_snapshot - .parent_snapshot(self.table.metadata()) - .unwrap(); - let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); - let current_partition_spec = self.table.metadata().default_partition_spec(); - - // Write data files - let mut writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - None, - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_data(); - writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/1.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(100) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long(100))])) - .key_metadata(None) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - writer - .add_delete_entry( - ManifestEntry::builder() - .status(ManifestStatus::Deleted) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/2.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(100) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long(200))])) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - writer - .add_existing_entry( - ManifestEntry::builder() - .status(ManifestStatus::Existing) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/3.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(100) - .record_count(1) - .partition(Struct::from_iter([Some(Literal::long(300))])) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - let data_file_manifest = writer.write_manifest_file().await.unwrap(); + assert_eq!(resolve_field_id(&schema, "id", true).unwrap(), Some(1)); + assert_eq!(resolve_field_id(&schema, "DATA", false).unwrap(), Some(3)); + let error = resolve_field_id(&schema, "Id", false).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.message().contains("case-insensitively"), "{error}"); + } - // Write to manifest list - let mut manifest_list_write = ManifestListWriter::v2( - self.table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap(), - current_snapshot.snapshot_id(), - current_snapshot.parent_snapshot_id(), - current_snapshot.sequence_number(), - ); - manifest_list_write - .add_manifests(vec![data_file_manifest].into_iter()) - .unwrap(); - manifest_list_write.close().await.unwrap(); - - // prepare data - let schema = { - let fields = vec![ - arrow_schema::Field::new("x", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - arrow_schema::Field::new("y", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - arrow_schema::Field::new("z", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - arrow_schema::Field::new("a", arrow_schema::DataType::Utf8, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "4".to_string(), - )])), - arrow_schema::Field::new("dbl", arrow_schema::DataType::Float64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "5".to_string(), - )])), - arrow_schema::Field::new("i32", arrow_schema::DataType::Int32, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "6".to_string(), - )])), - arrow_schema::Field::new("i64", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "7".to_string(), - )])), - arrow_schema::Field::new("bool", arrow_schema::DataType::Boolean, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "8".to_string(), - )])), - ]; - Arc::new(arrow_schema::Schema::new(fields)) - }; - // x: [1, 1, 1, 1, ...] - let col1 = Arc::new(Int64Array::from_iter_values(vec![1; 1024])) as ArrayRef; - - let mut values = vec![2; 512]; - values.append(vec![3; 200].as_mut()); - values.append(vec![4; 300].as_mut()); - values.append(vec![5; 12].as_mut()); - - // y: [2, 2, 2, 2, ..., 3, 3, 3, 3, ..., 4, 4, 4, 4, ..., 5, 5, 5, 5] - let col2 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - - let mut values = vec![3; 512]; - values.append(vec![4; 512].as_mut()); - - // z: [3, 3, 3, 3, ..., 4, 4, 4, 4] - let col3 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - - // a: ["Apache", "Apache", "Apache", ..., "Iceberg", "Iceberg", "Iceberg"] - let mut values = vec!["Apache"; 512]; - values.append(vec!["Iceberg"; 512].as_mut()); - let col4 = Arc::new(StringArray::from_iter_values(values)) as ArrayRef; - - // dbl: - let mut values = vec![100.0f64; 512]; - values.append(vec![150.0f64; 12].as_mut()); - values.append(vec![200.0f64; 500].as_mut()); - let col5 = Arc::new(Float64Array::from_iter_values(values)) as ArrayRef; - - // i32: - let mut values = vec![100i32; 512]; - values.append(vec![150i32; 12].as_mut()); - values.append(vec![200i32; 500].as_mut()); - let col6 = Arc::new(Int32Array::from_iter_values(values)) as ArrayRef; - - // i64: - let mut values = vec![100i64; 512]; - values.append(vec![150i64; 12].as_mut()); - values.append(vec![200i64; 500].as_mut()); - let col7 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - - // bool: - let mut values = vec![false; 512]; - values.append(vec![true; 512].as_mut()); - let values: BooleanArray = values.into(); - let col8 = Arc::new(values) as ArrayRef; - - let to_write = RecordBatch::try_new(schema.clone(), vec![ - col1, col2, col3, col4, col5, col6, col7, col8, + #[test] + fn test_case_insensitive_scan_with_ambiguous_schema() { + let fixture = TableTestFixture::new(); + let mut metadata = fixture.table.metadata().clone(); + let schema = Schema::builder() + .with_schema_id(metadata.current_schema_id()) + .with_fields(vec![ + NestedField::optional(1, "id", PrimitiveType::Int).into(), + NestedField::optional(2, "ID", PrimitiveType::Int).into(), + NestedField::optional(3, "data", PrimitiveType::Int).into(), ]) + .build() .unwrap(); + metadata + .schemas + .insert(metadata.current_schema_id(), Arc::new(schema)); + let table = fixture.table.with_metadata(Arc::new(metadata)); - // Write the Parquet files - let props = WriterProperties::builder() - .set_compression(Compression::SNAPPY) - .build(); - - for n in 1..=3 { - let file = File::create(format!("{}/{}.parquet", &self.table_location, n)).unwrap(); - let mut writer = - ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap(); - - writer.write(&to_write).expect("Writing batch"); - - // writer must be closed to write footer - writer.close().unwrap(); - } - } - - pub async fn setup_unpartitioned_manifest_files(&mut self) { - let current_snapshot = self.table.metadata().current_snapshot().unwrap(); - let parent_snapshot = current_snapshot - .parent_snapshot(self.table.metadata()) - .unwrap(); - let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); - let current_partition_spec = Arc::new(PartitionSpec::unpartition_spec()); - - // Write data files using an empty partition for unpartitioned tables. - let mut writer = ManifestWriterBuilder::new( - self.next_manifest_file(), - Some(current_snapshot.snapshot_id()), - None, - current_schema.clone(), - current_partition_spec.as_ref().clone(), - ) - .build_v2_data(); - - // Create an empty partition value. - let empty_partition = Struct::empty(); - - writer - .add_entry( - ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/1.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(100) - .record_count(1) - .partition(empty_partition.clone()) - .key_metadata(None) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - - writer - .add_delete_entry( - ManifestEntry::builder() - .status(ManifestStatus::Deleted) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/2.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(100) - .record_count(1) - .partition(empty_partition.clone()) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - - writer - .add_existing_entry( - ManifestEntry::builder() - .status(ManifestStatus::Existing) - .snapshot_id(parent_snapshot.snapshot_id()) - .sequence_number(parent_snapshot.sequence_number()) - .file_sequence_number(parent_snapshot.sequence_number()) - .data_file( - DataFileBuilder::default() - .partition_spec_id(0) - .content(DataContentType::Data) - .file_path(format!("{}/3.parquet", &self.table_location)) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(100) - .record_count(1) - .partition(empty_partition.clone()) - .build() - .unwrap(), - ) - .build(), - ) - .unwrap(); - - let data_file_manifest = writer.write_manifest_file().await.unwrap(); - - // Write to manifest list - let mut manifest_list_write = ManifestListWriter::v2( - self.table - .file_io() - .new_output(current_snapshot.manifest_list()) - .unwrap(), - current_snapshot.snapshot_id(), - current_snapshot.parent_snapshot_id(), - current_snapshot.sequence_number(), - ); - manifest_list_write - .add_manifests(vec![data_file_manifest].into_iter()) - .unwrap(); - manifest_list_write.close().await.unwrap(); - - // prepare data for parquet files - let schema = { - let fields = vec![ - arrow_schema::Field::new("x", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - arrow_schema::Field::new("y", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - arrow_schema::Field::new("z", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - arrow_schema::Field::new("a", arrow_schema::DataType::Utf8, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "4".to_string(), - )])), - arrow_schema::Field::new("dbl", arrow_schema::DataType::Float64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "5".to_string(), - )])), - arrow_schema::Field::new("i32", arrow_schema::DataType::Int32, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "6".to_string(), - )])), - arrow_schema::Field::new("i64", arrow_schema::DataType::Int64, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "7".to_string(), - )])), - arrow_schema::Field::new("bool", arrow_schema::DataType::Boolean, false) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "8".to_string(), - )])), - ]; - Arc::new(arrow_schema::Schema::new(fields)) - }; - - // Build the arrays for the RecordBatch - let col1 = Arc::new(Int64Array::from_iter_values(vec![1; 1024])) as ArrayRef; - - let mut values = vec![2; 512]; - values.append(vec![3; 200].as_mut()); - values.append(vec![4; 300].as_mut()); - values.append(vec![5; 12].as_mut()); - let col2 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - - let mut values = vec![3; 512]; - values.append(vec![4; 512].as_mut()); - let col3 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - - let mut values = vec!["Apache"; 512]; - values.append(vec!["Iceberg"; 512].as_mut()); - let col4 = Arc::new(StringArray::from_iter_values(values)) as ArrayRef; - - let mut values = vec![100.0f64; 512]; - values.append(vec![150.0f64; 12].as_mut()); - values.append(vec![200.0f64; 500].as_mut()); - let col5 = Arc::new(Float64Array::from_iter_values(values)) as ArrayRef; - - let mut values = vec![100i32; 512]; - values.append(vec![150i32; 12].as_mut()); - values.append(vec![200i32; 500].as_mut()); - let col6 = Arc::new(Int32Array::from_iter_values(values)) as ArrayRef; - - let mut values = vec![100i64; 512]; - values.append(vec![150i64; 12].as_mut()); - values.append(vec![200i64; 500].as_mut()); - let col7 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - - let mut values = vec![false; 512]; - values.append(vec![true; 512].as_mut()); - let values: BooleanArray = values.into(); - let col8 = Arc::new(values) as ArrayRef; - - let to_write = RecordBatch::try_new(schema.clone(), vec![ - col1, col2, col3, col4, col5, col6, col7, col8, - ]) + table + .scan() + .with_case_sensitive(false) + .select(["DATA"]) + .build() .unwrap(); - - // Write the Parquet files - let props = WriterProperties::builder() - .set_compression(Compression::SNAPPY) - .build(); - - for n in 1..=3 { - let file = File::create(format!("{}/{}.parquet", &self.table_location, n)).unwrap(); - let mut writer = - ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap(); - - writer.write(&to_write).expect("Writing batch"); - - // writer must be closed to write footer - writer.close().unwrap(); - } - } + let error = table + .scan() + .with_case_sensitive(false) + .select(["Id"]) + .build() + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.message().contains("case-insensitively"), "{error}"); } - #[test] - fn test_table_scan_columns() { - let table = TableTestFixture::new().table; - - let table_scan = table.scan().select(["x", "y"]).build().unwrap(); - assert_eq!( - Some(vec!["x".to_string(), "y".to_string()]), - table_scan.column_names - ); + #[tokio::test] + async fn test_case_insensitive_scan_resolves_mismatched_column_case() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; - let table_scan = table + // The schema declares lowercase "x" and "z"; a case-insensitive scan must + // resolve the upper-cased names in both the projection and the filter. + let table_scan = fixture + .table .scan() - .select(["x", "y"]) - .select(["z"]) + .with_case_sensitive(false) + .select(["X", "Z"]) + .with_filter(Reference::new("Y").greater_than(Datum::long(1))) .build() .unwrap(); - assert_eq!(Some(vec!["z".to_string()]), table_scan.column_names); - } - - #[test] - fn test_select_all() { - let table = TableTestFixture::new().table; - - let table_scan = table.scan().select_all().build().unwrap(); - assert!(table_scan.column_names.is_none()); - } - #[test] - fn test_select_no_exist_column() { - let table = TableTestFixture::new().table; + let batches: Vec<_> = table_scan + .to_arrow(ArrowReaderOptions::default()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); - let table_scan = table.scan().select(["x", "y", "z", "a", "b"]).build(); - assert!(table_scan.is_err()); + assert_eq!(batches[0].num_columns(), 2); + assert_eq!( + batches[0] + .column_by_name("x") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 1 + ); + assert_eq!( + batches[0] + .column_by_name("z") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 3 + ); } - #[test] - fn test_table_scan_default_snapshot_id() { + #[tokio::test] + async fn test_table_scan_default_snapshot_id() { let table = TableTestFixture::new().table; let table_scan = table.scan().build().unwrap(); @@ -1215,14 +1035,13 @@ pub mod tests { assert!(table_scan.is_err()); } - #[test] - fn test_table_scan_with_snapshot_id() { + #[tokio::test] + async fn test_table_scan_with_snapshot_id() { let table = TableTestFixture::new().table; let table_scan = table .scan() .snapshot_id(3051729675574597004) - .with_row_selection_enabled(true) .build() .unwrap(); assert_eq!( @@ -1231,570 +1050,3192 @@ pub mod tests { ); } - #[tokio::test] - async fn test_plan_files_on_table_without_any_snapshots() { + #[test] + fn test_table_scan_as_of_time_after_rollback() { + for elapsed_ms in [0, 1000] { + let table = TableTestFixture::new().table; + let mut metadata = table.metadata().clone(); + let original = metadata.history()[0].snapshot_id; + let previous = metadata.history()[1].clone(); + let rollback_time = previous.timestamp_ms + elapsed_ms; + metadata.snapshot_log.push(crate::spec::SnapshotLog { + timestamp_ms: rollback_time, + snapshot_id: original, + }); + metadata.current_snapshot_id = Some(original); + metadata.refs.get_mut(MAIN_BRANCH).unwrap().snapshot_id = original; + let table = table.with_metadata(Arc::new(metadata)); + let scan = table.scan().as_of_time(rollback_time).build().unwrap(); + // A rollback at the same timestamp keeps the earlier history entry. + let expected = if elapsed_ms == 0 { + previous.snapshot_id + } else { + original + }; + assert_eq!(scan.snapshot().unwrap().snapshot_id(), expected); + } + } + + #[test] + fn test_table_scan_as_of_time_rejects_empty_history() { let table = TableTestFixture::new_empty().table; - let batch_stream = table.scan().build().unwrap().to_arrow().await.unwrap(); - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert!(batches.is_empty()); + assert!(table.scan().build().unwrap().snapshot().is_none()); + let err = table.scan().as_of_time(0).build().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("No snapshot history")); } - #[tokio::test] - async fn test_plan_files_no_deletions() { - let mut fixture = TableTestFixture::new(); - fixture.setup_manifest_files().await; + #[test] + fn test_table_scan_as_of_time_rejects_before_history() { + let table = TableTestFixture::new().table; + let before = table.metadata().history()[0].timestamp_ms - 1; + let err = table.scan().as_of_time(before).build().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains(&before.to_string())); + } - // Create table scan for current snapshot and plan files - let table_scan = fixture - .table + #[test] + fn test_table_scan_as_of_time_rejects_all_qualifying_snapshots_expired() { + let table = TableTestFixture::new_with_deep_history().table; + let mut metadata = table.metadata().clone(); + let entry = metadata.history()[3].clone(); + let expired_ids: Vec<_> = metadata + .history() + .iter() + .filter(|log| log.timestamp_ms <= entry.timestamp_ms) + .map(|log| log.snapshot_id) + .collect(); + assert_eq!(expired_ids.len(), 4); + for snapshot_id in expired_ids { + metadata.snapshots.remove(&snapshot_id); + } + let table = table.with_metadata(Arc::new(metadata)); + // History resolution succeeds even though every qualifying snapshot expired. + assert_eq!( + snapshot_id_as_of_time(&table.metadata_ref(), entry.timestamp_ms).unwrap(), + entry.snapshot_id + ); + assert!(table.scan().build().unwrap().snapshot().is_some()); + let err = table .scan() - .with_row_selection_enabled(true) + .as_of_time(entry.timestamp_ms) .build() - .unwrap(); + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + format!( + "Snapshot with id {} resolved from timestamp {} ms not found; it may have expired", + entry.snapshot_id, entry.timestamp_ms + ) + ); + } - let mut tasks = table_scan - .plan_files() - .await - .unwrap() - .try_fold(vec![], |mut acc, task| async move { - acc.push(task); - Ok(acc) - }) - .await - .unwrap(); + #[test] + fn test_table_scan_last_snapshot_selection_wins() { + let table = TableTestFixture::new().table; + let first = &table.metadata().history()[0]; + let last = &table.metadata().history()[1]; + let cases = [ + ( + table + .scan() + .snapshot_id(last.snapshot_id) + .as_of_time(first.timestamp_ms), + first.snapshot_id, + ), + ( + table + .scan() + .as_of_time(first.timestamp_ms) + .snapshot_id(last.snapshot_id), + last.snapshot_id, + ), + // An overwritten selector is not resolved or validated. + ( + table.scan().snapshot_id(-1).as_of_time(first.timestamp_ms), + first.snapshot_id, + ), + ( + table + .scan() + .as_of_time(i64::MIN) + .snapshot_id(last.snapshot_id), + last.snapshot_id, + ), + ( + table + .scan() + .snapshot_id(first.snapshot_id) + .snapshot_id(last.snapshot_id), + last.snapshot_id, + ), + ( + table + .scan() + .as_of_time(i64::MAX) + .as_of_time(first.timestamp_ms), + first.snapshot_id, + ), + ]; + for (builder, expected) in cases { + let scan = builder.build().unwrap(); + assert_eq!(scan.snapshot().unwrap().snapshot_id(), expected); + } + } - assert_eq!(tasks.len(), 2); + #[test] + fn test_table_scan_invalid_last_snapshot_selection() { + let table = TableTestFixture::new().table; + let first = &table.metadata().history()[0]; + let cases = [ + ( + table + .scan() + .snapshot_id(first.snapshot_id) + .as_of_time(first.timestamp_ms - 1), + "No snapshot history", + ), + ( + table.scan().as_of_time(first.timestamp_ms).snapshot_id(-1), + "Snapshot with id -1 not found", + ), + ]; + for (builder, expected) in cases { + let err = builder.build().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains(expected)); + } + } - tasks.sort_by_key(|t| t.data_file_path.to_string()); + #[test] + fn test_table_scan_as_of_time_uses_snapshot_schema() { + for select_current in [false, true] { + let table = TableTestFixture::new().table; + let mut metadata = table.metadata().clone(); + let entry = metadata.history()[usize::from(select_current)].clone(); + // Simulate a schema-only update after this snapshot: schema 0 has x only, + // while the current table schema also contains y and other columns. + Arc::make_mut(metadata.snapshots.get_mut(&entry.snapshot_id).unwrap()).schema_id = + Some(0); + let table = table.with_metadata(Arc::new(metadata)); + let scan = table + .scan() + .as_of_time(entry.timestamp_ms) + .select(["x"]) + .with_filter(Reference::new("x").greater_than(Datum::long(0))) + .build() + .unwrap(); + let context = scan.plan_context.as_ref().unwrap(); + assert_eq!(context.snapshot_schema.schema_id(), 0); + assert!(context.snapshot_bound_predicate.is_some()); + assert!( + table + .scan() + .as_of_time(entry.timestamp_ms) + .select(["y"]) + .build() + .is_err() + ); + assert!( + table + .scan() + .as_of_time(entry.timestamp_ms) + .with_filter(Reference::new("y").greater_than(Datum::long(0))) + .build() + .is_err() + ); + } + } - // Check first task is added data file - assert_eq!( - tasks[0].data_file_path, - format!("{}/1.parquet", &fixture.table_location) + #[test] + fn test_table_scan_as_of_time_schema_compatibility() { + let table = TableTestFixture::new().table; + let entry = table.metadata().history()[0].clone(); + // Older snapshots may omit schema-id and use the current-schema fallback. + assert!( + table + .metadata() + .snapshot_by_id(entry.snapshot_id) + .unwrap() + .schema_id() + .is_none() ); - - // Check second task is existing data file + let scan = table.scan().as_of_time(entry.timestamp_ms).build().unwrap(); assert_eq!( - tasks[1].data_file_path, - format!("{}/3.parquet", &fixture.table_location) + scan.plan_context.as_ref().unwrap().snapshot_schema, + *table.metadata().current_schema() ); + + let mut metadata = table.metadata().clone(); + Arc::make_mut(metadata.snapshots.get_mut(&entry.snapshot_id).unwrap()).schema_id = + Some(1234); + let table = table.with_metadata(Arc::new(metadata)); + let err = table + .scan() + .as_of_time(entry.timestamp_ms) + .build() + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("Schema with id 1234 not found")); } #[tokio::test] - async fn test_open_parquet_no_deletions() { + async fn test_table_scan_as_of_time_reads_historical_rows() { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; + let entry = fixture.table.metadata().history()[0].clone(); - // Create table scan for current snapshot and plan files - let table_scan = fixture + // The older snapshot has only x, while the current snapshot has eight columns. + let mut metadata = fixture.table.metadata().clone(); + Arc::make_mut(metadata.snapshots.get_mut(&entry.snapshot_id).unwrap()).schema_id = Some(0); + fixture.table = fixture.table.with_metadata(Arc::new(metadata)); + let snapshot = fixture .table + .metadata() + .snapshot_by_id(entry.snapshot_id) + .unwrap(); + let schema = snapshot.schema(fixture.table.metadata()).unwrap(); + let arrow_schema = Arc::new(crate::arrow::schema_to_arrow_schema(&schema).unwrap()); + + // Write distinct historical data: x = 100, versus x = 1 in the current files. + let path = format!("{}/historical.parquet", fixture.table_location); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int64Array::from( + vec![100], + ))]) + .unwrap(); + let mut writer = + ArrowWriter::try_new(File::create(&path).unwrap(), arrow_schema, None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let mut manifest_writer = ManifestWriterBuilder::new( + fixture.next_manifest_file(), + Some(snapshot.snapshot_id()), + schema, + fixture + .table + .metadata() + .default_partition_spec() + .as_ref() + .clone(), + ) + .build_v2_data(); + manifest_writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(fs::metadata(&path).unwrap().len()) + .file_path(path) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let manifest = manifest_writer.write_manifest_file().await.unwrap(); + let output = fixture + .table + .file_io() + .new_output(snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut list_writer = ManifestListWriter::v2( + output, + snapshot.snapshot_id(), + snapshot.parent_snapshot_id(), + snapshot.sequence_number(), + ); + list_writer.add_manifests([manifest].into_iter()).unwrap(); + list_writer.close().await.unwrap(); + + let table = fixture.table; + let [by_time, by_id] = [ + table.scan().as_of_time(entry.timestamp_ms), + table.scan().snapshot_id(entry.snapshot_id), + ] + .map(|scan| { + scan.with_filter(Reference::new("x").greater_than_or_equal_to(Datum::long(100))) + .build() + .unwrap() + }); + let current = table.scan().build().unwrap(); + let mut planned_files = Vec::new(); + let mut rows = Vec::new(); + let mut schemas = Vec::new(); + for scan in [by_time, by_id, current] { + let mut tasks: Vec<_> = scan + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + tasks.sort_by_key(|task| task.data_file_path().to_string()); + planned_files.push(tasks); + let batches: Vec<_> = scan + .to_arrow(ArrowReaderOptions::default()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert!(!batches.is_empty()); + schemas.push(batches[0].schema()); + let mut values: Vec = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("x") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .iter() + .copied() + }) + .collect(); + values.sort_unstable(); + rows.push(values); + } + assert!(!planned_files[0].is_empty()); + assert_eq!(planned_files[0], planned_files[1]); + assert_eq!(rows[0], vec![100]); + assert_eq!(rows[0], rows[1]); + // The current snapshot contains two live fixture files. + assert_eq!(rows[2], vec![1; 2 * ROWS_PER_FIXTURE_FILE]); + assert_eq!(schemas[0], schemas[1]); + assert_eq!(schemas[0].fields().len(), 1); + assert_eq!(schemas[2].fields().len(), 8); + } + + /// A scan of the current state reads the current snapshot with the table's + /// current schema, not the older one that snapshot was written with. + #[test] + fn test_scan_of_current_state_uses_current_schema() { + let table = TableTestFixture::new_with_column_added_after_last_write().table; + + let table_scan = table.scan().build().unwrap(); + assert_eq!(table_scan.snapshot(), table.metadata().current_snapshot()); + assert_eq!( + table_scan.plan_context.as_ref().unwrap().snapshot_schema, + *table.metadata().current_schema() + ); + } + + /// A scan pinned to a snapshot uses the schema that snapshot was written + /// with, even when it is the current snapshot. + #[test] + fn test_time_travel_scan_uses_snapshot_schema() { + let table = TableTestFixture::new_with_column_added_after_last_write().table; + let snapshot = table.metadata().current_snapshot().unwrap(); + + let table_scan = table .scan() - .with_row_selection_enabled(true) + .snapshot_id(snapshot.snapshot_id()) .build() .unwrap(); + assert_eq!(table_scan.snapshot(), Some(snapshot)); + assert_eq!( + table_scan.plan_context.as_ref().unwrap().snapshot_schema, + snapshot.schema(table.metadata()).unwrap() + ); + } - let batch_stream = table_scan.to_arrow().await.unwrap(); + fn table_with_property(key: &str, value: &str) -> Table { + let fixture = TableTestFixture::new(); + let mut metadata = fixture.table.metadata().clone(); + metadata + .properties + .insert(key.to_string(), value.to_string()); + Table::builder() + .metadata(metadata) + .identifier(fixture.table.identifier().clone()) + .file_io(fixture.table.file_io().clone()) + .metadata_location(fixture.table.metadata_location().unwrap().to_string()) + .runtime(test_runtime()) + .build() + .unwrap() + } - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + #[test] + fn test_table_scan_without_name_mapping_property() { + let table = TableTestFixture::new().table; - let col = batches[0].column_by_name("x").unwrap(); + let table_scan = table.scan().build().unwrap(); + assert!( + table_scan + .plan_context + .as_ref() + .unwrap() + .name_mapping + .is_none() + ); + } - let int64_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(int64_arr.value(0), 1); + #[test] + fn test_table_scan_with_name_mapping_property() { + let mapping_json = r#"[{"field-id":1,"names":["id","record_id"]}]"#; + let table = + table_with_property(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING, mapping_json); + + let table_scan = table.scan().build().unwrap(); + let mapping = table_scan + .plan_context + .as_ref() + .unwrap() + .name_mapping + .as_ref() + .expect("name_mapping should be parsed from the table property"); + let fields = mapping.fields(); + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].field_id(), Some(1)); + assert_eq!(fields[0].names(), &[ + "id".to_string(), + "record_id".to_string() + ]); + } + + #[test] + fn test_table_scan_with_malformed_name_mapping_property() { + let table = table_with_property( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING, + "{ not valid json", + ); + + let err = table + .scan() + .build() + .expect_err("malformed name mapping should fail to parse"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); } #[tokio::test] - async fn test_open_parquet_no_deletions_by_separate_reader() { + async fn test_plan_files_carries_name_mapping_into_file_scan_task() { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - // Create table scan for current snapshot and plan files - let table_scan = fixture - .table - .scan() - .with_row_selection_enabled(true) + let mapping_json = r#"[{"field-id":1,"names":["id","record_id"]}]"#; + let mut metadata = fixture.table.metadata().clone(); + metadata.properties.insert( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(), + mapping_json.to_string(), + ); + let table = Table::builder() + .metadata(metadata) + .identifier(fixture.table.identifier().clone()) + .file_io(fixture.table.file_io().clone()) + .metadata_location(fixture.table.metadata_location().unwrap().to_string()) + .runtime(test_runtime()) .build() .unwrap(); - let mut plan_task: Vec<_> = table_scan + let tasks: Vec<_> = table + .scan() + .build() + .unwrap() .plan_files() .await .unwrap() .try_collect() .await .unwrap(); - assert_eq!(plan_task.len(), 2); - let reader = ArrowReaderBuilder::new(fixture.table.file_io().clone()).build(); - let batch_stream = reader - .clone() - .read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))]))) - .unwrap(); - let batch_1: Vec<_> = batch_stream.try_collect().await.unwrap(); + assert!(!tasks.is_empty(), "expected at least one FileScanTask"); + for task in &tasks { + let mapping = task + .name_mapping() + .expect("name_mapping should reach the FileScanTask"); + assert_eq!(mapping.fields().len(), 1); + assert_eq!(mapping.fields()[0].field_id(), Some(1)); + } + } + + #[tokio::test] + async fn test_plan_files_carries_sort_order_into_file_scan_task() { + let mut fixture = TableTestFixture::new(); + + // Inject the reserved unsorted order (id 0) inline rather than editing the shared + // testdata fixture, so the id-0 file below exercises the `!is_unsorted()` filter + // branch instead of the `and_then` short-circuit an absent entry would take. + let mut metadata = fixture.table.metadata().clone(); + metadata + .sort_orders + .insert(0, Arc::new(SortOrder::unsorted_order())); + fixture.table = fixture.table.with_metadata(Arc::new(metadata)); + + let expected_sort_order = fixture + .table + .metadata() + .sort_order_by_id(3) + .unwrap() + .clone(); + + // sort_order_ids: resolvable (3), absent, unresolvable (99), reserved unsorted (0). + fixture + .setup_manifest_files_with_sort_order_ids([Some(3), None, Some(99), Some(0)]) + .await; + + let tasks: Vec<_> = fixture + .table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + assert_eq!(tasks.len(), 4, "expected all four FileScanTasks"); + + // Aggregates catch a systemic regression (every entry resolving to id 3, or + // resolution dropping entirely) that the per-file checks below would each still pass. + assert_eq!( + tasks.iter().filter(|t| t.sort_order().is_some()).count(), + 1, + "exactly one file resolves to a sort order" + ); + assert_eq!( + tasks.iter().filter(|t| t.sort_order_id().is_some()).count(), + 3, + "three files carry a raw sort_order_id" + ); + + let resolved = tasks + .iter() + .find(|t| t.data_file_path().ends_with("1.parquet")) + .unwrap(); + assert_eq!(resolved.sort_order_id(), Some(3)); + assert_eq!( + resolved.sort_order(), + Some(&expected_sort_order), + "sort_order_id 3 should resolve to the table's sort order at id 3" + ); + + let missing = tasks + .iter() + .find(|t| t.data_file_path().ends_with("2.parquet")) + .unwrap(); + assert_eq!(missing.sort_order_id(), None); + assert!( + missing.sort_order().is_none(), + "a file with no sort_order_id carries no sort_order" + ); + + let unresolvable = tasks + .iter() + .find(|t| t.data_file_path().ends_with("3.parquet")) + .unwrap(); + assert_eq!( + unresolvable.sort_order_id(), + Some(99), + "the raw id is preserved even when it does not resolve" + ); + assert!( + unresolvable.sort_order().is_none(), + "an unresolvable sort_order_id resolves to no sort_order" + ); + + let unsorted = tasks + .iter() + .find(|t| t.data_file_path().ends_with("4.parquet")) + .unwrap(); + assert_eq!(unsorted.sort_order_id(), Some(0)); + assert!( + unsorted.sort_order().is_none(), + "the reserved unsorted order (id 0) resolves to no sort_order" + ); + } + + /// Statistics for long columns `field_ids`, with values derived from each id so that a + /// column's statistics do not depend on which other columns are present. + fn column_stats(field_ids: &[i32]) -> FileScanTaskColumnStats { + let counts = |count: fn(u64) -> u64| -> HashMap { + field_ids.iter().map(|&id| (id, count(id as u64))).collect() + }; + let bounds = |offset: i64| -> HashMap { + field_ids + .iter() + .map(|&id| (id, Datum::long(offset + i64::from(id)))) + .collect() + }; + FileScanTaskColumnStats::builder() + .with_column_sizes(counts(|id| 8 * id)) + .with_value_counts(counts(|id| 10 + id)) + .with_null_value_counts(counts(|id| id)) + .with_nan_value_counts(counts(|id| 2 * id)) + .with_lower_bounds(bounds(0)) + .with_upper_bounds(bounds(100)) + .build() + } + + async fn column_stats_by_file(scan: TableScan) -> Vec> { + let mut tasks: Vec<_> = scan + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + tasks.sort_by(|a, b| a.data_file_path().cmp(b.data_file_path())); + tasks + .into_iter() + .map(|task| task.column_stats().cloned()) + .collect() + } + + #[tokio::test] + async fn test_plan_files_retains_only_requested_column_stats() { + let mut fixture = TableTestFixture::new(); + // The second file has no statistics for `x` (field id 1). + fixture + .setup_manifest_files_with_column_stats(&[(None, vec![ + column_stats(&[1, 2, 7]), + column_stats(&[2, 7]), + ])]) + .await; + + let scan = fixture + .table + .scan() + .with_column_stats(["x", "i64"]) + .build() + .unwrap(); + + assert_eq!(column_stats_by_file(scan).await, vec![ + Some(column_stats(&[1, 7])), + Some(column_stats(&[7])) + ]); + } + + #[tokio::test] + async fn test_plan_files_carries_column_stats_only_when_requested() { + let mut fixture = TableTestFixture::new(); + fixture + .setup_manifest_files_with_column_stats(&[(None, vec![column_stats(&[1])])]) + .await; + let table = &fixture.table; + + assert_eq!( + column_stats_by_file(table.scan().build().unwrap()).await, + vec![None] + ); + assert_eq!( + column_stats_by_file( + table + .scan() + .with_column_stats(Vec::::new()) + .build() + .unwrap() + ) + .await, + vec![Some(FileScanTaskColumnStats::default())] + ); + } + + #[tokio::test] + async fn test_plan_files_resolves_column_stats_names_with_scan_case_sensitivity() { + let mut fixture = TableTestFixture::new(); + fixture + .setup_manifest_files_with_column_stats(&[(None, vec![column_stats(&[1, 2])])]) + .await; + + let scan = fixture + .table + .scan() + .with_case_sensitive(false) + .with_column_stats(["X"]) + .build() + .unwrap(); + + assert_eq!(column_stats_by_file(scan).await, vec![Some(column_stats( + &[1] + ))]); + } + + #[test] + fn test_build_rejects_unknown_column_stats_name() { + let table = TableTestFixture::new().table; + + let err = table + .scan() + .with_column_stats(["unknown"]) + .build() + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_plan_files_promotes_bounds_to_scan_schema_type() { + let mut fixture = TableTestFixture::new(); + let y_bounds = |lower: Datum, upper: Datum| { + FileScanTaskColumnStats::builder() + .with_lower_bounds(HashMap::from([(2, lower)])) + .with_upper_bounds(HashMap::from([(2, upper)])) + .build() + }; + // `y` is a `long` in the table schema. The first manifest was written when it was an `int`. + let int_schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", PrimitiveType::Long).into(), + NestedField::required(2, "y", PrimitiveType::Int).into(), + ]) + .build() + .unwrap(), + ); + let written_as_long = y_bounds(Datum::long(6), Datum::long(9)); + fixture + .setup_manifest_files_with_column_stats(&[ + (Some(int_schema), vec![y_bounds( + Datum::int(5), + Datum::int(7), + )]), + (None, vec![written_as_long.clone()]), + ]) + .await; + + let scan = fixture + .table + .scan() + .with_column_stats(["y"]) + .build() + .unwrap(); + + assert_eq!(column_stats_by_file(scan).await, vec![ + Some(y_bounds(Datum::long(5), Datum::long(7))), + Some(written_as_long) + ]); + } + + #[tokio::test] + async fn test_plan_files_on_table_without_any_snapshots() { + let table = TableTestFixture::new_empty().table; + let batch_stream = table + .scan() + .build() + .unwrap() + .to_arrow(ArrowReaderOptions::default()) + .await + .unwrap(); + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + assert!(batches.is_empty()); + } + + #[tokio::test] + async fn test_plan_files_no_deletions() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Create table scan for current snapshot and plan files + let table_scan = fixture.table.scan().build().unwrap(); + + let mut tasks = table_scan + .plan_files() + .await + .unwrap() + .try_fold(vec![], |mut acc, task| async move { + acc.push(task); + Ok(acc) + }) + .await + .unwrap(); + + assert_eq!(tasks.len(), 2); + + tasks.sort_by_key(|t| t.data_file_path().to_string()); + + // Check first task is added data file + assert_eq!( + tasks[0].data_file_path(), + format!("{}/1.parquet", &fixture.table_location) + ); + + // Check second task is existing data file + assert_eq!( + tasks[1].data_file_path(), + format!("{}/3.parquet", &fixture.table_location) + ); + } + + #[tokio::test] + async fn test_scan_widens_old_partition_so_pruning_and_deletes_match() { + // Guards the reader-side normalization through a real scan: a data manifest + // written when the partition source `x` was `int` must have its `int(7)` + // partition widened to `long(7)` during planning, so (a) a `x = 7` (long) + // filter keeps the file and (b) the equality delete keyed by `long(7)` + // attaches. Without the normalization the int/long mismatch drops the delete + // (and prunes the file under the filter). + let mut fixture = TableTestFixture::new(); + fixture.setup_promoted_partition_with_eq_delete(false).await; + + let tasks: Vec = fixture + .table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(tasks.len(), 1); + assert_eq!( + tasks[0].deletes().len(), + 1, + "equality delete must attach after widening the old int partition to long" + ); + assert_eq!( + tasks[0].partition(), + Some(&Struct::from_iter([Some(Literal::long(7))])), + "the widened long(7) tuple must reach FileScanTask::partition()" + ); + + let tasks: Vec = fixture + .table + .scan() + .with_filter(Reference::new("x").equal_to(Datum::long(7))) + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + tasks.len(), + 1, + "filter on the promoted type must not prune the file" + ); + assert_eq!(tasks[0].deletes().len(), 1); + + let tasks: Vec = fixture + .table + .scan() + .with_filter(Reference::new("x").equal_to(Datum::long(8))) + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert!( + tasks.is_empty(), + "a file in partition x=7 must be pruned by an x=8 filter" + ); + } + + #[tokio::test] + async fn test_scan_widens_old_partition_with_pre_promotion_delete() { + // Data file and equality delete both written before the promotion (int + // partition). They matched on main; after both sides widen to long they must + // still match, so the delete stays attached. + let mut fixture = TableTestFixture::new(); + fixture.setup_promoted_partition_with_eq_delete(true).await; + + let tasks: Vec = fixture + .table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(tasks.len(), 1); + assert_eq!(tasks[0].deletes().len(), 1); + } + + #[tokio::test] + async fn test_scan_widens_old_partition_so_deletion_vector_matches() { + // A v3 deletion vector written after the promotion (long partition) references + // an older data file (int partition). The widening must make the partitions + // agree so the DV attaches, instead of failing the DV partition check. + let mut fixture = TableTestFixture::new(); + fixture.setup_promoted_partition_with_dv().await; + + let tasks: Vec = fixture + .table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(tasks.len(), 1); + assert_eq!(tasks[0].deletes().len(), 1); + } + + #[tokio::test] + async fn test_scan_keeps_eq_delete_after_dropping_promoted_source() { + // Partition source `x` promoted int -> long, then dropped. The data file (int) + // and its equality delete (long) must still normalize to the same long partition + // so the delete stays attached; otherwise it silently detaches. + let mut fixture = TableTestFixture::new(); + fixture.setup_promoted_partition_with_eq_delete(false).await; + fixture.drop_partition_source(1); + let table = fixture.table; + + let tasks: Vec = table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(tasks.len(), 1); + assert_eq!(tasks[0].deletes().len(), 1); + } + + #[tokio::test] + async fn test_scan_keeps_deletion_vector_after_dropping_promoted_source() { + // Same as above for a deletion vector: without widening the dropped numeric + // source the DV partition check errors (long(7) vs int(7)). + let mut fixture = TableTestFixture::new(); + fixture.setup_promoted_partition_with_dv().await; + fixture.drop_partition_source(1); + let table = fixture.table; + + let tasks: Vec = table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(tasks.len(), 1); + assert_eq!(tasks[0].deletes().len(), 1); + } + + #[tokio::test] + async fn test_scan_keeps_eq_delete_when_dropped_source_is_date() { + // A promoted numeric source `x` sits beside a dropped `date` source `d`. The gate + // fires on `x`'s int -> long widening, but `d` (stored as an int) must be copied + // through, not widened to long, or the data file's partition diverges from the + // equality delete's and detaches it. + let mut fixture = TableTestFixture::new(); + fixture + .setup_promoted_partition_with_eq_delete_and_dropped_date() + .await; + + let tasks: Vec = fixture + .table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(tasks.len(), 1); + assert_eq!(tasks[0].deletes().len(), 1); + } + + #[tokio::test] + async fn test_plan_files_carries_row_lineage_into_file_scan_task() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + let mut tasks: Vec<_> = fixture + .table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + assert_eq!(tasks.len(), 2); + tasks.sort_by_key(|task| task.data_file_path().to_string()); + + // The added file inherits the current snapshot's data sequence number, + // the existing file keeps the one it was written with. + assert_eq!( + tasks[0].data_file_path(), + format!("{}/1.parquet", &fixture.table_location) + ); + assert_eq!(tasks[0].data_sequence_number(), Some(1)); + assert_eq!( + tasks[1].data_file_path(), + format!("{}/3.parquet", &fixture.table_location) + ); + assert_eq!(tasks[1].data_sequence_number(), Some(0)); + + // first_row_id is a v3 concept; a v2 manifest carries none. + assert!(tasks.iter().all(|task| task.first_row_id().is_none())); + } + + #[tokio::test] + async fn test_plan_files_carries_row_lineage_from_v3_manifest() { + let mut fixture = TableTestFixture::new(); + fixture.setup_v3_manifest_files().await; + + let task = fixture + .table + .scan() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect::>() + .await + .unwrap() + .into_iter() + .next() + .expect("expected one FileScanTask"); + + // The manifest-level first_row_id (42) is inherited onto the entry on + // read, then carried onto the task. + assert_eq!(task.first_row_id(), Some(42)); + // The data sequence number is threaded through the same v3 read path. + assert_eq!(task.data_sequence_number(), Some(1)); + } + + #[tokio::test] + async fn test_filtered_scan_with_dropped_partition_source_column() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Baseline: the same filtered scan against the table before evolution. + let baseline = scan_y_gte_5(&fixture.table).await; + assert!(!baseline.is_empty()); + assert!(baseline.iter().all(|y| *y >= 5)); + + // Evolve the table so that the manifests reference a historical spec whose source + // column is no longer in the current schema: make an unpartitioned spec the + // default, then drop the original spec's source column from the schema. + let current_schema = fixture.table.metadata().current_schema(); + let evolved_schema = Schema::builder() + .with_fields( + current_schema + .as_struct() + .fields() + .iter() + .filter(|field| field.id != 1) + .cloned(), + ) + .with_identifier_field_ids(vec![2]) + .build() + .unwrap(); + let evolved = + TableMetadataBuilder::new_from_metadata(fixture.table.metadata().clone(), None) + .add_default_partition_spec(UnboundPartitionSpec::builder().build()) + .unwrap() + .add_current_schema(evolved_schema) + .unwrap() + .build() + .unwrap() + .metadata; + + // a commit after the evolution carries the previous manifests forward: the new + // snapshot uses the evolved schema while its manifests still use historical spec 0 + let parent = evolved.current_snapshot().unwrap().clone(); + let snapshot = Snapshot::builder() + .with_snapshot_id(parent.snapshot_id() + 1) + .with_parent_snapshot_id(Some(parent.snapshot_id())) + .with_sequence_number(evolved.last_sequence_number() + 1) + .with_timestamp_ms(evolved.last_updated_ms + 1) + .with_schema_id(evolved.current_schema_id()) + .with_manifest_list(parent.manifest_list()) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .build(); + let metadata = TableMetadataBuilder::new_from_metadata(evolved, None) + .set_branch_snapshot(snapshot, MAIN_BRANCH) + .unwrap() + .build() + .unwrap() + .metadata; + let table = fixture.table.clone().with_metadata(Arc::new(metadata)); + + // Planning and reading must succeed, and the results must match the table before + // evolution: no rows wrongly pruned and none returned unfiltered. + let evolved = scan_y_gte_5(&table).await; + assert_eq!(evolved, baseline); + } + + async fn scan_y_gte_5(table: &Table) -> Vec { + let table_scan = table + .scan() + .select(["y"]) + .with_filter(Reference::new("y").greater_than_or_equal_to(Datum::long(5))) + .build() + .unwrap(); + let batches: Vec<_> = table_scan + .to_arrow(ArrowReaderOptions::default()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + let mut values: Vec = batches + .iter() + .flat_map(|batch| { + let col = batch.column_by_name("y").unwrap(); + let arr = col.as_any().downcast_ref::().unwrap(); + (0..arr.len()).map(|i| arr.value(i)).collect::>() + }) + .collect(); + values.sort_unstable(); + values + } + + #[tokio::test] + async fn test_open_parquet_no_deletions() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Create table scan for current snapshot and plan files + let table_scan = fixture.table.scan().build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + let col = batches[0].column_by_name("x").unwrap(); + + let int64_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(int64_arr.value(0), 1); + } + + #[tokio::test] + async fn reader_options_preserve_reads_after_dropping_scan() { + let mut fixture = TableTestFixture::new(); + fixture.setup_multi_row_group_manifest(&[150, 299]).await; + let scan = fixture + .table + .scan() + .select(["y"]) + .with_filter(Reference::new("y").greater_than(Datum::long(1100))) + .build() + .unwrap(); + let options = ArrowReaderOptions::builder() + .with_batch_size(17) + .with_data_file_concurrency_limit(1) + .with_parquet_options( + ParquetReadOptions::builder() + .with_metadata_size_hint(4096) + .with_range_fetch_concurrency(2) + .build(), + ) + .with_row_selection_enabled(true) + .with_bloom_filter_enabled(true) + .build(); + let expected: Vec<_> = scan + .to_arrow(options.clone()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let tasks: Vec<_> = scan + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let serialized = serde_json::to_value(&options).unwrap(); + assert_eq!( + serialized, + serde_json::json!({ + "batch_size": 17, "data_file_concurrency_limit": 1, + "row_group_filtering_enabled": true, "row_selection_enabled": true, + "bloom_filter_enabled": true, + "parquet_options": { + "metadata_size_hint": 4096, "range_coalesce_bytes": 1048576, "range_fetch_concurrency": 2, + "preload_column_index": true, "preload_offset_index": true, "preload_page_index": false, + }, + }) + ); + let deserialized: ArrowReaderOptions = serde_json::from_value(serialized).unwrap(); + assert_eq!(deserialized, options); + let options = deserialized; + drop(scan); + let reader = + ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::try_current().unwrap()) + .with_options(options) + .build(); + let actual: Vec<_> = reader + .read(Box::pin(stream::iter(tasks.into_iter().map(Ok)))) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + assert!(!actual.is_empty()); + assert!(actual.iter().all(|batch| batch.num_rows() <= 17)); + assert_eq!(actual, expected); + } + + #[tokio::test] + async fn test_open_parquet_no_deletions_by_separate_reader() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Create table scan for current snapshot and plan files + let table_scan = fixture.table.scan().build().unwrap(); + + let mut plan_task: Vec<_> = table_scan + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(plan_task.len(), 2); + + let reader = ArrowReaderBuilder::new( + fixture.table.file_io().clone(), + fixture.table.runtime().clone(), + ) + .build(); + let batch_stream = reader + .clone() + .read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))]))) + .unwrap() + .stream(); + let batch_1: Vec<_> = batch_stream.try_collect().await.unwrap(); + + let reader = ArrowReaderBuilder::new( + fixture.table.file_io().clone(), + fixture.table.runtime().clone(), + ) + .build(); + let batch_stream = reader + .read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))]))) + .unwrap() + .stream(); + let batch_2: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batch_1, batch_2); + } + + #[tokio::test] + async fn test_open_parquet_with_projection() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Create table scan for current snapshot and plan files + let table_scan = fixture.table.scan().select(["x", "z"]).build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches[0].num_columns(), 2); + + let col1 = batches[0].column_by_name("x").unwrap(); + let int64_arr = col1.as_any().downcast_ref::().unwrap(); + assert_eq!(int64_arr.value(0), 1); + + let col2 = batches[0].column_by_name("z").unwrap(); + let int64_arr = col2.as_any().downcast_ref::().unwrap(); + assert_eq!(int64_arr.value(0), 3); + + // test empty scan + let table_scan = fixture.table.scan().select_empty().build().unwrap(); + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches[0].num_columns(), 0); + assert_eq!(batches[0].num_rows(), 1024); + } + + #[tokio::test] + async fn test_filter_on_arrow_lt() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: y < 3 + let mut builder = fixture.table.scan(); + let predicate = Reference::new("y").less_than(Datum::long(3)); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches[0].num_rows(), 512); + + let col = batches[0].column_by_name("x").unwrap(); + let int64_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(int64_arr.value(0), 1); + + let col = batches[0].column_by_name("y").unwrap(); + let int64_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(int64_arr.value(0), 2); + } + + #[tokio::test] + async fn test_filter_on_arrow_gt_eq() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: y >= 5 + let mut builder = fixture.table.scan(); + let predicate = Reference::new("y").greater_than_or_equal_to(Datum::long(5)); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches[0].num_rows(), 12); + + let col = batches[0].column_by_name("x").unwrap(); + let int64_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(int64_arr.value(0), 1); + + let col = batches[0].column_by_name("y").unwrap(); + let int64_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(int64_arr.value(0), 5); + } + + #[tokio::test] + async fn test_filter_double_eq() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: dbl == 150.0 + let mut builder = fixture.table.scan(); + let predicate = Reference::new("dbl").equal_to(Datum::double(150.0f64)); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].num_rows(), 12); + + let col = batches[0].column_by_name("dbl").unwrap(); + let f64_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(f64_arr.value(1), 150.0f64); + } + + #[tokio::test] + async fn test_filter_int_eq() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: i32 == 150 + let mut builder = fixture.table.scan(); + let predicate = Reference::new("i32").equal_to(Datum::int(150i32)); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].num_rows(), 12); + + let col = batches[0].column_by_name("i32").unwrap(); + let i32_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(i32_arr.value(1), 150i32); + } + + #[tokio::test] + async fn test_filter_long_eq() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: i64 == 150 + let mut builder = fixture.table.scan(); + let predicate = Reference::new("i64").equal_to(Datum::long(150i64)); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].num_rows(), 12); + + let col = batches[0].column_by_name("i64").unwrap(); + let i64_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(i64_arr.value(1), 150i64); + } + + #[tokio::test] + async fn test_filter_bool_eq() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: bool == true + let mut builder = fixture.table.scan(); + let predicate = Reference::new("bool").equal_to(Datum::bool(true)); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches.len(), 2); + assert_eq!(batches[0].num_rows(), 512); + + let col = batches[0].column_by_name("bool").unwrap(); + let bool_arr = col.as_any().downcast_ref::().unwrap(); + assert!(bool_arr.value(1)); + } + + #[tokio::test] + async fn test_filter_on_arrow_is_null() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: y is null + let mut builder = fixture.table.scan(); + let predicate = Reference::new("y").is_null(); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + assert_eq!(batches.len(), 0); + } + + #[tokio::test] + async fn test_filter_on_arrow_is_not_null() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: y is not null + let mut builder = fixture.table.scan(); + let predicate = Reference::new("y").is_not_null(); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + assert_eq!(batches[0].num_rows(), 1024); + } + + #[tokio::test] + async fn test_filter_on_arrow_lt_and_gt() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: y < 5 AND z >= 4 + let mut builder = fixture.table.scan(); + let predicate = Reference::new("y") + .less_than(Datum::long(5)) + .and(Reference::new("z").greater_than_or_equal_to(Datum::long(4))); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + assert_eq!(batches[0].num_rows(), 500); + + let col = batches[0].column_by_name("x").unwrap(); + let expected_x = Arc::new(Int64Array::from_iter_values(vec![1; 500])) as ArrayRef; + assert_eq!(col, &expected_x); + + let col = batches[0].column_by_name("y").unwrap(); + let mut values = vec![]; + values.append(vec![3; 200].as_mut()); + values.append(vec![4; 300].as_mut()); + let expected_y = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; + assert_eq!(col, &expected_y); + + let col = batches[0].column_by_name("z").unwrap(); + let expected_z = Arc::new(Int64Array::from_iter_values(vec![4; 500])) as ArrayRef; + assert_eq!(col, &expected_z); + } + + #[tokio::test] + async fn test_filter_on_arrow_lt_or_gt() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: y < 5 AND z >= 4 + let mut builder = fixture.table.scan(); + let predicate = Reference::new("y") + .less_than(Datum::long(5)) + .or(Reference::new("z").greater_than_or_equal_to(Datum::long(4))); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + assert_eq!(batches[0].num_rows(), 1024); - let reader = ArrowReaderBuilder::new(fixture.table.file_io().clone()).build(); - let batch_stream = reader - .read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))]))) + let col = batches[0].column_by_name("x").unwrap(); + let expected_x = Arc::new(Int64Array::from_iter_values(vec![1; 1024])) as ArrayRef; + assert_eq!(col, &expected_x); + + let col = batches[0].column_by_name("y").unwrap(); + let mut values = vec![2; 512]; + values.append(vec![3; 200].as_mut()); + values.append(vec![4; 300].as_mut()); + values.append(vec![5; 12].as_mut()); + let expected_y = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; + assert_eq!(col, &expected_y); + + let col = batches[0].column_by_name("z").unwrap(); + let mut values = vec![3; 512]; + values.append(vec![4; 512].as_mut()); + let expected_z = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; + assert_eq!(col, &expected_z); + } + + #[tokio::test] + async fn test_filter_on_arrow_startswith() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: a STARTSWITH "Ice" + let mut builder = fixture.table.scan(); + let predicate = Reference::new("a").starts_with(Datum::string("Ice")); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches[0].num_rows(), 512); + + let col = batches[0].column_by_name("a").unwrap(); + let string_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(string_arr.value(0), "Iceberg"); + } + + #[tokio::test] + async fn test_filter_on_arrow_not_startswith() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: a NOT STARTSWITH "Ice" + let mut builder = fixture.table.scan(); + let predicate = Reference::new("a").not_starts_with(Datum::string("Ice")); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches[0].num_rows(), 512); + + let col = batches[0].column_by_name("a").unwrap(); + let string_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(string_arr.value(0), "Apache"); + } + + #[tokio::test] + async fn test_filter_on_arrow_in() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: a IN ("Sioux", "Iceberg") + let mut builder = fixture.table.scan(); + let predicate = + Reference::new("a").is_in([Datum::string("Sioux"), Datum::string("Iceberg")]); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches[0].num_rows(), 512); + + let col = batches[0].column_by_name("a").unwrap(); + let string_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(string_arr.value(0), "Iceberg"); + } + + #[tokio::test] + async fn test_filter_on_arrow_not_in() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Filter: a NOT IN ("Sioux", "Iceberg") + let mut builder = fixture.table.scan(); + let predicate = + Reference::new("a").is_not_in([Datum::string("Sioux"), Datum::string("Iceberg")]); + builder = builder.with_filter(predicate); + let table_scan = builder.build().unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches[0].num_rows(), 512); + + let col = batches[0].column_by_name("a").unwrap(); + let string_arr = col.as_any().downcast_ref::().unwrap(); + assert_eq!(string_arr.value(0), "Apache"); + } + + fn file_scan_task_test_schema(primitive_type: PrimitiveType) -> Arc { + Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::required( + 1, + "x", + Type::Primitive(primitive_type), + ))]) + .build() + .unwrap(), + ) + } + + fn assert_file_scan_task_serde_round_trip(task: FileScanTask) { + // Regression test for https://github.com/apache/iceberg-rust/issues/3089. + let serialized = serde_json::to_string(&task).unwrap(); + let deserialized: FileScanTask = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(task, deserialized); + } + + fn file_scan_task_with_partition( + primitive_type: PrimitiveType, + transform: Transform, + partition_value: Literal, + ) -> FileScanTask { + let schema = file_scan_task_test_schema(primitive_type); + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .add_partition_field("x", "x_partition", transform) + .unwrap() + .build() + .unwrap(), + ); + FileScanTask::builder() + .with_data_file_path("data_file_path".to_string()) + .with_file_size_in_bytes(123) + .with_start(10) + .with_length(100) + .with_project_field_ids(vec![1]) + .with_schema(schema) + .with_data_file_format(DataFileFormat::Parquet) + .with_partition(Some(Struct::from_iter([Some(partition_value)]))) + .with_partition_spec(Some(partition_spec)) + .with_case_sensitive(true) + .build() + .unwrap() + } + + #[test] + fn test_file_scan_task_serde_without_predicate() { + let task = FileScanTask::builder() + .with_data_file_path("data_file_path".to_string()) + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(100) + .with_project_field_ids(vec![1, 2, 3]) + .with_schema(file_scan_task_test_schema(PrimitiveType::Binary)) + .with_record_count(Some(100)) + .with_first_row_id(Some(1000)) + .with_data_sequence_number(Some(5)) + .with_data_file_format(DataFileFormat::Parquet) + .with_case_sensitive(false) + .build() + .unwrap(); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_predicate() { + let task = FileScanTask::builder() + .with_data_file_path("data_file_path".to_string()) + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(100) + .with_project_field_ids(vec![1, 2, 3]) + .with_predicate(Some(BoundPredicate::AlwaysTrue)) + .with_schema(file_scan_task_test_schema(PrimitiveType::Binary)) + .with_data_file_format(DataFileFormat::Avro) + .with_case_sensitive(false) + .build() + .unwrap(); + + let serialized = serde_json::to_value(&task).unwrap(); + assert!(serialized.get("record_count").is_none()); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_unpartitioned_file_scan_task_serde() { + let task = FileScanTask::builder() + .with_data_file_path("data_file_path".to_string()) + .with_file_size_in_bytes(0) + .with_start(0) + .with_length(100) + .with_project_field_ids(vec![1, 2, 3]) + .with_schema(file_scan_task_test_schema(PrimitiveType::Binary)) + .with_data_file_format(DataFileFormat::Parquet) + .with_partition(Some(Struct::empty())) + .with_case_sensitive(false) + .build() + .unwrap(); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_all_optional_fields() { + let schema = file_scan_task_test_schema(PrimitiveType::Long); + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .add_partition_field("x", "x", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + let unified_partition_type = Arc::new(partition_spec.partition_type(&schema).unwrap()); + let sort_order = Arc::new( + SortOrder::builder() + .with_order_id(1) + .with_sort_field( + SortField::builder() + .source_id(1) + .transform(Transform::Identity) + .direction(SortDirection::Ascending) + .null_order(NullOrder::First) + .build(), + ) + .build(&schema) + .unwrap(), + ); + let task = FileScanTask::builder() + .with_data_file_path("data_file_path".to_string()) + .with_file_size_in_bytes(123) + .with_start(10) + .with_length(100) + .with_project_field_ids(vec![1]) + .with_schema(schema) + .with_data_file_format(DataFileFormat::Parquet) + .with_deletes(vec![ + FileScanTaskDeleteFile::builder() + .with_file_path("delete_file_path".to_string()) + .with_file_size_in_bytes(23) + .with_file_type(DataContentType::EqualityDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_equality_ids(Some(vec![1])) + .with_referenced_data_file(Some("data_file_path".to_string())) + .with_content_offset(Some(12)) + .with_content_size_in_bytes(Some(34)) + .with_record_count(Some(5)) + .with_key_metadata(Some(vec![4, 5, 6].into_boxed_slice())) + .build() + .unwrap(), + ]) + .with_partition(Some(Struct::from_iter([Some(Literal::long(42))]))) + .with_partition_spec(Some(partition_spec)) + .with_name_mapping(Some(Arc::new(NameMapping::new(vec![MappedField::new( + Some(1), + vec!["x".to_string()], + vec![], + )])))) + .with_unified_partition_type(Some(unified_partition_type)) + .with_sort_order_id(Some(1)) + .with_sort_order(Some(sort_order)) + .with_column_stats(Some( + FileScanTaskColumnStats::builder() + .with_column_sizes(HashMap::from([(1, 80), (2, 96)])) + .with_value_counts(HashMap::from([(1, 10), (2, 10)])) + .with_null_value_counts(HashMap::from([(1, 0), (2, 1)])) + .with_nan_value_counts(HashMap::from([(3, 2)])) + .with_lower_bounds(HashMap::from([ + (1, Datum::string("apple")), + (2, Datum::decimal_from_str("-12.34").unwrap()), + (3, Datum::float(100.1_f32)), + (4, Datum::double(f64::NEG_INFINITY)), + ])) + .with_upper_bounds(HashMap::from([ + (1, Datum::string("pear")), + (2, Datum::decimal_from_str("56.78").unwrap()), + (3, Datum::float(f32::INFINITY)), + (4, Datum::double(f64::INFINITY)), + ])) + .build(), + )) + .with_case_sensitive(true) + .with_key_metadata(Some(vec![1, 2, 3].into_boxed_slice())) + .build() + .unwrap(); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_omits_absent_column_stats() { + let task = file_scan_task_with_partition( + PrimitiveType::Long, + Transform::Identity, + Literal::long(42), + ); + + let serialized = serde_json::to_value(&task).unwrap(); + assert!(serialized.get("column_stats").is_none()); + + let deserialized: FileScanTask = serde_json::from_value(serialized).unwrap(); + assert_eq!(deserialized.column_stats(), None); + } + + #[test] + fn test_file_scan_task_serde_with_date_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::Date, + Transform::Identity, + Literal::date(19_000), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_timestamp_ns_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::TimestampNs, + Transform::Identity, + Literal::timestamp_nano(1_510_871_468_123_456_789), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_timestamptz_ns_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::TimestamptzNs, + Transform::Identity, + Literal::timestamptz_nano(1_510_871_468_123_456_789), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_decimal_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::Decimal { + precision: 9, + scale: 2, + }, + Transform::Identity, + Literal::decimal(12_345), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_uuid_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::Uuid, + Transform::Identity, + Literal::uuid(Uuid::from_u128(0x12345678_90ab_cdef_1234_567890abcdef)), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_fixed_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::Fixed(4), + Transform::Identity, + Literal::fixed([1, 2, 3, 4]), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[test] + fn test_file_scan_task_serde_with_bucket_partition() { + let task = file_scan_task_with_partition( + PrimitiveType::String, + Transform::Bucket(4), + Literal::int(2), + ); + assert_file_scan_task_serde_round_trip(task); + } + + #[tokio::test] + async fn test_select_with_file_column() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Select regular columns plus the _file column + let table_scan = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_FILE]) + .build() + .unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + // Verify we have 2 columns: x and _file + assert_eq!(batches[0].num_columns(), 2); + + // Verify the x column exists and has correct data + let x_col = batches[0].column_by_name("x").unwrap(); + let x_arr = x_col.as_primitive::(); + assert_eq!(x_arr.value(0), 1); + + // Verify the _file column exists + let file_col = batches[0].column_by_name(RESERVED_COL_NAME_FILE); + assert!( + file_col.is_some(), + "_file column should be present in the batch" + ); + + // Verify the _file column contains a file path + let file_col = file_col.unwrap(); + assert!( + matches!( + file_col.data_type(), + arrow_schema::DataType::RunEndEncoded(_, _) + ), + "_file column should use RunEndEncoded type" + ); + + // Decode the RunArray to verify it contains the file path + let run_array = file_col + .as_any() + .downcast_ref::>() + .expect("_file column should be a RunArray"); + + let values = run_array.values(); + let string_values = values.as_string::(); + assert_eq!(string_values.len(), 1, "Should have a single file path"); + + let file_path = string_values.value(0); + assert!( + file_path.ends_with(".parquet"), + "File path should end with .parquet, got: {file_path}" + ); + } + + #[tokio::test] + async fn test_select_file_column_position() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Select columns in specific order: x, _file, z + let table_scan = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_FILE, "z"]) + .build() + .unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + assert_eq!(batches[0].num_columns(), 3); + + // Verify column order: x at position 0, _file at position 1, z at position 2 + let schema = batches[0].schema(); + assert_eq!(schema.field(0).name(), "x"); + assert_eq!(schema.field(1).name(), RESERVED_COL_NAME_FILE); + assert_eq!(schema.field(2).name(), "z"); + + // Verify columns by name also works + assert!(batches[0].column_by_name("x").is_some()); + assert!(batches[0].column_by_name(RESERVED_COL_NAME_FILE).is_some()); + assert!(batches[0].column_by_name("z").is_some()); + } + + #[tokio::test] + async fn test_select_file_column_only() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Select only the _file column + let table_scan = fixture + .table + .scan() + .select([RESERVED_COL_NAME_FILE]) + .build() + .unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await .unwrap(); - let batch_2: Vec<_> = batch_stream.try_collect().await.unwrap(); + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batch_1, batch_2); + // Should have exactly 1 column + assert_eq!(batches[0].num_columns(), 1); + + // Verify it's the _file column + let schema = batches[0].schema(); + assert_eq!(schema.field(0).name(), RESERVED_COL_NAME_FILE); + + // Verify the batch has the correct number of rows + // The scan reads files 1.parquet and 3.parquet (2.parquet is deleted) + // Each file has 1024 rows, so total is 2048 rows + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 2048); } #[tokio::test] - async fn test_open_parquet_with_projection() { + async fn test_file_column_with_multiple_files() { + use std::collections::HashSet; + let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - // Create table scan for current snapshot and plan files + // Select x and _file columns let table_scan = fixture .table .scan() - .select(["x", "z"]) - .with_row_selection_enabled(true) + .select(["x", RESERVED_COL_NAME_FILE]) .build() .unwrap(); - let batch_stream = table_scan.to_arrow().await.unwrap(); - + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches[0].num_columns(), 2); + // Collect all unique file paths from the batches + let mut file_paths = HashSet::new(); + for batch in &batches { + let file_col = batch.column_by_name(RESERVED_COL_NAME_FILE).unwrap(); + let run_array = file_col + .as_any() + .downcast_ref::>() + .expect("_file column should be a RunArray"); + + let values = run_array.values(); + let string_values = values.as_string::(); + for i in 0..string_values.len() { + file_paths.insert(string_values.value(i).to_string()); + } + } - let col1 = batches[0].column_by_name("x").unwrap(); - let int64_arr = col1.as_any().downcast_ref::().unwrap(); - assert_eq!(int64_arr.value(0), 1); + // We should have multiple files (the test creates 1.parquet and 3.parquet) + assert!(!file_paths.is_empty(), "Should have at least one file path"); - let col2 = batches[0].column_by_name("z").unwrap(); - let int64_arr = col2.as_any().downcast_ref::().unwrap(); - assert_eq!(int64_arr.value(0), 3); + // All paths should end with .parquet + for path in &file_paths { + assert!( + path.ends_with(".parquet"), + "All file paths should end with .parquet, got: {path}" + ); + } + } - // test empty scan - let table_scan = fixture.table.scan().select_empty().build().unwrap(); - let batch_stream = table_scan.to_arrow().await.unwrap(); + #[tokio::test] + async fn test_file_column_at_start() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; + + // Select _file at the start + let table_scan = fixture + .table + .scan() + .select([RESERVED_COL_NAME_FILE, "x", "y"]) + .build() + .unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches[0].num_columns(), 0); - assert_eq!(batches[0].num_rows(), 1024); + assert_eq!(batches[0].num_columns(), 3); + + // Verify _file is at position 0 + let schema = batches[0].schema(); + assert_eq!(schema.field(0).name(), RESERVED_COL_NAME_FILE); + assert_eq!(schema.field(1).name(), "x"); + assert_eq!(schema.field(2).name(), "y"); } #[tokio::test] - async fn test_filter_on_arrow_lt() { + async fn test_file_column_at_end() { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - // Filter: y < 3 - let mut builder = fixture.table.scan(); - let predicate = Reference::new("y").less_than(Datum::long(3)); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); - - let batch_stream = table_scan.to_arrow().await.unwrap(); + // Select _file at the end + let table_scan = fixture + .table + .scan() + .select(["x", "y", RESERVED_COL_NAME_FILE]) + .build() + .unwrap(); + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches[0].num_rows(), 512); - - let col = batches[0].column_by_name("x").unwrap(); - let int64_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(int64_arr.value(0), 1); + assert_eq!(batches[0].num_columns(), 3); - let col = batches[0].column_by_name("y").unwrap(); - let int64_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(int64_arr.value(0), 2); + // Verify _file is at position 2 (the end) + let schema = batches[0].schema(); + assert_eq!(schema.field(0).name(), "x"); + assert_eq!(schema.field(1).name(), "y"); + assert_eq!(schema.field(2).name(), RESERVED_COL_NAME_FILE); } #[tokio::test] - async fn test_filter_on_arrow_gt_eq() { + async fn test_select_with_repeated_column_names() { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - // Filter: y >= 5 - let mut builder = fixture.table.scan(); - let predicate = Reference::new("y").greater_than_or_equal_to(Datum::long(5)); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); - - let batch_stream = table_scan.to_arrow().await.unwrap(); + // Select with repeated column names - both regular columns and virtual columns + // Repeated columns should appear multiple times in the result (duplicates are allowed) + let table_scan = fixture + .table + .scan() + .select([ + "x", + RESERVED_COL_NAME_FILE, + "x", // x repeated + "y", + RESERVED_COL_NAME_FILE, // _file repeated + "y", // y repeated + ]) + .build() + .unwrap(); + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches[0].num_rows(), 12); + // Verify we have exactly 6 columns (duplicates are allowed and preserved) + assert_eq!( + batches[0].num_columns(), + 6, + "Should have exactly 6 columns with duplicates" + ); - let col = batches[0].column_by_name("x").unwrap(); - let int64_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(int64_arr.value(0), 1); + let schema = batches[0].schema(); - let col = batches[0].column_by_name("y").unwrap(); - let int64_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(int64_arr.value(0), 5); + // Verify columns appear in the exact order requested: x, _file, x, y, _file, y + assert_eq!(schema.field(0).name(), "x", "Column 0 should be x"); + assert_eq!( + schema.field(1).name(), + RESERVED_COL_NAME_FILE, + "Column 1 should be _file" + ); + assert_eq!( + schema.field(2).name(), + "x", + "Column 2 should be x (duplicate)" + ); + assert_eq!(schema.field(3).name(), "y", "Column 3 should be y"); + assert_eq!( + schema.field(4).name(), + RESERVED_COL_NAME_FILE, + "Column 4 should be _file (duplicate)" + ); + assert_eq!( + schema.field(5).name(), + "y", + "Column 5 should be y (duplicate)" + ); + + // Verify all columns have correct data types + assert!( + matches!(schema.field(0).data_type(), arrow_schema::DataType::Int64), + "Column x should be Int64" + ); + assert!( + matches!(schema.field(2).data_type(), arrow_schema::DataType::Int64), + "Column x (duplicate) should be Int64" + ); + assert!( + matches!(schema.field(3).data_type(), arrow_schema::DataType::Int64), + "Column y should be Int64" + ); + assert!( + matches!(schema.field(5).data_type(), arrow_schema::DataType::Int64), + "Column y (duplicate) should be Int64" + ); + assert!( + matches!( + schema.field(1).data_type(), + arrow_schema::DataType::RunEndEncoded(_, _) + ), + "_file column should use RunEndEncoded type" + ); + assert!( + matches!( + schema.field(4).data_type(), + arrow_schema::DataType::RunEndEncoded(_, _) + ), + "_file column (duplicate) should use RunEndEncoded type" + ); } - #[tokio::test] - async fn test_filter_double_eq() { - let mut fixture = TableTestFixture::new(); - fixture.setup_manifest_files().await; + /// Builds a minimal single-snapshot table (no manifests on disk) whose schema + /// contains a column with the given name, so scan planning resolves column names + /// against a real schema. `TableScan::build()` only reads metadata, not manifest + /// files, so a snapshot pointing at a dummy manifest list is enough. Used to + /// reproduce issue #2837. + fn table_with_data_column(column_name: &str) -> Table { + use crate::spec::{ + FormatVersion, MAIN_BRANCH, Operation, Snapshot, SnapshotReference, SnapshotRetention, + SortOrder, Summary, TableMetadataBuilder, UnboundPartitionSpec, + }; - // Filter: dbl == 150.0 - let mut builder = fixture.table.scan(); - let predicate = Reference::new("dbl").equal_to(Datum::double(150.0f64)); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); + let schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, column_name, Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); - let batch_stream = table_scan.to_arrow().await.unwrap(); + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_timestamp_ms(1) + .with_sequence_number(0) + .with_schema_id(0) + .with_manifest_list("/snap-1.avro") + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .build(); - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + let metadata = TableMetadataBuilder::new( + schema, + UnboundPartitionSpec::builder().with_spec_id(0).build(), + SortOrder::unsorted_order(), + "s3://bucket/table".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .add_snapshot(snapshot) + .unwrap() + .set_ref(MAIN_BRANCH, SnapshotReference { + snapshot_id: 1, + retention: SnapshotRetention::Branch { + min_snapshots_to_keep: None, + max_snapshot_age_ms: None, + max_ref_age_ms: None, + }, + }) + .unwrap() + .build() + .unwrap() + .metadata; + + Table::builder() + .metadata(metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(FileIO::new_with_fs()) + .runtime(test_runtime()) + .build() + .unwrap() + } - assert_eq!(batches.len(), 2); - assert_eq!(batches[0].num_rows(), 12); + /// A user data column named `pos` (a delete-file internal column name that is not a + /// data-table metadata column) must be projectable rather than shadowed. Regression + /// test for issue #2837. + #[test] + fn test_scan_projects_data_column_named_like_delete_file_column() { + for column_name in ["pos", "file_path"] { + let table = table_with_data_column(column_name); + + // Projecting the data column must succeed and resolve to its real field id (2), + // not the reserved delete-file field id. + let table_scan = table + .scan() + .select([column_name]) + .build() + .unwrap_or_else(|e| panic!("scan of data column `{column_name}` failed: {e}")); - let col = batches[0].column_by_name("dbl").unwrap(); - let f64_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(f64_arr.value(1), 150.0f64); + assert_eq!( + table_scan.plan_context.as_ref().unwrap().field_ids.as_ref(), + &[2] + ); + + // The default projection (all columns) must resolve to the real field ids + // too, not shadow the data column with a reserved delete-file id. + let default_scan = table.scan().build().unwrap(); + assert_eq!( + default_scan + .plan_context + .as_ref() + .unwrap() + .field_ids + .as_ref(), + &[1, 2] + ); + } + } + + /// Projecting a genuinely absent column still fails with a clear "not found" error + /// rather than being silently accepted as a metadata column. + #[test] + fn test_scan_rejects_unknown_column_named_like_delete_file_column() { + // This table has no `pos` column (only `id` and `file_path`). + let table = table_with_data_column("file_path"); + + let err = table + .scan() + .select(["pos"]) + .build() + .expect_err("projecting an absent column should fail"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("not found")); } #[tokio::test] - async fn test_filter_int_eq() { + async fn test_scan_deadlock() { let mut fixture = TableTestFixture::new(); - fixture.setup_manifest_files().await; + fixture.setup_deadlock_manifests().await; + + // Create table scan with concurrency limit 1 + // This sets channel size to 1. + // Data manifest has 10 entries -> will block producer. + // Delete manifest is 2nd in list -> won't be processed. + // Consumer 2 (Data) not started -> blocked. + // Consumer 1 (Delete) waiting -> blocked. + let table_scan = fixture + .table + .scan() + .with_concurrency_limit(1) + .build() + .unwrap(); - // Filter: i32 == 150 - let mut builder = fixture.table.scan(); - let predicate = Reference::new("i32").equal_to(Datum::int(150i32)); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); + // This should timeout/hang if deadlock exists + // We can use tokio::time::timeout + let result = tokio::time::timeout(std::time::Duration::from_secs(5), async { + table_scan + .plan_files() + .await + .unwrap() + .try_collect::>() + .await + }) + .await; + + // Assert it finished (didn't timeout) + assert!(result.is_ok(), "Scan timed out - deadlock detected"); + } + + #[tokio::test] + async fn test_select_with_spec_id_column() { + let mut fixture = TableTestFixture::new(); + fixture.setup_manifest_files().await; - let batch_stream = table_scan.to_arrow().await.unwrap(); + // Select regular columns plus the _spec_id column + let table_scan = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_SPEC_ID, "z"]) + .build() + .unwrap(); + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches.len(), 2); - assert_eq!(batches[0].num_rows(), 12); + // Verify we have 3 columns: x, _spec_id, and z + assert_eq!(batches[0].num_columns(), 3); - let col = batches[0].column_by_name("i32").unwrap(); - let i32_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(i32_arr.value(1), 150i32); + // Verify the x column exists and has correct data + let col1 = batches[0].column_by_name("x").unwrap(); + let int64_arr = col1.as_any().downcast_ref::().unwrap(); + assert_eq!(int64_arr.value(0), 1); + + // Verify the _spec_id column exists + let spec_id_col = batches[0].column_by_name(RESERVED_COL_NAME_SPEC_ID); + assert!( + spec_id_col.is_some(), + "_spec_id column should be present in the batch" + ); + + // Verify the _spec_id data type + let spec_id_col = spec_id_col.unwrap(); + assert!( + matches!( + spec_id_col.data_type(), + arrow_schema::DataType::RunEndEncoded(_, _) + ), + "_spec_id column should use RunEndEncoded type" + ); + + // Decode the RunArray to verify it contains the spec id + let run_array = spec_id_col + .as_any() + .downcast_ref::>() + .expect("_spec_id column should be a RunArray"); + + let values = run_array.values(); + let int_values = values.as_primitive::(); + assert_eq!(int_values.len(), 1, "Should have a single _spec_id"); + + let spec_id = int_values.value(0); + assert_eq!(spec_id, 0, "_spec_id should be 0, got: {spec_id}"); + + // Verify 'z' column exists + assert!(batches[0].column_by_name("z").is_some()); } #[tokio::test] - async fn test_filter_long_eq() { + async fn test_select_with_last_updated_sequence_number_column() { + // A v2 fixture: data files have a null first_row_id. Per the spec's Row + // Lineage read rules, a file with a null first_row_id produces a null + // _last_updated_sequence_number for all rows; both lineage columns are + // gated on first_row_id. let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - // Filter: i64 == 150 - let mut builder = fixture.table.scan(); - let predicate = Reference::new("i64").equal_to(Datum::long(150i64)); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); + let table_scan = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER]) + .build() + .unwrap(); + + let batches: Vec<_> = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap() + .try_collect() + .await + .unwrap(); - let batch_stream = table_scan.to_arrow().await.unwrap(); + // Every row's value is null (v2 files have no first_row_id). + assert_last_updated_seq_all(&batches, None); + } - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + #[tokio::test] + async fn test_select_with_last_updated_sequence_number_column_v3() { + // A v3 fixture: the data file inherits a first_row_id and a data sequence + // number through manifest read. End to end, the projected + // _last_updated_sequence_number materializes to the file's data sequence + // number for every row, exercising the full inherit, populate and + // materialize wiring, not just a hand-built task. + let mut fixture = TableTestFixture::new(); + fixture.setup_v3_manifest_files().await; - assert_eq!(batches.len(), 2); - assert_eq!(batches[0].num_rows(), 12); + // The added file inherits the current snapshot's sequence number; derive it + // from the fixture rather than hardcoding so the assertion tracks the fixture. + let expected_seq = fixture + .table + .metadata() + .current_snapshot() + .unwrap() + .sequence_number(); - let col = batches[0].column_by_name("i64").unwrap(); - let i64_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(i64_arr.value(1), 150i64); + let table_scan = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER]) + .build() + .unwrap(); + + let batches: Vec<_> = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + assert_last_updated_seq_all(&batches, Some(expected_seq)); } #[tokio::test] - async fn test_filter_bool_eq() { - let mut fixture = TableTestFixture::new(); - fixture.setup_manifest_files().await; + async fn test_select_with_spec_id_column_from_unpartitioned_table() { + let mut fixture = TableTestFixture::new_unpartitioned(); + fixture.setup_unpartitioned_manifest_files().await; - // Filter: bool == true - let mut builder = fixture.table.scan(); - let predicate = Reference::new("bool").equal_to(Datum::bool(true)); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); + // Select regular columns plus the _spec_id column + let table_scan = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_SPEC_ID]) + .build() + .unwrap(); + + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); + let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + + // Verify we have 2 columns: x and _spec_id + assert_eq!(batches[0].num_columns(), 2); + + // Verify the _spec_id column exists + let spec_id_col = batches[0].column_by_name(RESERVED_COL_NAME_SPEC_ID); + assert!( + spec_id_col.is_some(), + "_spec_id column should be present in the batch" + ); - let batch_stream = table_scan.to_arrow().await.unwrap(); + // Verify the _spec_id data type + let spec_id_col = spec_id_col.unwrap(); + assert!( + matches!( + spec_id_col.data_type(), + arrow_schema::DataType::RunEndEncoded(_, _) + ), + "_spec_id column should use RunEndEncoded type" + ); - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + // Decode the RunArray to verify it contains the spec id + let run_array = spec_id_col + .as_any() + .downcast_ref::>() + .expect("_spec_id column should be a RunArray"); - assert_eq!(batches.len(), 2); - assert_eq!(batches[0].num_rows(), 512); + let values = run_array.values(); + let int_values = values.as_primitive::(); + assert_eq!(int_values.len(), 1, "Should have a single _spec_id"); - let col = batches[0].column_by_name("bool").unwrap(); - let bool_arr = col.as_any().downcast_ref::().unwrap(); - assert!(bool_arr.value(1)); + let spec_id = int_values.value(0); + assert_eq!(spec_id, 0, "_spec_id should be 0, got: {spec_id}"); } #[tokio::test] - async fn test_filter_on_arrow_is_null() { - let mut fixture = TableTestFixture::new(); - fixture.setup_manifest_files().await; - - // Filter: y is null - let mut builder = fixture.table.scan(); - let predicate = Reference::new("y").is_null(); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); + async fn test_select_with_spec_id_column_with_partition_evolution() { + let mut fixture = TableTestFixture::new_with_partition_evolution(); + fixture + .setup_manifest_files_with_partition_evolution() + .await; - let batch_stream = table_scan.to_arrow().await.unwrap(); + // Select regular columns plus the _spec_id column + let table_scan = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_SPEC_ID, "z"]) + .build() + .unwrap(); + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches.len(), 0); - } - #[tokio::test] - async fn test_filter_on_arrow_is_not_null() { - let mut fixture = TableTestFixture::new(); - fixture.setup_manifest_files().await; + // Verify the x column exists and has correct data + let col1 = batches[0].column_by_name("x").unwrap(); + let int64_arr = col1.as_any().downcast_ref::().unwrap(); + assert_eq!(int64_arr.value(0), 1); - // Filter: y is not null - let mut builder = fixture.table.scan(); - let predicate = Reference::new("y").is_not_null(); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); + // Verify the _spec_id column exists + let spec_id_col = batches[0].column_by_name(RESERVED_COL_NAME_SPEC_ID); + assert!( + spec_id_col.is_some(), + "_spec_id column should be present in the batch" + ); - let batch_stream = table_scan.to_arrow().await.unwrap(); + // Verify the _spec_id data type + let spec_id_col = spec_id_col.unwrap(); + assert!( + matches!( + spec_id_col.data_type(), + arrow_schema::DataType::RunEndEncoded(_, _) + ), + "_spec_id column should use RunEndEncoded type" + ); - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches[0].num_rows(), 1024); + // Decode the RunArray to verify it contains the spec id + let run_array = spec_id_col + .as_any() + .downcast_ref::>() + .expect("_spec_id column should be a RunArray"); + + let values = run_array.values(); + let int_values = values.as_primitive::(); + assert_eq!(int_values.len(), 1, "Should have a single _spec_id"); + + let spec_id = int_values.value(0); + assert_eq!(spec_id, 2, "_spec_id should be 2, got: {spec_id}"); } #[tokio::test] - async fn test_filter_on_arrow_lt_and_gt() { + async fn test_select_with_pos_and_file_columns() { + use arrow_array::cast::AsArray; + let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - // Filter: y < 5 AND z >= 4 - let mut builder = fixture.table.scan(); - let predicate = Reference::new("y") - .less_than(Datum::long(5)) - .and(Reference::new("z").greater_than_or_equal_to(Datum::long(4))); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); - - let batch_stream = table_scan.to_arrow().await.unwrap(); + // Select regular columns plus the _pos column + let table_scan = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_POS, RESERVED_COL_NAME_FILE]) + .build() + .unwrap(); + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches[0].num_rows(), 500); - - let col = batches[0].column_by_name("x").unwrap(); - let expected_x = Arc::new(Int64Array::from_iter_values(vec![1; 500])) as ArrayRef; - assert_eq!(col, &expected_x); - - let col = batches[0].column_by_name("y").unwrap(); - let mut values = vec![]; - values.append(vec![3; 200].as_mut()); - values.append(vec![4; 300].as_mut()); - let expected_y = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - assert_eq!(col, &expected_y); + assert_eq!(batches.len(), 2); - let col = batches[0].column_by_name("z").unwrap(); - let expected_z = Arc::new(Int64Array::from_iter_values(vec![4; 500])) as ArrayRef; - assert_eq!(col, &expected_z); + // Examine batches are 1.paruqet and 3.parquet, 2.parquet is deleted. + for batch in batches.iter() { + // Verify we have 3 columns: x, _pos and _file + assert_eq!(batch.num_columns(), 3); + + // Verify the x column exists and has correct data + let x_col = batch.column_by_name("x").unwrap(); + let x_arr = x_col.as_primitive::(); + assert_eq!(x_arr.value(0), 1); + + // The _pos column exists and verify it is Int64Array with the expected values + let pos_col = batch.column(1); + let pos_array: &Int64Array = pos_col + .as_any() + .downcast_ref::() + .expect("_pos column should be a Int64Array"); + assert_eq!(*pos_array, Int64Array::from_iter_values(0i64..1024)); + + // Verify the _file column exists + let file_col = batch.column_by_name(RESERVED_COL_NAME_FILE); + assert!( + file_col.is_some(), + "_file column should be present in the batch" + ); + } } #[tokio::test] - async fn test_filter_on_arrow_lt_or_gt() { + async fn test_pos_column_at_start_with_filters() { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - // Filter: y < 5 AND z >= 4 - let mut builder = fixture.table.scan(); + // y is in [4, 5) let predicate = Reference::new("y") - .less_than(Datum::long(5)) - .or(Reference::new("z").greater_than_or_equal_to(Datum::long(4))); - builder = builder + .greater_than(Datum::long(4i64)) + .and(Reference::new("y").less_than_or_equal_to(Datum::long(5i64))); + // Select _pos at the start + let table_scan = fixture + .table + .scan() + .select([RESERVED_COL_NAME_POS, "x", "y"]) .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); - - let batch_stream = table_scan.to_arrow().await.unwrap(); + .build() + .unwrap(); + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches[0].num_rows(), 1024); - - let col = batches[0].column_by_name("x").unwrap(); - let expected_x = Arc::new(Int64Array::from_iter_values(vec![1; 1024])) as ArrayRef; - assert_eq!(col, &expected_x); - - let col = batches[0].column_by_name("y").unwrap(); - let mut values = vec![2; 512]; - values.append(vec![3; 200].as_mut()); - values.append(vec![4; 300].as_mut()); - values.append(vec![5; 12].as_mut()); - let expected_y = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - assert_eq!(col, &expected_y); + assert_eq!(batches.len(), 2); - let col = batches[0].column_by_name("z").unwrap(); - let mut values = vec![3; 512]; - values.append(vec![4; 512].as_mut()); - let expected_z = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; - assert_eq!(col, &expected_z); + // Examine batches are 1.paruqet and 3.parquet, 2.parquet is deleted. + for batch in batches.iter() { + assert_eq!(batch.num_columns(), 3); + assert_eq!(batch.num_rows(), 12); + + // Verify _pos is at position 0 + let schema = batch.schema(); + assert_eq!(schema.field(0).name(), RESERVED_COL_NAME_POS); + assert_eq!(schema.field(1).name(), "x"); + assert_eq!(schema.field(2).name(), "y"); + + let pos_col = batch.column(0); + let pos_array: &Int64Array = pos_col + .as_any() + .downcast_ref::() + .expect("_pos column should be a Int64Array"); + assert_eq!(*pos_array, Int64Array::from_iter_values(1012i64..1024)); + } } #[tokio::test] - async fn test_filter_on_arrow_startswith() { + async fn test_repeated_pos_column_with_filter() { let mut fixture = TableTestFixture::new(); fixture.setup_manifest_files().await; - // Filter: a STARTSWITH "Ice" - let mut builder = fixture.table.scan(); - let predicate = Reference::new("a").starts_with(Datum::string("Ice")); - builder = builder + // a NOT STARTSWITH "Apa" + let predicate = Reference::new("a").not_starts_with(Datum::string("Apa")); + // Select '_pos' columns twice + let table_scan = fixture + .table + .scan() + .select([RESERVED_COL_NAME_POS, "a", RESERVED_COL_NAME_POS, "x"]) .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); + .build() + .unwrap(); - let batch_stream = table_scan.to_arrow().await.unwrap(); + let batch_stream = table_scan + .to_arrow( + ArrowReaderOptions::builder() + .with_row_selection_enabled(true) + .build(), + ) + .await + .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + assert_eq!(batches.len(), 2); - assert_eq!(batches[0].num_rows(), 512); - - let col = batches[0].column_by_name("a").unwrap(); - let string_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(string_arr.value(0), "Iceberg"); + // Examine batches are 1.paruqet and 3.parquet, 2.parquet is deleted. + for batch in batches.iter() { + assert_eq!(batch.num_rows(), 512); + + // fetch the 1st _pos column by name and verify it is Int64Array with the expected values + let pos_col = batch + .column_by_name("_pos") + .expect("_pos column should be present in the batch"); + let pos_array: &Int64Array = pos_col + .as_any() + .downcast_ref::() + .expect("_pos column should be a Int64Array"); + assert_eq!(*pos_array, Int64Array::from_iter_values(512i64..1024)); + + // fetch the 2nd _pos column by index and verify it is Int64Array with the expected values + let pos_col = batch.column(2); + let pos_array: &Int64Array = pos_col + .as_any() + .downcast_ref::() + .expect("_pos column should be a Int64Array"); + assert_eq!(*pos_array, Int64Array::from_iter_values(512i64..1024)); + } } + /// End-to-end through `TableScan`: a data file with three row groups planned + /// as a single whole-file `FileScanTask` must yield contiguous, file-absolute + /// `_pos` values (0..300) across the row-group boundaries. #[tokio::test] - async fn test_filter_on_arrow_not_startswith() { + async fn test_pos_across_row_groups_via_table_scan() { let mut fixture = TableTestFixture::new(); - fixture.setup_manifest_files().await; + fixture.setup_multi_row_group_manifest(&[]).await; - // Filter: a NOT STARTSWITH "Ice" - let mut builder = fixture.table.scan(); - let predicate = Reference::new("a").not_starts_with(Datum::string("Ice")); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); - - let batch_stream = table_scan.to_arrow().await.unwrap(); - - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + // Planning must produce exactly one whole-file task with _pos projected and + // no delete files, confirming TableScan does not sub-split the file. + let tasks: Vec<_> = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_POS]) + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(tasks.len(), 1, "expected a single FileScanTask"); + let task = &tasks[0]; + assert!( + task.project_field_ids().contains(&RESERVED_FIELD_ID_POS), + "_pos field id must be projected into the FileScanTask" + ); + assert_eq!(task.start(), 0, "TableScan should plan whole-file tasks"); + assert_eq!(task.length(), task.file_size_in_bytes()); + assert!(task.deletes().is_empty()); - assert_eq!(batches[0].num_rows(), 512); + // Reading that task yields absolute _pos 0..300 in order. + let batches: Vec<_> = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_POS]) + .build() + .unwrap() + .to_arrow(ArrowReaderOptions::default()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); - let col = batches[0].column_by_name("a").unwrap(); - let string_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(string_arr.value(0), "Apache"); + let pos: Vec = batches + .iter() + .flat_map(|b| { + b.column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_any() + .downcast_ref::() + .expect("_pos column should be a Int64Array") + .values() + .to_vec() + }) + .collect(); + assert_eq!(pos, (0..300).collect::>()); + + // Sanity: x == 1000 + _pos, proving _pos aligns with the actual rows read. + let x: Vec = batches + .iter() + .flat_map(|b| { + b.column_by_name("x") + .unwrap() + .as_primitive::() + .values() + .to_vec() + }) + .collect(); + assert_eq!(x, (1000..1300).collect::>()); } + /// A positional delete file registered in the manifest must be attached to the planned + /// `FileScanTask` and applied on read, while surviving `_pos` values stay file-absolute. #[tokio::test] - async fn test_filter_on_arrow_in() { + async fn test_pos_with_positional_deletes_via_table_scan() { let mut fixture = TableTestFixture::new(); - fixture.setup_manifest_files().await; - - // Filter: a IN ("Sioux", "Iceberg") - let mut builder = fixture.table.scan(); - let predicate = - Reference::new("a").is_in([Datum::string("Sioux"), Datum::string("Iceberg")]); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); + // Delete file-absolute positions 150 (middle row group) and 299 (last row). + fixture.setup_multi_row_group_manifest(&[150, 299]).await; - let batch_stream = table_scan.to_arrow().await.unwrap(); + // Planning must attach the positional delete file to the task. + let tasks: Vec<_> = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_POS]) + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(tasks.len(), 1); + assert_eq!( + tasks[0].deletes().len(), + 1, + "positional delete file should be planned into the task" + ); + assert_eq!( + tasks[0].deletes()[0].file_type(), + DataContentType::PositionDeletes + ); - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); + // Reading applies the deletes; _pos must skip 150 and 299 and stay absolute. + let batches: Vec<_> = fixture + .table + .scan() + .select(["x", RESERVED_COL_NAME_POS]) + .build() + .unwrap() + .to_arrow(ArrowReaderOptions::default()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); - assert_eq!(batches[0].num_rows(), 512); + let pos: Vec = batches + .iter() + .flat_map(|b| { + b.column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_any() + .downcast_ref::() + .expect("_pos column should be a Int64Array") + .values() + .to_vec() + }) + .collect(); - let col = batches[0].column_by_name("a").unwrap(); - let string_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(string_arr.value(0), "Iceberg"); + let total: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + total, 298, + "two rows should be removed by positional deletes" + ); + assert!(!pos.contains(&150) && !pos.contains(&299), "got {pos:?}"); + let expected: Vec = (0..150).chain(151..299).collect(); + assert_eq!(pos, expected); } + /// A filter that only matches the middle row group (`y` in [1100, 1200)) and + /// prunes the other two row groups by statistics, so only the middle row group + /// is read. `_pos` must report the file-absolute positions 100..200 for those + /// rows, not values reset to 0..100. + /// + /// `y` is a non-partition column, so pruning here is driven purely by Parquet + /// row-group statistics (the TableTestFixture's partition column is `x`). #[tokio::test] - async fn test_filter_on_arrow_not_in() { + async fn test_pos_reads_only_middle_row_group_via_filter() { let mut fixture = TableTestFixture::new(); - fixture.setup_manifest_files().await; - - // Filter: a NOT IN ("Sioux", "Iceberg") - let mut builder = fixture.table.scan(); - let predicate = - Reference::new("a").is_not_in([Datum::string("Sioux"), Datum::string("Iceberg")]); - builder = builder - .with_filter(predicate) - .with_row_selection_enabled(true); - let table_scan = builder.build().unwrap(); - - let batch_stream = table_scan.to_arrow().await.unwrap(); - - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - - assert_eq!(batches[0].num_rows(), 512); + fixture.setup_multi_row_group_manifest(&[]).await; - let col = batches[0].column_by_name("a").unwrap(); - let string_arr = col.as_any().downcast_ref::().unwrap(); - assert_eq!(string_arr.value(0), "Apache"); - } + // Middle row group holds y = 1100..1200 at file positions 100..200. + let predicate = Reference::new("y") + .greater_than_or_equal_to(Datum::long(1100)) + .and(Reference::new("y").less_than(Datum::long(1200))); - #[test] - fn test_file_scan_task_serialize_deserialize() { - let test_fn = |task: FileScanTask| { - let serialized = serde_json::to_string(&task).unwrap(); - let deserialized: FileScanTask = serde_json::from_str(&serialized).unwrap(); - - assert_eq!(task.data_file_path, deserialized.data_file_path); - assert_eq!(task.start, deserialized.start); - assert_eq!(task.length, deserialized.length); - assert_eq!(task.project_field_ids, deserialized.project_field_ids); - assert_eq!(task.predicate, deserialized.predicate); - assert_eq!(task.schema, deserialized.schema); - }; + let batches: Vec<_> = fixture + .table + .scan() + .select(["y", RESERVED_COL_NAME_POS]) + .with_filter(predicate) + .build() + .unwrap() + .to_arrow(ArrowReaderOptions::default()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); - // without predicate - let schema = Arc::new( - Schema::builder() - .with_fields(vec![Arc::new(NestedField::required( - 1, - "x", - Type::Primitive(PrimitiveType::Binary), - ))]) - .build() - .unwrap(), + let total: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total, 100, "only the middle row group should be read"); + + let pos: Vec = batches + .iter() + .flat_map(|b| { + b.column_by_name(RESERVED_COL_NAME_POS) + .expect("_pos column should be present") + .as_any() + .downcast_ref::() + .expect("_pos column should be a Int64Array") + .values() + .to_vec() + }) + .collect(); + assert_eq!( + pos, + (100..200).collect::>(), + "_pos must be file-absolute for the middle row group" ); - let task = FileScanTask { - data_file_path: "data_file_path".to_string(), - start: 0, - length: 100, - project_field_ids: vec![1, 2, 3], - predicate: None, - schema: schema.clone(), - record_count: Some(100), - data_file_format: DataFileFormat::Parquet, - deletes: vec![], - }; - test_fn(task); - - // with predicate - let task = FileScanTask { - data_file_path: "data_file_path".to_string(), - start: 0, - length: 100, - project_field_ids: vec![1, 2, 3], - predicate: Some(BoundPredicate::AlwaysTrue), - schema, - record_count: None, - data_file_format: DataFileFormat::Avro, - deletes: vec![], - }; - test_fn(task); + + // Cross-check: y == 1000 + _pos for every surviving row. + let y: Vec = batches + .iter() + .flat_map(|b| { + b.column_by_name("y") + .unwrap() + .as_primitive::() + .values() + .to_vec() + }) + .collect(); + assert_eq!(y, (1100..1200).collect::>()); } } diff --git a/crates/iceberg/src/scan/task.rs b/crates/iceberg/src/scan/task.rs index 32fe3ae309..1aa27c359e 100644 --- a/crates/iceberg/src/scan/task.rs +++ b/crates/iceberg/src/scan/task.rs @@ -15,53 +15,198 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; +use std::sync::Arc; + use futures::stream::BoxStream; use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; -use crate::Result; +use crate::error::invalid_data; use crate::expr::BoundPredicate; -use crate::spec::{DataContentType, DataFileFormat, ManifestEntryRef, Schema, SchemaRef}; +use crate::spec::{ + DataContentType, DataFile, DataFileFormat, Datum, ManifestEntryRef, NameMapping, PartitionSpec, + PrimitiveType, Schema, SchemaRef, SortOrderRef, Struct, StructType, +}; +use crate::{Error, Result}; /// A stream of [`FileScanTask`]. pub type FileScanTaskStream = BoxStream<'static, Result>; /// A task to scan part of file. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Deserialize, PartialEq, TypedBuilder)] +#[serde(try_from = "crate::scan::task::_serde::FileScanTaskSerde")] +#[builder( + field_defaults(setter(prefix = "with_")), + build_method(into = Result) +)] pub struct FileScanTask { + /// The total size of the data file in bytes, from the manifest entry. + /// Used to skip a stat/HEAD request when reading Parquet footers. + file_size_in_bytes: u64, /// The start offset of the file to scan. - pub start: u64, + start: u64, /// The length of the file to scan. - pub length: u64, + length: u64, /// The number of records in the file to scan. /// /// This is an optional field, and only available if we are /// reading the entire data file. - pub record_count: Option, + #[builder(default)] + record_count: Option, + + /// The first row id assigned to the data file. + /// + /// Used to derive the `_row_id` metadata column: for a row without an + /// explicit `_row_id`, it is this value plus the row's ordinal position. + #[builder(default)] + first_row_id: Option, + + /// The data sequence number of the file, as opposed to its file sequence + /// number: the sequence number preserved when a file is carried forward + /// across a rewrite. May be null for an existing entry in a malformed + /// manifest that lacks one. + /// + /// Used to derive the `_last_updated_sequence_number` metadata column. + #[builder(default)] + data_sequence_number: Option, /// The data file path corresponding to the task. - pub data_file_path: String, + data_file_path: String, /// The format of the file to scan. - pub data_file_format: DataFileFormat, + data_file_format: DataFileFormat, /// The schema of the file to scan. - pub schema: SchemaRef, + schema: SchemaRef, /// The field ids to project. - pub project_field_ids: Vec, + project_field_ids: Vec, /// The predicate to filter. - #[serde(skip_serializing_if = "Option::is_none")] - pub predicate: Option, + #[builder(default)] + predicate: Option, /// The list of delete files that may need to be applied to this data file - pub deletes: Vec, + #[builder(default)] + deletes: Vec, + + /// Partition data from the manifest entry, used to identify which columns can use + /// constant values from partition metadata vs. reading from the data file. + /// Per the Iceberg spec, only identity-transformed partition fields should use constants. + #[builder(default)] + partition: Option, + + /// The partition spec for this file, used to distinguish identity transforms + /// (which use partition metadata constants) from non-identity transforms like + /// bucket/truncate (which must read source columns from the data file). + #[builder(default)] + partition_spec: Option>, + + /// Name mapping from table metadata (property: schema.name-mapping.default), + /// used to resolve field IDs from column names when Parquet files lack field IDs + /// or have field ID conflicts. + #[builder(default)] + name_mapping: Option>, + + /// The unified partition type across all specs in the table. + /// When `RESERVED_FIELD_ID_PARTITION` is in the projected field IDs, the reader + /// uses this type along with the task's partition_spec and partition data to + /// materialize the `_partition` struct column at read time. + /// + /// This is a table-level value (same for all tasks in a scan), stored per-task + /// so that readers are self-contained without needing back-pointers to table + /// metadata. The cost is one Arc clone per task. + #[builder(default)] + unified_partition_type: Option>, + + /// The raw `sort_order_id` recorded on the data file (spec field 140), carried through + /// unresolved. See [`DataFile::sort_order_id()`](crate::spec::DataFile::sort_order_id). + /// `None` if the file has no sort order id. + /// + /// This preserves the distinction that [`sort_order`](Self::sort_order) collapses: a + /// non-zero id that does not resolve against the table's sort orders still marks the file + /// as physically sorted, which a consumer such as the DataFusion sorted-scan optimizer + /// needs even when the order definition itself is unavailable. + #[builder(default)] + sort_order_id: Option, + + /// The sort order that this file's rows are sorted by, resolved from the data file's + /// [`sort_order_id`](Self::sort_order_id) against the table's known sort orders. `Some` + /// only when the id resolves to an order that has sort fields. `None` if the file has no + /// sort order id, the id does not resolve, or it resolves to an order with no sort fields + /// (the reserved unsorted order, id 0 per the spec). Use [`sort_order_id`](Self::sort_order_id) + /// to tell those cases apart. + #[builder(default)] + sort_order: Option, + + #[builder(default, setter(transform = |stats: Option| stats.map(Box::new)))] + column_stats: Option>, + + /// Whether this scan task should treat column names as case-sensitive when binding predicates. + case_sensitive: bool, + + /// Key metadata for encrypted data files (Parquet Modular Encryption). + /// When present, the reader uses this to build `FileDecryptionProperties`. + /// + /// Note on the trust boundary: for the standard encryption scheme this + /// carries `StandardKeyMetadata`, whose payload is the *plaintext* DEK. + /// Because `FileScanTask` implements [`Serialize`], that plaintext DEK is part + /// of the serialized scan plan should these tasks ever be serialized and sent + /// over the network. + #[builder(default)] + key_metadata: Option>, } impl FileScanTask { + /// Returns the total size of the data file in bytes. + pub fn file_size_in_bytes(&self) -> u64 { + self.file_size_in_bytes + } + + /// Returns the start offset of the file to scan. + pub fn start(&self) -> u64 { + self.start + } + + /// Returns the length of the file to scan. + pub fn length(&self) -> u64 { + self.length + } + + /// Returns the number of records in the file when the whole file is scanned. + pub fn record_count(&self) -> Option { + self.record_count + } + + /// Returns the first row id assigned to the data file. + pub fn first_row_id(&self) -> Option { + self.first_row_id + } + + /// Returns the data sequence number of the file. + pub fn data_sequence_number(&self) -> Option { + self.data_sequence_number + } + /// Returns the data file path of this file scan task. pub fn data_file_path(&self) -> &str { &self.data_file_path } + /// Returns the format of the data file. + pub fn data_file_format(&self) -> DataFileFormat { + self.data_file_format + } + + /// Returns the schema of this file scan task as a reference. + pub fn schema(&self) -> &Schema { + &self.schema + } + + /// Returns the schema of this file scan task as a [`SchemaRef`]. + pub fn schema_ref(&self) -> SchemaRef { + self.schema.clone() + } + /// Returns the project field id of this file scan task. pub fn project_field_ids(&self) -> &[i32] { &self.project_field_ids @@ -72,14 +217,100 @@ impl FileScanTask { self.predicate.as_ref() } - /// Returns the schema of this file scan task as a reference - pub fn schema(&self) -> &Schema { - &self.schema + /// Clears the row predicate of this file scan task. + /// + /// The COW rewrite path uses this after candidate selection: candidates are + /// chosen with the predicate during planning, but each chosen file must then + /// be read in full so the rewrite sees every surviving row. Delete-file + /// application is unaffected. + pub(crate) fn clear_predicate(&mut self) { + self.predicate = None; } - /// Returns the schema of this file scan task as a SchemaRef - pub fn schema_ref(&self) -> SchemaRef { - self.schema.clone() + /// Returns the delete files that may need to be applied to the data file. + pub fn deletes(&self) -> &[FileScanTaskDeleteFile] { + &self.deletes + } + + /// Returns the partition data from the manifest entry. + pub fn partition(&self) -> Option<&Struct> { + self.partition.as_ref() + } + + /// Returns the partition spec for the data file. + pub fn partition_spec(&self) -> Option<&Arc> { + self.partition_spec.as_ref() + } + + /// Returns the name mapping used to resolve field ids. + pub fn name_mapping(&self) -> Option<&Arc> { + self.name_mapping.as_ref() + } + + /// Returns the unified partition type across all table partition specs. + pub fn unified_partition_type(&self) -> Option<&Arc> { + self.unified_partition_type.as_ref() + } + + /// Returns the raw sort order id recorded on the data file, unresolved. + pub fn sort_order_id(&self) -> Option { + self.sort_order_id + } + + /// Returns the sort order this file's rows are sorted by, if resolved. + pub fn sort_order(&self) -> Option<&SortOrderRef> { + self.sort_order.as_ref() + } + + /// Returns the data file's column statistics for the columns requested with + /// [`TableScanBuilder::with_column_stats`](crate::scan::TableScanBuilder::with_column_stats), + /// or `None` if none were requested. + pub fn column_stats(&self) -> Option<&FileScanTaskColumnStats> { + self.column_stats.as_deref() + } + + /// Returns whether names are treated as case-sensitive. + pub fn case_sensitive(&self) -> bool { + self.case_sensitive + } + + /// Returns the key metadata for the encrypted data file. + pub fn key_metadata(&self) -> Option<&[u8]> { + self.key_metadata.as_deref() + } + + fn validate(&self) -> Result<()> { + match (self.partition.as_ref(), self.partition_spec.as_deref()) { + (None, None) => Ok(()), + (None, Some(partition_spec)) if partition_spec.is_unpartitioned() => Ok(()), + (None, Some(_)) => Err(invalid_data!( + "FileScanTask with a partitioned spec requires partition values" + )), + (Some(partition), None) if partition.fields().is_empty() => Ok(()), + (Some(_), None) => Err(invalid_data!( + "Non-empty FileScanTask partition requires a partition spec" + )), + (Some(partition), Some(partition_spec)) + if partition.fields().len() != partition_spec.fields().len() => + { + Err(invalid_data!( + "FileScanTask partition has {} fields but partition spec has {} fields", + partition.fields().len(), + partition_spec.fields().len() + )) + } + (Some(_), Some(partition_spec)) => { + partition_spec.partition_type(&self.schema)?; + Ok(()) + } + } + } +} + +impl From for Result { + fn from(task: FileScanTask) -> Self { + task.validate()?; + Ok(task) } } @@ -89,29 +320,1093 @@ pub(crate) struct DeleteFileContext { pub(crate) partition_spec_id: i32, } -impl From<&DeleteFileContext> for FileScanTaskDeleteFile { - fn from(ctx: &DeleteFileContext) -> Self { - FileScanTaskDeleteFile { - file_path: ctx.manifest_entry.file_path().to_string(), - file_type: ctx.manifest_entry.content_type(), - partition_spec_id: ctx.partition_spec_id, - equality_ids: ctx.manifest_entry.data_file.equality_ids.clone(), - } +impl TryFrom<&DeleteFileContext> for FileScanTaskDeleteFile { + type Error = Error; + + fn try_from(ctx: &DeleteFileContext) -> Result { + // The manifest stores these as i64. Convert here so a negative byte offset is + // rejected once at the boundary, for every delete-file kind rather than only + // deletion vectors, instead of being carried inward and re-checked per consumer. + let file_path = ctx.manifest_entry.file_path(); + let to_offset = |value: Option, field: &str| -> Result> { + value + .map(|value| { + u64::try_from(value).map_err(|_| { + invalid_data!("delete file {file_path} has negative {field} {value}") + }) + }) + .transpose() + }; + + FileScanTaskDeleteFile::builder() + .with_file_path(ctx.manifest_entry.file_path().to_string()) + .with_file_size_in_bytes(ctx.manifest_entry.file_size_in_bytes()) + .with_file_type(ctx.manifest_entry.content_type()) + .with_file_format(ctx.manifest_entry.data_file().file_format()) + .with_partition_spec_id(ctx.partition_spec_id) + .with_equality_ids(ctx.manifest_entry.data_file.equality_ids.clone()) + .with_referenced_data_file(ctx.manifest_entry.data_file.referenced_data_file.clone()) + .with_content_offset(to_offset( + ctx.manifest_entry.data_file.content_offset, + "content_offset", + )?) + .with_content_size_in_bytes(to_offset( + ctx.manifest_entry.data_file.content_size_in_bytes, + "content_size_in_bytes", + )?) + .with_record_count(Some(ctx.manifest_entry.record_count())) + .with_key_metadata( + ctx.manifest_entry + .data_file + .key_metadata + .as_deref() + .map(Box::from), + ) + .build() } } /// A task to scan part of file. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, TypedBuilder)] +#[serde(try_from = "crate::scan::task::_serde::FileScanTaskDeleteFileSerde")] +#[builder( + field_defaults(setter(prefix = "with_")), + build_method(into = Result) +)] pub struct FileScanTaskDeleteFile { /// The delete file path - pub file_path: String, + file_path: String, + + /// The total size of the delete file in bytes, from the manifest entry. + file_size_in_bytes: u64, /// delete file type - pub file_type: DataContentType, + file_type: DataContentType, + + /// The delete file's format, from the manifest entry. A `PositionDeletes` entry written as + /// `Puffin` is a V3 deletion vector; one written as `Parquet` is a position delete file. + file_format: DataFileFormat, /// partition id - pub partition_spec_id: i32, + partition_spec_id: i32, /// equality ids for equality deletes (null for anything other than equality-deletes) - pub equality_ids: Option>, + #[builder(default)] + equality_ids: Option>, + + /// For a deletion vector, the location of the data file whose rows it deletes. Required for + /// deletion vectors, and may also be set on a position delete file scoped to one data file. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default)] + referenced_data_file: Option, + + /// For a deletion vector, the offset of the blob within its Puffin file. Set only for + /// deletion vectors, where it locates the blob for direct access. Unsigned because it is a + /// byte offset: a negative value is rejected once, when the task is built from a manifest + /// entry, rather than re-checked by every consumer. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default)] + content_offset: Option, + + /// For a deletion vector, the length in bytes of the blob within its Puffin file. + /// Required together with `content_offset`; both are absent for non-DV delete files. + /// Unsigned for the same reason as `content_offset`. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default)] + content_size_in_bytes: Option, + + /// The number of records in the delete file, from the manifest entry; for a deletion vector, + /// the cardinality of its bitmap. `None` only for a task not built from a manifest entry. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default)] + record_count: Option, + + /// Key metadata for an encrypted delete file. When present, the reader uses this to + /// decrypt the file: for a Parquet equality or position delete file, this builds + /// `FileDecryptionProperties` (Parquet Modular Encryption); for a deletion vector, whose + /// Puffin file has no native encryption, this wraps the range read in an + /// `EncryptedInputFile` (AGS1 stream encryption). + /// + /// Same plaintext-DEK trust boundary as [`FileScanTask::key_metadata`]: + /// this is serialized into the scan plan and crosses the planner -> worker + /// channel in the clear for the standard encryption scheme. + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + #[builder(default)] + key_metadata: Option>, +} + +mod _serde { + use std::sync::Arc; + + use serde::{Deserialize, Serialize}; + + use super::{FileScanTask, FileScanTaskColumnStats, FileScanTaskDeleteFile}; + use crate::error::invalid_data; + use crate::expr::BoundPredicate; + use crate::spec::{ + DataContentType, DataFileFormat, Literal, NameMapping, PartitionSpec, RawLiteral, + SchemaRef, SortOrderRef, StructType, Type, + }; + use crate::{Error, Result}; + + #[derive(Deserialize)] + pub(super) struct FileScanTaskSerde { + file_size_in_bytes: u64, + start: u64, + length: u64, + record_count: Option, + first_row_id: Option, + data_sequence_number: Option, + data_file_path: String, + data_file_format: DataFileFormat, + schema: SchemaRef, + project_field_ids: Vec, + predicate: Option, + deletes: Vec, + #[serde(default)] + partition: Option, + #[serde(default)] + partition_spec: Option>, + #[serde(default)] + name_mapping: Option>, + #[serde(default)] + unified_partition_type: Option>, + #[serde(default)] + sort_order_id: Option, + #[serde(default)] + sort_order: Option, + #[serde(default)] + column_stats: Option, + case_sensitive: bool, + #[serde(default)] + key_metadata: Option>, + } + + #[derive(Serialize)] + struct FileScanTaskRefSerde<'a> { + file_size_in_bytes: u64, + start: u64, + length: u64, + #[serde(skip_serializing_if = "Option::is_none")] + record_count: Option, + #[serde(skip_serializing_if = "Option::is_none")] + first_row_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + data_sequence_number: Option, + data_file_path: &'a str, + data_file_format: DataFileFormat, + schema: &'a SchemaRef, + project_field_ids: &'a [i32], + #[serde(skip_serializing_if = "Option::is_none")] + predicate: Option<&'a BoundPredicate>, + deletes: &'a [FileScanTaskDeleteFile], + #[serde(skip_serializing_if = "Option::is_none")] + partition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + partition_spec: Option<&'a Arc>, + #[serde(skip_serializing_if = "Option::is_none")] + name_mapping: Option<&'a Arc>, + #[serde(skip_serializing_if = "Option::is_none")] + unified_partition_type: Option<&'a Arc>, + #[serde(skip_serializing_if = "Option::is_none")] + sort_order_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + sort_order: Option<&'a SortOrderRef>, + #[serde(skip_serializing_if = "Option::is_none")] + column_stats: Option<&'a FileScanTaskColumnStats>, + case_sensitive: bool, + #[serde(skip_serializing_if = "Option::is_none")] + key_metadata: Option<&'a [u8]>, + } + + fn partition_type( + partition_spec: Option<&PartitionSpec>, + schema: &crate::spec::Schema, + ) -> Result { + let partition_type = match partition_spec { + Some(partition_spec) => partition_spec.partition_type(schema)?, + None => PartitionSpec::unpartition_spec().partition_type(schema)?, + }; + Ok(Type::Struct(partition_type)) + } + + impl<'a> TryFrom<&'a FileScanTask> for FileScanTaskRefSerde<'a> { + type Error = Error; + + fn try_from(value: &'a FileScanTask) -> Result { + let partition = value + .partition + .as_ref() + .map(|partition| { + let partition_type = + partition_type(value.partition_spec.as_deref(), &value.schema)?; + RawLiteral::try_from(Literal::Struct(partition.clone()), &partition_type) + }) + .transpose()?; + + Ok(Self { + file_size_in_bytes: value.file_size_in_bytes, + start: value.start, + length: value.length, + record_count: value.record_count, + first_row_id: value.first_row_id, + data_sequence_number: value.data_sequence_number, + data_file_path: &value.data_file_path, + data_file_format: value.data_file_format, + schema: &value.schema, + project_field_ids: &value.project_field_ids, + predicate: value.predicate.as_ref(), + deletes: &value.deletes, + partition, + partition_spec: value.partition_spec.as_ref(), + name_mapping: value.name_mapping.as_ref(), + unified_partition_type: value.unified_partition_type.as_ref(), + sort_order_id: value.sort_order_id, + sort_order: value.sort_order.as_ref(), + column_stats: value.column_stats.as_deref(), + case_sensitive: value.case_sensitive, + key_metadata: value.key_metadata.as_deref(), + }) + } + } + + impl Serialize for FileScanTask { + fn serialize(&self, serializer: S) -> std::result::Result + where S: serde::Serializer { + FileScanTaskRefSerde::try_from(self) + .map_err(serde::ser::Error::custom)? + .serialize(serializer) + } + } + + impl TryFrom for FileScanTask { + type Error = Error; + + fn try_from(value: FileScanTaskSerde) -> Result { + let partition = value + .partition + .map(|partition| { + let partition_type = + partition_type(value.partition_spec.as_deref(), &value.schema)?; + match partition.try_into(&partition_type)? { + Some(Literal::Struct(partition)) => Ok(partition), + _ => Err(invalid_data!("FileScanTask partition must be a struct")), + } + }) + .transpose()?; + + Self::builder() + .with_file_size_in_bytes(value.file_size_in_bytes) + .with_start(value.start) + .with_length(value.length) + .with_record_count(value.record_count) + .with_first_row_id(value.first_row_id) + .with_data_sequence_number(value.data_sequence_number) + .with_data_file_path(value.data_file_path) + .with_data_file_format(value.data_file_format) + .with_schema(value.schema) + .with_project_field_ids(value.project_field_ids) + .with_predicate(value.predicate) + .with_deletes(value.deletes) + .with_partition(partition) + .with_partition_spec(value.partition_spec) + .with_name_mapping(value.name_mapping) + .with_unified_partition_type(value.unified_partition_type) + .with_sort_order_id(value.sort_order_id) + .with_sort_order(value.sort_order) + .with_column_stats(value.column_stats) + .with_case_sensitive(value.case_sensitive) + .with_key_metadata(value.key_metadata) + .build() + } + } + + #[derive(Deserialize)] + pub(super) struct FileScanTaskDeleteFileSerde { + file_path: String, + file_size_in_bytes: u64, + file_type: DataContentType, + file_format: DataFileFormat, + partition_spec_id: i32, + equality_ids: Option>, + #[serde(default)] + referenced_data_file: Option, + #[serde(default)] + content_offset: Option, + #[serde(default)] + content_size_in_bytes: Option, + #[serde(default)] + record_count: Option, + #[serde(default)] + key_metadata: Option>, + } + + impl TryFrom for FileScanTaskDeleteFile { + type Error = Error; + + fn try_from(value: FileScanTaskDeleteFileSerde) -> Result { + Self::builder() + .with_file_path(value.file_path) + .with_file_size_in_bytes(value.file_size_in_bytes) + .with_file_type(value.file_type) + .with_file_format(value.file_format) + .with_partition_spec_id(value.partition_spec_id) + .with_equality_ids(value.equality_ids) + .with_referenced_data_file(value.referenced_data_file) + .with_content_offset(value.content_offset) + .with_content_size_in_bytes(value.content_size_in_bytes) + .with_record_count(value.record_count) + .with_key_metadata(value.key_metadata) + .build() + } + } + + /// Serializes column bounds as their type and the spec's single-value binary encoding, + /// because the JSON encoding of [`RawLiteral`] does not round-trip every float value. + pub(super) mod bounds { + use std::collections::HashMap; + + use serde::ser::SerializeMap; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_bytes::ByteBuf; + + use crate::spec::{Datum, PrimitiveType}; + + #[derive(Serialize, Deserialize)] + struct BoundSerde { + r#type: PrimitiveType, + bytes: ByteBuf, + } + + pub(in crate::scan::task) fn serialize( + bounds: &HashMap, + serializer: S, + ) -> Result { + let mut map = serializer.serialize_map(Some(bounds.len()))?; + for (field_id, datum) in bounds { + let bound = BoundSerde { + r#type: datum.data_type().clone(), + bytes: datum.to_bytes().map_err(serde::ser::Error::custom)?, + }; + map.serialize_entry(field_id, &bound)?; + } + map.end() + } + + pub(in crate::scan::task) fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(field_id, bound)| { + Datum::try_from_bytes(&bound.bytes, bound.r#type) + .map(|datum| (field_id, datum)) + .map_err(serde::de::Error::custom) + }) + .collect() + } + } +} + +impl FileScanTaskDeleteFile { + /// Returns the delete file path. + pub fn file_path(&self) -> &str { + &self.file_path + } + + /// Returns the total size of the delete file in bytes. + pub fn file_size_in_bytes(&self) -> u64 { + self.file_size_in_bytes + } + + /// Returns the delete file content type. + pub fn file_type(&self) -> DataContentType { + self.file_type + } + + /// Returns the delete file format. + pub fn file_format(&self) -> DataFileFormat { + self.file_format + } + + /// Returns the partition spec id. + pub fn partition_spec_id(&self) -> i32 { + self.partition_spec_id + } + + /// Returns the equality field ids for an equality delete file. + pub fn equality_ids(&self) -> Option<&[i32]> { + self.equality_ids.as_deref() + } + + /// Returns the referenced data file path. + pub fn referenced_data_file(&self) -> Option<&str> { + self.referenced_data_file.as_deref() + } + + /// Returns the deletion vector blob offset. + pub fn content_offset(&self) -> Option { + self.content_offset + } + + /// Returns the deletion vector blob size in bytes. + pub fn content_size_in_bytes(&self) -> Option { + self.content_size_in_bytes + } + + /// Returns the number of records in the delete file. + pub fn record_count(&self) -> Option { + self.record_count + } + + /// Returns the key metadata for the encrypted delete file. + pub fn key_metadata(&self) -> Option<&[u8]> { + self.key_metadata.as_deref() + } + + fn is_deletion_vector(&self) -> bool { + self.file_type == DataContentType::PositionDeletes + && self.file_format == DataFileFormat::Puffin + } + + fn validate(&self) -> Result<()> { + // Negative offsets/sizes are unrepresentable: both fields are u64, and the + // conversion from the manifest entry's i64 rejects a negative there. + + // Spec: `content_size_in_bytes` is required whenever `content_offset` is + // present, for every delete-file kind (not only deletion vectors). + if self.content_offset.is_some() && self.content_size_in_bytes.is_none() { + return Err(invalid_data!( + "delete file {} is missing content_size_in_bytes while content_offset is present", + self.file_path + )); + } + + // Presence of the deletion-vector coordinates is required only for + // deletion vectors; other delete files legitimately carry none of these. + if !self.is_deletion_vector() { + return Ok(()); + } + + let missing = if self.referenced_data_file.is_none() { + Some("referenced_data_file") + } else if self.content_offset.is_none() { + Some("content_offset") + } else if self.content_size_in_bytes.is_none() { + Some("content_size_in_bytes") + } else if self.record_count.is_none() { + Some("record_count") + } else { + None + }; + + if let Some(field) = missing { + return Err(invalid_data!( + "deletion vector {} is missing {field}", + self.file_path + )); + } + + Ok(()) + } +} + +impl From for Result { + fn from(task: FileScanTaskDeleteFile) -> Self { + task.validate()?; + Ok(task) + } +} + +/// Column statistics of a [`FileScanTask`]'s data file, keyed by field id and limited to the +/// columns requested for the scan. +/// +/// A requested column missing from a map has unknown statistics of that kind. Bounds are +/// inclusive, and all statistics cover the whole file, not the scan's filter, delete files or +/// task splits. +/// +/// A manifest written before a type promotion stores the narrower type, so bounds are promoted to +/// the column's type in the scan schema: `int` to `long`, `float` to `double`, and a decimal to a +/// wider precision with the same scale. A bound of any other type is omitted. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, TypedBuilder)] +#[builder(field_defaults(default, setter(prefix = "with_")))] +pub struct FileScanTaskColumnStats { + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + column_sizes: HashMap, + + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + value_counts: HashMap, + + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + null_value_counts: HashMap, + + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + nan_value_counts: HashMap, + + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::scan::task::_serde::bounds" + )] + lower_bounds: HashMap, + + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::scan::task::_serde::bounds" + )] + upper_bounds: HashMap, +} + +/// The field id and type of each column whose statistics a scan keeps. +pub(crate) type ColumnStatsColumns = Arc<[(i32, PrimitiveType)]>; + +/// Returns `bound` typed as `column_type`, or `None` if it cannot be promoted to it. +fn bound_as_column_type(bound: &Datum, column_type: &PrimitiveType) -> Option { + if bound.data_type() == column_type { + return Some(bound.clone()); + } + if let Some(promoted) = bound.promote_to(column_type) { + return Some(promoted); + } + match (bound.data_type(), column_type) { + ( + PrimitiveType::Decimal { + precision: from_precision, + scale: from_scale, + }, + PrimitiveType::Decimal { precision, scale }, + ) if from_scale == scale && from_precision <= precision => { + Some(Datum::new(column_type.clone(), bound.literal().clone())) + } + _ => None, + } +} + +impl FileScanTaskColumnStats { + pub(crate) fn from_data_file(data_file: &DataFile, columns: &[(i32, PrimitiveType)]) -> Self { + fn retain( + map: &HashMap, + columns: &[(i32, PrimitiveType)], + ) -> HashMap { + if map.is_empty() { + return HashMap::new(); + } + let mut retained = HashMap::with_capacity(columns.len().min(map.len())); + retained.extend( + columns + .iter() + .filter_map(|(id, _)| map.get(id).map(|value| (*id, value.clone()))), + ); + retained + } + + fn retain_bounds( + bounds: &HashMap, + columns: &[(i32, PrimitiveType)], + ) -> HashMap { + if bounds.is_empty() { + return HashMap::new(); + } + let mut retained = HashMap::with_capacity(columns.len().min(bounds.len())); + retained.extend(columns.iter().filter_map(|(id, column_type)| { + let bound = bound_as_column_type(bounds.get(id)?, column_type)?; + Some((*id, bound)) + })); + retained + } + + Self { + column_sizes: retain(data_file.column_sizes(), columns), + value_counts: retain(data_file.value_counts(), columns), + null_value_counts: retain(data_file.null_value_counts(), columns), + nan_value_counts: retain(data_file.nan_value_counts(), columns), + lower_bounds: retain_bounds(data_file.lower_bounds(), columns), + upper_bounds: retain_bounds(data_file.upper_bounds(), columns), + } + } + + /// Returns the size in bytes of each column's data in the file, per field id. + pub fn column_sizes(&self) -> &HashMap { + &self.column_sizes + } + + /// Returns the number of values per field id, including nulls and NaNs. + pub fn value_counts(&self) -> &HashMap { + &self.value_counts + } + + /// Returns the number of null values per field id. + pub fn null_value_counts(&self) -> &HashMap { + &self.null_value_counts + } + + /// Returns the number of NaN values per field id. + pub fn nan_value_counts(&self) -> &HashMap { + &self.nan_value_counts + } + + /// Returns the inclusive lower bound per field id. + pub fn lower_bounds(&self) -> &HashMap { + &self.lower_bounds + } + + /// Returns the inclusive upper bound per field id. + pub fn upper_bounds(&self) -> &HashMap { + &self.upper_bounds + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + use crate::ErrorKind; + use crate::spec::{ + DataFileBuilder, Literal, ManifestEntry, ManifestStatus, NestedField, PrimitiveLiteral, + PrimitiveType, Transform, Type, + }; + + fn build_file_scan_task( + schema: SchemaRef, + partition: Option, + partition_spec: Option>, + ) -> Result { + FileScanTask::builder() + .with_file_size_in_bytes(100) + .with_start(0) + .with_length(100) + .with_data_file_path("data_file_path".to_string()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(schema) + .with_project_field_ids(vec![]) + .with_partition(partition) + .with_partition_spec(partition_spec) + .with_case_sensitive(false) + .build() + } + + fn schema_and_spec( + primitive_type: PrimitiveType, + transform: Transform, + ) -> (SchemaRef, Arc) { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::required( + 1, + "x", + Type::Primitive(primitive_type), + ))]) + .build() + .unwrap(), + ); + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .add_partition_field("x", "x_partition", transform) + .unwrap() + .build() + .unwrap(), + ); + (schema, partition_spec) + } + + fn build_delete_file_task( + file_type: DataContentType, + file_format: DataFileFormat, + ) -> Result { + FileScanTaskDeleteFile::builder() + .with_file_path("delete-file".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(file_type) + .with_file_format(file_format) + .with_partition_spec_id(0) + .build() + } + + fn assert_delete_file_builder_error( + result: Result, + expected_message: &str, + ) { + match result { + Ok(task) => panic!( + "expected delete file builder to fail with `{expected_message}`, but got Ok({task:?})" + ), + Err(err) => { + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), expected_message); + } + } + } + + #[test] + fn test_file_scan_task_builder_rejects_non_empty_partition_without_spec() { + // Regression test for https://github.com/apache/iceberg-rust/issues/3130. + let err = build_file_scan_task( + Arc::new(Schema::builder().build().unwrap()), + Some(Struct::from_iter([Some(Literal::long(42))])), + None, + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + "Non-empty FileScanTask partition requires a partition spec" + ); + } + + #[test] + fn test_file_scan_task_builder_accepts_empty_partition_without_spec() { + build_file_scan_task( + Arc::new(Schema::builder().build().unwrap()), + Some(Struct::empty()), + None, + ) + .unwrap(); + } + + #[test] + fn test_file_scan_task_builder_rejects_partitioned_spec_without_partition() { + let (schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity); + + let err = build_file_scan_task(schema, None, Some(partition_spec)).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + "FileScanTask with a partitioned spec requires partition values" + ); + } + + #[test] + fn test_file_scan_task_builder_accepts_unpartitioned_spec_without_partition() { + build_file_scan_task( + Arc::new(Schema::builder().build().unwrap()), + None, + Some(Arc::new(PartitionSpec::unpartition_spec())), + ) + .unwrap(); + } + + #[test] + fn test_file_scan_task_builder_rejects_partition_arity_mismatch() { + let (schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity); + + let err = + build_file_scan_task(schema, Some(Struct::empty()), Some(partition_spec)).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("partition has 0 fields")); + assert!(err.message().contains("partition spec has 1 fields")); + } + + #[test] + fn test_file_scan_task_builder_accepts_dropped_partition_source_column() { + let (_historical_schema, partition_spec) = + schema_and_spec(PrimitiveType::Long, Transform::Identity); + let current_schema = Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::required( + 2, + "y", + Type::Primitive(PrimitiveType::String), + ))]) + .build() + .unwrap(), + ); + + build_file_scan_task( + current_schema, + Some(Struct::from_iter([Some(Literal::long(42))])), + Some(partition_spec), + ) + .unwrap(); + } + + #[test] + fn test_file_scan_task_builder_rejects_partition_spec_incompatible_with_schema() { + let (_historical_schema, partition_spec) = + schema_and_spec(PrimitiveType::Timestamp, Transform::Day); + let current_schema = Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::required( + 1, + "x", + Type::Primitive(PrimitiveType::String), + ))]) + .build() + .unwrap(), + ); + + let err = build_file_scan_task( + current_schema, + Some(Struct::from_iter([Some(Literal::date(20_000))])), + Some(partition_spec), + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_delete_file_builder_accepts_valid_deletion_vector() { + let task = FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(7) + .with_referenced_data_file(Some("data.parquet".to_string())) + .with_content_offset(Some(11)) + .with_content_size_in_bytes(Some(13)) + .with_record_count(Some(3)) + .with_key_metadata(Some(vec![17, 19].into_boxed_slice())) + .build() + .unwrap(); + + assert_eq!(task.file_path(), "dv.puffin"); + assert_eq!(task.file_size_in_bytes(), 100); + assert_eq!(task.file_type(), DataContentType::PositionDeletes); + assert_eq!(task.file_format(), DataFileFormat::Puffin); + assert_eq!(task.partition_spec_id(), 7); + assert_eq!(task.equality_ids(), None); + assert_eq!(task.referenced_data_file(), Some("data.parquet")); + assert_eq!(task.content_offset(), Some(11)); + assert_eq!(task.content_size_in_bytes(), Some(13)); + assert_eq!(task.record_count(), Some(3)); + assert_eq!(task.key_metadata(), Some([17, 19].as_slice())); + } + + #[test] + fn test_delete_file_builder_rejects_dv_missing_referenced_data_file() { + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_content_offset(Some(7)) + .with_content_size_in_bytes(Some(11)) + .with_record_count(Some(3)) + .build(), + "deletion vector dv.puffin is missing referenced_data_file", + ); + } + + #[test] + fn test_delete_file_builder_rejects_dv_missing_content_offset() { + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some("data.parquet".to_string())) + .with_content_size_in_bytes(Some(11)) + .with_record_count(Some(3)) + .build(), + "deletion vector dv.puffin is missing content_offset", + ); + } + + #[test] + fn test_delete_file_builder_rejects_dv_missing_content_size() { + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some("data.parquet".to_string())) + .with_content_offset(Some(7)) + .with_record_count(Some(3)) + .build(), + "delete file dv.puffin is missing content_size_in_bytes while content_offset is present", + ); + } + + #[test] + fn test_delete_file_builder_rejects_dv_missing_record_count() { + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("dv.puffin".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Puffin) + .with_partition_spec_id(0) + .with_referenced_data_file(Some("data.parquet".to_string())) + .with_content_offset(Some(7)) + .with_content_size_in_bytes(Some(11)) + .build(), + "deletion vector dv.puffin is missing record_count", + ); + } + + #[test] + fn test_delete_file_builder_rejects_non_dv_offset_without_size() { + // Spec pairing rule applies to every delete-file kind: a Parquet + // position-delete file that carries a content_offset must also carry a + // content_size_in_bytes, even though it is not a deletion vector. + assert_delete_file_builder_error( + FileScanTaskDeleteFile::builder() + .with_file_path("position-deletes.parquet".to_string()) + .with_file_size_in_bytes(100) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_partition_spec_id(0) + .with_content_offset(Some(7)) + .build(), + "delete file position-deletes.parquet is missing content_size_in_bytes while content_offset is present", + ); + } + + fn delete_manifest_entry( + file_path: &str, + file_format: DataFileFormat, + content_offset: Option, + content_size_in_bytes: Option, + ) -> DeleteFileContext { + let mut data_file = DataFileBuilder::default() + .content(DataContentType::PositionDeletes) + .file_path(file_path.to_string()) + .file_format(file_format) + .partition(Struct::empty()) + .record_count(3) + .file_size_in_bytes(100) + .column_sizes(HashMap::new()) + .value_counts(HashMap::new()) + .null_value_counts(HashMap::new()) + .partition_spec_id(0) + .referenced_data_file(Some("data.parquet".to_string())) + .build() + .unwrap(); + data_file.content_offset = content_offset; + data_file.content_size_in_bytes = content_size_in_bytes; + + DeleteFileContext { + manifest_entry: Arc::new( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(data_file) + .build(), + ), + partition_spec_id: 0, + } + } + + fn assert_delete_context_error(ctx: &DeleteFileContext, expected_message: &str) { + match FileScanTaskDeleteFile::try_from(ctx) { + Ok(task) => panic!( + "expected conversion to fail with `{expected_message}`, but got Ok({task:?})" + ), + Err(error) => { + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!( + error.message().contains(expected_message), + "expected `{expected_message}`, got `{}`", + error.message() + ); + } + } + } + + /// The manifest entry supplies `content_offset` / `content_size_in_bytes` as + /// `i64`; the conversion into a task rejects a negative value as it enters, + /// since the task fields are `u64`. + #[test] + fn test_delete_context_rejects_negative_dv_offset() { + assert_delete_context_error( + &delete_manifest_entry("dv.puffin", DataFileFormat::Puffin, Some(-1), Some(11)), + "delete file dv.puffin has negative content_offset -1", + ); + } + + #[test] + fn test_delete_context_rejects_negative_dv_size() { + assert_delete_context_error( + &delete_manifest_entry("dv.puffin", DataFileFormat::Puffin, Some(7), Some(-1)), + "delete file dv.puffin has negative content_size_in_bytes -1", + ); + } + + /// The negative-coordinate check applies to every delete-file kind, including + /// a Parquet position-delete file rather than only deletion vectors. + #[test] + fn test_delete_context_rejects_negative_coordinates_for_non_dv() { + assert_delete_context_error( + &delete_manifest_entry( + "position-deletes.parquet", + DataFileFormat::Parquet, + Some(-1), + None, + ), + "delete file position-deletes.parquet has negative content_offset -1", + ); + assert_delete_context_error( + &delete_manifest_entry( + "position-deletes.parquet", + DataFileFormat::Parquet, + None, + Some(-1), + ), + "delete file position-deletes.parquet has negative content_size_in_bytes -1", + ); + } + + #[test] + fn test_bound_as_column_type() { + let decimal = |precision, scale| PrimitiveType::Decimal { precision, scale }; + let decimal_datum = + |precision, scale| Datum::new(decimal(precision, scale), PrimitiveLiteral::Int128(150)); + let cases = [ + (Datum::int(-5), PrimitiveType::Long, Some(Datum::long(-5))), + ( + Datum::float(f32::NEG_INFINITY), + PrimitiveType::Double, + Some(Datum::double(f64::NEG_INFINITY)), + ), + ( + Datum::float(1.5_f32), + PrimitiveType::Double, + Some(Datum::double(1.5)), + ), + ( + Datum::string("a"), + PrimitiveType::String, + Some(Datum::string("a")), + ), + ( + decimal_datum(9, 2), + decimal(18, 2), + Some(decimal_datum(18, 2)), + ), + (decimal_datum(9, 2), decimal(18, 3), None), + (decimal_datum(38, 2), decimal(18, 2), None), + (Datum::long(6), PrimitiveType::Int, None), + (Datum::string("a"), PrimitiveType::Long, None), + ]; + + for (bound, column_type, expected) in cases { + assert_eq!( + bound_as_column_type(&bound, &column_type), + expected, + "{bound:?} as {column_type:?}" + ); + } + } + + #[test] + fn test_delete_file_builder_accepts_non_dv_delete_without_dv_fields() { + build_delete_file_task(DataContentType::PositionDeletes, DataFileFormat::Parquet).unwrap(); + build_delete_file_task(DataContentType::EqualityDeletes, DataFileFormat::Parquet).unwrap(); + } } diff --git a/crates/iceberg/src/sensitive.rs b/crates/iceberg/src/sensitive.rs new file mode 100644 index 0000000000..f7f92efd62 --- /dev/null +++ b/crates/iceberg/src/sensitive.rs @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! This module contains types to keep sensitive data in memory. + +use std::fmt; + +use zeroize::Zeroizing; + +/// A string-like type containing sensitive information such as passwords or tokens. +/// +/// It is redacted from debug logs and automatically zeroized. +/// +/// # Example +/// ``` +/// use iceberg::sensitive::SensitiveString; +/// +/// let sensitive_value = "my-pw-12345"; +/// let sensitive_string = SensitiveString::from(sensitive_value.to_string()); +/// +/// // Not contained in debug logs. +/// assert!(!format!("{:?}", sensitive_string).contains(sensitive_value)); +/// ``` +/// +/// # Display +/// [`SensitiveString`] does **not** implement [`std::fmt::Display`] to prevent bugs like: +/// +/// ```compile_fail +/// # use iceberg::sensitive::SensitiveString; +/// // We don't want to send a redacted `Bearer: *****`. +/// let auth_header = format!("Bearer {}", SensitiveString::from("token".to_string())); +/// ``` +/// +/// Instead use an explicit [`SensitiveString::expose`] when you need it: +/// +/// ``` +/// # use iceberg::sensitive::SensitiveString; +/// let auth_header = format!( +/// "Bearer: {}", +/// SensitiveString::from("token".to_string()).expose() +/// ); +/// ``` +#[derive(Clone, PartialEq, Eq)] +pub struct SensitiveString(Zeroizing); + +impl SensitiveString { + /// Returns the raw value of the sensitive string. + pub fn expose(&self) -> &str { + &self.0 + } + + /// Returns `true` if the string value is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Debug for SensitiveString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("SensitiveString([REDACTED])") + } +} + +impl From for SensitiveString { + fn from(value: String) -> Self { + Self(Zeroizing::new(value)) + } +} + +/// Wrapper for sensitive byte data (encryption keys, DEKs, etc.) that: +/// - Zeroizes memory on drop +/// - Redacts content in [`Debug`] and [`std::fmt::Display`] output +/// - Provides only `&[u8]` access via [`as_bytes()`](Self::as_bytes) +/// - Uses `Box<[u8]>` (immutable boxed slice) since key bytes never grow +/// +/// Use this type for any struct field that holds plaintext key material. +/// Because its [`Debug`] impl always prints `[N bytes REDACTED]`, structs +/// containing `SensitiveBytes` can safely derive or implement `Debug` +/// without risk of leaking key material. +#[derive(Clone, PartialEq, Eq)] +pub struct SensitiveBytes(Zeroizing>); + +impl SensitiveBytes { + /// Wraps the given bytes as sensitive material. + pub fn new(bytes: impl Into>) -> Self { + Self(Zeroizing::new(bytes.into())) + } + + /// Returns the underlying bytes. + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + + /// Returns the number of bytes. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if the byte slice is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Debug for SensitiveBytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[{} bytes REDACTED]", self.0.len()) + } +} + +impl fmt::Display for SensitiveBytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[{} bytes REDACTED]", self.0.len()) + } +} + +#[cfg(test)] +mod tests { + use crate::sensitive::{SensitiveBytes, SensitiveString}; + + #[test] + fn test_sensitive_string_redacts_debug_value() { + let sensitive_value = "my-pw-12346"; + + let logged = format!("{:?}", SensitiveString::from(sensitive_value.to_string())); + assert!(!logged.contains(sensitive_value)); + } + + #[test] + fn test_sensitive_bytes_redacts_debug_value() { + let sensitive_value = b"my-secret-bytes"; + + let logged = format!("{:?}", SensitiveBytes::new(&sensitive_value[..])); + assert!( + !logged + .as_bytes() + .windows(sensitive_value.len()) + .any(|window| window == sensitive_value) + ); + } + + #[test] + fn test_sensitive_bytes_redacts_display_value() { + let sensitive_value = b"my-secret-bytes"; + + let logged = format!("{}", SensitiveBytes::new(&sensitive_value[..])); + assert!( + !logged + .as_bytes() + .windows(sensitive_value.len()) + .any(|window| window == sensitive_value) + ); + } +} diff --git a/crates/iceberg/src/spec/datatypes.rs b/crates/iceberg/src/spec/datatypes.rs index 7c03001f88..1345cac2ad 100644 --- a/crates/iceberg/src/spec/datatypes.rs +++ b/crates/iceberg/src/spec/datatypes.rs @@ -19,21 +19,21 @@ * Data Types */ use std::collections::HashMap; -use std::convert::identity; use std::fmt; use std::ops::Index; use std::sync::{Arc, OnceLock}; use ::serde::de::{MapAccess, Visitor}; use serde::de::{Error, IntoDeserializer}; -use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value as JsonValue; +use super::geospatial::{GeographyType, GeometryType, parse_geography, parse_geometry}; use super::values::Literal; use crate::ensure_data_valid; use crate::error::Result; -use crate::spec::PrimitiveLiteral; use crate::spec::datatypes::_decimal::{MAX_PRECISION, REQUIRED_LENGTH}; +use crate::spec::{FormatVersion, PrimitiveLiteral}; /// Field name for list type. pub const LIST_FIELD_NAME: &str = "element"; @@ -90,15 +90,18 @@ pub enum Type { List(ListType), /// Map type Map(MapType), + /// Variant Type + Variant(VariantType), } impl fmt::Display for Type { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - Type::Primitive(primitive) => write!(f, "{}", primitive), - Type::Struct(s) => write!(f, "{}", s), + Type::Primitive(primitive) => write!(f, "{primitive}"), + Type::Struct(s) => write!(f, "{s}"), Type::List(_) => write!(f, "list"), Type::Map(_) => write!(f, "map"), + Type::Variant(_) => write!(f, "variant"), } } } @@ -122,6 +125,36 @@ impl Type { matches!(self, Type::Struct(_) | Type::List(_) | Type::Map(_)) } + /// Whether the type is variant type. + #[inline(always)] + pub fn is_variant(&self) -> bool { + matches!(self, Type::Variant(_)) + } + + /// Minimum [`FormatVersion`] required to support this type, **without** taking + /// nested field types into account. + /// + /// `Unknown` / `TimestampNs` / `TimestamptzNs` / `Geometry` / `Geography` / `Variant` require + /// [`FormatVersion::V3`]; every other type is valid from [`FormatVersion::V1`]. Mirrors Java's + /// `Schema.MIN_FORMAT_VERSIONS` (a shallow lookup keyed by type id), so it + /// intentionally does not recurse: callers needing the floor for a whole schema + /// iterate its flattened fields (see [`Schema::calc_min_compatible_format`]). + /// + /// [`Schema::calc_min_compatible_format`]: crate::spec::Schema::calc_min_compatible_format + pub(crate) fn min_format_version(&self) -> FormatVersion { + match self { + Type::Primitive( + PrimitiveType::Unknown + | PrimitiveType::TimestampNs + | PrimitiveType::TimestamptzNs + | PrimitiveType::Geometry(_) + | PrimitiveType::Geography(_), + ) + | Type::Variant(_) => FormatVersion::V3, + _ => FormatVersion::V1, + } + } + /// Convert Type to reference of PrimitiveType pub fn as_primitive_type(&self) -> Option<&PrimitiveType> { if let Type::Primitive(primitive_type) = self { @@ -140,7 +173,7 @@ impl Type { } } - /// Return max precision for decimal given [`num_bytes`] bytes. + /// Return max precision for decimal given `num_bytes` bytes. #[inline(always)] pub fn decimal_max_precision(num_bytes: u32) -> Result { ensure_data_valid!( @@ -150,7 +183,7 @@ impl Type { Ok(MAX_PRECISION[num_bytes as usize - 1]) } - /// Returns minimum bytes required for decimal with [`precision`]. + /// Returns minimum bytes required for decimal with `precision`. #[inline(always)] pub fn decimal_required_bytes(precision: u32) -> Result { ensure_data_valid!( @@ -163,10 +196,7 @@ impl Type { /// Creates decimal type. #[inline(always)] pub fn decimal(precision: u32, scale: u32) -> Result { - ensure_data_valid!( - precision > 0 && precision <= MAX_DECIMAL_PRECISION, - "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}", - ); + validate_decimal_precision(precision)?; Ok(Type::Primitive(PrimitiveType::Decimal { precision, scale })) } @@ -247,6 +277,12 @@ pub enum PrimitiveType { Fixed(u64), /// Arbitrary-length byte array. Binary, + /// Default / null column type used when a more specific type is not known. + Unknown, + /// Geometry values encoded as well-known binary. + Geometry(GeometryType), + /// Geography values encoded as well-known binary. + Geography(GeographyType), } impl PrimitiveType { @@ -270,6 +306,8 @@ impl PrimitiveType { | (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(_)) | (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(_)) | (PrimitiveType::Binary, PrimitiveLiteral::Binary(_)) + | (PrimitiveType::Geometry(_), PrimitiveLiteral::Binary(_)) + | (PrimitiveType::Geography(_), PrimitiveLiteral::Binary(_)) ) } } @@ -298,6 +336,16 @@ impl<'de> Deserialize<'de> for PrimitiveType { deserialize_decimal(s.into_deserializer()) } else if s.starts_with("fixed") { deserialize_fixed(s.into_deserializer()) + } else if s + .get(..8) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("geometry")) + { + parse_geometry(&s).map_err(D::Error::custom) + } else if s + .get(..9) + .is_some_and(|prefix| prefix.eq_ignore_ascii_case("geography")) + { + parse_geography(&s).map_err(D::Error::custom) } else { PrimitiveType::deserialize(s.into_deserializer()) } @@ -312,24 +360,48 @@ impl Serialize for PrimitiveType { serialize_decimal(precision, scale, serializer) } PrimitiveType::Fixed(l) => serialize_fixed(l, serializer), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { + serializer.serialize_str(&self.to_string()) + } _ => PrimitiveType::serialize(self, serializer), } } } +fn validate_decimal_precision(precision: u32) -> Result<()> { + // Leave scale unconstrained for Java parity; the spec leaves scale > precision open. + ensure_data_valid!(precision > 0, "Decimal precision must be greater than zero",); + ensure_data_valid!( + precision <= MAX_DECIMAL_PRECISION, + "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}", + ); + Ok(()) +} + fn deserialize_decimal<'de, D>(deserializer: D) -> std::result::Result where D: Deserializer<'de> { let s = String::deserialize(deserializer)?; + let malformed = || D::Error::custom(format!("Invalid decimal type: {s}")); + let (precision, scale) = s - .trim_start_matches(r"decimal(") - .trim_end_matches(')') + .strip_prefix("decimal(") + .and_then(|inner| inner.strip_suffix(')')) + .ok_or_else(malformed)? .split_once(',') - .ok_or_else(|| D::Error::custom("Decimal requires precision and scale: {s}"))?; + .ok_or_else(|| D::Error::custom(format!("Decimal requires precision and scale: {s}")))?; - Ok(PrimitiveType::Decimal { - precision: precision.trim().parse().map_err(D::Error::custom)?, - scale: scale.trim().parse().map_err(D::Error::custom)?, - }) + let (precision, scale) = (precision.trim(), scale.trim()); + if [precision, scale] + .iter() + .any(|token| token.is_empty() || !token.bytes().all(|byte| byte.is_ascii_digit())) + { + return Err(malformed()); + } + let precision: u32 = precision.parse().map_err(D::Error::custom)?; + let scale: u32 = scale.parse().map_err(D::Error::custom)?; + validate_decimal_precision(precision).map_err(D::Error::custom)?; + + Ok(PrimitiveType::Decimal { precision, scale }) } fn serialize_decimal( @@ -340,17 +412,21 @@ fn serialize_decimal( where S: Serializer, { - serializer.serialize_str(&format!("decimal({precision},{scale})")) + serializer.serialize_str(&format!("decimal({precision}, {scale})")) } fn deserialize_fixed<'de, D>(deserializer: D) -> std::result::Result where D: Deserializer<'de> { - let fixed = String::deserialize(deserializer)? - .trim_start_matches(r"fixed[") - .trim_end_matches(']') - .to_owned(); + let s = String::deserialize(deserializer)?; + + let length = s + .strip_prefix("fixed[") + .and_then(|inner| inner.strip_suffix(']')) + .map(str::trim) + .filter(|length| !length.is_empty() && length.bytes().all(|byte| byte.is_ascii_digit())) + .ok_or_else(|| D::Error::custom(format!("Invalid fixed type: {s}")))?; - fixed + length .parse() .map(PrimitiveType::Fixed) .map_err(D::Error::custom) @@ -364,13 +440,14 @@ where S: Serializer { impl fmt::Display for PrimitiveType { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { + PrimitiveType::Unknown => write!(f, "unknown"), PrimitiveType::Boolean => write!(f, "boolean"), PrimitiveType::Int => write!(f, "int"), PrimitiveType::Long => write!(f, "long"), PrimitiveType::Float => write!(f, "float"), PrimitiveType::Double => write!(f, "double"), PrimitiveType::Decimal { precision, scale } => { - write!(f, "decimal({},{})", precision, scale) + write!(f, "decimal({precision}, {scale})") } PrimitiveType::Date => write!(f, "date"), PrimitiveType::Time => write!(f, "time"), @@ -380,8 +457,15 @@ impl fmt::Display for PrimitiveType { PrimitiveType::TimestamptzNs => write!(f, "timestamptz_ns"), PrimitiveType::String => write!(f, "string"), PrimitiveType::Uuid => write!(f, "uuid"), - PrimitiveType::Fixed(size) => write!(f, "fixed({})", size), + PrimitiveType::Fixed(size) => write!(f, "fixed({size})"), PrimitiveType::Binary => write!(f, "binary"), + PrimitiveType::Geometry(geometry) => write!(f, "geometry({})", geometry.crs()), + PrimitiveType::Geography(geography) => write!( + f, + "geography({}, {})", + geography.crs(), + geography.algorithm().as_str() + ), } } } @@ -423,17 +507,24 @@ impl<'de> Deserialize<'de> for StructType { let mut fields = None; while let Some(key) = map.next_key()? { match key { - Field::Type => (), + Field::Type => { + let type_val: String = map.next_value()?; + if type_val != "struct" { + return Err(Error::custom(format!( + "expected type 'struct', got '{type_val}'" + ))); + } + } Field::Fields => { if fields.is_some() { - return Err(serde::de::Error::duplicate_field("fields")); + return Err(Error::duplicate_field("fields")); } fields = Some(map.next_value()?); } } } let fields: Vec = - fields.ok_or_else(|| de::Error::missing_field("fields"))?; + fields.ok_or_else(|| Error::missing_field("fields"))?; Ok(StructType::new(fields)) } @@ -520,7 +611,7 @@ impl fmt::Display for StructType { } #[derive(Debug, PartialEq, Serialize, Deserialize, Eq, Clone)] -#[serde(from = "SerdeNestedField", into = "SerdeNestedField")] +#[serde(try_from = "SerdeNestedField", into = "SerdeNestedField")] /// A struct is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema. /// Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type. /// Fields may have an optional comment or doc string. Fields can have default values. @@ -557,25 +648,30 @@ struct SerdeNestedField { pub write_default: Option, } -impl From for NestedField { - fn from(value: SerdeNestedField) -> Self { - NestedField { +impl TryFrom for NestedField { + type Error = crate::Error; + + fn try_from(value: SerdeNestedField) -> Result { + let initial_default = value + .initial_default + .map(|default| Literal::try_from_json(default, &value.field_type)) + .transpose()? + .flatten(); + let write_default = value + .write_default + .map(|default| Literal::try_from_json(default, &value.field_type)) + .transpose()? + .flatten(); + + Ok(NestedField { id: value.id, name: value.name, required: value.required, - initial_default: value.initial_default.and_then(|x| { - Literal::try_from_json(x, &value.field_type) - .ok() - .and_then(identity) - }), - write_default: value.write_default.and_then(|x| { - Literal::try_from_json(x, &value.field_type) - .ok() - .and_then(identity) - }), + initial_default, + write_default, field_type: value.field_type, doc: value.doc, - } + }) } } @@ -600,12 +696,12 @@ pub type NestedFieldRef = Arc; impl NestedField { /// Construct a new field. - pub fn new(id: i32, name: impl ToString, field_type: Type, required: bool) -> Self { + pub fn new(id: i32, name: impl ToString, field_type: impl Into, required: bool) -> Self { Self { id, name: name.to_string(), required, - field_type: Box::new(field_type), + field_type: Box::new(field_type.into()), doc: None, initial_default: None, write_default: None, @@ -613,27 +709,27 @@ impl NestedField { } /// Construct a required field. - pub fn required(id: i32, name: impl ToString, field_type: Type) -> Self { + pub fn required(id: i32, name: impl ToString, field_type: impl Into) -> Self { Self::new(id, name, field_type, true) } /// Construct an optional field. - pub fn optional(id: i32, name: impl ToString, field_type: Type) -> Self { + pub fn optional(id: i32, name: impl ToString, field_type: impl Into) -> Self { Self::new(id, name, field_type, false) } /// Construct list type's element field. - pub fn list_element(id: i32, field_type: Type, required: bool) -> Self { + pub fn list_element(id: i32, field_type: impl Into, required: bool) -> Self { Self::new(id, LIST_FIELD_NAME, field_type, required) } /// Construct map type's key field. - pub fn map_key_element(id: i32, field_type: Type) -> Self { + pub fn map_key_element(id: i32, field_type: impl Into) -> Self { Self::required(id, MAP_KEY_FIELD_NAME, field_type) } /// Construct map type's value field. - pub fn map_value_element(id: i32, field_type: Type, required: bool) -> Self { + pub fn map_value_element(id: i32, field_type: impl Into, required: bool) -> Self { Self::new(id, MAP_VALUE_FIELD_NAME, field_type, required) } @@ -673,7 +769,7 @@ impl fmt::Display for NestedField { } write!(f, "{} ", self.field_type)?; if let Some(doc) = &self.doc { - write!(f, "{}", doc)?; + write!(f, "{doc}")?; } Ok(()) } @@ -703,6 +799,7 @@ pub(super) mod _serde { use crate::spec::datatypes::Type::Map; use crate::spec::datatypes::{ ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType, Type, + VariantType, }; /// List type for serialization and deserialization @@ -730,6 +827,7 @@ pub(super) mod _serde { value: Cow<'a, Type>, }, Primitive(PrimitiveType), + Variant(VariantType), } impl From> for Type { @@ -768,6 +866,7 @@ pub(super) mod _serde { Self::Struct(StructType::new(fields.into_owned())) } SerdeType::Primitive(p) => Self::Primitive(p), + SerdeType::Variant(v) => Self::Variant(v), } } } @@ -781,7 +880,7 @@ pub(super) mod _serde { element_required: list.element_field.required, element: Cow::Borrowed(&list.element_field.field_type), }, - Type::Map(map) => SerdeType::Map { + Map(map) => SerdeType::Map { r#type: "map".to_string(), key_id: map.key_field.id, key: Cow::Borrowed(&map.key_field.field_type), @@ -794,6 +893,7 @@ pub(super) mod _serde { fields: Cow::Borrowed(&s.fields), }, Type::Primitive(p) => SerdeType::Primitive(p.clone()), + Type::Variant(v) => SerdeType::Variant(*v), } } } @@ -819,14 +919,79 @@ impl MapType { value_field, } } + + /// Construct an optional map type with the given key and value fields. + pub fn optional( + key_id: i32, + key_type: impl Into, + value_id: i32, + value_type: impl Into, + ) -> Self { + Self { + key_field: NestedField::map_key_element(key_id, key_type).into(), + value_field: NestedField::map_value_element(value_id, value_type, false).into(), + } + } + + /// Construct a required map type with the given key and value fields. + pub fn required( + key_id: i32, + key_type: impl Into, + value_id: i32, + value_type: impl Into, + ) -> Self { + Self { + key_field: NestedField::map_key_element(key_id, key_type).into(), + value_field: NestedField::map_value_element(value_id, value_type, true).into(), + } + } +} + +/// Variant type - can hold semi-structured data of any type. +/// This is an Iceberg V3 feature. +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +pub struct VariantType; + +impl fmt::Display for VariantType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "variant") + } +} + +impl From for Type { + fn from(_: VariantType) -> Self { + Type::Variant(VariantType) + } +} + +impl Serialize for VariantType { + fn serialize(&self, serializer: S) -> std::result::Result + where S: Serializer { + serializer.serialize_str("variant") + } +} + +impl<'de> Deserialize<'de> for VariantType { + fn deserialize(deserializer: D) -> std::result::Result + where D: Deserializer<'de> { + let s = String::deserialize(deserializer)?; + if s == "variant" { + Ok(VariantType) + } else { + Err(D::Error::custom(format!("expected 'variant', got '{s}'"))) + } + } } #[cfg(test)] mod tests { + use std::hash::{Hash, Hasher}; + use pretty_assertions::assert_eq; use uuid::Uuid; use super::*; + use crate::spec::geospatial::{DEFAULT_GEOSPATIAL_CRS, EdgeInterpolationAlgorithm}; use crate::spec::values::PrimitiveLiteral; fn check_type_serde(json: &str, expected_type: Type) { @@ -846,12 +1011,13 @@ mod tests { { "type": "struct", "fields": [ + {"id": 17, "name": "unknown_field", "required": false, "type": "unknown"}, {"id": 1, "name": "bool_field", "required": true, "type": "boolean"}, {"id": 2, "name": "int_field", "required": true, "type": "int"}, {"id": 3, "name": "long_field", "required": true, "type": "long"}, {"id": 4, "name": "float_field", "required": true, "type": "float"}, {"id": 5, "name": "double_field", "required": true, "type": "double"}, - {"id": 6, "name": "decimal_field", "required": true, "type": "decimal(9,2)"}, + {"id": 6, "name": "decimal_field", "required": true, "type": "decimal(9, 2)"}, {"id": 7, "name": "date_field", "required": true, "type": "date"}, {"id": 8, "name": "time_field", "required": true, "type": "time"}, {"id": 9, "name": "timestamp_field", "required": true, "type": "timestamp"}, @@ -870,6 +1036,12 @@ mod tests { record, Type::Struct(StructType { fields: vec![ + NestedField::optional( + 17, + "unknown_field", + Type::Primitive(PrimitiveType::Unknown), + ) + .into(), NestedField::required(1, "bool_field", Type::Primitive(PrimitiveType::Boolean)) .into(), NestedField::required(2, "int_field", Type::Primitive(PrimitiveType::Int)) @@ -948,6 +1120,67 @@ mod tests { ) } + #[test] + fn primitive_type_geospatial() { + let cases = vec![ + ( + r#""geometry""#, + PrimitiveType::Geometry(GeometryType::default()), + "geometry(OGC:CRS84)", + ), + ( + r#""GeOmEtRy ( EPSG:3857 )""#, + PrimitiveType::Geometry(GeometryType::new(Some("EPSG:3857".to_string())).unwrap()), + "geometry(EPSG:3857)", + ), + ( + r#""GEOGRAPHY""#, + PrimitiveType::Geography(GeographyType::default()), + "geography(OGC:CRS84, spherical)", + ), + ( + r#""GeOgRaPhY ( OGC:CRS27 , KARNEY )""#, + PrimitiveType::Geography( + GeographyType::new( + Some("OGC:CRS27".to_string()), + EdgeInterpolationAlgorithm::Karney, + ) + .unwrap(), + ), + "geography(OGC:CRS27, karney)", + ), + ]; + + for (json, expected, display) in cases { + let actual: PrimitiveType = serde_json::from_str(json).unwrap(); + assert_eq!( + actual, expected, + "parsed primitive type did not match expectation" + ); + assert_eq!(actual.to_string(), display, "display impl did not match"); + assert_eq!( + serde_json::to_string(&actual).unwrap(), + format!(r#""{display}""#), + "JSON serialization did not match expectation" + ); + } + + let equivalent_default = GeometryType::new(Some("EPSG:4326".to_string())).unwrap(); + assert_eq!(equivalent_default.crs(), "EPSG:4326"); + assert_ne!(equivalent_default, GeometryType::default()); + let lowercase_default = + GeometryType::new(Some(DEFAULT_GEOSPATIAL_CRS.to_ascii_lowercase())).unwrap(); + assert_eq!(lowercase_default, GeometryType::default()); + let mut lowercase_hash = std::collections::hash_map::DefaultHasher::new(); + lowercase_default.hash(&mut lowercase_hash); + let mut default_hash = std::collections::hash_map::DefaultHasher::new(); + GeometryType::default().hash(&mut default_hash); + assert_eq!(lowercase_hash.finish(), default_hash.finish()); + assert!( + serde_json::from_str::(r#""geography(OGC:CRS27,unknown)""#).is_err() + ); + } + #[test] fn struct_type() { let record = r#" @@ -1128,6 +1361,16 @@ mod tests { .into(), }), ); + + check_type_serde( + record, + Type::Map(MapType::optional( + 4, + Type::Primitive(PrimitiveType::String), + 5, + Type::Primitive(PrimitiveType::Double), + )), + ); } #[test] @@ -1156,6 +1399,40 @@ mod tests { .into(), }), ); + + check_type_serde( + record, + Type::Map(MapType::optional( + 4, + Type::Primitive(PrimitiveType::Int), + 5, + Type::Primitive(PrimitiveType::String), + )), + ); + } + + #[test] + fn map_required_int() { + let record = r#" + { + "type": "map", + "key-id": 4, + "key": "int", + "value-id": 5, + "value-required": true, + "value": "string" + } + "#; + + check_type_serde( + record, + Type::Map(MapType::required( + 4, + Type::Primitive(PrimitiveType::Int), + 5, + Type::Primitive(PrimitiveType::String), + )), + ); } #[test] @@ -1176,6 +1453,182 @@ mod tests { assert_eq!(16, Type::decimal_required_bytes(38).unwrap()); } + #[test] + fn test_reject_malformed_decimal_and_fixed_type_strings() { + for invalid in [ + r#""decimal(50, 2)""#, + r#""decimal(0, 0)""#, + r#""decimal(decimal(5, 2)))""#, + r#""decimal(decimal(5, 2)""#, + r#""decimal(5, 2""#, + r#""decimal(5, 2)))))""#, + r#""decimal(5, 2, 3)""#, + r#""decimal(-5, 2)""#, + r#""decimal(5, -2)""#, + r#""decimal(-5, -2)""#, + r#""decimal(-2, -5)""#, + r#""decimal((5, 2))""#, + r#""decimal[5, 2]""#, + r#""decimal()""#, + r#""fixed[fixed[16]]]""#, + r#""fixed[16""#, + r#""fixed[16]]]""#, + r#""fixed[[16]]""#, + r#""fixed[[16]""#, + r#""fixed(16)""#, + r#""fixed[]""#, + ] { + assert!( + serde_json::from_str::(invalid).is_err(), + "expected {invalid} to be rejected" + ); + } + } + + #[test] + fn test_reject_leading_plus_in_type_strings() { + for invalid in [ + r#""decimal(+5, 2)""#, + r#""decimal(5, +2)""#, + r#""decimal(+5, +2)""#, + r#""fixed[+16]""#, + ] { + assert!(serde_json::from_str::(invalid).is_err(), "{invalid}"); + assert!( + serde_json::from_str::(invalid).is_err(), + "{invalid}" + ); + } + } + + #[test] + fn test_decimal_constructor_json_roundtrip() { + for (precision, scale) in [(1, 0), (5, 5), (5, 8), (38, 38)] { + let decimal = Type::decimal(precision, scale).unwrap(); + let serialized = serde_json::to_string(&decimal).unwrap(); + let reparsed: Type = serde_json::from_str(&serialized).unwrap(); + assert_eq!(reparsed, decimal); + let primitive: PrimitiveType = serde_json::from_str(&serialized).unwrap(); + assert_eq!(Type::Primitive(primitive), decimal); + } + for precision in [0, 39] { + assert!(Type::decimal(precision, 0).is_err()); + let json = format!(r#""decimal({precision}, 0)""#); + assert!(serde_json::from_str::(&json).is_err()); + assert!(serde_json::from_str::(&json).is_err()); + } + } + + #[test] + fn test_empty_decimal_and_fixed_tokens() { + for invalid in [ + r#""decimal(, 2)""#, + r#""decimal(5,)""#, + r#""decimal( , )""#, + r#""fixed[]""#, + r#""fixed[ ]""#, + ] { + assert!(serde_json::from_str::(invalid).is_err(), "{invalid}"); + assert!( + serde_json::from_str::(invalid).is_err(), + "{invalid}" + ); + } + } + + #[test] + fn test_decimal_and_fixed_overflow_errors() { + for invalid in [ + r#""decimal(9999999999, 2)""#, + r#""decimal(5, 9999999999)""#, + r#""fixed[18446744073709551616]""#, + ] { + assert!(serde_json::from_str::(invalid).is_err(), "{invalid}"); + let error = serde_json::from_str::(invalid).unwrap_err(); + assert!( + error + .to_string() + .contains("number too large to fit in target type"), + "unexpected error for {invalid}: {error}" + ); + } + } + + #[test] + fn test_decimal_zero_precision_error() { + let error = serde_json::from_str::(r#""decimal(0, 0)""#).unwrap_err(); + assert!( + error + .to_string() + .contains("Decimal precision must be greater than zero"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_accept_valid_decimal_and_fixed_type_strings() { + for (json, canonical, expected) in [ + ( + r#""decimal(9, 2)""#, + r#""decimal(9, 2)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 2, + }), + ), + ( + r#""decimal(38, 10)""#, + r#""decimal(38, 10)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 38, + scale: 10, + }), + ), + ( + r#""decimal(5,2)""#, + r#""decimal(5, 2)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 5, + scale: 2, + }), + ), + ( + r#""decimal(5, 0)""#, + r#""decimal(5, 0)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 5, + scale: 0, + }), + ), + ( + r#""decimal(5, 5)""#, + r#""decimal(5, 5)""#, + Type::Primitive(PrimitiveType::Decimal { + precision: 5, + scale: 5, + }), + ), + ( + r#""fixed[16]""#, + r#""fixed[16]""#, + Type::Primitive(PrimitiveType::Fixed(16)), + ), + ] { + check_type_serde_roundtrip_value(json, canonical, expected); + } + } + + fn check_type_serde_roundtrip_value(json: &str, canonical: &str, expected_type: Type) { + let parsed: Type = serde_json::from_str(json).unwrap(); + assert_eq!(parsed, expected_type); + let serialized = serde_json::to_string(&parsed).unwrap(); + assert_eq!(serialized, canonical); + let primitive: PrimitiveType = serde_json::from_str(json).unwrap(); + assert_eq!(serde_json::to_string(&primitive).unwrap(), canonical); + let reparsed: Type = serde_json::from_str(&serialized).unwrap(); + assert_eq!(reparsed, expected_type); + } + #[test] fn test_primitive_type_compatible() { let pairs = vec![ @@ -1203,9 +1656,100 @@ mod tests { ), (PrimitiveType::Fixed(8), PrimitiveLiteral::Binary(vec![1])), (PrimitiveType::Binary, PrimitiveLiteral::Binary(vec![1])), + ( + PrimitiveType::Geometry(GeometryType::default()), + PrimitiveLiteral::Binary(vec![1]), + ), + ( + PrimitiveType::Geography(GeographyType::default()), + PrimitiveLiteral::Binary(vec![1]), + ), ]; for (ty, literal) in pairs { assert!(ty.compatible(&literal)); } + + assert!(!PrimitiveType::Unknown.compatible(&PrimitiveLiteral::Int(1))); + } + + #[test] + fn variant_type_serde() { + let json = r#"{"id": 1, "name": "v", "required": true, "type": "variant"}"#; + let field: NestedField = serde_json::from_str(json).unwrap(); + assert_eq!(*field.field_type, Type::Variant(VariantType)); + + let serialized = serde_json::to_string(&field).unwrap(); + let roundtrip: NestedField = serde_json::from_str(&serialized).unwrap(); + assert_eq!(field, roundtrip); + } + + #[test] + fn nested_field_rejects_invalid_map_defaults() { + for default_name in ["initial-default", "write-default"] { + let json = format!( + r#"{{ + "id": 1, + "name": "properties", + "required": false, + "type": {{ + "type": "map", + "key-id": 2, + "key": "string", + "value-id": 3, + "value-required": false, + "value": "int" + }}, + "{default_name}": {{"keys": ["a", "b"], "values": [1]}} + }}"# + ); + + let error = serde_json::from_str::(&json).unwrap_err(); + assert!(error.to_string().contains("must have the same length")); + } + } + + #[test] + fn struct_type_with_type_field() { + // Test that StructType properly deserializes JSON with "type":"struct" field + // This was previously broken because the deserializer wasn't consuming the type field value + let json = r#" + { + "type": "struct", + "fields": [ + {"id": 1, "name": "field1", "required": true, "type": "string"} + ] + } + "#; + + let struct_type: StructType = serde_json::from_str(json) + .expect("Should successfully deserialize StructType with type field"); + + assert_eq!(struct_type.fields().len(), 1); + assert_eq!(struct_type.fields()[0].name, "field1"); + } + + #[test] + fn struct_type_rejects_wrong_type() { + // Test that StructType validation rejects incorrect type field values + let json = r#" + { + "type": "list", + "fields": [ + {"id": 1, "name": "field1", "required": true, "type": "string"} + ] + } + "#; + + let result = serde_json::from_str::(json); + assert!( + result.is_err(), + "Should reject StructType with wrong type field" + ); + assert!( + result + .unwrap_err() + .to_string() + .contains("expected type 'struct'") + ); } } diff --git a/crates/iceberg/src/spec/encrypted_key.rs b/crates/iceberg/src/spec/encrypted_key.rs index db19a023b2..6908ade1d5 100644 --- a/crates/iceberg/src/spec/encrypted_key.rs +++ b/crates/iceberg/src/spec/encrypted_key.rs @@ -26,16 +26,16 @@ use serde::{Deserialize, Serialize}; pub struct EncryptedKey { /// Unique identifier for the key #[builder(setter(into))] - key_id: String, + pub(crate) key_id: String, /// Encrypted key metadata as binary data #[builder(setter(into))] - encrypted_key_metadata: Vec, + pub(crate) encrypted_key_metadata: Vec, /// Identifier of the entity that encrypted this key - #[builder(setter(into))] - encrypted_by_id: String, + #[builder(default, setter(into, strip_option))] + pub(crate) encrypted_by_id: Option, /// Additional properties associated with the key #[builder(default)] - properties: HashMap, + pub(crate) properties: HashMap, } impl EncryptedKey { @@ -50,8 +50,8 @@ impl EncryptedKey { } /// Returns the ID of the entity that encrypted this key - pub fn encrypted_by_id(&self) -> &str { - &self.encrypted_by_id + pub fn encrypted_by_id(&self) -> Option<&str> { + self.encrypted_by_id.as_deref() } /// Returns the properties map @@ -72,7 +72,7 @@ pub(super) mod _serde { pub(super) struct EncryptedKeySerde { pub key_id: String, pub encrypted_key_metadata: String, // Base64 encoded - pub encrypted_by_id: String, + pub encrypted_by_id: Option, #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub properties: HashMap, } diff --git a/crates/iceberg/src/spec/geospatial.rs b/crates/iceberg/src/spec/geospatial.rs new file mode 100644 index 0000000000..df2ea5a2ec --- /dev/null +++ b/crates/iceberg/src/spec/geospatial.rs @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Geospatial types defined by the Iceberg specification. + +use std::hash::{Hash, Hasher}; + +use serde::{Deserialize, Serialize}; + +use super::PrimitiveType; +use crate::error::{Result, invalid_data}; + +pub(crate) const DEFAULT_GEOSPATIAL_CRS: &str = "OGC:CRS84"; + +#[derive(Debug, Serialize, Deserialize, Clone)] +struct Crs(String); + +impl Crs { + fn new(crs: Option) -> Result { + let crs = crs.unwrap_or_else(|| DEFAULT_GEOSPATIAL_CRS.to_string()); + let crs = crs.trim().to_string(); + if crs.is_empty() || crs.contains([',', ')']) { + return Err(crate::Error::new( + crate::ErrorKind::DataInvalid, + "Geospatial CRS must be non-empty and must not contain ',' or ')'", + )); + } + + Ok(Self(crs)) + } +} + +impl Default for Crs { + fn default() -> Self { + Self(DEFAULT_GEOSPATIAL_CRS.to_string()) + } +} + +impl PartialEq for Crs { + fn eq(&self, other: &Self) -> bool { + self.0.eq_ignore_ascii_case(&other.0) + } +} + +impl Eq for Crs {} + +impl Hash for Crs { + fn hash(&self, state: &mut H) { + for byte in self.0.bytes() { + state.write_u8(byte.to_ascii_lowercase()); + } + } +} + +/// Iceberg geometry type. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash, Default)] +pub struct GeometryType { + crs: Crs, +} + +impl GeometryType { + /// Creates a geometry type with an optional coordinate reference system. + pub fn new(crs: Option) -> Result { + Ok(Self { + crs: Crs::new(crs)?, + }) + } + + /// Returns the coordinate reference system. + pub fn crs(&self) -> &str { + &self.crs.0 + } +} + +/// Iceberg geography edge interpolation algorithm. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Copy, Hash, Default)] +#[serde(rename_all = "lowercase")] +pub enum EdgeInterpolationAlgorithm { + /// Spherical edge interpolation. + #[default] + Spherical, + /// Vincenty edge interpolation. + Vincenty, + /// Thomas edge interpolation. + Thomas, + /// Andoyer edge interpolation. + Andoyer, + /// Karney edge interpolation. + Karney, +} + +impl EdgeInterpolationAlgorithm { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Spherical => "spherical", + Self::Vincenty => "vincenty", + Self::Thomas => "thomas", + Self::Andoyer => "andoyer", + Self::Karney => "karney", + } + } + + fn parse(value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "spherical" => Ok(Self::Spherical), + "vincenty" => Ok(Self::Vincenty), + "thomas" => Ok(Self::Thomas), + "andoyer" => Ok(Self::Andoyer), + "karney" => Ok(Self::Karney), + _ => Err(invalid_data!( + "Unknown geography edge interpolation algorithm: {value}" + )), + } + } +} + +/// Iceberg geography type. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)] +pub struct GeographyType { + crs: Crs, + algorithm: EdgeInterpolationAlgorithm, +} + +impl Default for GeographyType { + fn default() -> Self { + Self { + crs: Crs::default(), + algorithm: EdgeInterpolationAlgorithm::Spherical, + } + } +} + +impl GeographyType { + /// Creates a geography type with an optional coordinate reference system and edge interpolation algorithm. + pub fn new(crs: Option, algorithm: EdgeInterpolationAlgorithm) -> Result { + Ok(Self { + crs: Crs::new(crs)?, + algorithm, + }) + } + + /// Returns the coordinate reference system. + pub fn crs(&self) -> &str { + &self.crs.0 + } + + /// Returns the edge interpolation algorithm. + pub fn algorithm(&self) -> EdgeInterpolationAlgorithm { + self.algorithm + } +} + +fn parse_geospatial_params<'a>(value: &'a str, type_name: &str) -> Result> { + if value.eq_ignore_ascii_case(type_name) { + return Ok(vec![]); + } + + let params = value + .get(..type_name.len()) + .filter(|prefix| prefix.eq_ignore_ascii_case(type_name)) + .map(|_| &value[type_name.len()..]) + .map(str::trim_start) + .and_then(|s| s.strip_prefix('(')) + .and_then(|s| s.strip_suffix(')')) + .ok_or_else(|| invalid_data!("Invalid {type_name} type: {value}"))?; + + if params.trim().is_empty() { + return Err(invalid_data!("{type_name} requires a non-empty CRS")); + } + + Ok(params.split(',').map(str::trim).collect()) +} + +pub(super) fn parse_geometry(value: &str) -> Result { + let params = parse_geospatial_params(value.trim(), "geometry")?; + let geometry = match params.as_slice() { + [] => GeometryType::default(), + [crs] if !crs.is_empty() => GeometryType::new(Some((*crs).to_string()))?, + _ => return Err(invalid_data!("Invalid geometry type: {value}")), + }; + + Ok(PrimitiveType::Geometry(geometry)) +} + +pub(super) fn parse_geography(value: &str) -> Result { + let params = parse_geospatial_params(value.trim(), "geography")?; + let geography = match params.as_slice() { + [] => GeographyType::default(), + [crs] if !crs.is_empty() => GeographyType::new( + Some((*crs).to_string()), + EdgeInterpolationAlgorithm::Spherical, + )?, + [crs, algorithm] if !crs.is_empty() && !algorithm.is_empty() => GeographyType::new( + Some((*crs).to_string()), + EdgeInterpolationAlgorithm::parse(algorithm)?, + )?, + _ => return Err(invalid_data!("Invalid geography type: {value}")), + }; + + Ok(PrimitiveType::Geography(geography)) +} diff --git a/crates/iceberg/src/spec/manifest/_serde.rs b/crates/iceberg/src/spec/manifest/_serde.rs index 7738af46d4..f6793be2f2 100644 --- a/crates/iceberg/src/spec/manifest/_serde.rs +++ b/crates/iceberg/src/spec/manifest/_serde.rs @@ -15,14 +15,24 @@ // specific language governing permissions and limitations // under the License. +//! Serde forms of manifest entries. +//! +//! `Manifest::parse_avro` deserializes these structs from the writer schema +//! without a reader schema, so they decide which manifests read. A field that +//! a writer may omit, per the spec's read rules for every format version the +//! struct reads, must be an `Option` or have `#[serde(default)]`. +//! `test_parse_manifest_without_each_field` and +//! `test_parse_v1_manifest_without_each_field` check each field. + use std::collections::HashMap; use serde_derive::{Deserialize, Serialize}; use serde_with::serde_as; use super::{Datum, ManifestEntry, Schema, Struct}; -use crate::spec::{FormatVersion, Literal, RawLiteral, StructType, Type}; -use crate::{Error, ErrorKind}; +use crate::error::invalid_data; +use crate::spec::{FormatVersion, Literal, RawLiteral, Type}; +use crate::{Error, metadata_columns}; #[derive(Serialize, Deserialize)] pub(super) struct ManifestEntryV2 { @@ -34,7 +44,7 @@ pub(super) struct ManifestEntryV2 { } impl ManifestEntryV2 { - pub fn try_from(value: ManifestEntry, partition_type: &StructType) -> Result { + pub fn try_from(value: ManifestEntry, partition_type: &Type) -> Result { Ok(Self { status: value.status as i32, snapshot_id: value.snapshot_id, @@ -47,7 +57,7 @@ impl ManifestEntryV2 { pub fn try_into( self, partition_spec_id: i32, - partition_type: &StructType, + partition_type: &Type, schema: &Schema, ) -> Result { Ok(ManifestEntry { @@ -57,6 +67,7 @@ impl ManifestEntryV2 { file_sequence_number: self.file_sequence_number, data_file: self .data_file + .project_partition_by_name(partition_type)? .try_into(partition_spec_id, partition_type, schema)?, }) } @@ -65,12 +76,14 @@ impl ManifestEntryV2 { #[derive(Serialize, Deserialize)] pub(super) struct ManifestEntryV1 { status: i32, + // Required in v1 by the spec. Java declares it optional and writes null when + // snapshot ID inheritance is enabled, which #3371 tracks. pub snapshot_id: i64, data_file: DataFileSerde, } impl ManifestEntryV1 { - pub fn try_from(value: ManifestEntry, partition_type: &StructType) -> Result { + pub fn try_from(value: ManifestEntry, partition_type: &Type) -> Result { Ok(Self { status: value.status as i32, snapshot_id: value.snapshot_id.unwrap_or_default(), @@ -81,7 +94,7 @@ impl ManifestEntryV1 { pub fn try_into( self, partition_spec_id: i32, - partition_type: &StructType, + partition_type: &Type, schema: &Schema, ) -> Result { Ok(ManifestEntry { @@ -91,6 +104,7 @@ impl ManifestEntryV1 { file_sequence_number: Some(0), data_file: self .data_file + .project_partition_by_name(partition_type)? .try_into(partition_spec_id, partition_type, schema)?, }) } @@ -127,7 +141,7 @@ pub(super) struct DataFileSerde { impl DataFileSerde { pub fn try_from( value: super::DataFile, - partition_type: &StructType, + partition_type: &Type, format_version: FormatVersion, ) -> Result { let block_size_in_bytes = if format_version == FormatVersion::V1 { @@ -139,10 +153,7 @@ impl DataFileSerde { content: value.content as i32, file_path: value.file_path, file_format: value.file_format.to_string().to_ascii_uppercase(), - partition: RawLiteral::try_from( - Literal::Struct(value.partition), - &Type::Struct(partition_type.clone()), - )?, + partition: RawLiteral::try_from(Literal::Struct(value.partition), partition_type)?, record_count: value.record_count.try_into()?, file_size_in_bytes: value.file_size_in_bytes.try_into()?, block_size_in_bytes, @@ -153,7 +164,7 @@ impl DataFileSerde { lower_bounds: Some(to_bytes_entry(value.lower_bounds)?), upper_bounds: Some(to_bytes_entry(value.upper_bounds)?), key_metadata: value.key_metadata.map(serde_bytes::ByteBuf::from), - split_offsets: Some(value.split_offsets), + split_offsets: value.split_offsets, equality_ids: value.equality_ids, sort_order_id: value.sort_order_id, first_row_id: value.first_row_id, @@ -163,23 +174,30 @@ impl DataFileSerde { }) } + /// Matches the partition value's fields to `partition_type` by name. Manifest + /// entries are deserialized with the writer's partition fields and order, + /// which can differ from the partition spec's. + fn project_partition_by_name(mut self, partition_type: &Type) -> Result { + if let Type::Struct(struct_type) = partition_type { + self.partition = self.partition.project_by_name(struct_type)?; + } + Ok(self) + } + pub fn try_into( self, partition_spec_id: i32, - partition_type: &StructType, + partition_type: &Type, schema: &Schema, ) -> Result { let partition = self .partition - .try_into(&Type::Struct(partition_type.clone()))? + .try_into(partition_type)? .map(|v| { if let Literal::Struct(v) = v { Ok(v) } else { - Err(Error::new( - ErrorKind::DataInvalid, - "partition value is not a struct", - )) + Err(invalid_data!("partition value is not a struct")) } }) .transpose()? @@ -222,7 +240,7 @@ impl DataFileSerde { .transpose()? .unwrap_or_default(), key_metadata: self.key_metadata.map(|v| v.to_vec()), - split_offsets: self.split_offsets.unwrap_or_default(), + split_offsets: self.split_offsets, equality_ids: self.equality_ids, sort_order_id: self.sort_order_id, partition_spec_id, @@ -245,20 +263,20 @@ struct BytesEntry { fn parse_bytes_entry(v: Vec, schema: &Schema) -> Result, Error> { let mut m = HashMap::with_capacity(v.len()); for entry in v { - // We ignore the entry if the field is not found in the schema, due to schema evolution. - if let Some(field) = schema.field_by_id(entry.key) { + // First try to find the field in the schema, or check if it's a reserved metadata field + let field = schema + .field_by_id(entry.key) + .or_else(|| metadata_columns::get_metadata_field(entry.key).ok()); + + if let Some(field) = field { let data_type = field .field_type .as_primitive_type() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("field {} is not a primitive type", field.name), - ) - })? + .ok_or_else(|| invalid_data!("field {} is not a primitive type", field.name))? .clone(); m.insert(entry.key, Datum::try_from_bytes(&entry.value, data_type)?); } + // We ignore the entry if the field is not found in schema or metadata columns (schema evolution). } Ok(m) } @@ -320,9 +338,10 @@ mod tests { use std::io::Cursor; use std::sync::Arc; + use crate::avro::{assert_deflate_codec, define_named_types_repeatedly}; use crate::spec::manifest::_serde::{I64Entry, parse_i64_entry}; use crate::spec::{ - DataContentType, DataFile, DataFileFormat, Datum, FormatVersion, NestedField, + DataContentType, DataFile, DataFileFormat, Datum, FormatVersion, Literal, NestedField, PrimitiveType, Schema, Struct, StructType, Type, read_data_files_from_avro, write_data_files_to_avro, }; @@ -380,7 +399,7 @@ mod tests { lower_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]), upper_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: Some(0), partition_spec_id: 0, @@ -404,6 +423,7 @@ mod tests { FormatVersion::V2, ) .unwrap(); + assert_deflate_codec(&buffer); let actual_data_file = read_data_files_from_avro( &mut Cursor::new(buffer), @@ -417,6 +437,52 @@ mod tests { assert_eq!(data_files, actual_data_file); } + /// apache-avro 0.22 checks the writer schema for repeated names only when + /// the reader has no reader schema, so this path reads 0.21 output without + /// the header rewrite that `Manifest::parse_avro` needs. + #[test] + fn test_read_data_files_with_repeated_named_type_definitions() { + let decimal = Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }); + let schema = Schema::builder() + .with_fields(schema().as_struct().fields().to_vec()) + .with_fields(vec![ + NestedField::optional(4, "a", decimal.clone()).into(), + NestedField::optional(5, "b", decimal.clone()).into(), + ]) + .build() + .unwrap(); + let partition_type = StructType::new(vec![ + NestedField::optional(1000, "a", decimal.clone()).into(), + NestedField::optional(1001, "b", decimal).into(), + ]); + let mut data_files = data_files(); + data_files[0].partition = + Struct::from_iter([Some(Literal::decimal(12345)), Some(Literal::decimal(-678))]); + let mut buffer = Vec::new(); + write_data_files_to_avro( + &mut buffer, + data_files.clone(), + &partition_type, + FormatVersion::V2, + ) + .unwrap(); + let legacy = define_named_types_repeatedly(&buffer); + + let actual = read_data_files_from_avro( + &mut Cursor::new(legacy), + &schema, + 0, + &partition_type, + FormatVersion::V2, + ) + .unwrap(); + + assert_eq!(actual, data_files); + } + #[tokio::test] async fn test_data_file_serialize_deserialize_v1_data_on_v2_reader() { let schema = schema(); @@ -430,6 +496,7 @@ mod tests { FormatVersion::V1, ) .unwrap(); + assert_deflate_codec(&buffer); let actual_data_file = read_data_files_from_avro( &mut Cursor::new(buffer), @@ -487,7 +554,7 @@ mod tests { let v2_entry = v1_entry .try_into( 0, // partition_spec_id - &StructType::new(vec![]), + &Type::Struct(StructType::new(vec![])), &schema(), ) .unwrap(); @@ -567,7 +634,7 @@ mod tests { let data_file = v1_style_data_file .try_into( 0, // partition_spec_id - &StructType::new(vec![]), + &Type::Struct(StructType::new(vec![])), &schema(), ) .unwrap(); @@ -590,4 +657,35 @@ mod tests { assert_eq!(data_file.file_size_in_bytes, 2048); assert_eq!(data_file.partition_spec_id, 0); } + + #[test] + fn test_parse_bytes_entry_with_metadata_column() { + use crate::metadata_columns::RESERVED_FIELD_ID_POS; + use crate::spec::manifest::_serde::{BytesEntry, parse_bytes_entry}; + + // Create a minimal schema that doesn't include the _pos metadata column + let test_schema = schema(); + + // Create a BytesEntry with the _pos field ID (reserved metadata field) + // The _pos field is a Long (i64) in the metadata column definition + let pos_value: i64 = 42; + let bytes_entry = BytesEntry { + key: RESERVED_FIELD_ID_POS, + value: serde_bytes::ByteBuf::from(pos_value.to_le_bytes().to_vec()), + }; + + // Parse the bytes entry - should use metadata column definition for _pos + let result = parse_bytes_entry(vec![bytes_entry], &test_schema).unwrap(); + + // Verify that the _pos field was parsed correctly using metadata column definition + assert!( + result.contains_key(&RESERVED_FIELD_ID_POS), + "_pos metadata field should be parsed" + ); + assert_eq!( + result.get(&RESERVED_FIELD_ID_POS), + Some(&Datum::long(pos_value)), + "_pos should be parsed as long with correct value" + ); + } } diff --git a/crates/iceberg/src/spec/manifest/data_file.rs b/crates/iceberg/src/spec/manifest/data_file.rs index 931f9441e9..0b67a0f12f 100644 --- a/crates/iceberg/src/spec/manifest/data_file.rs +++ b/crates/iceberg/src/spec/manifest/data_file.rs @@ -19,15 +19,20 @@ use std::collections::HashMap; use std::io::{Read, Write}; use std::str::FromStr; -use apache_avro::{Reader as AvroReader, Writer as AvroWriter, from_value, to_value}; +use apache_avro::{Reader as AvroReader, from_value, to_value}; use serde_derive::{Deserialize, Serialize}; use serde_with::{DeserializeFromStr, SerializeDisplay}; use super::_serde::DataFileSerde; -use super::{Datum, FormatVersion, Schema, data_file_schema_v1, data_file_schema_v2}; -use crate::error::Result; -use crate::spec::{Struct, StructType}; -use crate::{Error, ErrorKind}; +use super::{ + Datum, FormatVersion, Schema, data_file_schema_v1, data_file_schema_v2, data_file_schema_v3, +}; +use crate::Error; +use crate::avro::new_writer; +use crate::error::{Result, invalid_data}; +use crate::spec::{ + DEFAULT_PARTITION_SPEC_ID, Literal, NestedFieldRef, PrimitiveType, Struct, StructType, Type, +}; /// Data file carries data file path, partition tuple, metrics, … #[derive(Debug, PartialEq, Clone, Eq, Builder)] @@ -49,6 +54,7 @@ pub struct DataFile { /// /// Partition data tuple, schema based on the partition spec output using /// partition field ids for the struct field ids + #[builder(default = "Struct::empty()")] pub(crate) partition: Struct, /// field id: 103 /// @@ -124,9 +130,10 @@ pub struct DataFile { /// element field id: 133 /// /// Split offsets for the data file. For example, all row group offsets - /// in a Parquet file. Must be sorted ascending + /// in a Parquet file. Must be sorted ascending. Optional field that + /// should be serialized as null when not present. #[builder(default)] - pub(crate) split_offsets: Vec, + pub(crate) split_offsets: Option>, /// field id: 135 /// element field id: 136 /// @@ -151,11 +158,12 @@ pub struct DataFile { /// field id: 142 /// /// The _row_id for the first row in the data file. - /// For more details, refer to https://github.com/apache/iceberg/blob/main/format/spec.md#first-row-id-inheritance + /// For more details, refer to #[builder(default)] pub(crate) first_row_id: Option, /// This field is not included in spec. It is just store in memory representation used /// in process. + #[builder(default = "DEFAULT_PARTITION_SPEC_ID")] pub(crate) partition_spec_id: i32, /// field id: 143 /// @@ -179,7 +187,71 @@ pub struct DataFile { pub(crate) content_size_in_bytes: Option, } +fn promote_partition_value( + value: Option<&Literal>, + current_field: &NestedFieldRef, + written_field: &NestedFieldRef, +) -> Option { + match value { + Some(Literal::Primitive(primitive)) => { + let promoted = match current_field.field_type.as_primitive_type() { + // The source column was dropped, so there is no current type to widen to. + // Widen a numeric to its promotion ceiling so this value and its deletes + // normalize the same way; keep every other type (e.g. a `date`, stored as + // an `int`) at its written type so deletes that skip normalization match. + Some(PrimitiveType::Unknown) => { + match written_field.field_type.as_primitive_type() { + Some(PrimitiveType::Int | PrimitiveType::Float) => primitive.widen(), + _ => None, + } + } + Some(target) => primitive.promote_to(target), + None => None, + }; + promoted.map(Literal::Primitive).or_else(|| value.cloned()) + } + other => other.cloned(), + } +} + impl DataFile { + /// Returns this file's partition tuple normalized to the current schema: a field + /// promoted (`int` -> `long`, `float` -> `double`) since the manifest was written is + /// widened to its `current` type; a dropped numeric source, which has no `current` + /// type, is widened to its promotion ceiling by its `written` type; all other values + /// are copied through. Column bounds are widened separately, lazily, where they are + /// compared. + /// + /// `current` and `written` are the file's spec resolved against the current schema and + /// against the manifest's schema, so they match the stored tuple in length. + /// + /// # Errors + /// Returns `DataInvalid` if the stored tuple and `current` have different lengths; the + /// lengths should always match, since both come from the same spec. + pub(crate) fn promoted_partition( + &self, + current: &StructType, + written: &StructType, + ) -> Result { + if self.partition.fields().len() != current.fields().len() { + return Err(invalid_data!( + "Partition tuple has {} values but the spec partition type has {} fields", + self.partition.fields().len(), + current.fields().len() + )); + } + + Ok(self + .partition + .iter() + .zip(current.fields()) + .zip(written.fields()) + .map(|((value, current_field), written_field)| { + promote_partition_value(value, current_field, written_field) + }) + .collect()) + } + /// Get the content type of the data file (data, equality deletes, or position deletes) pub fn content_type(&self) -> DataContentType { self.content @@ -243,8 +315,9 @@ impl DataFile { } /// Get the split offsets of the data file. /// For example, all row group offsets in a Parquet file. - pub fn split_offsets(&self) -> &[i64] { - &self.split_offsets + /// Returns `None` if no split offsets are present. + pub fn split_offsets(&self) -> Option<&[i64]> { + self.split_offsets.as_deref() } /// Get the equality ids of the data file. /// Field ids used to determine row equality in equality delete files. @@ -293,17 +366,21 @@ pub fn write_data_files_to_avro( let avro_schema = match version { FormatVersion::V1 => data_file_schema_v1(partition_type).unwrap(), FormatVersion::V2 => data_file_schema_v2(partition_type).unwrap(), + FormatVersion::V3 => data_file_schema_v3(partition_type).unwrap(), }; - let mut writer = AvroWriter::new(&avro_schema, writer); + let mut writer = new_writer(&avro_schema, writer)?; + // Wrap once and reuse across files so the field-name lookup is not rebuilt + // for each data file. + let partition_struct_type = Type::Struct(partition_type.clone()); for data_file in data_files { let value = to_value(DataFileSerde::try_from( data_file, - partition_type, + &partition_struct_type, FormatVersion::V1, )?)? .resolve(&avro_schema)?; - writer.append(value)?; + writer.append_value(value)?; } Ok(writer.flush()?) @@ -320,15 +397,21 @@ pub fn read_data_files_from_avro( let avro_schema = match version { FormatVersion::V1 => data_file_schema_v1(partition_type).unwrap(), FormatVersion::V2 => data_file_schema_v2(partition_type).unwrap(), + FormatVersion::V3 => data_file_schema_v3(partition_type).unwrap(), }; - let reader = AvroReader::with_schema(&avro_schema, reader)?; + let reader = AvroReader::builder(reader) + .reader_schema(&avro_schema) + .build()?; + // Wrap once and reuse across files so the field-name lookup is not rebuilt + // for each data file. + let partition_struct_type = Type::Struct(partition_type.clone()); reader .into_iter() .map(|value| { from_value::(&value?)?.try_into( partition_spec_id, - partition_type, + &partition_struct_type, schema, ) }) @@ -356,10 +439,7 @@ impl TryFrom for DataContentType { 0 => Ok(DataContentType::Data), 1 => Ok(DataContentType::PositionDeletes), 2 => Ok(DataContentType::EqualityDeletes), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("data content type {} is invalid", v), - )), + _ => Err(invalid_data!("data content type {v} is invalid")), } } } @@ -386,10 +466,7 @@ impl FromStr for DataFileFormat { "orc" => Ok(Self::Orc), "parquet" => Ok(Self::Parquet), "puffin" => Ok(Self::Puffin), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Unsupported data file format: {}", s), - )), + _ => Err(invalid_data!("Unsupported data file format: {s}")), } } } @@ -407,7 +484,12 @@ impl std::fmt::Display for DataFileFormat { #[cfg(test)] mod test { - use crate::spec::DataContentType; + use crate::ErrorKind; + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Literal, NestedField, PrimitiveType, + Struct, StructType, Type, + }; + #[test] fn test_data_content_type_default() { assert_eq!(DataContentType::default(), DataContentType::Data); @@ -417,4 +499,161 @@ mod test { fn test_data_content_type_default_value() { assert_eq!(DataContentType::default() as i32, 0); } + + #[test] + fn promoted_partition_widens_int_and_float() { + // Partition columns promoted int -> long and float -> double; a file + // written before the promotion still carries the narrower types. + let current = StructType::new(vec![ + NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(1001, "c", Type::Primitive(PrimitiveType::Double)).into(), + ]); + let written = StructType::new(vec![ + NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(1001, "c", Type::Primitive(PrimitiveType::Float)).into(), + ]); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("x.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .partition(Struct::from_iter([ + Some(Literal::int(7)), + Some(Literal::float(1.5_f32)), + ])) + .record_count(1) + .file_size_in_bytes(1) + .partition_spec_id(0) + .build() + .unwrap(); + + assert_eq!( + data_file.promoted_partition(¤t, &written).unwrap(), + Struct::from_iter([Some(Literal::long(7)), Some(Literal::double(1.5))]) + ); + } + + #[test] + fn promoted_partition_widens_dropped_numeric_source() { + // A dropped partition source resolves to `Unknown`, so there is no current + // type; a numeric still widens to its ceiling by its written type, keeping it + // aligned with deletes written while the source still existed. + let current = StructType::new(vec![ + NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Unknown)).into(), + ]); + let written = StructType::new(vec![ + NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Int)).into(), + ]); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("x.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .partition(Struct::from_iter([Some(Literal::int(7))])) + .record_count(1) + .file_size_in_bytes(1) + .partition_spec_id(0) + .build() + .unwrap(); + + assert_eq!( + data_file.promoted_partition(¤t, &written).unwrap(), + Struct::from_iter([Some(Literal::long(7))]) + ); + } + + #[test] + fn promoted_partition_preserves_dropped_date_source() { + // A `date` is stored as an `int`, so widening a dropped date the way a dropped + // int widens would turn it into a `long` and diverge from delete manifests that + // skip normalization. A dropped non-numeric source is copied through unchanged. + let current = StructType::new(vec![ + NestedField::required(1000, "d", Type::Primitive(PrimitiveType::Unknown)).into(), + ]); + let written = StructType::new(vec![ + NestedField::required(1000, "d", Type::Primitive(PrimitiveType::Date)).into(), + ]); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("x.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .partition(Struct::from_iter([Some(Literal::date(20_000))])) + .record_count(1) + .file_size_in_bytes(1) + .partition_spec_id(0) + .build() + .unwrap(); + + assert_eq!( + data_file.promoted_partition(¤t, &written).unwrap(), + data_file.partition + ); + } + + #[test] + fn promoted_partition_leaves_matching_types_unchanged() { + let partition_type = StructType::new(vec![ + NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Long)).into(), + ]); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("x.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .partition(Struct::from_iter([Some(Literal::long(7))])) + .record_count(1) + .file_size_in_bytes(1) + .partition_spec_id(0) + .build() + .unwrap(); + + assert_eq!( + data_file + .promoted_partition(&partition_type, &partition_type) + .unwrap(), + data_file.partition + ); + } + + #[test] + fn promoted_partition_rejects_tuple_length_mismatch() { + let data_file = |values: Vec>| { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("x.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .partition(Struct::from_iter(values)) + .record_count(1) + .file_size_in_bytes(1) + .partition_spec_id(0) + .build() + .unwrap() + }; + let one_field = StructType::new(vec![ + NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Long)).into(), + ]); + let two_fields = StructType::new(vec![ + NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(1001, "c", Type::Primitive(PrimitiveType::Long)).into(), + ]); + + let longer = data_file(vec![Some(Literal::int(7)), Some(Literal::int(8))]); + assert_eq!( + longer + .promoted_partition(&one_field, &one_field) + .unwrap_err() + .kind(), + ErrorKind::DataInvalid + ); + + let shorter = data_file(vec![Some(Literal::int(7))]); + assert_eq!( + shorter + .promoted_partition(&two_fields, &two_fields) + .unwrap_err() + .kind(), + ErrorKind::DataInvalid + ); + } } diff --git a/crates/iceberg/src/spec/manifest/entry.rs b/crates/iceberg/src/spec/manifest/entry.rs index 7ba9efb3b9..c8ed0ea1b9 100644 --- a/crates/iceberg/src/spec/manifest/entry.rs +++ b/crates/iceberg/src/spec/manifest/entry.rs @@ -21,13 +21,13 @@ use apache_avro::Schema as AvroSchema; use once_cell::sync::Lazy; use typed_builder::TypedBuilder; +use crate::Error; use crate::avro::schema_to_avro_schema; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::{ DataContentType, DataFile, INITIAL_SEQUENCE_NUMBER, ListType, Literal, ManifestFile, MapType, NestedField, NestedFieldRef, PrimitiveLiteral, PrimitiveType, Schema, StructType, Type, }; -use crate::{Error, ErrorKind}; /// Reference to [`ManifestEntry`]. pub type ManifestEntryRef = Arc; @@ -170,16 +170,24 @@ impl TryFrom for ManifestStatus { 0 => Ok(ManifestStatus::Existing), 1 => Ok(ManifestStatus::Added), 2 => Ok(ManifestStatus::Deleted), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("manifest status {} is invalid", v), - )), + _ => Err(invalid_data!("manifest status {v} is invalid")), } } } use super::DataFileFormat; +/// Field ID of `manifest_entry.data_file`. +pub(crate) const DATA_FILE_FIELD_ID: i32 = 2; +/// Field ID of `manifest_entry.sequence_number`. +pub(crate) const SEQUENCE_NUMBER_FIELD_ID: i32 = 3; +/// Field ID of `manifest_entry.file_sequence_number`. +pub(crate) const FILE_SEQUENCE_NUMBER_FIELD_ID: i32 = 4; +/// Field ID of `data_file.partition`. +pub(crate) const DATA_FILE_PARTITION_FIELD_ID: i32 = 102; +/// Field ID of `data_file.content`. +pub(crate) const DATA_FILE_CONTENT_FIELD_ID: i32 = 134; + static STATUS: Lazy = { Lazy::new(|| { Arc::new(NestedField::required( @@ -213,7 +221,7 @@ static SNAPSHOT_ID_V2: Lazy = { static SEQUENCE_NUMBER: Lazy = { Lazy::new(|| { Arc::new(NestedField::optional( - 3, + SEQUENCE_NUMBER_FIELD_ID, "sequence_number", Type::Primitive(PrimitiveType::Long), )) @@ -223,7 +231,7 @@ static SEQUENCE_NUMBER: Lazy = { static FILE_SEQUENCE_NUMBER: Lazy = { Lazy::new(|| { Arc::new(NestedField::optional( - 4, + FILE_SEQUENCE_NUMBER_FIELD_ID, "file_sequence_number", Type::Primitive(PrimitiveType::Long), )) @@ -233,9 +241,13 @@ static FILE_SEQUENCE_NUMBER: Lazy = { static CONTENT: Lazy = { Lazy::new(|| { Arc::new( - NestedField::required(134, "content", Type::Primitive(PrimitiveType::Int)) - // 0 refers to DataContentType::DATA - .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(0))), + NestedField::required( + DATA_FILE_CONTENT_FIELD_ID, + "content", + Type::Primitive(PrimitiveType::Int), + ) + // 0 refers to DataContentType::DATA + .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(0))), ) }) }; @@ -509,13 +521,67 @@ static CONTENT_SIZE_IN_BYTES: Lazy = { }) }; +fn data_file_fields_v3(partition_type: &StructType) -> Vec { + vec![ + CONTENT.clone(), + FILE_PATH.clone(), + FILE_FORMAT.clone(), + Arc::new(NestedField::required( + DATA_FILE_PARTITION_FIELD_ID, + "partition", + Type::Struct(partition_type.clone()), + )), + RECORD_COUNT.clone(), + FILE_SIZE_IN_BYTES.clone(), + COLUMN_SIZES.clone(), + VALUE_COUNTS.clone(), + NULL_VALUE_COUNTS.clone(), + NAN_VALUE_COUNTS.clone(), + LOWER_BOUNDS.clone(), + UPPER_BOUNDS.clone(), + KEY_METADATA.clone(), + SPLIT_OFFSETS.clone(), + EQUALITY_IDS.clone(), + SORT_ORDER_ID.clone(), + FIRST_ROW_ID.clone(), + REFERENCE_DATA_FILE.clone(), + CONTENT_OFFSET.clone(), + CONTENT_SIZE_IN_BYTES.clone(), + ] +} + +/// Fields of the `manifest_entry` struct in the latest format version. A +/// [`ManifestEntry`] of any version fits them: earlier versions lack only +/// optional fields, apart from the deprecated v1 `block_size_in_bytes`, which +/// [`DataFile`] doesn't keep. +pub(crate) fn manifest_entry_fields(partition_type: &StructType) -> Vec { + vec![ + STATUS.clone(), + SNAPSHOT_ID_V2.clone(), + SEQUENCE_NUMBER.clone(), + FILE_SEQUENCE_NUMBER.clone(), + Arc::new(NestedField::required( + DATA_FILE_FIELD_ID, + "data_file", + Type::Struct(StructType::new(data_file_fields_v3(partition_type))), + )), + ] +} + +pub(super) fn data_file_schema_v3(partition_type: &StructType) -> Result { + let schema = Schema::builder() + .with_fields(data_file_fields_v3(partition_type)) + .build()?; + schema_to_avro_schema("data_file", &schema) +} + fn data_file_fields_v2(partition_type: &StructType) -> Vec { vec![ CONTENT.clone(), FILE_PATH.clone(), FILE_FORMAT.clone(), Arc::new(NestedField::required( - 102, + DATA_FILE_PARTITION_FIELD_ID, "partition", Type::Struct(partition_type.clone()), )), @@ -533,6 +599,8 @@ fn data_file_fields_v2(partition_type: &StructType) -> Vec { SORT_ORDER_ID.clone(), FIRST_ROW_ID.clone(), REFERENCE_DATA_FILE.clone(), + // Why are the following two fields here in the existing v2 schema? + // In the spec, they are not even listed as optional for v2. CONTENT_OFFSET.clone(), CONTENT_SIZE_IN_BYTES.clone(), ] @@ -552,7 +620,7 @@ pub(super) fn manifest_schema_v2(partition_type: &StructType) -> Result Vec { FILE_PATH.clone(), FILE_FORMAT.clone(), Arc::new(NestedField::required( - 102, + DATA_FILE_PARTITION_FIELD_ID, "partition", Type::Struct(partition_type.clone()), )), @@ -597,7 +665,7 @@ pub(super) fn manifest_schema_v1(partition_type: &StructType) -> Result>) -> Result { let schema = Arc::new({ let bs = meta.get("schema").ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "schema is required in manifest metadata but not found", - ) + invalid_data!("schema is required in manifest metadata but not found") })?; serde_json::from_slice::(bs).map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse schema in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse schema in manifest metadata").with_source(err) })? }); let schema_id: i32 = meta .get("schema-id") .map(|bs| { String::from_utf8_lossy(bs).parse().map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse schema id in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse schema id in manifest metadata").with_source(err) }) }) .transpose()? @@ -75,28 +63,19 @@ impl ManifestMetadata { let partition_spec = { let fields = { let bs = meta.get("partition-spec").ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "partition-spec is required in manifest metadata but not found", - ) + invalid_data!("partition-spec is required in manifest metadata but not found") })?; serde_json::from_slice::>(bs).map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse partition spec in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse partition spec in manifest metadata") + .with_source(err) })? }; let spec_id = meta .get("partition-spec-id") .map(|bs| { String::from_utf8_lossy(bs).parse().map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse partition spec id in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse partition spec id in manifest metadata") + .with_source(err) }) }) .transpose()? @@ -108,11 +87,7 @@ impl ManifestMetadata { }; let format_version = if let Some(bs) = meta.get("format-version") { serde_json::from_slice::(bs).map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - "Fail to parse format version in manifest metadata", - ) - .with_source(err) + invalid_data!("Fail to parse format version in manifest metadata").with_source(err) })? } else { FormatVersion::V1 diff --git a/crates/iceberg/src/spec/manifest/mod.rs b/crates/iceberg/src/spec/manifest/mod.rs index a1a5612c36..f4269638a2 100644 --- a/crates/iceberg/src/spec/manifest/mod.rs +++ b/crates/iceberg/src/spec/manifest/mod.rs @@ -23,18 +23,26 @@ mod entry; pub use entry::*; mod metadata; pub use metadata::*; +mod reader; +pub use reader::*; mod writer; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; -use apache_avro::{Reader as AvroReader, from_value}; +use apache_avro::Reader as AvroReader; +use apache_avro::error::Details; pub use writer::*; use super::{ - Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType, Schema, Struct, + Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType, Schema, Struct, Type, UNASSIGNED_SEQUENCE_NUMBER, }; -use crate::error::Result; -use crate::{Error, ErrorKind}; +use crate::avro::{Resolved, define_named_types_once}; +use crate::error::{Error, Result, invalid_data}; + +/// Whether a manifest with repeated Avro named type definitions was logged at +/// warn level. A table written by an affected release can have many of them. +static WARNED_REPEATED_DEFINITIONS: AtomicBool = AtomicBool::new(false); /// A manifest contains metadata and a list of entries. #[derive(Debug, PartialEq, Eq, Clone)] @@ -44,9 +52,53 @@ pub struct Manifest { } impl Manifest { - /// Parse manifest metadata and entries from bytes of avro file. - pub(crate) fn try_from_avro_bytes(bs: &[u8]) -> Result<(ManifestMetadata, Vec)> { - let reader = AvroReader::new(bs)?; + /// Parse manifest metadata and entries from bytes of avro file. `location` + /// names the manifest in warnings. + pub(crate) fn try_from_avro_bytes( + bs: &[u8], + location: Option<&str>, + ) -> Result<(ManifestMetadata, Vec)> { + let rewritten; + let reader = match AvroReader::new(bs) { + Ok(reader) => reader, + // iceberg-rust repeated `decimal` definitions before + // `schema_to_avro_schema` defined each named type once, so this + // fallback stays while tables can contain manifests it wrote. + Err(e) if matches!(e.details(), Details::AmbiguousSchemaDefinition(_)) => { + let (bs, repeated) = match define_named_types_once(bs) { + Ok(Some(rewrite)) => rewrite, + Ok(None) => return Err(e.into()), + Err(rewrite_error) => { + return Err(Error::from(e).with_context( + "error defining each named type once", + rewrite_error.to_string(), + )); + } + }; + rewritten = bs; + let reader = AvroReader::new(rewritten.as_slice()).map_err(|retry_error| { + Error::from(e).with_context( + "error after defining each named type once", + retry_error.to_string(), + ) + })?; + let location = location.unwrap_or(""); + if WARNED_REPEATED_DEFINITIONS.swap(true, Ordering::Relaxed) { + tracing::debug!( + "Manifest {location} defines Avro named types {repeated:?} more than once." + ); + } else { + tracing::warn!( + "Manifest {location} defines Avro named types {repeated:?} more than once, \ + which the Avro specification doesn't allow. Reading it with each repeated \ + definition replaced by a reference to the first. Later manifests like \ + this are logged at debug level." + ); + } + reader + } + Err(e) => return Err(e.into()), + }; // Parse manifest metadata let meta = reader.user_metadata(); @@ -54,36 +106,33 @@ impl Manifest { // Parse manifest entries let partition_type = metadata.partition_spec.partition_type(&metadata.schema)?; + // Wrap the partition type once and share it across all entries: the + // per-entry conversion needs a `&Type`, and building it here keeps the + // lazily-populated field-name lookup from being rebuilt for every entry. + let partition_struct_type = Type::Struct(partition_type.clone()); let entries = match metadata.format_version { - FormatVersion::V1 => { - let schema = manifest_schema_v1(&partition_type)?; - let reader = AvroReader::with_schema(&schema, bs)?; - reader - .into_iter() - .map(|value| { - from_value::<_serde::ManifestEntryV1>(&value?)?.try_into( - metadata.partition_spec.spec_id(), - &partition_type, - &metadata.schema, - ) - }) - .collect::>>()? - } - FormatVersion::V2 => { - let schema = manifest_schema_v2(&partition_type)?; - let reader = AvroReader::with_schema(&schema, bs)?; - reader - .into_iter() - .map(|value| { - from_value::<_serde::ManifestEntryV2>(&value?)?.try_into( - metadata.partition_spec.spec_id(), - &partition_type, - &metadata.schema, - ) - }) - .collect::>>()? - } + FormatVersion::V1 => reader + .into_deser_iter::>() + .map(|entry| { + entry?.0.try_into( + metadata.partition_spec.spec_id(), + &partition_struct_type, + &metadata.schema, + ) + }) + .collect::>>()?, + // Manifest Schema & Manifest Entry did not change between V2 and V3 + FormatVersion::V2 | FormatVersion::V3 => reader + .into_deser_iter::>() + .map(|entry| { + entry?.0.try_into( + metadata.partition_spec.spec_id(), + &partition_struct_type, + &metadata.schema, + ) + }) + .collect::>>()?, }; Ok((metadata, entries)) @@ -91,7 +140,7 @@ impl Manifest { /// Parse manifest from bytes of avro file. pub fn parse_avro(bs: &[u8]) -> Result { - let (metadata, entries) = Self::try_from_avro_bytes(bs)?; + let (metadata, entries) = Self::try_from_avro_bytes(bs, None)?; Ok(Self::new(metadata, entries)) } @@ -126,14 +175,10 @@ pub fn serialize_data_file_to_json( partition_type: &super::StructType, format_version: FormatVersion, ) -> Result { - let serde = _serde::DataFileSerde::try_from(data_file, partition_type, format_version)?; - serde_json::to_string(&serde).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to serialize DataFile to JSON!".to_string(), - ) - .with_source(e) - }) + let partition_struct_type = Type::Struct(partition_type.clone()); + let serde = _serde::DataFileSerde::try_from(data_file, &partition_struct_type, format_version)?; + serde_json::to_string(&serde) + .map_err(|e| invalid_data!("Failed to serialize DataFile to JSON!").with_source(e)) } /// Deserialize a DataFile from a JSON string. @@ -143,15 +188,11 @@ pub fn deserialize_data_file_from_json( partition_type: &super::StructType, schema: &Schema, ) -> Result { - let serde = serde_json::from_str::<_serde::DataFileSerde>(json).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Failed to deserialize JSON to DataFile!".to_string(), - ) - .with_source(e) - })?; + let serde = serde_json::from_str::<_serde::DataFileSerde>(json) + .map_err(|e| invalid_data!("Failed to deserialize JSON to DataFile!").with_source(e))?; - serde.try_into(partition_spec_id, partition_type, schema) + let partition_struct_type = Type::Struct(partition_type.clone()); + serde.try_into(partition_spec_id, &partition_struct_type, schema) } #[cfg(test)] @@ -160,11 +201,16 @@ mod tests { use std::fs; use std::sync::Arc; - use serde_json::Value; + use apache_avro::types::Value as AvroValue; + use apache_avro::{Codec, Writer, to_value}; + use serde_json::{Value, to_vec}; use tempfile::TempDir; + use tracing::Level; + use tracing::subscriber::with_default; use super::*; - use crate::io::FileIOBuilder; + use crate::ErrorKind; + use crate::io::FileIO; use crate::spec::{Literal, NestedField, PrimitiveType, Struct, Transform, Type}; #[tokio::test] @@ -256,19 +302,18 @@ mod tests { snapshot_id: None, sequence_number: None, file_sequence_number: None, - data_file: DataFile {content:DataContentType::Data,file_path:"s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),file_format:DataFileFormat::Parquet,partition:Struct::empty(),record_count:1,file_size_in_bytes:5442,column_sizes:HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),value_counts:HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),null_value_counts:HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),nan_value_counts:HashMap::new(),lower_bounds:HashMap::new(),upper_bounds:HashMap::new(),key_metadata:None,split_offsets:vec![4],equality_ids:Some(Vec::new()),sort_order_id:None, partition_spec_id: 0,first_row_id: None,referenced_data_file: None,content_offset: None,content_size_in_bytes: None } + data_file: DataFile {content:DataContentType::Data,file_path:"s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),file_format:DataFileFormat::Parquet,partition:Struct::empty(),record_count:1,file_size_in_bytes:5442,column_sizes:HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),value_counts:HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),null_value_counts:HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),nan_value_counts:HashMap::new(),lower_bounds:HashMap::new(),upper_bounds:HashMap::new(),key_metadata:None,split_offsets:Some(vec![4]),equality_ids:Some(Vec::new()),sort_order_id:None, partition_spec_id: 0,first_row_id: None,referenced_data_file: None,content_offset: None,content_size_in_bytes: None } } ]; // write manifest to file let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path().join("test_manifest.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); + let io = FileIO::new_with_fs(); let output_file = io.new_output(path.to_str().unwrap()).unwrap(); let mut writer = ManifestWriterBuilder::new( output_file, Some(1), - None, metadata.schema.clone(), metadata.partition_spec.clone(), ) @@ -287,6 +332,127 @@ mod tests { assert_eq!(actual_manifest, Manifest::new(metadata, entries)); } + #[test] + fn test_parse_snappy_manifest_v2() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::optional( + 1, + "id", + Type::Primitive(PrimitiveType::Long), + ))]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .build() + .unwrap(); + + for (manifest_content, file_content, file_path) in [ + ( + ManifestContentType::Data, + DataContentType::Data, + "s3://bucket/table/data/data.parquet", + ), + ( + ManifestContentType::Deletes, + DataContentType::PositionDeletes, + "s3://bucket/table/data/delete.parquet", + ), + ] { + let metadata = ManifestMetadata { + schema_id: 0, + schema: schema.clone(), + partition_spec: partition_spec.clone(), + content: manifest_content, + format_version: FormatVersion::V2, + }; + let entry = ManifestEntry { + status: ManifestStatus::Added, + snapshot_id: Some(1), + sequence_number: None, + file_sequence_number: None, + data_file: DataFile { + content: file_content, + file_path: file_path.to_string(), + file_format: DataFileFormat::Parquet, + partition: Struct::empty(), + record_count: 1, + file_size_in_bytes: 1024, + column_sizes: HashMap::new(), + value_counts: HashMap::new(), + null_value_counts: HashMap::new(), + nan_value_counts: HashMap::new(), + lower_bounds: HashMap::new(), + upper_bounds: HashMap::new(), + key_metadata: None, + split_offsets: None, + equality_ids: None, + sort_order_id: None, + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + }, + }; + + let partition_type = metadata + .partition_spec + .partition_type(&metadata.schema) + .unwrap(); + let avro_schema = manifest_schema_v2(&partition_type).unwrap(); + let mut writer = Writer::with_codec(&avro_schema, Vec::new(), Codec::Snappy).unwrap(); + writer + .add_user_metadata("schema".to_string(), to_vec(&metadata.schema).unwrap()) + .unwrap(); + writer + .add_user_metadata( + "schema-id".to_string(), + metadata.schema.schema_id().to_string(), + ) + .unwrap(); + writer + .add_user_metadata( + "partition-spec".to_string(), + to_vec(&metadata.partition_spec.fields()).unwrap(), + ) + .unwrap(); + writer + .add_user_metadata( + "partition-spec-id".to_string(), + metadata.partition_spec.spec_id().to_string(), + ) + .unwrap(); + writer + .add_user_metadata( + "format-version".to_string(), + (metadata.format_version as u8).to_string(), + ) + .unwrap(); + writer + .add_user_metadata("content".to_string(), metadata.content.to_string()) + .unwrap(); + let value = to_value( + _serde::ManifestEntryV2::try_from( + entry.clone(), + &Type::Struct(partition_type.clone()), + ) + .unwrap(), + ) + .unwrap() + .resolve(&avro_schema) + .unwrap(); + writer.append_value(value).unwrap(); + let bs = writer.into_inner().unwrap(); + + let parsed_manifest = Manifest::parse_avro(&bs).unwrap(); + + assert_eq!(parsed_manifest, Manifest::new(metadata, vec![entry])); + } + } + #[tokio::test] async fn test_parse_manifest_v2_partition() { let schema = Arc::new( @@ -434,7 +600,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: Some(Vec::new()), sort_order_id: None, partition_spec_id: 0, @@ -448,12 +614,11 @@ mod tests { // write manifest to file and check the return manifest file. let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path().join("test_manifest.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); + let io = FileIO::new_with_fs(); let output_file = io.new_output(path.to_str().unwrap()).unwrap(); let mut writer = ManifestWriterBuilder::new( output_file, Some(2), - None, metadata.schema.clone(), metadata.partition_spec.clone(), ) @@ -531,7 +696,7 @@ mod tests { lower_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]), upper_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: Some(0), partition_spec_id: 0, @@ -545,12 +710,11 @@ mod tests { // write manifest to file let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path().join("test_manifest.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); + let io = FileIO::new_with_fs(); let output_file = io.new_output(path.to_str().unwrap()).unwrap(); let mut writer = ManifestWriterBuilder::new( output_file, Some(3), - None, metadata.schema.clone(), metadata.partition_spec.clone(), ) @@ -639,7 +803,7 @@ mod tests { (3, Datum::string("x")) ]), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: Some(0), partition_spec_id: 0, @@ -654,12 +818,11 @@ mod tests { // write manifest to file let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path().join("test_manifest.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); + let io = FileIO::new_with_fs(); let output_file = io.new_output(path.to_str().unwrap()).unwrap(); let mut writer = ManifestWriterBuilder::new( output_file, Some(2), - None, metadata.schema.clone(), metadata.partition_spec.clone(), ) @@ -748,7 +911,7 @@ mod tests { (3, Datum::string("x")) ]), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -762,12 +925,11 @@ mod tests { // write manifest to file let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path().join("test_manifest.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); + let io = FileIO::new_with_fs(); let output_file = io.new_output(path.to_str().unwrap()).unwrap(); let mut writer = ManifestWriterBuilder::new( output_file, Some(2), - None, metadata.schema.clone(), metadata.partition_spec.clone(), ) @@ -839,7 +1001,7 @@ mod tests { (2, Datum::int(2)), ]), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -908,7 +1070,7 @@ mod tests { partition: Struct::from_iter( vec![ Some(Literal::int(2021)), - Some(Literal::float(1.0)), + Some(Literal::float(1.0_f32)), Some(Literal::double(2.0)), ] ), @@ -921,7 +1083,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -943,7 +1105,7 @@ mod tests { partition: Struct::from_iter( vec![ Some(Literal::int(1111)), - Some(Literal::float(15.5)), + Some(Literal::float(15.5_f32)), Some(Literal::double(25.5)), ] ), @@ -956,7 +1118,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -991,7 +1153,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -1026,7 +1188,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -1041,12 +1203,11 @@ mod tests { // write manifest to file let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path().join("test_manifest.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); + let io = FileIO::new_with_fs(); let output_file = io.new_output(path.to_str().unwrap()).unwrap(); let mut writer = ManifestWriterBuilder::new( output_file, Some(1), - None, metadata.schema.clone(), metadata.partition_spec.clone(), ) @@ -1072,11 +1233,11 @@ mod tests { assert_eq!( partitions[1].clone().lower_bound.unwrap(), - Datum::float(1.0).to_bytes().unwrap() + Datum::float(1.0_f32).to_bytes().unwrap() ); assert_eq!( partitions[1].clone().upper_bound.unwrap(), - Datum::float(15.5).to_bytes().unwrap() + Datum::float(15.5_f32).to_bytes().unwrap() ); assert!(partitions[1].clone().contains_null); assert_eq!(partitions[1].clone().contains_nan, Some(true)); @@ -1181,7 +1342,7 @@ mod tests { "lower_bounds": [], "upper_bounds": [], "key_metadata": null, - "split_offsets": [], + "split_offsets": null, "equality_ids": null, "sort_order_id": null, "first_row_id": null, @@ -1212,7 +1373,7 @@ mod tests { "lower_bounds": [], "upper_bounds": [], "key_metadata": null, - "split_offsets": [], + "split_offsets": null, "equality_ids": null, "sort_order_id": null, "first_row_id": null, @@ -1247,4 +1408,870 @@ mod tests { assert_eq!(deserialized_data_file1, original_data_file1); assert_eq!(deserialized_data_file2, original_data_file2); } + + /// Metadata for the writer schema tests: a V2 data manifest whose spec has + /// identity partitions on a long, a string, and a double column. + fn writer_schema_test_metadata() -> ManifestMetadata { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "score", Type::Primitive(PrimitiveType::Double)) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .add_partition_field("name", "name", Transform::Identity) + .unwrap() + .add_partition_field("score", "score", Transform::Identity) + .unwrap() + .build() + .unwrap(); + ManifestMetadata { + schema_id: 0, + schema, + partition_spec, + content: ManifestContentType::Data, + format_version: FormatVersion::V2, + } + } + + /// A V2 manifest entry writer schema with Java's record names and only the + /// required `data_file` fields. + fn v2_writer_schema(partition_fields: Value) -> Value { + serde_json::json!({ + "type": "record", + "name": "manifest_entry", + "fields": [ + {"name": "status", "type": "int", "field-id": 0}, + {"name": "snapshot_id", "type": ["null", "long"], "default": null, "field-id": 1}, + {"name": "sequence_number", "type": ["null", "long"], "default": null, "field-id": 3}, + {"name": "file_sequence_number", "type": ["null", "long"], "default": null, "field-id": 4}, + {"name": "data_file", "field-id": 2, "type": { + "type": "record", + "name": "r2", + "fields": [ + {"name": "content", "type": "int", "field-id": 134}, + {"name": "file_path", "type": "string", "field-id": 100}, + {"name": "file_format", "type": "string", "field-id": 101}, + {"name": "partition", "field-id": 102, "type": { + "type": "record", + "name": "r102", + "fields": partition_fields, + }}, + {"name": "record_count", "type": "long", "field-id": 103}, + {"name": "file_size_in_bytes", "type": "long", "field-id": 104}, + ], + }}, + ], + }) + } + + /// The field named by `path` in a record schema, descending through nested + /// record types. + fn writer_schema_field<'a>(record: &'a mut Value, path: &[&str]) -> &'a mut Value { + let (name, rest) = path.split_first().unwrap(); + let field = record["fields"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|field| field["name"] == *name) + .unwrap(); + if rest.is_empty() { + field + } else { + writer_schema_field(&mut field["type"], rest) + } + } + + fn v2_entry(partition: Value) -> Value { + serde_json::json!({ + "status": 1, + "snapshot_id": 7, + "sequence_number": null, + "file_sequence_number": null, + "data_file": { + "content": 0, + "file_path": "s3://bucket/table/data/a.parquet", + "file_format": "PARQUET", + "partition": partition, + "record_count": 10, + "file_size_in_bytes": 100, + }, + }) + } + + fn expected_entry(partition: Struct) -> ManifestEntry { + ManifestEntry { + status: ManifestStatus::Added, + snapshot_id: Some(7), + sequence_number: None, + file_sequence_number: None, + data_file: DataFile { + content: DataContentType::Data, + file_path: "s3://bucket/table/data/a.parquet".to_string(), + file_format: DataFileFormat::Parquet, + partition, + record_count: 10, + file_size_in_bytes: 100, + column_sizes: HashMap::new(), + value_counts: HashMap::new(), + null_value_counts: HashMap::new(), + nan_value_counts: HashMap::new(), + lower_bounds: HashMap::new(), + upper_bounds: HashMap::new(), + key_metadata: None, + split_offsets: None, + equality_ids: None, + sort_order_id: None, + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + }, + } + } + + /// Writes `entries` as a manifest with the given writer schema, encoding each + /// JSON value with the writer schema's types. + fn write_with_writer_schema( + metadata: &ManifestMetadata, + writer_schema: &Value, + entries: Vec, + ) -> Vec { + write_avro_values_with_writer_schema( + metadata, + writer_schema, + entries + .into_iter() + .map(|entry| AvroValue::try_from(entry).unwrap()) + .collect(), + ) + } + + /// Like [`write_with_writer_schema`], for entries that JSON can't express. + fn write_avro_values_with_writer_schema( + metadata: &ManifestMetadata, + writer_schema: &Value, + entries: Vec, + ) -> Vec { + let avro_schema = apache_avro::Schema::parse(writer_schema).unwrap(); + let mut writer = Writer::new(&avro_schema, Vec::new()).unwrap(); + for (key, value) in [ + ("schema", to_vec(&metadata.schema).unwrap()), + ("schema-id", metadata.schema_id.to_string().into_bytes()), + ( + "partition-spec", + to_vec(&metadata.partition_spec.fields()).unwrap(), + ), + ( + "partition-spec-id", + metadata.partition_spec.spec_id().to_string().into_bytes(), + ), + ( + "format-version", + (metadata.format_version as u8).to_string().into_bytes(), + ), + ("content", metadata.content.to_string().into_bytes()), + ] { + writer.add_user_metadata(key.to_string(), value).unwrap(); + } + for entry in entries { + writer + .append_value(entry.resolve(&avro_schema).unwrap()) + .unwrap(); + } + writer.into_inner().unwrap() + } + + /// Removes the field named by `path` from a record schema and from a record + /// value of that schema. + fn remove_writer_field(schema: &mut Value, entry: &mut AvroValue, path: &[&str]) { + let (name, rest) = path.split_first().unwrap(); + let AvroValue::Record(values) = entry else { + unreachable!("the entry is a record"); + }; + let fields = schema["fields"].as_array_mut().unwrap(); + if rest.is_empty() { + fields.retain(|field| field["name"] != *name); + values.retain(|(value_name, _)| value_name != name); + } else { + let field = fields.iter_mut().find(|field| field["name"] == *name); + let value = values.iter_mut().find(|(value_name, _)| value_name == name); + remove_writer_field(&mut field.unwrap()["type"], &mut value.unwrap().1, rest); + } + } + + type ResetField = fn(&mut ManifestEntry); + + /// Removes each field in `optional` and `required` in turn from a manifest + /// that `full_schema` and `full_value` write as `full_entry`. Checks that the + /// manifest reads with an optional field reset and fails naming a required one. + fn assert_reads_without_each_field( + metadata: &ManifestMetadata, + full_schema: &Value, + full_value: &AvroValue, + full_entry: &ManifestEntry, + optional: &[(&[&str], ResetField)], + required: &[&[&str]], + ) { + for (path, reset) in optional { + let (mut schema, mut value) = (full_schema.clone(), full_value.clone()); + remove_writer_field(&mut schema, &mut value, path); + let bs = write_avro_values_with_writer_schema(metadata, &schema, vec![value]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let mut expected = full_entry.clone(); + reset(&mut expected); + assert_eq!( + manifest, + Manifest::new(metadata.clone(), vec![expected]), + "{path:?}" + ); + } + + for path in required { + let (mut schema, mut value) = (full_schema.clone(), full_value.clone()); + remove_writer_field(&mut schema, &mut value, path); + let bs = write_avro_values_with_writer_schema(metadata, &schema, vec![value]); + + let err = Manifest::parse_avro(&bs).unwrap_err(); + + assert!( + err.to_string().contains(path.last().unwrap()), + "{path:?}: {err}" + ); + } + } + + #[test] + fn test_parse_manifest_without_each_field() { + // Fields that the spec doesn't require in every version must read as + // their default when the writer omits them. + let metadata = writer_schema_test_metadata(); + let partition_type = metadata + .partition_spec + .partition_type(&metadata.schema) + .unwrap(); + let full_schema = + serde_json::to_value(manifest_schema_v2(&partition_type).unwrap()).unwrap(); + let mut full_entry = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + full_entry.sequence_number = Some(3); + full_entry.file_sequence_number = Some(4); + let data_file = &mut full_entry.data_file; + data_file.content = DataContentType::PositionDeletes; + data_file.column_sizes = HashMap::from([(1, 40)]); + data_file.value_counts = HashMap::from([(1, 10)]); + data_file.null_value_counts = HashMap::from([(1, 1)]); + data_file.nan_value_counts = HashMap::from([(3, 2)]); + data_file.lower_bounds = HashMap::from([(1, Datum::long(1))]); + data_file.upper_bounds = HashMap::from([(1, Datum::long(9))]); + data_file.key_metadata = Some(vec![1, 2]); + data_file.split_offsets = Some(vec![4]); + data_file.equality_ids = Some(vec![1]); + data_file.sort_order_id = Some(0); + data_file.first_row_id = Some(100); + data_file.referenced_data_file = Some("s3://bucket/table/data/b.parquet".to_string()); + data_file.content_offset = Some(4); + data_file.content_size_in_bytes = Some(8); + let full_value = to_value( + _serde::ManifestEntryV2::try_from( + full_entry.clone(), + &Type::Struct(partition_type.clone()), + ) + .unwrap(), + ) + .unwrap(); + + let optional: [(&[&str], ResetField); 18] = [ + (&["snapshot_id"], |e| e.snapshot_id = None), + (&["sequence_number"], |e| e.sequence_number = None), + (&["file_sequence_number"], |e| e.file_sequence_number = None), + (&["data_file", "content"], |e| { + e.data_file.content = DataContentType::Data + }), + (&["data_file", "column_sizes"], |e| { + e.data_file.column_sizes.clear() + }), + (&["data_file", "value_counts"], |e| { + e.data_file.value_counts.clear() + }), + (&["data_file", "null_value_counts"], |e| { + e.data_file.null_value_counts.clear() + }), + (&["data_file", "nan_value_counts"], |e| { + e.data_file.nan_value_counts.clear() + }), + (&["data_file", "lower_bounds"], |e| { + e.data_file.lower_bounds.clear() + }), + (&["data_file", "upper_bounds"], |e| { + e.data_file.upper_bounds.clear() + }), + (&["data_file", "key_metadata"], |e| { + e.data_file.key_metadata = None + }), + (&["data_file", "split_offsets"], |e| { + e.data_file.split_offsets = None + }), + (&["data_file", "equality_ids"], |e| { + e.data_file.equality_ids = None + }), + (&["data_file", "sort_order_id"], |e| { + e.data_file.sort_order_id = None + }), + (&["data_file", "first_row_id"], |e| { + e.data_file.first_row_id = None + }), + (&["data_file", "referenced_data_file"], |e| { + e.data_file.referenced_data_file = None + }), + (&["data_file", "content_offset"], |e| { + e.data_file.content_offset = None + }), + (&["data_file", "content_size_in_bytes"], |e| { + e.data_file.content_size_in_bytes = None + }), + ]; + let required: [&[&str]; 7] = [ + &["status"], + &["data_file"], + &["data_file", "file_path"], + &["data_file", "file_format"], + &["data_file", "partition"], + &["data_file", "record_count"], + &["data_file", "file_size_in_bytes"], + ]; + assert_reads_without_each_field( + &metadata, + &full_schema, + &full_value, + &full_entry, + &optional, + &required, + ); + } + + #[test] + fn test_parse_v1_manifest_without_each_field() { + let mut metadata = writer_schema_test_metadata(); + metadata.format_version = FormatVersion::V1; + let partition_type = metadata + .partition_spec + .partition_type(&metadata.schema) + .unwrap(); + let full_schema = + serde_json::to_value(manifest_schema_v1(&partition_type).unwrap()).unwrap(); + let mut full_entry = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + // V1 has no sequence numbers, and every V1 entry reads with 0. + full_entry.sequence_number = Some(0); + full_entry.file_sequence_number = Some(0); + let data_file = &mut full_entry.data_file; + data_file.column_sizes = HashMap::from([(1, 40)]); + data_file.value_counts = HashMap::from([(1, 10)]); + data_file.null_value_counts = HashMap::from([(1, 1)]); + data_file.nan_value_counts = HashMap::from([(3, 2)]); + data_file.lower_bounds = HashMap::from([(1, Datum::long(1))]); + data_file.upper_bounds = HashMap::from([(1, Datum::long(9))]); + data_file.key_metadata = Some(vec![1, 2]); + data_file.split_offsets = Some(vec![4]); + data_file.sort_order_id = Some(0); + let full_value = to_value( + _serde::ManifestEntryV1::try_from( + full_entry.clone(), + &Type::Struct(partition_type.clone()), + ) + .unwrap(), + ) + .unwrap(); + + let optional: [(&[&str], ResetField); 10] = [ + // Required in V1, but deprecated and not read. + (&["data_file", "block_size_in_bytes"], |_| {}), + (&["data_file", "column_sizes"], |e| { + e.data_file.column_sizes.clear() + }), + (&["data_file", "value_counts"], |e| { + e.data_file.value_counts.clear() + }), + (&["data_file", "null_value_counts"], |e| { + e.data_file.null_value_counts.clear() + }), + (&["data_file", "nan_value_counts"], |e| { + e.data_file.nan_value_counts.clear() + }), + (&["data_file", "lower_bounds"], |e| { + e.data_file.lower_bounds.clear() + }), + (&["data_file", "upper_bounds"], |e| { + e.data_file.upper_bounds.clear() + }), + (&["data_file", "key_metadata"], |e| { + e.data_file.key_metadata = None + }), + (&["data_file", "split_offsets"], |e| { + e.data_file.split_offsets = None + }), + (&["data_file", "sort_order_id"], |e| { + e.data_file.sort_order_id = None + }), + ]; + let required: [&[&str]; 8] = [ + &["status"], + &["snapshot_id"], + &["data_file"], + &["data_file", "file_path"], + &["data_file", "file_format"], + &["data_file", "partition"], + &["data_file", "record_count"], + &["data_file", "file_size_in_bytes"], + ]; + assert_reads_without_each_field( + &metadata, + &full_schema, + &full_value, + &full_entry, + &optional, + &required, + ); + } + + #[test] + fn test_parse_manifest_matches_partition_fields_by_name() { + // The writer orders the partition fields differently from the spec, omits + // `name`, and adds a field the spec doesn't have. + let metadata = writer_schema_test_metadata(); + let writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + {"name": "extra", "type": ["null", "int"], "default": null, "field-id": 1003}, + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + ])); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry( + serde_json::json!({"score": 2.5, "extra": 9, "id": 5}), + )]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let partition = + Struct::from_iter([Some(Literal::long(5)), None, Some(Literal::double(2.5))]); + assert_eq!( + manifest, + Manifest::new(metadata, vec![expected_entry(partition)]) + ); + } + + #[test] + fn test_parse_manifest_promotes_writer_types() { + // Avro promotes int to long and float to double. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "int"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "float"], "default": null, "field-id": 1002}, + ])); + for field in ["record_count", "file_size_in_bytes"] { + writer_schema_field(&mut writer_schema, &["data_file", field])["type"] = + serde_json::json!("int"); + } + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry( + serde_json::json!({"id": 5, "name": "a", "score": 2.5}), + )]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let partition = Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ]); + assert_eq!( + manifest, + Manifest::new(metadata, vec![expected_entry(partition)]) + ); + } + + #[test] + fn test_parse_manifest_reads_required_writer_fields_as_optional() { + // Fields that are optional in the reader schema are written without a + // union. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": "long", "field-id": 1000}, + {"name": "name", "type": "string", "field-id": 1001}, + {"name": "score", "type": "double", "field-id": 1002}, + ])); + *writer_schema_field(&mut writer_schema, &["snapshot_id"]) = + serde_json::json!({"name": "snapshot_id", "type": "long", "field-id": 1}); + writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({"name": "sort_order_id", "type": "int", "field-id": 140})); + let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5})); + entry["data_file"]["sort_order_id"] = serde_json::json!(3); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let mut expected = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + expected.data_file.sort_order_id = Some(3); + assert_eq!(manifest, Manifest::new(metadata, vec![expected])); + } + + #[test] + fn test_parse_manifest_reads_union_writer_fields_as_required() { + // A field that is required in the reader schema is written as a union with + // null. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + ])); + writer_schema_field(&mut writer_schema, &["data_file", "record_count"])["type"] = + serde_json::json!(["null", "long"]); + let partition = serde_json::json!({"id": 5, "name": "a", "score": 2.5}); + let bs = + write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry(partition.clone())]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let expected = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + assert_eq!(manifest, Manifest::new(metadata.clone(), vec![expected])); + + let mut entry = v2_entry(partition); + entry["data_file"]["record_count"] = Value::Null; + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let err = Manifest::parse_avro(&bs).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_parse_manifest_with_fixed_uuid_partition() { + // The spec stores a uuid in Avro as a 16-byte fixed with logical type uuid. + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "u", Type::Primitive(PrimitiveType::Uuid)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("u", "u", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let metadata = ManifestMetadata { + schema_id: 0, + schema, + partition_spec, + content: ManifestContentType::Data, + format_version: FormatVersion::V2, + }; + let writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "u", "default": null, "field-id": 1000, "type": ["null", { + "type": "fixed", "name": "uuid_fixed", "size": 16, "logicalType": "uuid", + }]}, + ])); + let uuid = uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7); + // `Value::resolve` doesn't convert a JSON string to a fixed uuid, so the + // partition value is set as an Avro value. + let mut entry = AvroValue::try_from(v2_entry(serde_json::json!({}))).unwrap(); + let AvroValue::Map(fields) = &mut entry else { + unreachable!("a JSON object converts to an Avro map"); + }; + let Some(AvroValue::Map(data_file)) = fields.get_mut("data_file") else { + unreachable!("a JSON object converts to an Avro map"); + }; + data_file.insert( + "partition".to_string(), + AvroValue::Map(HashMap::from([("u".to_string(), AvroValue::Uuid(uuid))])), + ); + let bs = write_avro_values_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let expected = expected_entry(Struct::from_iter([Some(Literal::uuid(uuid))])); + assert_eq!(manifest, Manifest::new(metadata, vec![expected])); + } + + #[test] + fn test_parse_manifest_ignores_record_names_and_unknown_fields() { + // Record names differ from the ones Java writes, and the writer has fields + // the reader schema doesn't. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + ])); + writer_schema["name"] = serde_json::json!("entry"); + writer_schema["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({"name": "unknown", "type": "string"})); + let data_file = &mut writer_schema_field(&mut writer_schema, &["data_file"])["type"]; + data_file["name"] = serde_json::json!("file"); + data_file["fields"][3]["type"]["name"] = serde_json::json!("part"); + let data_file_fields = data_file["fields"].as_array_mut().unwrap(); + data_file_fields.push(serde_json::json!({ + "name": "column_sizes", + "field-id": 108, + "default": null, + "type": ["null", { + "type": "array", + "logicalType": "map", + "items": { + "type": "record", + "name": "sizes", + "fields": [ + {"name": "key", "type": "int", "field-id": 117}, + {"name": "value", "type": "long", "field-id": 118}, + ], + }, + }], + })); + data_file_fields.push( + serde_json::json!({"name": "block_size_in_bytes", "type": "long", "field-id": 105}), + ); + let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5})); + entry["unknown"] = serde_json::json!("ignored"); + entry["data_file"]["column_sizes"] = serde_json::json!([{"key": 1, "value": 40}]); + entry["data_file"]["block_size_in_bytes"] = serde_json::json!(64); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let mut expected = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + expected.data_file.column_sizes = HashMap::from([(1, 40)]); + assert_eq!(manifest, Manifest::new(metadata, vec![expected])); + } + + #[test] + fn test_parse_manifest_equality_ids_written_as_long() { + // PyIceberg wrote `equality_ids` as `array` before + // apache/iceberg-python#3842. + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + ])); + writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": "equality_ids", + "field-id": 135, + "default": null, + "type": ["null", {"type": "array", "items": "long", "element-id": 136}], + })); + let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5})); + entry["data_file"]["equality_ids"] = serde_json::json!([1, 2]); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let mut expected = expected_entry(Struct::from_iter([ + Some(Literal::long(5)), + Some(Literal::string("a")), + Some(Literal::double(2.5)), + ])); + expected.data_file.equality_ids = Some(vec![1, 2]); + assert_eq!(manifest, Manifest::new(metadata, vec![expected])); + } + + #[test] + fn test_parse_manifest_rejects_equality_id_above_int_range() { + let metadata = writer_schema_test_metadata(); + let mut writer_schema = v2_writer_schema(serde_json::json!([ + {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000}, + {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001}, + {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002}, + ])); + writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": "equality_ids", + "field-id": 135, + "default": null, + "type": ["null", {"type": "array", "items": "long", "element-id": 136}], + })); + let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5})); + entry["data_file"]["equality_ids"] = serde_json::json!([i64::from(i32::MAX) + 1]); + let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]); + + let err = Manifest::parse_avro(&bs).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[test] + fn test_parse_manifest_written_by_pyiceberg() { + let bs = fs::read(format!( + "{}/testdata/manifests/pyiceberg-v2-data.avro", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "category", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::optional(3, "score", Type::Primitive(PrimitiveType::Double)) + .into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("category", "category", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let metadata = ManifestMetadata { + schema_id: 0, + schema, + partition_spec, + content: ManifestContentType::Data, + format_version: FormatVersion::V2, + }; + let entries = (0..2) + .map(|i: i64| { + let category = format!("c{i}"); + ManifestEntry { + status: ManifestStatus::Added, + snapshot_id: Some(7), + sequence_number: Some(1), + file_sequence_number: Some(1), + data_file: DataFile { + content: DataContentType::Data, + file_path: format!( + "s3://bucket/table/data/category={category}/0000{i}.parquet" + ), + file_format: DataFileFormat::Parquet, + partition: Struct::from_iter([Some(Literal::string(&category))]), + record_count: 100 + i as u64, + file_size_in_bytes: 1000 + i as u64, + column_sizes: HashMap::from([ + (1, 10 + i as u64), + (2, 20 + i as u64), + (3, 30 + i as u64), + ]), + value_counts: HashMap::from([ + (1, 100 + i as u64), + (2, 100 + i as u64), + (3, 100 + i as u64), + ]), + null_value_counts: HashMap::from([(1, 0), (2, i as u64), (3, 1)]), + nan_value_counts: HashMap::from([(3, i as u64)]), + lower_bounds: HashMap::from([ + (1, Datum::long(i)), + (2, Datum::string(&category)), + ]), + upper_bounds: HashMap::from([ + (1, Datum::long(i + 50)), + (2, Datum::string(&category)), + ]), + key_metadata: None, + split_offsets: Some(vec![4]), + equality_ids: None, + sort_order_id: Some(0), + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + }, + } + }) + .collect(); + assert_eq!(manifest, Manifest::new(metadata, entries)); + } + + #[test] + fn test_parse_manifest_with_repeated_named_type_definitions() { + let bs = fs::read(format!( + "{}/testdata/manifests/repeated-decimal-type-definitions.avro", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + + let manifest = Manifest::parse_avro(&bs).unwrap(); + + assert_eq!( + *manifest.entries()[0].data_file().partition(), + Struct::from_iter([Some(Literal::decimal(12345)), Some(Literal::decimal(-678))]) + ); + } + + #[test] + fn test_parse_manifest_with_repeated_named_type_definitions_reports_original_error() { + let bs = fs::read(format!( + "{}/testdata/manifests/repeated-decimal-type-definitions.avro", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + // Cut the header's sync marker in half, so the rewritten header parses + // its schema but fails to read the marker. The file ends with the marker. + let marker = &bs[bs.len() - 16..]; + let header_marker = bs.windows(16).position(|w| w == marker).unwrap(); + let truncated = &bs[..header_marker + 8]; + let log_dir = TempDir::new().unwrap(); + let log_path = log_dir.path().join("log"); + let subscriber = tracing_subscriber::fmt() + .with_max_level(Level::DEBUG) + .with_writer(Arc::new(fs::File::create(&log_path).unwrap())) + .finish(); + + let err = with_default(subscriber, || Manifest::parse_avro(truncated)).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + let message = err.to_string(); + assert!( + message.contains("Two named schema defined for same fullname"), + "{message}" + ); + assert!(message.contains("Failed to read marker bytes"), "{message}"); + // The fallback logs only manifests that it reads. + let log = fs::read_to_string(&log_path).unwrap(); + assert!(!log.contains("more than once"), "{log}"); + } } diff --git a/crates/iceberg/src/spec/manifest/reader.rs b/crates/iceberg/src/spec/manifest/reader.rs new file mode 100644 index 0000000000..14348ca49f --- /dev/null +++ b/crates/iceberg/src/spec/manifest/reader.rs @@ -0,0 +1,446 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::Manifest; +use crate::encryption::{EncryptedInputFile, StandardKeyMetadata}; +use crate::error::{Result, invalid_data}; +use crate::io::FileIO; +use crate::spec::{ManifestContentType, ManifestEntry, ManifestFile}; + +/// Reads a manifest file referenced by a manifest list entry, transparently +/// decrypting it when the entry records key metadata. +pub struct ManifestReader { + file_io: FileIO, +} + +impl ManifestReader { + /// Create a manifest reader. + pub(crate) fn new(file_io: FileIO) -> Self { + Self { file_io } + } + + /// Read, decrypt, parse and return the manifest described by + /// `manifest_file`. + pub async fn read(self, manifest_file: &ManifestFile) -> Result { + let input_file = self.file_io.new_input(&manifest_file.manifest_path)?; + let key_metadata = manifest_file + .key_metadata + .as_deref() + .map(StandardKeyMetadata::decode) + .transpose()?; + let bytes = match key_metadata { + Some(key_metadata) => { + EncryptedInputFile::new(input_file, key_metadata) + .read() + .await? + } + None => input_file.read().await?, + }; + + let (metadata, mut entries) = + Manifest::try_from_avro_bytes(&bytes, Some(&manifest_file.manifest_path))?; + + for entry in &mut entries { + entry.inherit_data(manifest_file); + } + + self.assign_first_row_ids(manifest_file, &mut entries)?; + + Ok(Manifest::new(metadata, entries)) + } + + /// Assigns `first_row_id` to the live data-file entries, following the + /// row-lineage inheritance rules in + /// . + /// + /// With a manifest-level `first_row_id`, each live entry lacking one is + /// assigned the running id, which then advances by that entry's record + /// count; entries that already carry a `first_row_id` keep it and do not + /// advance the counter. Without a manifest-level `first_row_id`, any + /// inherited per-entry value is cleared so callers never observe a stale id. + fn assign_first_row_ids( + &self, + manifest_file: &ManifestFile, + entries: &mut [ManifestEntry], + ) -> Result<()> { + // A `first_row_id` is only valid on data manifests. Delete files always + // have a null `first_row_id`, so there is nothing to assign or clear; a + // stray value on a delete manifest is a spec violation by the writer, + // which we surface without failing the read. + if manifest_file.content != ManifestContentType::Data { + if let Some(manifest_first_row_id) = manifest_file.first_row_id { + tracing::warn!( + "Ignoring first_row_id {manifest_first_row_id} on delete manifest {}", + manifest_file.manifest_path + ); + } + + return Ok(()); + } + + let Some(manifest_first_row_id) = manifest_file.first_row_id else { + // A data manifest with no manifest-level `first_row_id` predates row + // lineage; clear any per-entry value inherited from an earlier read. + for entry in entries { + entry.data_file.first_row_id = None; + } + + return Ok(()); + }; + + let mut next_row_id = i64::try_from(manifest_first_row_id).map_err(|_| { + invalid_data!("Invalid first_row_id: {manifest_first_row_id} (exceeds i64::MAX)") + })?; + + for entry in entries { + if !entry.is_alive() { + continue; + } + + if entry.data_file.first_row_id.is_none() { + let file_first_row_id = next_row_id; + entry.data_file.first_row_id = Some(file_first_row_id); + let record_count = entry.data_file.record_count; + next_row_id = file_first_row_id.checked_add_unsigned(record_count).ok_or_else(|| { + invalid_data!("Row ID overflow assigning first_row_id in {}. File first_row_id: {file_first_row_id}, record count: {record_count}", + manifest_file.manifest_path) + })?; + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use super::*; + use crate::ErrorKind; + use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata}; + use crate::io::FileIO; + use crate::spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, ManifestEntry, ManifestStatus, + ManifestWriterBuilder, NestedField, PartitionSpec, PrimitiveType, Schema, Struct, Type, + }; + + #[tokio::test] + async fn test_read_plaintext_manifest_inherits_entries() { + let schema = test_schema(); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .build() + .unwrap(); + + let io = FileIO::new_with_memory(); + let path = "memory:///table/metadata/plain.avro"; + let mut writer = ManifestWriterBuilder::new( + io.new_output(path).unwrap(), + Some(1), + schema.clone(), + partition_spec, + ) + .build_v2_data(); + writer.add_entry(test_entry()).unwrap(); + // Writing the manifest yields the manifest list entry describing it. + let manifest_file = writer.write_manifest_file().await.unwrap(); + + let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap(); + assert_eq!(manifest.entries().len(), 1); + assert_eq!( + manifest.entries()[0].data_file().file_path(), + "memory:///table/data/00000.parquet" + ); + // Entries must inherit values from the manifest list entry. + assert_eq!( + manifest.entries()[0].sequence_number(), + Some(manifest_file.sequence_number) + ); + assert_eq!( + manifest.entries()[0].snapshot_id(), + Some(manifest_file.added_snapshot_id) + ); + } + + /// Per spec, inheritance does not apply to v1: a v1 manifest's entries read + /// with sequence number 0 even when the manifest list entry referencing them + /// carries a nonzero one. + #[tokio::test] + async fn test_read_v1_manifest_defaults_sequence_number_to_zero() { + let schema = test_schema(); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .build() + .unwrap(); + + let io = FileIO::new_with_memory(); + let path = "memory:///table/metadata/v1.avro"; + let mut writer = ManifestWriterBuilder::new( + io.new_output(path).unwrap(), + Some(1), + schema.clone(), + partition_spec, + ) + .build_v1(); + writer.add_entry(test_entry()).unwrap(); + let mut manifest_file = writer.write_manifest_file().await.unwrap(); + manifest_file.sequence_number = 36; + + let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap(); + assert_eq!(manifest.entries()[0].sequence_number(), Some(0)); + assert_eq!(manifest.entries()[0].file_sequence_number, Some(0)); + } + + #[tokio::test] + async fn test_read_encrypted_manifest_roundtrip() { + let schema = test_schema(); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .build() + .unwrap(); + + let io = FileIO::new_with_memory(); + let path = "memory:///table/metadata/encrypted.avro"; + let encrypted_output = + EncryptedOutputFile::new(io.new_output(path).unwrap(), key_metadata()); + + let mut writer = ManifestWriterBuilder::new_from_encrypted( + encrypted_output, + Some(1), + schema.clone(), + partition_spec, + ) + .unwrap() + .build_v3_data(); + writer.add_entry(test_entry()).unwrap(); + // The returned manifest list entry records the key metadata. + let manifest_file = writer.write_manifest_file().await.unwrap(); + assert!(manifest_file.key_metadata.is_some()); + + // Reading with the recorded key metadata must recover the entry. + let manifest = ManifestReader::new(io.clone()) + .read(&manifest_file) + .await + .unwrap(); + assert_eq!(manifest.entries().len(), 1); + assert_eq!( + manifest.entries()[0].data_file().file_path(), + "memory:///table/data/00000.parquet" + ); + + // Without the key metadata the encrypted bytes must not read as plaintext. + let mut plaintext_entry = manifest_file.clone(); + plaintext_entry.key_metadata = None; + assert!( + ManifestReader::new(io) + .read(&plaintext_entry) + .await + .is_err(), + "encrypted manifest must not parse as plaintext" + ); + } + + fn test_schema() -> Arc { + Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::optional( + 1, + "id", + Type::Primitive(PrimitiveType::Long), + ))]) + .build() + .unwrap(), + ) + } + + fn test_entry() -> ManifestEntry { + ManifestEntry { + status: ManifestStatus::Added, + snapshot_id: None, + sequence_number: None, + file_sequence_number: None, + data_file: DataFile { + content: DataContentType::Data, + file_path: "memory:///table/data/00000.parquet".to_string(), + file_format: DataFileFormat::Parquet, + partition: Struct::empty(), + record_count: 1, + file_size_in_bytes: 4096, + column_sizes: HashMap::new(), + value_counts: HashMap::new(), + null_value_counts: HashMap::new(), + nan_value_counts: HashMap::new(), + lower_bounds: HashMap::new(), + upper_bounds: HashMap::new(), + key_metadata: None, + split_offsets: None, + equality_ids: None, + sort_order_id: None, + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + }, + } + } + + fn key_metadata() -> StandardKeyMetadata { + StandardKeyMetadata::try_new(b"0123456789abcdef").unwrap() + } + + /// A reader with an unused in-memory `FileIO`, for exercising the pure + /// row-id assignment logic without touching storage. + fn test_reader() -> ManifestReader { + ManifestReader::new(FileIO::new_with_memory()) + } + + /// Builds a data-file manifest entry with the given status, record count, + /// and pre-existing `first_row_id`. + fn data_entry( + status: ManifestStatus, + record_count: u64, + first_row_id: Option, + ) -> ManifestEntry { + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("s3://bucket/table/data/00000.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(4096) + .record_count(record_count) + .first_row_id(first_row_id) + .build() + .unwrap(); + + ManifestEntry::builder() + .status(status) + .data_file(data_file) + .build() + } + + /// Builds a manifest file with the given content type and manifest-level + /// `first_row_id`. Other fields are irrelevant to row-id assignment. + fn manifest_file(content: ManifestContentType, first_row_id: Option) -> ManifestFile { + ManifestFile { + manifest_path: "memory:///m.avro".to_string(), + manifest_length: 0, + partition_spec_id: 0, + content, + sequence_number: 0, + min_sequence_number: 0, + added_snapshot_id: 0, + added_files_count: None, + existing_files_count: None, + deleted_files_count: None, + added_rows_count: None, + existing_rows_count: None, + deleted_rows_count: None, + partitions: None, + key_metadata: None, + first_row_id, + } + } + + #[test] + fn test_assign_first_row_ids_interleaved() { + let manifest = manifest_file(ManifestContentType::Data, Some(10)); + let mut entries = vec![ + data_entry(ManifestStatus::Added, 3, None), + // A pre-assigned entry between two assigned ones: it keeps its id and + // must not advance the running counter. + data_entry(ManifestStatus::Added, 5, Some(100)), + // A deleted entry with a pre-set id: it is skipped, so the id is + // preserved verbatim and does not advance the counter. + data_entry(ManifestStatus::Deleted, 7, Some(999)), + data_entry(ManifestStatus::Existing, 2, None), + ]; + + test_reader() + .assign_first_row_ids(&manifest, &mut entries) + .unwrap(); + + assert_eq!(entries[0].data_file.first_row_id, Some(10)); + assert_eq!(entries[1].data_file.first_row_id, Some(100)); + assert_eq!(entries[2].data_file.first_row_id, Some(999)); + // 10 + 3 = 13; the preserved and deleted entries in between do not move it. + assert_eq!(entries[3].data_file.first_row_id, Some(13)); + } + + #[test] + fn test_assign_first_row_ids_clears_without_manifest_first_row_id() { + // A data manifest with no manifest-level first_row_id predates row + // lineage: any per-entry value inherited from an earlier read is cleared + // so callers never observe a stale id. + let manifest = manifest_file(ManifestContentType::Data, None); + let mut entries = vec![ + data_entry(ManifestStatus::Added, 3, None), + data_entry(ManifestStatus::Existing, 5, Some(100)), + ]; + + test_reader() + .assign_first_row_ids(&manifest, &mut entries) + .unwrap(); + + assert_eq!(entries[0].data_file.first_row_id, None); + assert_eq!(entries[1].data_file.first_row_id, None); + } + + #[test] + fn test_assign_first_row_ids_ignores_delete_manifest() { + // A stray first_row_id on a delete manifest is a writer-side spec + // violation; the read ignores it rather than failing, and does not + // assign ids to the entries. + let manifest = manifest_file(ManifestContentType::Deletes, Some(10)); + let mut entries = vec![data_entry(ManifestStatus::Added, 3, None)]; + + test_reader() + .assign_first_row_ids(&manifest, &mut entries) + .unwrap(); + + assert_eq!(entries[0].data_file.first_row_id, None); + } + + #[test] + fn test_assign_first_row_ids_rejects_oversized_manifest_first_row_id() { + // A manifest-level first_row_id above i64::MAX cannot be represented as the + // signed running counter and must be rejected. + let manifest = manifest_file(ManifestContentType::Data, Some(i64::MAX as u64 + 1)); + let mut entries = vec![data_entry(ManifestStatus::Added, 3, None)]; + + let err = test_reader() + .assign_first_row_ids(&manifest, &mut entries) + .expect_err("an oversized manifest first_row_id must be rejected"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("Invalid first_row_id")); + } + + #[test] + fn test_assign_first_row_ids_rejects_counter_overflow() { + // Advancing the running counter past i64::MAX must be rejected rather than + // wrapping to a negative value that would corrupt subsequent assignments. + let manifest = manifest_file(ManifestContentType::Data, Some(i64::MAX as u64)); + let mut entries = vec![data_entry(ManifestStatus::Added, 1, None)]; + + let err = test_reader() + .assign_first_row_ids(&manifest, &mut entries) + .expect_err("counter overflow past i64::MAX must be rejected"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("Row ID overflow")); + } +} diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index 673f8b5d2a..e822709c5d 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -17,7 +17,7 @@ use std::cmp::min; -use apache_avro::{Writer as AvroWriter, to_value}; +use apache_avro::to_value; use bytes::Bytes; use itertools::Itertools; use serde_json::to_vec; @@ -26,43 +26,94 @@ use super::{ Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType, UNASSIGNED_SEQUENCE_NUMBER, }; -use crate::error::Result; -use crate::io::OutputFile; +use crate::avro::new_writer; +use crate::encryption::EncryptedOutputFile; +use crate::error::{Result, invalid_data}; +use crate::io::{FileMetadata, FileWrite, OutputFile}; use crate::spec::manifest::_serde::{ManifestEntryV1, ManifestEntryV2}; use crate::spec::manifest::{manifest_schema_v1, manifest_schema_v2}; use crate::spec::{ DataContentType, DataFile, FieldSummary, ManifestEntry, ManifestFile, ManifestMetadata, - ManifestStatus, PrimitiveLiteral, SchemaRef, StructType, UNASSIGNED_SNAPSHOT_ID, + ManifestStatus, PrimitiveLiteral, SchemaRef, StructType, Type, }; -use crate::{Error, ErrorKind}; + +/// Placeholder for snapshot ID. The field with this value must be replaced +/// with the actual snapshot ID before it is committed. +const UNASSIGNED_SNAPSHOT_ID: i64 = -1; + +/// Retains the output until close provides the size required by encrypted key metadata. +pub(crate) enum ManifestOutput { + Plain(OutputFile), + Encrypted(EncryptedOutputFile), +} + +impl ManifestOutput { + async fn writer(&self) -> Result> { + match self { + Self::Plain(output) => output.writer().await, + Self::Encrypted(output) => output.writer().await, + } + } + + fn encoded_key_metadata(&self, file_metadata: &FileMetadata) -> Result>> { + match self { + Self::Plain(_) => Ok(None), + Self::Encrypted(output) => Ok(Some( + output + .key_metadata_with_saved_file_metadata(file_metadata) + .encode()? + .into_vec(), + )), + } + } +} /// The builder used to create a [`ManifestWriter`]. pub struct ManifestWriterBuilder { - output: OutputFile, + output: ManifestOutput, + location: String, snapshot_id: Option, - key_metadata: Option>, schema: SchemaRef, partition_spec: PartitionSpec, } impl ManifestWriterBuilder { - /// Create a new builder. + /// Create a new builder for unencrypted manifests. pub fn new( output: OutputFile, snapshot_id: Option, - key_metadata: Option>, schema: SchemaRef, partition_spec: PartitionSpec, ) -> Self { + let location = output.location().to_owned(); Self { - output, + output: ManifestOutput::Plain(output), + location, snapshot_id, - key_metadata, schema, partition_spec, } } + /// Create a new builder from an [`EncryptedOutputFile`]. + /// + /// Use this when writing manifests with transparent encryption. + pub fn new_from_encrypted( + encrypted_output: EncryptedOutputFile, + snapshot_id: Option, + schema: SchemaRef, + partition_spec: PartitionSpec, + ) -> Result { + let location = encrypted_output.location().to_owned(); + Ok(Self { + output: ManifestOutput::Encrypted(encrypted_output), + location, + snapshot_id, + schema, + partition_spec, + }) + } + /// Build a [`ManifestWriter`] for format version 1. pub fn build_v1(self) -> ManifestWriter { let metadata = ManifestMetadata::builder() @@ -72,7 +123,7 @@ impl ManifestWriterBuilder { .format_version(FormatVersion::V1) .content(ManifestContentType::Data) .build(); - ManifestWriter::new(self.output, self.snapshot_id, self.key_metadata, metadata) + ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None) } /// Build a [`ManifestWriter`] for format version 2, data content. @@ -84,7 +135,8 @@ impl ManifestWriterBuilder { .format_version(FormatVersion::V2) .content(ManifestContentType::Data) .build(); - ManifestWriter::new(self.output, self.snapshot_id, self.key_metadata, metadata) + + ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None) } /// Build a [`ManifestWriter`] for format version 2, deletes content. @@ -96,13 +148,46 @@ impl ManifestWriterBuilder { .format_version(FormatVersion::V2) .content(ManifestContentType::Deletes) .build(); - ManifestWriter::new(self.output, self.snapshot_id, self.key_metadata, metadata) + ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None) + } + + /// Build a [`ManifestWriter`] for format version 2, data content. + pub fn build_v3_data(self) -> ManifestWriter { + let metadata = ManifestMetadata::builder() + .schema_id(self.schema.schema_id()) + .schema(self.schema) + .partition_spec(self.partition_spec) + .format_version(FormatVersion::V3) + .content(ManifestContentType::Data) + .build(); + ManifestWriter::new( + self.output, + self.location, + self.snapshot_id, + metadata, + // First row id is assigned by the [`ManifestListWriter`] when the manifest + // is added to the list. + None, + ) + } + + /// Build a [`ManifestWriter`] for format version 3, deletes content. + pub fn build_v3_deletes(self) -> ManifestWriter { + let metadata = ManifestMetadata::builder() + .schema_id(self.schema.schema_id()) + .schema(self.schema) + .partition_spec(self.partition_spec) + .format_version(FormatVersion::V3) + .content(ManifestContentType::Deletes) + .build(); + ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None) } } /// A manifest writer. pub struct ManifestWriter { - output: OutputFile, + output: ManifestOutput, + location: String, snapshot_id: Option, @@ -112,11 +197,10 @@ pub struct ManifestWriter { existing_rows: u64, deleted_files: u32, deleted_rows: u64, + first_row_id: Option, min_seq_num: Option, - key_metadata: Option>, - manifest_entries: Vec, metadata: ManifestMetadata, @@ -125,13 +209,15 @@ pub struct ManifestWriter { impl ManifestWriter { /// Create a new manifest writer. pub(crate) fn new( - output: OutputFile, + output: ManifestOutput, + location: String, snapshot_id: Option, - key_metadata: Option>, metadata: ManifestMetadata, + first_row_id: Option, ) -> Self { Self { output, + location, snapshot_id, added_files: 0, added_rows: 0, @@ -139,8 +225,8 @@ impl ManifestWriter { existing_rows: 0, deleted_files: 0, deleted_rows: 0, + first_row_id, min_seq_num: None, - key_metadata, manifest_entries: Vec::new(), metadata, } @@ -168,13 +254,10 @@ impl ManifestWriter { match self.metadata.content { ManifestContentType::Data => { if data_file.content != DataContentType::Data { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Date file at path {} with manifest content type `data`, should have DataContentType `Data`, but has `{:?}`", - data_file.file_path(), - data_file.content - ), + return Err(invalid_data!( + "Date file at path {} with manifest content type `data`, should have DataContentType `Data`, but has `{:?}`", + data_file.file_path(), + data_file.content )); } } @@ -182,13 +265,10 @@ impl ManifestWriter { if data_file.content != DataContentType::EqualityDeletes && data_file.content != DataContentType::PositionDeletes { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Date file at path {} with manifest content type `deletes`, should have DataContentType `Data`, but has `{:?}`", - data_file.file_path(), - data_file.content - ), + return Err(invalid_data!( + "Date file at path {} with manifest content type `deletes`, should have DataContentType `Data`, but has `{:?}`", + data_file.file_path(), + data_file.content )); } } @@ -308,9 +388,8 @@ impl ManifestWriter { if (entry.status == ManifestStatus::Deleted || entry.status == ManifestStatus::Existing) && (entry.sequence_number.is_none() || entry.file_sequence_number.is_none()) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Manifest entry with status Existing or Deleted should have sequence number", + return Err(invalid_data!( + "Manifest entry with status Existing or Deleted should have sequence number" )); } @@ -329,10 +408,10 @@ impl ManifestWriter { self.existing_rows += entry.data_file.record_count; } } - if entry.is_alive() { - if let Some(seq_num) = entry.sequence_number { - self.min_seq_num = Some(self.min_seq_num.map_or(seq_num, |v| min(v, seq_num))); - } + if entry.is_alive() + && let Some(seq_num) = entry.sequence_number + { + self.min_seq_num = Some(self.min_seq_num.map_or(seq_num, |v| min(v, seq_num))); } self.manifest_entries.push(entry); Ok(()) @@ -345,18 +424,20 @@ impl ManifestWriter { .metadata .partition_spec .partition_type(&self.metadata.schema)?; + // Wrap once and reuse for every entry so the per-entry conversion does + // not rebuild the partition type's field-name lookup on each call. + let partition_struct_type = Type::Struct(partition_type.clone()); let table_schema = &self.metadata.schema; let avro_schema = match self.metadata.format_version { FormatVersion::V1 => manifest_schema_v1(&partition_type)?, - FormatVersion::V2 => manifest_schema_v2(&partition_type)?, + // Manifest schema did not change between V2 and V3 + FormatVersion::V2 | FormatVersion::V3 => manifest_schema_v2(&partition_type)?, }; - let mut avro_writer = AvroWriter::new(&avro_schema, Vec::new()); + let mut avro_writer = new_writer(&avro_schema, Vec::new())?; avro_writer.add_user_metadata( "schema".to_string(), - to_vec(table_schema).map_err(|err| { - Error::new(ErrorKind::DataInvalid, "Fail to serialize table schema") - .with_source(err) - })?, + to_vec(table_schema) + .map_err(|err| invalid_data!("Fail to serialize table schema").with_source(err))?, )?; avro_writer.add_user_metadata( "schema-id".to_string(), @@ -365,8 +446,7 @@ impl ManifestWriter { avro_writer.add_user_metadata( "partition-spec".to_string(), to_vec(&self.metadata.partition_spec.fields()).map_err(|err| { - Error::new(ErrorKind::DataInvalid, "Fail to serialize partition spec") - .with_source(err) + invalid_data!("Fail to serialize partition spec").with_source(err) })?, )?; avro_writer.add_user_metadata( @@ -377,31 +457,42 @@ impl ManifestWriter { "format-version".to_string(), (self.metadata.format_version as u8).to_string(), )?; - if self.metadata.format_version == FormatVersion::V2 { - avro_writer - .add_user_metadata("content".to_string(), self.metadata.content.to_string())?; + match self.metadata.format_version { + FormatVersion::V1 => {} + FormatVersion::V2 | FormatVersion::V3 => { + avro_writer + .add_user_metadata("content".to_string(), self.metadata.content.to_string())?; + } } let partition_summary = self.construct_partition_summaries(&partition_type)?; // Write manifest entries for entry in std::mem::take(&mut self.manifest_entries) { let value = match self.metadata.format_version { - FormatVersion::V1 => to_value(ManifestEntryV1::try_from(entry, &partition_type)?)? - .resolve(&avro_schema)?, - FormatVersion::V2 => to_value(ManifestEntryV2::try_from(entry, &partition_type)?)? - .resolve(&avro_schema)?, + FormatVersion::V1 => { + to_value(ManifestEntryV1::try_from(entry, &partition_struct_type)?)? + .resolve(&avro_schema)? + } + // Manifest entry format did not change between V2 and V3 + FormatVersion::V2 | FormatVersion::V3 => { + to_value(ManifestEntryV2::try_from(entry, &partition_struct_type)?)? + .resolve(&avro_schema)? + } }; - avro_writer.append(value)?; + avro_writer.append_value(value)?; } let content = avro_writer.into_inner()?; - let length = content.len(); - self.output.write(Bytes::from(content)).await?; + let mut writer = self.output.writer().await?; + writer.write(Bytes::from(content)).await?; + let file_metadata = writer.close().await?; + let key_metadata = self.output.encoded_key_metadata(&file_metadata)?; Ok(ManifestFile { - manifest_path: self.output.location().to_string(), - manifest_length: length as i64, + manifest_path: self.location, + // Manifest lengths are on-disk sizes, including encryption overhead. + manifest_length: file_metadata.size.try_into()?, partition_spec_id: self.metadata.partition_spec.spec_id(), content: self.metadata.content, // sequence_number and min_sequence_number with UNASSIGNED_SEQUENCE_NUMBER will be replace with @@ -416,7 +507,8 @@ impl ManifestWriter { existing_rows_count: Some(self.existing_rows), deleted_rows_count: Some(self.deleted_rows), partitions: Some(partition_summary), - key_metadata: self.key_metadata, + key_metadata, + first_row_id: self.first_row_id, }) } } @@ -447,10 +539,7 @@ impl PartitionFieldStats { return Ok(()); }; if !self.partition_type.compatible(&value) { - return Err(Error::new( - ErrorKind::DataInvalid, - "value is not compatible with type", - )); + return Err(invalid_data!("value is not compatible with type")); } let value = Datum::new(self.partition_type.clone(), value); @@ -492,8 +581,12 @@ mod tests { use tempfile::TempDir; use super::*; - use crate::io::FileIOBuilder; - use crate::spec::{DataFileFormat, Manifest, NestedField, PrimitiveType, Schema, Struct, Type}; + use crate::avro::{assert_deflate_codec, define_named_types_repeatedly}; + use crate::io::FileIO; + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Literal, Manifest, NestedField, + PrimitiveType, Schema, Struct, Transform, Type, + }; #[tokio::test] async fn test_add_delete_existing() { @@ -544,7 +637,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: Some(Vec::new()), - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -573,7 +666,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: Some(Vec::new()), - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -602,7 +695,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: Some(Vec::new()), - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -617,12 +710,11 @@ mod tests { // write manifest to file let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path().join("test_manifest.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); + let io = FileIO::new_with_fs(); let output_file = io.new_output(path.to_str().unwrap()).unwrap(); let mut writer = ManifestWriterBuilder::new( output_file, Some(3), - None, metadata.schema.clone(), metadata.partition_spec.clone(), ) @@ -633,9 +725,9 @@ mod tests { writer.write_manifest_file().await.unwrap(); // read back the manifest file and check the content - let actual_manifest = - Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice()) - .unwrap(); + let bytes = fs::read(path).expect("read_file must succeed"); + assert_deflate_codec(&bytes); + let actual_manifest = Manifest::parse_avro(&bytes).unwrap(); // The snapshot id is assigned when the entry is added and delete to the manifest. Existing entries are keep original. entries[0].snapshot_id = Some(3); @@ -644,4 +736,191 @@ mod tests { entries[0].file_sequence_number = None; assert_eq!(actual_manifest, Manifest::new(metadata, entries)); } + + #[tokio::test] + async fn test_v3_delete_manifest_delete_file_roundtrip() { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + Arc::new(NestedField::optional( + 1, + "id", + Type::Primitive(PrimitiveType::Long), + )), + Arc::new(NestedField::optional( + 2, + "data", + Type::Primitive(PrimitiveType::String), + )), + ]) + .build() + .unwrap(), + ); + + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .build() + .unwrap(); + + // Create a position delete file entry + let delete_entry = ManifestEntry { + status: ManifestStatus::Added, + snapshot_id: None, + sequence_number: None, + file_sequence_number: None, + data_file: DataFile { + content: DataContentType::PositionDeletes, + file_path: "s3://bucket/table/data/delete-00000.parquet".to_string(), + file_format: DataFileFormat::Parquet, + partition: Struct::empty(), + record_count: 10, + file_size_in_bytes: 1024, + column_sizes: HashMap::new(), + value_counts: HashMap::new(), + null_value_counts: HashMap::new(), + nan_value_counts: HashMap::new(), + lower_bounds: HashMap::new(), + upper_bounds: HashMap::new(), + key_metadata: None, + split_offsets: None, + equality_ids: None, + sort_order_id: None, + partition_spec_id: 0, + first_row_id: None, + referenced_data_file: None, + content_offset: None, + content_size_in_bytes: None, + }, + }; + + // Write a V3 delete manifest + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().join("v3_delete_manifest.avro"); + let io = FileIO::new_with_fs(); + let output_file = io.new_output(path.to_str().unwrap()).unwrap(); + + let mut writer = ManifestWriterBuilder::new( + output_file, + Some(1), + schema.clone(), + partition_spec.clone(), + ) + .build_v3_deletes(); + + writer.add_entry(delete_entry).unwrap(); + let manifest_file = writer.write_manifest_file().await.unwrap(); + + // The returned ManifestFile correctly reports Deletes content + assert_eq!(manifest_file.content, ManifestContentType::Deletes); + + // Read back the manifest file + let bytes = fs::read(&path).expect("read_file must succeed"); + assert_deflate_codec(&bytes); + assert_eq!(manifest_file.manifest_length, bytes.len() as i64); + let actual_manifest = Manifest::parse_avro(&bytes).unwrap(); + + // Verify the content type is correctly preserved as Deletes + assert_eq!( + actual_manifest.metadata().content, + ManifestContentType::Deletes, + ); + } + + /// Writes a manifest partitioned by identity on two columns of `field_type`, + /// reads it back, and returns the manifest file. + async fn roundtrip_two_partition_fields_of_type( + field_type: Type, + values: [Literal; 2], + ) -> Vec { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "a", field_type.clone()).into(), + NestedField::optional(2, "b", field_type).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_partition_field("a", "a", Transform::Identity) + .unwrap() + .add_partition_field("b", "b", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition = Struct::from_iter(values.map(Some)); + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().join("manifest.avro"); + let output_file = FileIO::new_with_fs() + .new_output(path.to_str().unwrap()) + .unwrap(); + let mut writer = ManifestWriterBuilder::new(output_file, Some(1), schema, partition_spec) + .build_v2_data(); + writer + .add_file( + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("s3://bucket/table/data/a.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .partition(partition.clone()) + .record_count(1) + .file_size_in_bytes(1) + .partition_spec_id(0) + .build() + .unwrap(), + 1, + ) + .unwrap(); + writer.write_manifest_file().await.unwrap(); + let bs = fs::read(&path).unwrap(); + + // Also read the manifest as iceberg-rust wrote it before defining each + // named type once. + for bs in [bs.clone(), define_named_types_repeatedly(&bs)] { + let manifest = Manifest::parse_avro(&bs).unwrap(); + + assert_eq!(*manifest.entries()[0].data_file().partition(), partition); + } + bs + } + + #[tokio::test] + async fn test_write_manifest_header_marks_int_keyed_maps() { + // Java and PyIceberg read an array of key-value records as a map only if + // it has `logicalType: map`. apache-avro 0.22 drops the attribute when it + // parses a schema (https://github.com/apache/avro-rs/issues/654), so this + // checks the schema JSON in the written header. + let bs = roundtrip_two_partition_fields_of_type(Type::Primitive(PrimitiveType::Int), [ + Literal::int(1), + Literal::int(2), + ]) + .await; + + let file = String::from_utf8_lossy(&bs); + // `column_sizes`, `value_counts`, `null_value_counts`, + // `nan_value_counts`, `lower_bounds`, and `upper_bounds`. + assert_eq!(file.matches(r#""logicalType":"map""#).count(), 6); + } + + #[tokio::test] + async fn test_write_manifest_with_repeated_decimal_partition_type() { + roundtrip_two_partition_fields_of_type( + Type::Primitive(PrimitiveType::Decimal { + precision: 10, + scale: 2, + }), + [Literal::decimal(12345), Literal::decimal(-678)], + ) + .await; + } + + #[tokio::test] + async fn test_write_manifest_with_repeated_fixed_partition_type() { + roundtrip_two_partition_fields_of_type(Type::Primitive(PrimitiveType::Fixed(4)), [ + Literal::fixed(vec![1, 2, 3, 4]), + Literal::fixed(vec![5, 6, 7, 8]), + ]) + .await; + } } diff --git a/crates/iceberg/src/spec/manifest_list.rs b/crates/iceberg/src/spec/manifest_list.rs deleted file mode 100644 index 43808bb249..0000000000 --- a/crates/iceberg/src/spec/manifest_list.rs +++ /dev/null @@ -1,1439 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! ManifestList for Iceberg. - -use std::collections::HashMap; -use std::str::FromStr; - -use apache_avro::types::Value; -use apache_avro::{Reader, Writer, from_value}; -use bytes::Bytes; -pub use serde_bytes::ByteBuf; -use serde_derive::{Deserialize, Serialize}; - -use self::_const_schema::{MANIFEST_LIST_AVRO_SCHEMA_V1, MANIFEST_LIST_AVRO_SCHEMA_V2}; -use self::_serde::{ManifestFileV1, ManifestFileV2}; -use super::{FormatVersion, Manifest}; -use crate::error::Result; -use crate::io::{FileIO, OutputFile}; -use crate::{Error, ErrorKind}; - -/// Placeholder for sequence number. The field with this value must be replaced with the actual sequence number before it write. -pub const UNASSIGNED_SEQUENCE_NUMBER: i64 = -1; - -/// Snapshots are embedded in table metadata, but the list of manifests for a -/// snapshot are stored in a separate manifest list file. -/// -/// A new manifest list is written for each attempt to commit a snapshot -/// because the list of manifests always changes to produce a new snapshot. -/// When a manifest list is written, the (optimistic) sequence number of the -/// snapshot is written for all new manifest files tracked by the list. -/// -/// A manifest list includes summary metadata that can be used to avoid -/// scanning all of the manifests in a snapshot when planning a table scan. -/// This includes the number of added, existing, and deleted files, and a -/// summary of values for each field of the partition spec used to write the -/// manifest. -#[derive(Debug, Clone, PartialEq)] -pub struct ManifestList { - /// Entries in a manifest list. - entries: Vec, -} - -impl ManifestList { - /// Parse manifest list from bytes. - pub fn parse_with_version(bs: &[u8], version: FormatVersion) -> Result { - match version { - FormatVersion::V1 => { - let reader = Reader::with_schema(&MANIFEST_LIST_AVRO_SCHEMA_V1, bs)?; - let values = Value::Array(reader.collect::, _>>()?); - from_value::<_serde::ManifestListV1>(&values)?.try_into() - } - FormatVersion::V2 => { - let reader = Reader::new(bs)?; - let values = Value::Array(reader.collect::, _>>()?); - from_value::<_serde::ManifestListV2>(&values)?.try_into() - } - } - } - - /// Get the entries in the manifest list. - pub fn entries(&self) -> &[ManifestFile] { - &self.entries - } - - /// Take ownership of the entries in the manifest list, consuming it - pub fn consume_entries(self) -> impl IntoIterator { - Box::new(self.entries.into_iter()) - } -} - -/// A manifest list writer. -pub struct ManifestListWriter { - format_version: FormatVersion, - output_file: OutputFile, - avro_writer: Writer<'static, Vec>, - sequence_number: i64, - snapshot_id: i64, -} - -impl std::fmt::Debug for ManifestListWriter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ManifestListWriter") - .field("format_version", &self.format_version) - .field("output_file", &self.output_file) - .field("avro_writer", &self.avro_writer.schema()) - .finish_non_exhaustive() - } -} - -impl ManifestListWriter { - /// Construct a v1 [`ManifestListWriter`] that writes to a provided [`OutputFile`]. - pub fn v1(output_file: OutputFile, snapshot_id: i64, parent_snapshot_id: Option) -> Self { - let mut metadata = HashMap::from_iter([ - ("snapshot-id".to_string(), snapshot_id.to_string()), - ("format-version".to_string(), "1".to_string()), - ]); - if let Some(parent_snapshot_id) = parent_snapshot_id { - metadata.insert( - "parent-snapshot-id".to_string(), - parent_snapshot_id.to_string(), - ); - } - Self::new(FormatVersion::V1, output_file, metadata, 0, snapshot_id) - } - - /// Construct a v2 [`ManifestListWriter`] that writes to a provided [`OutputFile`]. - pub fn v2( - output_file: OutputFile, - snapshot_id: i64, - parent_snapshot_id: Option, - sequence_number: i64, - ) -> Self { - let mut metadata = HashMap::from_iter([ - ("snapshot-id".to_string(), snapshot_id.to_string()), - ("sequence-number".to_string(), sequence_number.to_string()), - ("format-version".to_string(), "2".to_string()), - ]); - metadata.insert( - "parent-snapshot-id".to_string(), - parent_snapshot_id - .map(|v| v.to_string()) - .unwrap_or("null".to_string()), - ); - Self::new( - FormatVersion::V2, - output_file, - metadata, - sequence_number, - snapshot_id, - ) - } - - fn new( - format_version: FormatVersion, - output_file: OutputFile, - metadata: HashMap, - sequence_number: i64, - snapshot_id: i64, - ) -> Self { - let avro_schema = match format_version { - FormatVersion::V1 => &MANIFEST_LIST_AVRO_SCHEMA_V1, - FormatVersion::V2 => &MANIFEST_LIST_AVRO_SCHEMA_V2, - }; - let mut avro_writer = Writer::new(avro_schema, Vec::new()); - for (key, value) in metadata { - avro_writer - .add_user_metadata(key, value) - .expect("Avro metadata should be added to the writer before the first record."); - } - Self { - format_version, - output_file, - avro_writer, - sequence_number, - snapshot_id, - } - } - - /// Append manifests to be written. - pub fn add_manifests(&mut self, manifests: impl Iterator) -> Result<()> { - match self.format_version { - FormatVersion::V1 => { - for manifest in manifests { - let manifes: ManifestFileV1 = manifest.try_into()?; - self.avro_writer.append_ser(manifes)?; - } - } - FormatVersion::V2 => { - for mut manifest in manifests { - if manifest.sequence_number == UNASSIGNED_SEQUENCE_NUMBER { - if manifest.added_snapshot_id != self.snapshot_id { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Found unassigned sequence number for a manifest from snapshot {}.", - manifest.added_snapshot_id - ), - )); - } - manifest.sequence_number = self.sequence_number; - } - if manifest.min_sequence_number == UNASSIGNED_SEQUENCE_NUMBER { - if manifest.added_snapshot_id != self.snapshot_id { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Found unassigned sequence number for a manifest from snapshot {}.", - manifest.added_snapshot_id - ), - )); - } - manifest.min_sequence_number = self.sequence_number; - } - let manifest_entry: ManifestFileV2 = manifest.try_into()?; - self.avro_writer.append_ser(manifest_entry)?; - } - } - } - Ok(()) - } - - /// Write the manifest list to the output file. - pub async fn close(self) -> Result<()> { - let data = self.avro_writer.into_inner()?; - let mut writer = self.output_file.writer().await?; - writer.write(Bytes::from(data)).await?; - writer.close().await?; - Ok(()) - } -} - -/// This is a helper module that defines the schema field of the manifest list entry. -mod _const_schema { - use std::sync::Arc; - - use apache_avro::Schema as AvroSchema; - use once_cell::sync::Lazy; - - use crate::avro::schema_to_avro_schema; - use crate::spec::{ - ListType, NestedField, NestedFieldRef, PrimitiveType, Schema, StructType, Type, - }; - - static MANIFEST_PATH: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 500, - "manifest_path", - Type::Primitive(PrimitiveType::String), - )) - }) - }; - static MANIFEST_LENGTH: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 501, - "manifest_length", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static PARTITION_SPEC_ID: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 502, - "partition_spec_id", - Type::Primitive(PrimitiveType::Int), - )) - }) - }; - static CONTENT: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 517, - "content", - Type::Primitive(PrimitiveType::Int), - )) - }) - }; - static SEQUENCE_NUMBER: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 515, - "sequence_number", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static MIN_SEQUENCE_NUMBER: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 516, - "min_sequence_number", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static ADDED_SNAPSHOT_ID: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 503, - "added_snapshot_id", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static ADDED_FILES_COUNT_V2: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 504, - "added_files_count", - Type::Primitive(PrimitiveType::Int), - )) - }) - }; - static ADDED_FILES_COUNT_V1: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::optional( - 504, - "added_data_files_count", - Type::Primitive(PrimitiveType::Int), - )) - }) - }; - static EXISTING_FILES_COUNT_V2: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 505, - "existing_files_count", - Type::Primitive(PrimitiveType::Int), - )) - }) - }; - static EXISTING_FILES_COUNT_V1: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::optional( - 505, - "existing_data_files_count", - Type::Primitive(PrimitiveType::Int), - )) - }) - }; - static DELETED_FILES_COUNT_V2: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 506, - "deleted_files_count", - Type::Primitive(PrimitiveType::Int), - )) - }) - }; - static DELETED_FILES_COUNT_V1: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::optional( - 506, - "deleted_data_files_count", - Type::Primitive(PrimitiveType::Int), - )) - }) - }; - static ADDED_ROWS_COUNT_V2: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 512, - "added_rows_count", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static ADDED_ROWS_COUNT_V1: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::optional( - 512, - "added_rows_count", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static EXISTING_ROWS_COUNT_V2: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 513, - "existing_rows_count", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static EXISTING_ROWS_COUNT_V1: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::optional( - 513, - "existing_rows_count", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static DELETED_ROWS_COUNT_V2: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::required( - 514, - "deleted_rows_count", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static DELETED_ROWS_COUNT_V1: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::optional( - 514, - "deleted_rows_count", - Type::Primitive(PrimitiveType::Long), - )) - }) - }; - static PARTITIONS: Lazy = { - Lazy::new(|| { - // element type - let fields = vec![ - Arc::new(NestedField::required( - 509, - "contains_null", - Type::Primitive(PrimitiveType::Boolean), - )), - Arc::new(NestedField::optional( - 518, - "contains_nan", - Type::Primitive(PrimitiveType::Boolean), - )), - Arc::new(NestedField::optional( - 510, - "lower_bound", - Type::Primitive(PrimitiveType::Binary), - )), - Arc::new(NestedField::optional( - 511, - "upper_bound", - Type::Primitive(PrimitiveType::Binary), - )), - ]; - let element_field = Arc::new(NestedField::required( - 508, - "r_508", - Type::Struct(StructType::new(fields)), - )); - Arc::new(NestedField::optional( - 507, - "partitions", - Type::List(ListType { element_field }), - )) - }) - }; - static KEY_METADATA: Lazy = { - Lazy::new(|| { - Arc::new(NestedField::optional( - 519, - "key_metadata", - Type::Primitive(PrimitiveType::Binary), - )) - }) - }; - - static V1_SCHEMA: Lazy = { - Lazy::new(|| { - let fields = vec![ - MANIFEST_PATH.clone(), - MANIFEST_LENGTH.clone(), - PARTITION_SPEC_ID.clone(), - ADDED_SNAPSHOT_ID.clone(), - ADDED_FILES_COUNT_V1.clone().to_owned(), - EXISTING_FILES_COUNT_V1.clone(), - DELETED_FILES_COUNT_V1.clone(), - ADDED_ROWS_COUNT_V1.clone(), - EXISTING_ROWS_COUNT_V1.clone(), - DELETED_ROWS_COUNT_V1.clone(), - PARTITIONS.clone(), - KEY_METADATA.clone(), - ]; - Schema::builder().with_fields(fields).build().unwrap() - }) - }; - - static V2_SCHEMA: Lazy = { - Lazy::new(|| { - let fields = vec![ - MANIFEST_PATH.clone(), - MANIFEST_LENGTH.clone(), - PARTITION_SPEC_ID.clone(), - CONTENT.clone(), - SEQUENCE_NUMBER.clone(), - MIN_SEQUENCE_NUMBER.clone(), - ADDED_SNAPSHOT_ID.clone(), - ADDED_FILES_COUNT_V2.clone(), - EXISTING_FILES_COUNT_V2.clone(), - DELETED_FILES_COUNT_V2.clone(), - ADDED_ROWS_COUNT_V2.clone(), - EXISTING_ROWS_COUNT_V2.clone(), - DELETED_ROWS_COUNT_V2.clone(), - PARTITIONS.clone(), - KEY_METADATA.clone(), - ]; - Schema::builder().with_fields(fields).build().unwrap() - }) - }; - - pub(super) static MANIFEST_LIST_AVRO_SCHEMA_V1: Lazy = - Lazy::new(|| schema_to_avro_schema("manifest_file", &V1_SCHEMA).unwrap()); - - pub(super) static MANIFEST_LIST_AVRO_SCHEMA_V2: Lazy = - Lazy::new(|| schema_to_avro_schema("manifest_file", &V2_SCHEMA).unwrap()); -} - -/// Entry in a manifest list. -#[derive(Debug, PartialEq, Clone, Eq, Hash)] -pub struct ManifestFile { - /// field: 500 - /// - /// Location of the manifest file - pub manifest_path: String, - /// field: 501 - /// - /// Length of the manifest file in bytes - pub manifest_length: i64, - /// field: 502 - /// - /// ID of a partition spec used to write the manifest; must be listed - /// in table metadata partition-specs - pub partition_spec_id: i32, - /// field: 517 - /// - /// The type of files tracked by the manifest, either data or delete - /// files; 0 for all v1 manifests - pub content: ManifestContentType, - /// field: 515 - /// - /// The sequence number when the manifest was added to the table; use 0 - /// when reading v1 manifest lists - pub sequence_number: i64, - /// field: 516 - /// - /// The minimum data sequence number of all live data or delete files in - /// the manifest; use 0 when reading v1 manifest lists - pub min_sequence_number: i64, - /// field: 503 - /// - /// ID of the snapshot where the manifest file was added - pub added_snapshot_id: i64, - /// field: 504 - /// - /// Number of entries in the manifest that have status ADDED, when null - /// this is assumed to be non-zero - pub added_files_count: Option, - /// field: 505 - /// - /// Number of entries in the manifest that have status EXISTING (0), - /// when null this is assumed to be non-zero - pub existing_files_count: Option, - /// field: 506 - /// - /// Number of entries in the manifest that have status DELETED (2), - /// when null this is assumed to be non-zero - pub deleted_files_count: Option, - /// field: 512 - /// - /// Number of rows in all of files in the manifest that have status - /// ADDED, when null this is assumed to be non-zero - pub added_rows_count: Option, - /// field: 513 - /// - /// Number of rows in all of files in the manifest that have status - /// EXISTING, when null this is assumed to be non-zero - pub existing_rows_count: Option, - /// field: 514 - /// - /// Number of rows in all of files in the manifest that have status - /// DELETED, when null this is assumed to be non-zero - pub deleted_rows_count: Option, - /// field: 507 - /// element_field: 508 - /// - /// A list of field summaries for each partition field in the spec. Each - /// field in the list corresponds to a field in the manifest file’s - /// partition spec. - pub partitions: Option>, - /// field: 519 - /// - /// Implementation-specific key metadata for encryption - pub key_metadata: Option>, -} - -impl ManifestFile { - /// Checks if the manifest file has any added files. - pub fn has_added_files(&self) -> bool { - self.added_files_count.map(|c| c > 0).unwrap_or(true) - } - - /// Checks whether this manifest contains entries with DELETED status. - pub fn has_deleted_files(&self) -> bool { - self.deleted_files_count.map(|c| c > 0).unwrap_or(true) - } - - /// Checks if the manifest file has any existed files. - pub fn has_existing_files(&self) -> bool { - self.existing_files_count.map(|c| c > 0).unwrap_or(true) - } -} - -/// The type of files tracked by the manifest, either data or delete files; Data(0) for all v1 manifests -#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash, Default)] -pub enum ManifestContentType { - /// The manifest content is data. - #[default] - Data = 0, - /// The manifest content is deletes. - Deletes = 1, -} - -impl FromStr for ManifestContentType { - type Err = Error; - - fn from_str(s: &str) -> Result { - match s { - "data" => Ok(ManifestContentType::Data), - "deletes" => Ok(ManifestContentType::Deletes), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid manifest content type: {s}"), - )), - } - } -} - -impl std::fmt::Display for ManifestContentType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ManifestContentType::Data => write!(f, "data"), - ManifestContentType::Deletes => write!(f, "deletes"), - } - } -} - -impl TryFrom for ManifestContentType { - type Error = Error; - - fn try_from(value: i32) -> std::result::Result { - match value { - 0 => Ok(ManifestContentType::Data), - 1 => Ok(ManifestContentType::Deletes), - _ => Err(Error::new( - crate::ErrorKind::DataInvalid, - format!( - "Invalid manifest content type. Expected 0 or 1, got {}", - value - ), - )), - } - } -} - -impl ManifestFile { - /// Load [`Manifest`]. - /// - /// This method will also initialize inherited values of [`ManifestEntry`], such as `sequence_number`. - pub async fn load_manifest(&self, file_io: &FileIO) -> Result { - let avro = file_io.new_input(&self.manifest_path)?.read().await?; - - let (metadata, mut entries) = Manifest::try_from_avro_bytes(&avro)?; - - // Let entries inherit values from the manifest list entry. - for entry in &mut entries { - entry.inherit_data(self); - } - - Ok(Manifest::new(metadata, entries)) - } -} - -/// Field summary for partition field in the spec. -/// -/// Each field in the list corresponds to a field in the manifest file’s partition spec. -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, Hash)] -pub struct FieldSummary { - /// field: 509 - /// - /// Whether the manifest contains at least one partition with a null - /// value for the field - pub contains_null: bool, - /// field: 518 - /// Whether the manifest contains at least one partition with a NaN - /// value for the field - pub contains_nan: Option, - /// field: 510 - /// The minimum value for the field in the manifests - /// partitions. - pub lower_bound: Option, - /// field: 511 - /// The maximum value for the field in the manifests - /// partitions. - pub upper_bound: Option, -} - -/// This is a helper module that defines types to help with serialization/deserialization. -/// For deserialization the input first gets read into either the [ManifestFileV1] or [ManifestFileV2] struct -/// and then converted into the [ManifestFile] struct. Serialization works the other way around. -/// [ManifestFileV1] and [ManifestFileV2] are internal struct that are only used for serialization and deserialization. -pub(super) mod _serde { - pub use serde_bytes::ByteBuf; - use serde_derive::{Deserialize, Serialize}; - - use super::ManifestFile; - use crate::Error; - use crate::error::Result; - use crate::spec::FieldSummary; - - #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] - #[serde(transparent)] - pub(crate) struct ManifestListV2 { - entries: Vec, - } - - #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] - #[serde(transparent)] - pub(crate) struct ManifestListV1 { - entries: Vec, - } - - impl ManifestListV2 { - /// Converts the [ManifestListV2] into a [ManifestList]. - pub fn try_into(self) -> Result { - Ok(super::ManifestList { - entries: self - .entries - .into_iter() - .map(|v| v.try_into()) - .collect::>>()?, - }) - } - } - - impl TryFrom for ManifestListV2 { - type Error = Error; - - fn try_from(value: super::ManifestList) -> std::result::Result { - Ok(Self { - entries: value - .entries - .into_iter() - .map(|v| v.try_into()) - .collect::, _>>()?, - }) - } - } - - impl ManifestListV1 { - /// Converts the [ManifestListV1] into a [ManifestList]. - pub fn try_into(self) -> Result { - Ok(super::ManifestList { - entries: self - .entries - .into_iter() - .map(|v| v.try_into()) - .collect::>>()?, - }) - } - } - - impl TryFrom for ManifestListV1 { - type Error = Error; - - fn try_from(value: super::ManifestList) -> std::result::Result { - Ok(Self { - entries: value - .entries - .into_iter() - .map(|v| v.try_into()) - .collect::, _>>()?, - }) - } - } - - #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] - pub(super) struct ManifestFileV1 { - pub manifest_path: String, - pub manifest_length: i64, - pub partition_spec_id: i32, - pub added_snapshot_id: i64, - pub added_data_files_count: Option, - pub existing_data_files_count: Option, - pub deleted_data_files_count: Option, - pub added_rows_count: Option, - pub existing_rows_count: Option, - pub deleted_rows_count: Option, - pub partitions: Option>, - pub key_metadata: Option, - } - - // Aliases were added to fields that were renamed in Iceberg 1.5.0 (https://github.com/apache/iceberg/pull/5338), in order to support both conventions/versions. - // In the current implementation deserialization is done using field names, and therefore these fields may appear as either. - // see issue that raised this here: https://github.com/apache/iceberg-rust/issues/338 - #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] - pub(super) struct ManifestFileV2 { - pub manifest_path: String, - pub manifest_length: i64, - pub partition_spec_id: i32, - #[serde(default = "v2_default_content_for_v1")] - pub content: i32, - #[serde(default = "v2_default_sequence_number_for_v1")] - pub sequence_number: i64, - #[serde(default = "v2_default_min_sequence_number_for_v1")] - pub min_sequence_number: i64, - pub added_snapshot_id: i64, - #[serde(alias = "added_data_files_count", alias = "added_files_count")] - pub added_files_count: i32, - #[serde(alias = "existing_data_files_count", alias = "existing_files_count")] - pub existing_files_count: i32, - #[serde(alias = "deleted_data_files_count", alias = "deleted_files_count")] - pub deleted_files_count: i32, - pub added_rows_count: i64, - pub existing_rows_count: i64, - pub deleted_rows_count: i64, - pub partitions: Option>, - pub key_metadata: Option, - } - - impl ManifestFileV2 { - /// Converts the [ManifestFileV2] into a [ManifestFile]. - pub fn try_into(self) -> Result { - Ok(ManifestFile { - manifest_path: self.manifest_path, - manifest_length: self.manifest_length, - partition_spec_id: self.partition_spec_id, - content: self.content.try_into()?, - sequence_number: self.sequence_number, - min_sequence_number: self.min_sequence_number, - added_snapshot_id: self.added_snapshot_id, - added_files_count: Some(self.added_files_count.try_into()?), - existing_files_count: Some(self.existing_files_count.try_into()?), - deleted_files_count: Some(self.deleted_files_count.try_into()?), - added_rows_count: Some(self.added_rows_count.try_into()?), - existing_rows_count: Some(self.existing_rows_count.try_into()?), - deleted_rows_count: Some(self.deleted_rows_count.try_into()?), - partitions: self.partitions, - key_metadata: self.key_metadata.map(|b| b.into_vec()), - }) - } - } - - fn v2_default_content_for_v1() -> i32 { - super::ManifestContentType::Data as i32 - } - - fn v2_default_sequence_number_for_v1() -> i64 { - 0 - } - - fn v2_default_min_sequence_number_for_v1() -> i64 { - 0 - } - - impl ManifestFileV1 { - /// Converts the [ManifestFileV1] into a [ManifestFile]. - pub fn try_into(self) -> Result { - Ok(ManifestFile { - manifest_path: self.manifest_path, - manifest_length: self.manifest_length, - partition_spec_id: self.partition_spec_id, - added_snapshot_id: self.added_snapshot_id, - added_files_count: self - .added_data_files_count - .map(TryInto::try_into) - .transpose()?, - existing_files_count: self - .existing_data_files_count - .map(TryInto::try_into) - .transpose()?, - deleted_files_count: self - .deleted_data_files_count - .map(TryInto::try_into) - .transpose()?, - added_rows_count: self.added_rows_count.map(TryInto::try_into).transpose()?, - existing_rows_count: self - .existing_rows_count - .map(TryInto::try_into) - .transpose()?, - deleted_rows_count: self.deleted_rows_count.map(TryInto::try_into).transpose()?, - partitions: self.partitions, - key_metadata: self.key_metadata.map(|b| b.into_vec()), - // as ref: https://iceberg.apache.org/spec/#partitioning - // use 0 when reading v1 manifest lists - content: super::ManifestContentType::Data, - sequence_number: 0, - min_sequence_number: 0, - }) - } - } - - fn convert_to_serde_key_metadata(key_metadata: Option>) -> Option { - match key_metadata { - Some(metadata) if !metadata.is_empty() => Some(ByteBuf::from(metadata)), - _ => None, - } - } - - impl TryFrom for ManifestFileV2 { - type Error = Error; - - fn try_from(value: ManifestFile) -> std::result::Result { - let key_metadata = convert_to_serde_key_metadata(value.key_metadata); - Ok(Self { - manifest_path: value.manifest_path, - manifest_length: value.manifest_length, - partition_spec_id: value.partition_spec_id, - content: value.content as i32, - sequence_number: value.sequence_number, - min_sequence_number: value.min_sequence_number, - added_snapshot_id: value.added_snapshot_id, - added_files_count: value - .added_files_count - .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "added_data_files_count in ManifestFileV2 should be require", - ) - })? - .try_into()?, - existing_files_count: value - .existing_files_count - .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "existing_data_files_count in ManifestFileV2 should be require", - ) - })? - .try_into()?, - deleted_files_count: value - .deleted_files_count - .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "deleted_data_files_count in ManifestFileV2 should be require", - ) - })? - .try_into()?, - added_rows_count: value - .added_rows_count - .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "added_rows_count in ManifestFileV2 should be require", - ) - })? - .try_into()?, - existing_rows_count: value - .existing_rows_count - .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "existing_rows_count in ManifestFileV2 should be require", - ) - })? - .try_into()?, - deleted_rows_count: value - .deleted_rows_count - .ok_or_else(|| { - Error::new( - crate::ErrorKind::DataInvalid, - "deleted_rows_count in ManifestFileV2 should be require", - ) - })? - .try_into()?, - partitions: value.partitions, - key_metadata, - }) - } - } - - impl TryFrom for ManifestFileV1 { - type Error = Error; - - fn try_from(value: ManifestFile) -> std::result::Result { - let key_metadata = convert_to_serde_key_metadata(value.key_metadata); - Ok(Self { - manifest_path: value.manifest_path, - manifest_length: value.manifest_length, - partition_spec_id: value.partition_spec_id, - added_snapshot_id: value.added_snapshot_id, - added_data_files_count: value - .added_files_count - .map(TryInto::try_into) - .transpose()?, - existing_data_files_count: value - .existing_files_count - .map(TryInto::try_into) - .transpose()?, - deleted_data_files_count: value - .deleted_files_count - .map(TryInto::try_into) - .transpose()?, - added_rows_count: value.added_rows_count.map(TryInto::try_into).transpose()?, - existing_rows_count: value - .existing_rows_count - .map(TryInto::try_into) - .transpose()?, - deleted_rows_count: value - .deleted_rows_count - .map(TryInto::try_into) - .transpose()?, - partitions: value.partitions, - key_metadata, - }) - } - } -} - -#[cfg(test)] -mod test { - use std::fs; - - use apache_avro::{Reader, Schema}; - use tempfile::TempDir; - - use super::_serde::ManifestListV2; - use crate::io::FileIOBuilder; - use crate::spec::manifest_list::_serde::ManifestListV1; - use crate::spec::{ - Datum, FieldSummary, ManifestContentType, ManifestFile, ManifestList, ManifestListWriter, - UNASSIGNED_SEQUENCE_NUMBER, - }; - - #[tokio::test] - async fn test_parse_manifest_list_v1() { - let manifest_list = ManifestList { - entries: vec![ - ManifestFile { - manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(), - manifest_length: 5806, - partition_spec_id: 0, - content: ManifestContentType::Data, - sequence_number: 0, - min_sequence_number: 0, - added_snapshot_id: 1646658105718557341, - added_files_count: Some(3), - existing_files_count: Some(0), - deleted_files_count: Some(0), - added_rows_count: Some(3), - existing_rows_count: Some(0), - deleted_rows_count: Some(0), - partitions: Some(vec![]), - key_metadata: None, - } - ] - }; - - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); - - let tmp_dir = TempDir::new().unwrap(); - let file_name = "simple_manifest_list_v1.avro"; - let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name); - - let mut writer = ManifestListWriter::v1( - file_io.new_output(full_path.clone()).unwrap(), - 1646658105718557341, - Some(1646658105718557341), - ); - - writer - .add_manifests(manifest_list.entries.clone().into_iter()) - .unwrap(); - writer.close().await.unwrap(); - - let bs = fs::read(full_path).expect("read_file must succeed"); - - let parsed_manifest_list = - ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V1).unwrap(); - - assert_eq!(manifest_list, parsed_manifest_list); - } - - #[tokio::test] - async fn test_parse_manifest_list_v2() { - let manifest_list = ManifestList { - entries: vec![ - ManifestFile { - manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), - manifest_length: 6926, - partition_spec_id: 1, - content: ManifestContentType::Data, - sequence_number: 1, - min_sequence_number: 1, - added_snapshot_id: 377075049360453639, - added_files_count: Some(1), - existing_files_count: Some(0), - deleted_files_count: Some(0), - added_rows_count: Some(3), - existing_rows_count: Some(0), - deleted_rows_count: Some(0), - partitions: Some( - vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] - ), - key_metadata: None, - }, - ManifestFile { - manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m1.avro".to_string(), - manifest_length: 6926, - partition_spec_id: 2, - content: ManifestContentType::Data, - sequence_number: 1, - min_sequence_number: 1, - added_snapshot_id: 377075049360453639, - added_files_count: Some(1), - existing_files_count: Some(0), - deleted_files_count: Some(0), - added_rows_count: Some(3), - existing_rows_count: Some(0), - deleted_rows_count: Some(0), - partitions: Some( - vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1).to_bytes().unwrap())}] - ), - key_metadata: None, - } - ] - }; - - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); - - let tmp_dir = TempDir::new().unwrap(); - let file_name = "simple_manifest_list_v1.avro"; - let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name); - - let mut writer = ManifestListWriter::v2( - file_io.new_output(full_path.clone()).unwrap(), - 1646658105718557341, - Some(1646658105718557341), - 1, - ); - - writer - .add_manifests(manifest_list.entries.clone().into_iter()) - .unwrap(); - writer.close().await.unwrap(); - - let bs = fs::read(full_path).expect("read_file must succeed"); - - let parsed_manifest_list = - ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V2).unwrap(); - - assert_eq!(manifest_list, parsed_manifest_list); - } - - #[test] - fn test_serialize_manifest_list_v1() { - let manifest_list:ManifestListV1 = ManifestList { - entries: vec![ManifestFile { - manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(), - manifest_length: 5806, - partition_spec_id: 0, - content: ManifestContentType::Data, - sequence_number: 0, - min_sequence_number: 0, - added_snapshot_id: 1646658105718557341, - added_files_count: Some(3), - existing_files_count: Some(0), - deleted_files_count: Some(0), - added_rows_count: Some(3), - existing_rows_count: Some(0), - deleted_rows_count: Some(0), - partitions: None, - key_metadata: None, - }] - }.try_into().unwrap(); - let result = serde_json::to_string(&manifest_list).unwrap(); - assert_eq!( - result, - r#"[{"manifest_path":"/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro","manifest_length":5806,"partition_spec_id":0,"added_snapshot_id":1646658105718557341,"added_data_files_count":3,"existing_data_files_count":0,"deleted_data_files_count":0,"added_rows_count":3,"existing_rows_count":0,"deleted_rows_count":0,"partitions":null,"key_metadata":null}]"# - ); - } - - #[test] - fn test_serialize_manifest_list_v2() { - let manifest_list:ManifestListV2 = ManifestList { - entries: vec![ManifestFile { - manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), - manifest_length: 6926, - partition_spec_id: 1, - content: ManifestContentType::Data, - sequence_number: 1, - min_sequence_number: 1, - added_snapshot_id: 377075049360453639, - added_files_count: Some(1), - existing_files_count: Some(0), - deleted_files_count: Some(0), - added_rows_count: Some(3), - existing_rows_count: Some(0), - deleted_rows_count: Some(0), - partitions: Some( - vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] - ), - key_metadata: None, - }] - }.try_into().unwrap(); - let result = serde_json::to_string(&manifest_list).unwrap(); - assert_eq!( - result, - r#"[{"manifest_path":"s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro","manifest_length":6926,"partition_spec_id":1,"content":0,"sequence_number":1,"min_sequence_number":1,"added_snapshot_id":377075049360453639,"added_files_count":1,"existing_files_count":0,"deleted_files_count":0,"added_rows_count":3,"existing_rows_count":0,"deleted_rows_count":0,"partitions":[{"contains_null":false,"contains_nan":false,"lower_bound":[1,0,0,0,0,0,0,0],"upper_bound":[1,0,0,0,0,0,0,0]}],"key_metadata":null}]"# - ); - } - - #[tokio::test] - async fn test_manifest_list_writer_v1() { - let expected_manifest_list = ManifestList { - entries: vec![ManifestFile { - manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(), - manifest_length: 5806, - partition_spec_id: 1, - content: ManifestContentType::Data, - sequence_number: 0, - min_sequence_number: 0, - added_snapshot_id: 1646658105718557341, - added_files_count: Some(3), - existing_files_count: Some(0), - deleted_files_count: Some(0), - added_rows_count: Some(3), - existing_rows_count: Some(0), - deleted_rows_count: Some(0), - partitions: Some( - vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}], - ), - key_metadata: None, - }] - }; - - let temp_dir = TempDir::new().unwrap(); - let path = temp_dir.path().join("manifest_list_v1.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); - let output_file = io.new_output(path.to_str().unwrap()).unwrap(); - - let mut writer = ManifestListWriter::v1(output_file, 1646658105718557341, Some(0)); - writer - .add_manifests(expected_manifest_list.entries.clone().into_iter()) - .unwrap(); - writer.close().await.unwrap(); - - let bs = fs::read(path).unwrap(); - - let manifest_list = - ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V1).unwrap(); - assert_eq!(manifest_list, expected_manifest_list); - - temp_dir.close().unwrap(); - } - - #[tokio::test] - async fn test_manifest_list_writer_v2() { - let snapshot_id = 377075049360453639; - let seq_num = 1; - let mut expected_manifest_list = ManifestList { - entries: vec![ManifestFile { - manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), - manifest_length: 6926, - partition_spec_id: 1, - content: ManifestContentType::Data, - sequence_number: UNASSIGNED_SEQUENCE_NUMBER, - min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER, - added_snapshot_id: snapshot_id, - added_files_count: Some(1), - existing_files_count: Some(0), - deleted_files_count: Some(0), - added_rows_count: Some(3), - existing_rows_count: Some(0), - deleted_rows_count: Some(0), - partitions: Some( - vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] - ), - key_metadata: None, - }] - }; - - let temp_dir = TempDir::new().unwrap(); - let path = temp_dir.path().join("manifest_list_v2.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); - let output_file = io.new_output(path.to_str().unwrap()).unwrap(); - - let mut writer = ManifestListWriter::v2(output_file, snapshot_id, Some(0), seq_num); - writer - .add_manifests(expected_manifest_list.entries.clone().into_iter()) - .unwrap(); - writer.close().await.unwrap(); - - let bs = fs::read(path).unwrap(); - let manifest_list = - ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V2).unwrap(); - expected_manifest_list.entries[0].sequence_number = seq_num; - expected_manifest_list.entries[0].min_sequence_number = seq_num; - assert_eq!(manifest_list, expected_manifest_list); - - temp_dir.close().unwrap(); - } - - #[tokio::test] - async fn test_manifest_list_writer_v1_as_v2() { - let expected_manifest_list = ManifestList { - entries: vec![ManifestFile { - manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(), - manifest_length: 5806, - partition_spec_id: 1, - content: ManifestContentType::Data, - sequence_number: 0, - min_sequence_number: 0, - added_snapshot_id: 1646658105718557341, - added_files_count: Some(3), - existing_files_count: Some(0), - deleted_files_count: Some(0), - added_rows_count: Some(3), - existing_rows_count: Some(0), - deleted_rows_count: Some(0), - partitions: Some( - vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] - ), - key_metadata: None, - }] - }; - - let temp_dir = TempDir::new().unwrap(); - let path = temp_dir.path().join("manifest_list_v1.avro"); - let io = FileIOBuilder::new_fs_io().build().unwrap(); - let output_file = io.new_output(path.to_str().unwrap()).unwrap(); - - let mut writer = ManifestListWriter::v1(output_file, 1646658105718557341, Some(0)); - writer - .add_manifests(expected_manifest_list.entries.clone().into_iter()) - .unwrap(); - writer.close().await.unwrap(); - - let bs = fs::read(path).unwrap(); - - let manifest_list = - ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V2).unwrap(); - assert_eq!(manifest_list, expected_manifest_list); - - temp_dir.close().unwrap(); - } - - #[tokio::test] - async fn test_manifest_list_v2_deserializer_aliases() { - // reading avro manifest file generated by iceberg 1.4.0 - let avro_1_path = "testdata/manifests_lists/manifest-list-v2-1.avro"; - let bs_1 = fs::read(avro_1_path).unwrap(); - let avro_1_fields = read_avro_schema_fields_as_str(bs_1.clone()).await; - assert_eq!( - avro_1_fields, - "manifest_path, manifest_length, partition_spec_id, content, sequence_number, min_sequence_number, added_snapshot_id, added_data_files_count, existing_data_files_count, deleted_data_files_count, added_rows_count, existing_rows_count, deleted_rows_count, partitions" - ); - // reading avro manifest file generated by iceberg 1.5.0 - let avro_2_path = "testdata/manifests_lists/manifest-list-v2-2.avro"; - let bs_2 = fs::read(avro_2_path).unwrap(); - let avro_2_fields = read_avro_schema_fields_as_str(bs_2.clone()).await; - assert_eq!( - avro_2_fields, - "manifest_path, manifest_length, partition_spec_id, content, sequence_number, min_sequence_number, added_snapshot_id, added_files_count, existing_files_count, deleted_files_count, added_rows_count, existing_rows_count, deleted_rows_count, partitions" - ); - // deserializing both files to ManifestList struct - let _manifest_list_1 = - ManifestList::parse_with_version(&bs_1, crate::spec::FormatVersion::V2).unwrap(); - let _manifest_list_2 = - ManifestList::parse_with_version(&bs_2, crate::spec::FormatVersion::V2).unwrap(); - } - - async fn read_avro_schema_fields_as_str(bs: Vec) -> String { - let reader = Reader::new(&bs[..]).unwrap(); - let schema = reader.writer_schema(); - let fields: String = match schema { - Schema::Record(record) => record - .fields - .iter() - .map(|field| field.name.clone()) - .collect::>() - .join(", "), - _ => "".to_string(), - }; - fields - } - - #[test] - fn test_manifest_content_type_default() { - assert_eq!(ManifestContentType::default(), ManifestContentType::Data); - } - - #[test] - fn test_manifest_content_type_default_value() { - assert_eq!(ManifestContentType::default() as i32, 0); - } - - #[test] - fn test_manifest_file_v1_to_v2_projection() { - use crate::spec::manifest_list::_serde::ManifestFileV1; - - // Create a V1 manifest file object (without V2 fields) - let v1_manifest = ManifestFileV1 { - manifest_path: "/test/manifest.avro".to_string(), - manifest_length: 5806, - partition_spec_id: 0, - added_snapshot_id: 1646658105718557341, - added_data_files_count: Some(3), - existing_data_files_count: Some(0), - deleted_data_files_count: Some(0), - added_rows_count: Some(3), - existing_rows_count: Some(0), - deleted_rows_count: Some(0), - partitions: None, - key_metadata: None, - }; - - // Convert V1 to V2 - this should apply defaults for missing V2 fields - let v2_manifest: ManifestFile = v1_manifest.try_into().unwrap(); - - // Verify V1→V2 projection defaults are applied correctly - assert_eq!( - v2_manifest.content, - ManifestContentType::Data, - "V1 manifest content should default to Data (0)" - ); - assert_eq!( - v2_manifest.sequence_number, 0, - "V1 manifest sequence_number should default to 0" - ); - assert_eq!( - v2_manifest.min_sequence_number, 0, - "V1 manifest min_sequence_number should default to 0" - ); - - // Verify other fields are preserved correctly - assert_eq!(v2_manifest.manifest_path, "/test/manifest.avro"); - assert_eq!(v2_manifest.manifest_length, 5806); - assert_eq!(v2_manifest.partition_spec_id, 0); - assert_eq!(v2_manifest.added_snapshot_id, 1646658105718557341); - assert_eq!(v2_manifest.added_files_count, Some(3)); - assert_eq!(v2_manifest.existing_files_count, Some(0)); - assert_eq!(v2_manifest.deleted_files_count, Some(0)); - assert_eq!(v2_manifest.added_rows_count, Some(3)); - assert_eq!(v2_manifest.existing_rows_count, Some(0)); - assert_eq!(v2_manifest.deleted_rows_count, Some(0)); - assert_eq!(v2_manifest.partitions, None); - assert_eq!(v2_manifest.key_metadata, None); - } -} diff --git a/crates/iceberg/src/spec/manifest_list/_const_schema.rs b/crates/iceberg/src/spec/manifest_list/_const_schema.rs new file mode 100644 index 0000000000..ae417f3366 --- /dev/null +++ b/crates/iceberg/src/spec/manifest_list/_const_schema.rs @@ -0,0 +1,365 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// This is a helper module that defines the schema field of the manifest list entry. +use std::sync::Arc; + +use apache_avro::Schema as AvroSchema; +use once_cell::sync::Lazy; + +use crate::avro::schema_to_avro_schema; +use crate::spec::{ListType, NestedField, NestedFieldRef, PrimitiveType, Schema, StructType, Type}; + +/// Field ID of `manifest_file.sequence_number`. +pub(crate) const MANIFEST_SEQUENCE_NUMBER_FIELD_ID: i32 = 515; +/// Field ID of `manifest_file.min_sequence_number`. +pub(crate) const MIN_SEQUENCE_NUMBER_FIELD_ID: i32 = 516; +/// Field ID of `manifest_file.content`. +pub(crate) const MANIFEST_CONTENT_FIELD_ID: i32 = 517; + +static MANIFEST_PATH: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 500, + "manifest_path", + Type::Primitive(PrimitiveType::String), + )) + }) +}; +static MANIFEST_LENGTH: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 501, + "manifest_length", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static PARTITION_SPEC_ID: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 502, + "partition_spec_id", + Type::Primitive(PrimitiveType::Int), + )) + }) +}; +static CONTENT: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + MANIFEST_CONTENT_FIELD_ID, + "content", + Type::Primitive(PrimitiveType::Int), + )) + }) +}; +static SEQUENCE_NUMBER: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + MANIFEST_SEQUENCE_NUMBER_FIELD_ID, + "sequence_number", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static MIN_SEQUENCE_NUMBER: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + MIN_SEQUENCE_NUMBER_FIELD_ID, + "min_sequence_number", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static ADDED_SNAPSHOT_ID: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 503, + "added_snapshot_id", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static ADDED_FILES_COUNT_V2: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 504, + "added_files_count", + Type::Primitive(PrimitiveType::Int), + )) + }) +}; +static ADDED_FILES_COUNT_V1: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::optional( + 504, + "added_data_files_count", + Type::Primitive(PrimitiveType::Int), + )) + }) +}; +static EXISTING_FILES_COUNT_V2: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 505, + "existing_files_count", + Type::Primitive(PrimitiveType::Int), + )) + }) +}; +static EXISTING_FILES_COUNT_V1: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::optional( + 505, + "existing_data_files_count", + Type::Primitive(PrimitiveType::Int), + )) + }) +}; +static DELETED_FILES_COUNT_V2: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 506, + "deleted_files_count", + Type::Primitive(PrimitiveType::Int), + )) + }) +}; +static DELETED_FILES_COUNT_V1: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::optional( + 506, + "deleted_data_files_count", + Type::Primitive(PrimitiveType::Int), + )) + }) +}; +static ADDED_ROWS_COUNT_V2: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 512, + "added_rows_count", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static ADDED_ROWS_COUNT_V1: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::optional( + 512, + "added_rows_count", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static EXISTING_ROWS_COUNT_V2: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 513, + "existing_rows_count", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static EXISTING_ROWS_COUNT_V1: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::optional( + 513, + "existing_rows_count", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static DELETED_ROWS_COUNT_V2: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::required( + 514, + "deleted_rows_count", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static DELETED_ROWS_COUNT_V1: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::optional( + 514, + "deleted_rows_count", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; +static PARTITIONS: Lazy = { + Lazy::new(|| { + // element type + let fields = vec![ + Arc::new(NestedField::required( + 509, + "contains_null", + Type::Primitive(PrimitiveType::Boolean), + )), + Arc::new(NestedField::optional( + 518, + "contains_nan", + Type::Primitive(PrimitiveType::Boolean), + )), + Arc::new(NestedField::optional( + 510, + "lower_bound", + Type::Primitive(PrimitiveType::Binary), + )), + Arc::new(NestedField::optional( + 511, + "upper_bound", + Type::Primitive(PrimitiveType::Binary), + )), + ]; + let element_field = Arc::new(NestedField::required( + 508, + "r_508", + Type::Struct(StructType::new(fields)), + )); + Arc::new(NestedField::optional( + 507, + "partitions", + Type::List(ListType { element_field }), + )) + }) +}; +static KEY_METADATA: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::optional( + 519, + "key_metadata", + Type::Primitive(PrimitiveType::Binary), + )) + }) +}; +static FIRST_ROW_ID: Lazy = { + Lazy::new(|| { + Arc::new(NestedField::optional( + 520, + "first_row_id", + Type::Primitive(PrimitiveType::Long), + )) + }) +}; + +static V1_SCHEMA: Lazy = { + Lazy::new(|| { + let fields = vec![ + MANIFEST_PATH.clone(), + MANIFEST_LENGTH.clone(), + PARTITION_SPEC_ID.clone(), + ADDED_SNAPSHOT_ID.clone(), + ADDED_FILES_COUNT_V1.clone().to_owned(), + EXISTING_FILES_COUNT_V1.clone(), + DELETED_FILES_COUNT_V1.clone(), + ADDED_ROWS_COUNT_V1.clone(), + EXISTING_ROWS_COUNT_V1.clone(), + DELETED_ROWS_COUNT_V1.clone(), + PARTITIONS.clone(), + KEY_METADATA.clone(), + ]; + Schema::builder().with_fields(fields).build().unwrap() + }) +}; + +static V2_SCHEMA: Lazy = { + Lazy::new(|| { + let fields = vec![ + MANIFEST_PATH.clone(), + MANIFEST_LENGTH.clone(), + PARTITION_SPEC_ID.clone(), + CONTENT.clone(), + SEQUENCE_NUMBER.clone(), + MIN_SEQUENCE_NUMBER.clone(), + ADDED_SNAPSHOT_ID.clone(), + ADDED_FILES_COUNT_V2.clone(), + EXISTING_FILES_COUNT_V2.clone(), + DELETED_FILES_COUNT_V2.clone(), + ADDED_ROWS_COUNT_V2.clone(), + EXISTING_ROWS_COUNT_V2.clone(), + DELETED_ROWS_COUNT_V2.clone(), + PARTITIONS.clone(), + KEY_METADATA.clone(), + ]; + Schema::builder().with_fields(fields).build().unwrap() + }) +}; + +static V3_SCHEMA: Lazy = { + Lazy::new(|| { + let fields = vec![ + MANIFEST_PATH.clone(), + MANIFEST_LENGTH.clone(), + PARTITION_SPEC_ID.clone(), + CONTENT.clone(), + SEQUENCE_NUMBER.clone(), + MIN_SEQUENCE_NUMBER.clone(), + ADDED_SNAPSHOT_ID.clone(), + ADDED_FILES_COUNT_V2.clone(), + EXISTING_FILES_COUNT_V2.clone(), + DELETED_FILES_COUNT_V2.clone(), + ADDED_ROWS_COUNT_V2.clone(), + EXISTING_ROWS_COUNT_V2.clone(), + DELETED_ROWS_COUNT_V2.clone(), + PARTITIONS.clone(), + KEY_METADATA.clone(), + FIRST_ROW_ID.clone(), + ]; + Schema::builder().with_fields(fields).build().unwrap() + }) +}; + +/// Fields of the `manifest_file` struct in the latest format version, with the +/// file and row counts optional as in v1, so a manifest list of any version +/// fits them. +pub(crate) fn manifest_file_fields() -> Vec { + let optional = |field: &NestedFieldRef| { + Arc::new(NestedField { + required: false, + ..field.as_ref().clone() + }) + }; + vec![ + MANIFEST_PATH.clone(), + MANIFEST_LENGTH.clone(), + PARTITION_SPEC_ID.clone(), + CONTENT.clone(), + SEQUENCE_NUMBER.clone(), + MIN_SEQUENCE_NUMBER.clone(), + ADDED_SNAPSHOT_ID.clone(), + optional(&ADDED_FILES_COUNT_V2), + optional(&EXISTING_FILES_COUNT_V2), + optional(&DELETED_FILES_COUNT_V2), + optional(&ADDED_ROWS_COUNT_V2), + optional(&EXISTING_ROWS_COUNT_V2), + optional(&DELETED_ROWS_COUNT_V2), + PARTITIONS.clone(), + KEY_METADATA.clone(), + FIRST_ROW_ID.clone(), + ] +} + +pub(super) static MANIFEST_LIST_AVRO_SCHEMA_V1: Lazy = + Lazy::new(|| schema_to_avro_schema("manifest_file", &V1_SCHEMA).unwrap()); + +pub(super) static MANIFEST_LIST_AVRO_SCHEMA_V2: Lazy = + Lazy::new(|| schema_to_avro_schema("manifest_file", &V2_SCHEMA).unwrap()); + +pub(super) static MANIFEST_LIST_AVRO_SCHEMA_V3: Lazy = + Lazy::new(|| schema_to_avro_schema("manifest_file", &V3_SCHEMA).unwrap()); diff --git a/crates/iceberg/src/spec/manifest_list/_serde.rs b/crates/iceberg/src/spec/manifest_list/_serde.rs new file mode 100644 index 0000000000..3528dec7cf --- /dev/null +++ b/crates/iceberg/src/spec/manifest_list/_serde.rs @@ -0,0 +1,637 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// This is a helper module that defines types to help with serialization/deserialization. +/// For deserialization the input first gets read into either the [ManifestFileV1] or [ManifestFileV2] struct +/// and then converted into the [ManifestFile] struct. Serialization works the other way around. +/// [ManifestFileV1] and [ManifestFileV2] are internal struct that are only used for serialization and deserialization. +pub use serde_bytes::ByteBuf; +use serde_derive::{Deserialize, Serialize}; + +use super::ManifestFile; +use crate::Error; +use crate::error::{Result, invalid_data}; +use crate::spec::FieldSummary; + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(transparent)] +pub(crate) struct ManifestListV3 { + entries: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(transparent)] +pub(crate) struct ManifestListV2 { + entries: Vec, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(transparent)] +pub(crate) struct ManifestListV1 { + entries: Vec, +} + +impl ManifestListV3 { + /// Converts the [ManifestListV3] into a [ManifestList]. + pub fn try_into(self) -> Result { + Ok(super::ManifestList { + entries: self + .entries + .into_iter() + .map(|v| v.try_into()) + .collect::>>()?, + }) + } +} + +impl TryFrom for ManifestListV3 { + type Error = Error; + + fn try_from(value: super::ManifestList) -> std::result::Result { + Ok(Self { + entries: value + .entries + .into_iter() + .map(|v| v.try_into()) + .collect::, _>>()?, + }) + } +} + +impl ManifestListV2 { + /// Converts the [ManifestListV2] into a [ManifestList]. + pub fn try_into(self) -> Result { + Ok(super::ManifestList { + entries: self + .entries + .into_iter() + .map(|v| v.try_into()) + .collect::>>()?, + }) + } +} + +impl TryFrom for ManifestListV2 { + type Error = Error; + + fn try_from(value: super::ManifestList) -> std::result::Result { + Ok(Self { + entries: value + .entries + .into_iter() + .map(|v| v.try_into()) + .collect::, _>>()?, + }) + } +} + +impl ManifestListV1 { + /// Converts the [ManifestListV1] into a [ManifestList]. + pub fn try_into(self) -> Result { + Ok(super::ManifestList { + entries: self + .entries + .into_iter() + .map(|v| v.try_into()) + .collect::>>()?, + }) + } +} + +impl TryFrom for ManifestListV1 { + type Error = Error; + + fn try_from(value: super::ManifestList) -> std::result::Result { + Ok(Self { + entries: value + .entries + .into_iter() + .map(|v| v.try_into()) + .collect::, _>>()?, + }) + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(super) struct ManifestFileV1 { + pub manifest_path: String, + pub manifest_length: i64, + pub partition_spec_id: i32, + pub added_snapshot_id: i64, + pub added_data_files_count: Option, + pub existing_data_files_count: Option, + pub deleted_data_files_count: Option, + pub added_rows_count: Option, + pub existing_rows_count: Option, + pub deleted_rows_count: Option, + pub partitions: Option>, + pub key_metadata: Option, +} + +// Aliases were added to fields that were renamed in Iceberg 1.5.0 (https://github.com/apache/iceberg/pull/5338), in order to support both conventions/versions. +// In the current implementation deserialization is done using field names, and therefore these fields may appear as either. +// see issue that raised this here: https://github.com/apache/iceberg-rust/issues/338 +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(super) struct ManifestFileV2 { + pub manifest_path: String, + pub manifest_length: i64, + pub partition_spec_id: i32, + #[serde(default = "v2_default_content_for_v1")] + pub content: i32, + #[serde(default = "v2_default_sequence_number_for_v1")] + pub sequence_number: i64, + #[serde(default = "v2_default_min_sequence_number_for_v1")] + pub min_sequence_number: i64, + pub added_snapshot_id: i64, + #[serde(alias = "added_data_files_count", alias = "added_files_count")] + pub added_files_count: i32, + #[serde(alias = "existing_data_files_count", alias = "existing_files_count")] + pub existing_files_count: i32, + #[serde(alias = "deleted_data_files_count", alias = "deleted_files_count")] + pub deleted_files_count: i32, + pub added_rows_count: i64, + pub existing_rows_count: i64, + pub deleted_rows_count: i64, + pub partitions: Option>, + pub key_metadata: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +pub(super) struct ManifestFileV3 { + pub manifest_path: String, + pub manifest_length: i64, + pub partition_spec_id: i32, + #[serde(default = "v2_default_content_for_v1")] + pub content: i32, + #[serde(default = "v2_default_sequence_number_for_v1")] + pub sequence_number: i64, + #[serde(default = "v2_default_min_sequence_number_for_v1")] + pub min_sequence_number: i64, + pub added_snapshot_id: i64, + #[serde(alias = "added_data_files_count", alias = "added_files_count")] + pub added_files_count: i32, + #[serde(alias = "existing_data_files_count", alias = "existing_files_count")] + pub existing_files_count: i32, + #[serde(alias = "deleted_data_files_count", alias = "deleted_files_count")] + pub deleted_files_count: i32, + pub added_rows_count: i64, + pub existing_rows_count: i64, + pub deleted_rows_count: i64, + pub partitions: Option>, + pub key_metadata: Option, + pub first_row_id: Option, +} + +impl ManifestFileV3 { + /// Converts the [ManifestFileV3] into a [ManifestFile]. + pub fn try_into(self) -> Result { + let manifest_file = ManifestFile { + manifest_path: self.manifest_path, + manifest_length: self.manifest_length, + partition_spec_id: self.partition_spec_id, + content: self.content.try_into()?, + sequence_number: self.sequence_number, + min_sequence_number: self.min_sequence_number, + added_snapshot_id: self.added_snapshot_id, + added_files_count: Some(self.added_files_count.try_into()?), + existing_files_count: Some(self.existing_files_count.try_into()?), + deleted_files_count: Some(self.deleted_files_count.try_into()?), + added_rows_count: Some(self.added_rows_count.try_into()?), + existing_rows_count: Some(self.existing_rows_count.try_into()?), + deleted_rows_count: Some(self.deleted_rows_count.try_into()?), + partitions: self.partitions, + key_metadata: self.key_metadata.map(|b| b.into_vec()), + first_row_id: self.first_row_id.map(|n| n.try_into()).transpose()?, + }; + + Ok(manifest_file) + } +} + +impl ManifestFileV2 { + /// Converts the [ManifestFileV2] into a [ManifestFile]. + pub fn try_into(self) -> Result { + Ok(ManifestFile { + manifest_path: self.manifest_path, + manifest_length: self.manifest_length, + partition_spec_id: self.partition_spec_id, + content: self.content.try_into()?, + sequence_number: self.sequence_number, + min_sequence_number: self.min_sequence_number, + added_snapshot_id: self.added_snapshot_id, + added_files_count: Some(self.added_files_count.try_into()?), + existing_files_count: Some(self.existing_files_count.try_into()?), + deleted_files_count: Some(self.deleted_files_count.try_into()?), + added_rows_count: Some(self.added_rows_count.try_into()?), + existing_rows_count: Some(self.existing_rows_count.try_into()?), + deleted_rows_count: Some(self.deleted_rows_count.try_into()?), + partitions: self.partitions, + key_metadata: self.key_metadata.map(|b| b.into_vec()), + first_row_id: None, + }) + } +} + +fn v2_default_content_for_v1() -> i32 { + super::ManifestContentType::Data as i32 +} + +fn v2_default_sequence_number_for_v1() -> i64 { + 0 +} + +fn v2_default_min_sequence_number_for_v1() -> i64 { + 0 +} + +impl ManifestFileV1 { + /// Converts the [ManifestFileV1] into a [ManifestFile]. + pub fn try_into(self) -> Result { + Ok(ManifestFile { + manifest_path: self.manifest_path, + manifest_length: self.manifest_length, + partition_spec_id: self.partition_spec_id, + added_snapshot_id: self.added_snapshot_id, + added_files_count: self + .added_data_files_count + .map(TryInto::try_into) + .transpose()?, + existing_files_count: self + .existing_data_files_count + .map(TryInto::try_into) + .transpose()?, + deleted_files_count: self + .deleted_data_files_count + .map(TryInto::try_into) + .transpose()?, + added_rows_count: self.added_rows_count.map(TryInto::try_into).transpose()?, + existing_rows_count: self + .existing_rows_count + .map(TryInto::try_into) + .transpose()?, + deleted_rows_count: self.deleted_rows_count.map(TryInto::try_into).transpose()?, + partitions: self.partitions, + key_metadata: self.key_metadata.map(|b| b.into_vec()), + // as ref: https://iceberg.apache.org/spec/#partitioning + // use 0 when reading v1 manifest lists + content: super::ManifestContentType::Data, + sequence_number: 0, + min_sequence_number: 0, + first_row_id: None, + }) + } +} + +fn convert_to_serde_key_metadata(key_metadata: Option>) -> Option { + match key_metadata { + Some(metadata) if !metadata.is_empty() => Some(ByteBuf::from(metadata)), + _ => None, + } +} + +impl TryFrom for ManifestFileV3 { + type Error = Error; + + fn try_from(value: ManifestFile) -> std::result::Result { + let key_metadata = convert_to_serde_key_metadata(value.key_metadata); + Ok(Self { + manifest_path: value.manifest_path, + manifest_length: value.manifest_length, + partition_spec_id: value.partition_spec_id, + content: value.content as i32, + sequence_number: value.sequence_number, + min_sequence_number: value.min_sequence_number, + added_snapshot_id: value.added_snapshot_id, + added_files_count: value + .added_files_count + .ok_or_else(|| { + invalid_data!("added_data_files_count in ManifestFileV3 is required") + })? + .try_into()?, + existing_files_count: value + .existing_files_count + .ok_or_else(|| { + invalid_data!("existing_data_files_count in ManifestFileV3 is required") + })? + .try_into()?, + deleted_files_count: value + .deleted_files_count + .ok_or_else(|| { + invalid_data!("deleted_data_files_count in ManifestFileV3 is required") + })? + .try_into()?, + added_rows_count: value + .added_rows_count + .ok_or_else(|| invalid_data!("added_rows_count in ManifestFileV3 is required"))? + .try_into()?, + existing_rows_count: value + .existing_rows_count + .ok_or_else(|| invalid_data!("existing_rows_count in ManifestFileV3 is required"))? + .try_into()?, + deleted_rows_count: value + .deleted_rows_count + .ok_or_else(|| invalid_data!("deleted_rows_count in ManifestFileV3 is required"))? + .try_into()?, + partitions: value.partitions, + key_metadata, + first_row_id: value.first_row_id.map(|n| n.try_into()).transpose()?, + }) + } +} + +impl TryFrom for ManifestFileV2 { + type Error = Error; + + fn try_from(value: ManifestFile) -> std::result::Result { + let key_metadata = convert_to_serde_key_metadata(value.key_metadata); + Ok(Self { + manifest_path: value.manifest_path, + manifest_length: value.manifest_length, + partition_spec_id: value.partition_spec_id, + content: value.content as i32, + sequence_number: value.sequence_number, + min_sequence_number: value.min_sequence_number, + added_snapshot_id: value.added_snapshot_id, + added_files_count: value + .added_files_count + .ok_or_else(|| { + invalid_data!("added_data_files_count in ManifestFileV2 should be require") + })? + .try_into()?, + existing_files_count: value + .existing_files_count + .ok_or_else(|| { + invalid_data!("existing_data_files_count in ManifestFileV2 should be require") + })? + .try_into()?, + deleted_files_count: value + .deleted_files_count + .ok_or_else(|| { + invalid_data!("deleted_data_files_count in ManifestFileV2 should be require") + })? + .try_into()?, + added_rows_count: value + .added_rows_count + .ok_or_else(|| { + invalid_data!("added_rows_count in ManifestFileV2 should be require") + })? + .try_into()?, + existing_rows_count: value + .existing_rows_count + .ok_or_else(|| { + invalid_data!("existing_rows_count in ManifestFileV2 should be require") + })? + .try_into()?, + deleted_rows_count: value + .deleted_rows_count + .ok_or_else(|| { + invalid_data!("deleted_rows_count in ManifestFileV2 should be require") + })? + .try_into()?, + partitions: value.partitions, + key_metadata, + }) + } +} + +impl TryFrom for ManifestFileV1 { + type Error = Error; + + fn try_from(value: ManifestFile) -> std::result::Result { + let key_metadata = convert_to_serde_key_metadata(value.key_metadata); + Ok(Self { + manifest_path: value.manifest_path, + manifest_length: value.manifest_length, + partition_spec_id: value.partition_spec_id, + added_snapshot_id: value.added_snapshot_id, + added_data_files_count: value.added_files_count.map(TryInto::try_into).transpose()?, + existing_data_files_count: value + .existing_files_count + .map(TryInto::try_into) + .transpose()?, + deleted_data_files_count: value + .deleted_files_count + .map(TryInto::try_into) + .transpose()?, + added_rows_count: value.added_rows_count.map(TryInto::try_into).transpose()?, + existing_rows_count: value + .existing_rows_count + .map(TryInto::try_into) + .transpose()?, + deleted_rows_count: value + .deleted_rows_count + .map(TryInto::try_into) + .transpose()?, + partitions: value.partitions, + key_metadata, + }) + } +} + +#[cfg(test)] +mod test { + use std::fs; + + use apache_avro::{Reader, Schema}; + + use super::{ManifestListV1, ManifestListV2, ManifestListV3}; + use crate::spec::{Datum, FieldSummary, ManifestContentType, ManifestFile, ManifestList}; + + #[test] + fn test_serialize_manifest_list_v1() { + let manifest_list:ManifestListV1 = ManifestList { + entries: vec![ManifestFile { + manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(), + manifest_length: 5806, + partition_spec_id: 0, + content: ManifestContentType::Data, + sequence_number: 0, + min_sequence_number: 0, + added_snapshot_id: 1646658105718557341, + added_files_count: Some(3), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: None, + key_metadata: None, + first_row_id: None, + }] + }.try_into().unwrap(); + let result = serde_json::to_string(&manifest_list).unwrap(); + assert_eq!( + result, + r#"[{"manifest_path":"/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro","manifest_length":5806,"partition_spec_id":0,"added_snapshot_id":1646658105718557341,"added_data_files_count":3,"existing_data_files_count":0,"deleted_data_files_count":0,"added_rows_count":3,"existing_rows_count":0,"deleted_rows_count":0,"partitions":null,"key_metadata":null}]"# + ); + } + + #[test] + fn test_serialize_manifest_list_v2() { + let manifest_list:ManifestListV2 = ManifestList { + entries: vec![ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: 1, + min_sequence_number: 1, + added_snapshot_id: 377075049360453639, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: None, + }] + }.try_into().unwrap(); + let result = serde_json::to_string(&manifest_list).unwrap(); + assert_eq!( + result, + r#"[{"manifest_path":"s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro","manifest_length":6926,"partition_spec_id":1,"content":0,"sequence_number":1,"min_sequence_number":1,"added_snapshot_id":377075049360453639,"added_files_count":1,"existing_files_count":0,"deleted_files_count":0,"added_rows_count":3,"existing_rows_count":0,"deleted_rows_count":0,"partitions":[{"contains_null":false,"contains_nan":false,"lower_bound":[1,0,0,0,0,0,0,0],"upper_bound":[1,0,0,0,0,0,0,0]}],"key_metadata":null}]"# + ); + } + + #[test] + fn test_serialize_manifest_list_v3() { + let manifest_list: ManifestListV3 = ManifestList { + entries: vec![ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: 1, + min_sequence_number: 1, + added_snapshot_id: 377075049360453639, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: Some(10), + }] + }.try_into().unwrap(); + let result = serde_json::to_string(&manifest_list).unwrap(); + assert_eq!( + result, + r#"[{"manifest_path":"s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro","manifest_length":6926,"partition_spec_id":1,"content":0,"sequence_number":1,"min_sequence_number":1,"added_snapshot_id":377075049360453639,"added_files_count":1,"existing_files_count":0,"deleted_files_count":0,"added_rows_count":3,"existing_rows_count":0,"deleted_rows_count":0,"partitions":[{"contains_null":false,"contains_nan":false,"lower_bound":[1,0,0,0,0,0,0,0],"upper_bound":[1,0,0,0,0,0,0,0]}],"key_metadata":null,"first_row_id":10}]"# + ); + } + + #[tokio::test] + async fn test_manifest_list_v2_deserializer_aliases() { + // reading avro manifest file generated by iceberg 1.4.0 + let avro_1_path = "testdata/manifests_lists/manifest-list-v2-1.avro"; + let bs_1 = fs::read(avro_1_path).unwrap(); + let avro_1_fields = read_avro_schema_fields_as_str(bs_1.clone()).await; + assert_eq!( + avro_1_fields, + "manifest_path, manifest_length, partition_spec_id, content, sequence_number, min_sequence_number, added_snapshot_id, added_data_files_count, existing_data_files_count, deleted_data_files_count, added_rows_count, existing_rows_count, deleted_rows_count, partitions" + ); + // reading avro manifest file generated by iceberg 1.5.0 + let avro_2_path = "testdata/manifests_lists/manifest-list-v2-2.avro"; + let bs_2 = fs::read(avro_2_path).unwrap(); + let avro_2_fields = read_avro_schema_fields_as_str(bs_2.clone()).await; + assert_eq!( + avro_2_fields, + "manifest_path, manifest_length, partition_spec_id, content, sequence_number, min_sequence_number, added_snapshot_id, added_files_count, existing_files_count, deleted_files_count, added_rows_count, existing_rows_count, deleted_rows_count, partitions" + ); + // deserializing both files to ManifestList struct + let _manifest_list_1 = + ManifestList::parse_with_version(&bs_1, crate::spec::FormatVersion::V2).unwrap(); + let _manifest_list_2 = + ManifestList::parse_with_version(&bs_2, crate::spec::FormatVersion::V2).unwrap(); + } + + async fn read_avro_schema_fields_as_str(bs: Vec) -> String { + let reader = Reader::new(&bs[..]).unwrap(); + let schema = reader.writer_schema(); + let fields: String = match schema { + Schema::Record(record) => record + .fields + .iter() + .map(|field| field.name.clone()) + .collect::>() + .join(", "), + _ => "".to_string(), + }; + fields + } + + #[test] + fn test_manifest_file_v1_to_v2_projection() { + use super::ManifestFileV1; + + // Create a V1 manifest file object (without V2 fields) + let v1_manifest = ManifestFileV1 { + manifest_path: "/test/manifest.avro".to_string(), + manifest_length: 5806, + partition_spec_id: 0, + added_snapshot_id: 1646658105718557341, + added_data_files_count: Some(3), + existing_data_files_count: Some(0), + deleted_data_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: None, + key_metadata: None, + }; + + // Convert V1 to V2 - this should apply defaults for missing V2 fields + let v2_manifest: ManifestFile = v1_manifest.try_into().unwrap(); + + // Verify V1→V2 projection defaults are applied correctly + assert_eq!( + v2_manifest.content, + ManifestContentType::Data, + "V1 manifest content should default to Data (0)" + ); + assert_eq!( + v2_manifest.sequence_number, 0, + "V1 manifest sequence_number should default to 0" + ); + assert_eq!( + v2_manifest.min_sequence_number, 0, + "V1 manifest min_sequence_number should default to 0" + ); + + // Verify other fields are preserved correctly + assert_eq!(v2_manifest.manifest_path, "/test/manifest.avro"); + assert_eq!(v2_manifest.manifest_length, 5806); + assert_eq!(v2_manifest.partition_spec_id, 0); + assert_eq!(v2_manifest.added_snapshot_id, 1646658105718557341); + assert_eq!(v2_manifest.added_files_count, Some(3)); + assert_eq!(v2_manifest.existing_files_count, Some(0)); + assert_eq!(v2_manifest.deleted_files_count, Some(0)); + assert_eq!(v2_manifest.added_rows_count, Some(3)); + assert_eq!(v2_manifest.existing_rows_count, Some(0)); + assert_eq!(v2_manifest.deleted_rows_count, Some(0)); + assert_eq!(v2_manifest.partitions, None); + assert_eq!(v2_manifest.key_metadata, None); + } +} diff --git a/crates/iceberg/src/spec/manifest_list/manifest_file.rs b/crates/iceberg/src/spec/manifest_list/manifest_file.rs new file mode 100644 index 0000000000..24df6d4dca --- /dev/null +++ b/crates/iceberg/src/spec/manifest_list/manifest_file.rs @@ -0,0 +1,417 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::str::FromStr; + +use serde_derive::{Deserialize, Serialize}; + +use super::ByteBuf; +use crate::Error; +use crate::error::{Result, invalid_data}; + +/// Entry in a manifest list. +#[derive(Debug, PartialEq, Clone, Eq, Hash)] +pub struct ManifestFile { + /// field: 500 + /// + /// Location of the manifest file + pub manifest_path: String, + /// field: 501 + /// + /// Length of the manifest file in bytes + pub manifest_length: i64, + /// field: 502 + /// + /// ID of a partition spec used to write the manifest; must be listed + /// in table metadata partition-specs + pub partition_spec_id: i32, + /// field: 517 + /// + /// The type of files tracked by the manifest, either data or delete + /// files; 0 for all v1 manifests + pub content: ManifestContentType, + /// field: 515 + /// + /// The sequence number when the manifest was added to the table; use 0 + /// when reading v1 manifest lists + pub sequence_number: i64, + /// field: 516 + /// + /// The minimum data sequence number of all live data or delete files in + /// the manifest; use 0 when reading v1 manifest lists + pub min_sequence_number: i64, + /// field: 503 + /// + /// ID of the snapshot where the manifest file was added + pub added_snapshot_id: i64, + /// field: 504 + /// + /// Number of entries in the manifest that have status ADDED, when null + /// this is assumed to be non-zero + pub added_files_count: Option, + /// field: 505 + /// + /// Number of entries in the manifest that have status EXISTING (0), + /// when null this is assumed to be non-zero + pub existing_files_count: Option, + /// field: 506 + /// + /// Number of entries in the manifest that have status DELETED (2), + /// when null this is assumed to be non-zero + pub deleted_files_count: Option, + /// field: 512 + /// + /// Number of rows in all of files in the manifest that have status + /// ADDED, when null this is assumed to be non-zero + pub added_rows_count: Option, + /// field: 513 + /// + /// Number of rows in all of files in the manifest that have status + /// EXISTING, when null this is assumed to be non-zero + pub existing_rows_count: Option, + /// field: 514 + /// + /// Number of rows in all of files in the manifest that have status + /// DELETED, when null this is assumed to be non-zero + pub deleted_rows_count: Option, + /// field: 507 + /// element_field: 508 + /// + /// A list of field summaries for each partition field in the spec. Each + /// field in the list corresponds to a field in the manifest file’s + /// partition spec. + pub partitions: Option>, + /// field: 519 + /// + /// Implementation-specific key metadata for encryption + pub key_metadata: Option>, + /// field 520 + /// + /// The starting _row_id to assign to rows added by ADDED data files + pub first_row_id: Option, +} + +impl ManifestFile { + /// Checks if the manifest file has any added files. + pub fn has_added_files(&self) -> bool { + self.added_files_count.map(|c| c > 0).unwrap_or(true) + } + + /// Checks whether this manifest contains entries with DELETED status. + pub fn has_deleted_files(&self) -> bool { + self.deleted_files_count.map(|c| c > 0).unwrap_or(true) + } + + /// Checks if the manifest file has any existed files. + pub fn has_existing_files(&self) -> bool { + self.existing_files_count.map(|c| c > 0).unwrap_or(true) + } +} + +/// The type of files tracked by the manifest, either data or delete files; Data(0) for all v1 manifests +#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash, Default)] +pub enum ManifestContentType { + /// The manifest content is data. + #[default] + Data = 0, + /// The manifest content is deletes. + Deletes = 1, +} + +impl FromStr for ManifestContentType { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "data" => Ok(ManifestContentType::Data), + "deletes" => Ok(ManifestContentType::Deletes), + _ => Err(invalid_data!("Invalid manifest content type: {s}")), + } + } +} + +impl std::fmt::Display for ManifestContentType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ManifestContentType::Data => write!(f, "data"), + ManifestContentType::Deletes => write!(f, "deletes"), + } + } +} + +impl TryFrom for ManifestContentType { + type Error = Error; + + fn try_from(value: i32) -> std::result::Result { + match value { + 0 => Ok(ManifestContentType::Data), + 1 => Ok(ManifestContentType::Deletes), + _ => Err(invalid_data!( + "Invalid manifest content type. Expected 0 or 1, got {value}" + )), + } + } +} + +/// Field summary for partition field in the spec. +/// +/// Each field in the list corresponds to a field in the manifest file’s partition spec. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, Hash)] +pub struct FieldSummary { + /// field: 509 + /// + /// Whether the manifest contains at least one partition with a null + /// value for the field + pub contains_null: bool, + /// field: 518 + /// Whether the manifest contains at least one partition with a NaN + /// value for the field + pub contains_nan: Option, + /// field: 510 + /// The minimum value for the field in the manifests + /// partitions. + pub lower_bound: Option, + /// field: 511 + /// The maximum value for the field in the manifests + /// partitions. + pub upper_bound: Option, +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use super::{ManifestContentType, ManifestFile}; + use crate::ErrorKind; + use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata}; + use crate::io::FileIO; + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, ManifestEntry, ManifestReader, + ManifestStatus, ManifestWriterBuilder, NestedField, PartitionSpec, PrimitiveType, Schema, + SchemaRef, Type, + }; + + #[test] + fn test_manifest_content_type_default() { + assert_eq!(ManifestContentType::default(), ManifestContentType::Data); + } + + #[test] + fn test_manifest_content_type_default_value() { + assert_eq!(ManifestContentType::default() as i32, 0); + } + + /// A single-field schema used by the manifest-writing test helpers. + fn test_schema() -> SchemaRef { + Arc::new( + Schema::builder() + .with_fields(vec![Arc::new(NestedField::optional( + 1, + "id", + Type::Primitive(PrimitiveType::Long), + ))]) + .build() + .unwrap(), + ) + } + + /// Writes a single-entry v3 data manifest to `io` at `path`, without + /// encryption, and returns the resulting [`ManifestFile`]. + async fn write_manifest(io: &FileIO, path: &str) -> ManifestFile { + let schema = test_schema(); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .build() + .unwrap(); + + let output_file = io.new_output(path).unwrap(); + let mut writer = ManifestWriterBuilder::new(output_file, Some(1), schema, partition_spec) + .build_v3_data(); + + writer + .add_entry(data_entry(ManifestStatus::Added, 100, None)) + .unwrap(); + + writer.write_manifest_file().await.unwrap() + } + + /// Writes a single-entry v3 data manifest to `io` at `path`, encrypting it + /// with `key_metadata`, and returns the resulting [`ManifestFile`]. + async fn write_encrypted_manifest( + io: &FileIO, + path: &str, + key_metadata: StandardKeyMetadata, + ) -> ManifestFile { + let schema = test_schema(); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .build() + .unwrap(); + + let output_file = io.new_output(path).unwrap(); + let encrypted_output = EncryptedOutputFile::new(output_file, key_metadata); + + let mut writer = ManifestWriterBuilder::new_from_encrypted( + encrypted_output, + Some(1), + schema, + partition_spec, + ) + .expect("Expected a valid writer") + .build_v3_data(); + + writer + .add_entry(data_entry(ManifestStatus::Added, 100, None)) + .unwrap(); + + writer.write_manifest_file().await.unwrap() + } + + #[tokio::test] + async fn test_load_manifest_decrypts_when_key_metadata_present() { + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + let io = FileIO::new_with_memory(); + let path = "memory:///test/encrypted_manifest.avro"; + let manifest_file = write_encrypted_manifest(&io, path, key_metadata.clone()).await; + let size = io.new_input(path).unwrap().metadata().await.unwrap().size; + assert_eq!(manifest_file.manifest_length, size as i64); + assert_eq!( + StandardKeyMetadata::decode(manifest_file.key_metadata.as_ref().unwrap()).unwrap(), + key_metadata.with_file_length(size) + ); + + let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap(); + assert_eq!(manifest.entries().len(), 1); + assert_eq!( + manifest.entries()[0].file_path(), + "s3://bucket/table/data/00000.parquet" + ); + assert_eq!(manifest.entries()[0].data_file.record_count, 100); + } + + #[tokio::test] + async fn test_load_manifest_fails_with_wrong_key() { + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + + let io = FileIO::new_with_memory(); + let path = "memory:///test/wrong_key_manifest.avro"; + let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await; + + // Point the manifest file at key metadata carrying a different DEK (but + // the same AAD prefix). The bytes on disk were encrypted with the + // original key, so GCM authentication must fail rather than silently + // returning garbage. + let wrong_key_metadata = StandardKeyMetadata::try_new(b"fedcba9876543210") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!") + .with_file_length(manifest_file.manifest_length as u64); + manifest_file.key_metadata = Some(wrong_key_metadata.encode().unwrap().to_vec()); + + let err = ManifestReader::new(io) + .read(&manifest_file) + .await + .expect_err("read must fail when decrypting with the wrong key"); + assert_eq!(err.kind(), ErrorKind::Unexpected); + } + + #[tokio::test] + async fn test_load_manifest_fails_with_wrong_aad() { + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + + let io = FileIO::new_with_memory(); + let path = "memory:///test/wrong_aad_manifest.avro"; + let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await; + + // Point the manifest file at key metadata carrying the correct DEK but a + // different AAD prefix. The per-block AAD is `aad_prefix || block_index`, + // so GCM authentication must fail even though the key is right. + let wrong_aad_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"wrong-aad-prefix") + .with_file_length(manifest_file.manifest_length as u64); + manifest_file.key_metadata = Some(wrong_aad_metadata.encode().unwrap().to_vec()); + + let err = ManifestReader::new(io) + .read(&manifest_file) + .await + .expect_err("read must fail when decrypting with the wrong AAD prefix"); + assert_eq!(err.kind(), ErrorKind::Unexpected); + } + + /// Builds a data-file manifest entry with the given status, record count, + /// and pre-existing `first_row_id`. + fn data_entry( + status: ManifestStatus, + record_count: u64, + first_row_id: Option, + ) -> ManifestEntry { + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("s3://bucket/table/data/00000.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(4096) + .record_count(record_count) + .first_row_id(first_row_id) + .build() + .unwrap(); + + ManifestEntry::builder() + .status(status) + .data_file(data_file) + .build() + } + + #[tokio::test] + async fn test_load_manifest_reads_written_entries() { + let io = FileIO::new_with_memory(); + let path = "memory:///test/plaintext_manifest.avro"; + let manifest_file = write_manifest(&io, path).await; + assert_eq!(manifest_file.key_metadata, None); + + let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap(); + assert_eq!(manifest.entries().len(), 1); + assert_eq!( + manifest.entries()[0].file_path(), + "s3://bucket/table/data/00000.parquet" + ); + assert_eq!(manifest.entries()[0].data_file.record_count, 100); + } + + /// End-to-end: writing a v3 data manifest, stamping a manifest-level + /// `first_row_id`, and loading it must assign inherited `first_row_id`s to + /// the entries. This exercises the wiring in [`ManifestReader`] and the + /// write/read round-trip that leaves per-file `first_row_id` as `None`. + #[tokio::test] + async fn test_load_manifest_assigns_first_row_ids() { + let io = FileIO::new_with_memory(); + let path = "memory:///test/first_row_id_manifest.avro"; + let mut manifest_file = write_manifest(&io, path).await; + + // Stamp a manifest-level first_row_id, as the manifest-list writer would. + manifest_file.first_row_id = Some(1000); + + let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap(); + assert_eq!(manifest.entries().len(), 1); + assert_eq!(manifest.entries()[0].data_file().first_row_id(), Some(1000)); + } +} diff --git a/crates/iceberg/src/spec/manifest_list/mod.rs b/crates/iceberg/src/spec/manifest_list/mod.rs new file mode 100644 index 0000000000..8a5db38f35 --- /dev/null +++ b/crates/iceberg/src/spec/manifest_list/mod.rs @@ -0,0 +1,362 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! ManifestList for Iceberg. + +mod _const_schema; +pub(super) mod _serde; +mod manifest_file; +mod reader; +mod writer; + +use apache_avro::types::Value; +use apache_avro::{Reader, from_value}; +pub use manifest_file::*; +pub use reader::*; +pub use serde_bytes::ByteBuf; +pub use writer::*; + +use self::_const_schema::MANIFEST_LIST_AVRO_SCHEMA_V1; +pub(crate) use self::_const_schema::{ + MANIFEST_CONTENT_FIELD_ID, MANIFEST_SEQUENCE_NUMBER_FIELD_ID, MIN_SEQUENCE_NUMBER_FIELD_ID, + manifest_file_fields, +}; +use super::FormatVersion; +use crate::error::Result; + +/// Placeholder for sequence number. The field with this value must be replaced with the actual sequence number before it write. +pub const UNASSIGNED_SEQUENCE_NUMBER: i64 = -1; + +/// Snapshots are embedded in table metadata, but the list of manifests for a +/// snapshot are stored in a separate manifest list file. +/// +/// A new manifest list is written for each attempt to commit a snapshot +/// because the list of manifests always changes to produce a new snapshot. +/// When a manifest list is written, the (optimistic) sequence number of the +/// snapshot is written for all new manifest files tracked by the list. +/// +/// A manifest list includes summary metadata that can be used to avoid +/// scanning all of the manifests in a snapshot when planning a table scan. +/// This includes the number of added, existing, and deleted files, and a +/// summary of values for each field of the partition spec used to write the +/// manifest. +#[derive(Debug, Clone, PartialEq)] +pub struct ManifestList { + /// Entries in a manifest list. + entries: Vec, +} + +impl ManifestList { + /// Parse manifest list from bytes. + pub fn parse_with_version(bs: &[u8], version: FormatVersion) -> Result { + match version { + FormatVersion::V1 => { + let reader = Reader::builder(bs) + .reader_schema(&MANIFEST_LIST_AVRO_SCHEMA_V1) + .build()?; + let values = Value::Array(reader.collect::, _>>()?); + from_value::<_serde::ManifestListV1>(&values)?.try_into() + } + FormatVersion::V2 => { + let reader = Reader::new(bs)?; + let values = Value::Array(reader.collect::, _>>()?); + from_value::<_serde::ManifestListV2>(&values)?.try_into() + } + FormatVersion::V3 => { + let reader = Reader::new(bs)?; + let values = Value::Array(reader.collect::, _>>()?); + from_value::<_serde::ManifestListV3>(&values)?.try_into() + } + } + } + + /// Get the entries in the manifest list. + pub fn entries(&self) -> &[ManifestFile] { + &self.entries + } + + /// Take ownership of the entries in the manifest list, consuming it + pub fn consume_entries(self) -> impl IntoIterator { + Box::new(self.entries.into_iter()) + } +} + +#[cfg(test)] +mod test { + use std::fs; + + use apache_avro::{Codec, Writer}; + use tempfile::TempDir; + + use super::_const_schema::MANIFEST_LIST_AVRO_SCHEMA_V2; + use super::_serde::ManifestFileV2; + use super::*; + use crate::io::FileIO; + use crate::spec::{Datum, FieldSummary, ManifestContentType, ManifestFile}; + + #[tokio::test] + async fn test_parse_manifest_list_v1() { + let manifest_list = ManifestList { + entries: vec![ + ManifestFile { + manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(), + manifest_length: 5806, + partition_spec_id: 0, + content: ManifestContentType::Data, + sequence_number: 0, + min_sequence_number: 0, + added_snapshot_id: 1646658105718557341, + added_files_count: Some(3), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some(vec![]), + key_metadata: None, + first_row_id: None, + } + ] + }; + + let file_io = FileIO::new_with_fs(); + + let tmp_dir = TempDir::new().unwrap(); + let file_name = "simple_manifest_list_v1.avro"; + let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name); + + let mut writer = ManifestListWriter::v1( + file_io + .new_output(full_path.clone()) + .unwrap() + .writer() + .await + .unwrap(), + 1646658105718557341, + Some(1646658105718557341), + ); + + writer + .add_manifests(manifest_list.entries.clone().into_iter()) + .unwrap(); + writer.close().await.unwrap(); + + let bs = fs::read(full_path).expect("read_file must succeed"); + + let parsed_manifest_list = + ManifestList::parse_with_version(&bs, FormatVersion::V1).unwrap(); + + assert_eq!(manifest_list, parsed_manifest_list); + } + + #[tokio::test] + async fn test_parse_manifest_list_v2() { + let manifest_list = ManifestList { + entries: vec![ + ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: 1, + min_sequence_number: 1, + added_snapshot_id: 377075049360453639, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: None, + }, + ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m1.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 2, + content: ManifestContentType::Data, + sequence_number: 1, + min_sequence_number: 1, + added_snapshot_id: 377075049360453639, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1_f32).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1_f32).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: None, + } + ] + }; + + let file_io = FileIO::new_with_fs(); + + let tmp_dir = TempDir::new().unwrap(); + let file_name = "simple_manifest_list_v1.avro"; + let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name); + + let mut writer = ManifestListWriter::v2( + file_io + .new_output(full_path.clone()) + .unwrap() + .writer() + .await + .unwrap(), + 1646658105718557341, + Some(1646658105718557341), + 1, + ); + + writer + .add_manifests(manifest_list.entries.clone().into_iter()) + .unwrap(); + writer.close().await.unwrap(); + + let bs = fs::read(full_path).expect("read_file must succeed"); + + let parsed_manifest_list = + ManifestList::parse_with_version(&bs, FormatVersion::V2).unwrap(); + + assert_eq!(manifest_list, parsed_manifest_list); + } + + #[test] + fn test_parse_snappy_manifest_list_v2() { + let manifest_list = ManifestList { + entries: vec![ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/snappy-m0.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: 1, + min_sequence_number: 1, + added_snapshot_id: 377075049360453639, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some(vec![FieldSummary { + contains_null: false, + contains_nan: Some(false), + lower_bound: Some(Datum::long(1).to_bytes().unwrap()), + upper_bound: Some(Datum::long(1).to_bytes().unwrap()), + }]), + key_metadata: None, + first_row_id: None, + }], + }; + + let manifest_entry: ManifestFileV2 = manifest_list.entries[0].clone().try_into().unwrap(); + let mut writer = + Writer::with_codec(&MANIFEST_LIST_AVRO_SCHEMA_V2, Vec::new(), Codec::Snappy).unwrap(); + writer.append_ser(manifest_entry).unwrap(); + let bs = writer.into_inner().unwrap(); + + let parsed_manifest_list = + ManifestList::parse_with_version(&bs, FormatVersion::V2).unwrap(); + + assert_eq!(manifest_list, parsed_manifest_list); + } + + #[tokio::test] + async fn test_parse_manifest_list_v3() { + let manifest_list = ManifestList { + entries: vec![ + ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: 1, + min_sequence_number: 1, + added_snapshot_id: 377075049360453639, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: Some(10), + }, + ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m1.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 2, + content: ManifestContentType::Data, + sequence_number: 1, + min_sequence_number: 1, + added_snapshot_id: 377075049360453639, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1_f32).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1_f32).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: Some(13), + } + ] + }; + + let file_io = FileIO::new_with_fs(); + + let tmp_dir = TempDir::new().unwrap(); + let file_name = "simple_manifest_list_v3.avro"; + let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name); + + let mut writer = ManifestListWriter::v3( + file_io + .new_output(full_path.clone()) + .unwrap() + .writer() + .await + .unwrap(), + 377075049360453639, + Some(377075049360453639), + 1, + Some(10), + ); + + writer + .add_manifests(manifest_list.entries.clone().into_iter()) + .unwrap(); + writer.close().await.unwrap(); + + let bs = fs::read(full_path).expect("read_file must succeed"); + + let parsed_manifest_list = + ManifestList::parse_with_version(&bs, FormatVersion::V3).unwrap(); + + assert_eq!(manifest_list, parsed_manifest_list); + } +} diff --git a/crates/iceberg/src/spec/manifest_list/reader.rs b/crates/iceberg/src/spec/manifest_list/reader.rs new file mode 100644 index 0000000000..fa1a181cb7 --- /dev/null +++ b/crates/iceberg/src/spec/manifest_list/reader.rs @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use super::ManifestList; +use crate::encryption::{EncryptedInputFile, EncryptionManager}; +use crate::error::Result; +use crate::io::FileIO; +use crate::spec::{SnapshotRef, TableMetadataRef}; +use crate::{Error, ErrorKind}; + +/// A manifest list reader that encapsulates the logic for loading and parsing a [`ManifestList`] +/// from a snapshot. +pub struct ManifestListReader { + snapshot: SnapshotRef, + file_io: FileIO, + table_metadata: TableMetadataRef, + encryption_manager: Option>, +} + +impl ManifestListReader { + pub(crate) fn new( + snapshot: SnapshotRef, + file_io: FileIO, + table_metadata: TableMetadataRef, + encryption_manager: Option>, + ) -> Self { + Self { + snapshot, + file_io, + table_metadata, + encryption_manager, + } + } + + /// Loads and returns the [`ManifestList`] for this snapshot. + pub async fn load(&self) -> Result { + let manifest_list_content = match self.snapshot.encryption_key_id() { + Some(key_id) => { + let em = self.encryption_manager.as_ref().ok_or_else(|| { + Error::new( + ErrorKind::PreconditionFailed, + "Snapshot has encryption_key_id but no EncryptionManager configured on Table", + ) + })?; + let key_metadata = em.decrypt_manifest_list_key_metadata(key_id).await?; + let input = self.file_io.new_input(self.snapshot.manifest_list())?; + EncryptedInputFile::new(input, key_metadata).read().await? + } + None => { + self.file_io + .new_input(self.snapshot.manifest_list())? + .read() + .await? + } + }; + ManifestList::parse_with_version( + &manifest_list_content, + self.table_metadata.format_version(), + ) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::ManifestListReader; + use crate::encryption::kms::MemoryKeyManagementClient; + use crate::encryption::{EncryptionManager, SensitiveBytes}; + use crate::io::FileIO; + use crate::spec::{TableMetadata, TableMetadataRef}; + + fn encryption_test_metadata() -> TableMetadata { + let path = format!( + "{}/testdata/table_metadata/TableMetadataV3ValidEncryption.json", + env!("CARGO_MANIFEST_DIR"), + ); + let json = std::fs::read_to_string(path).unwrap(); + serde_json::from_str(&json).unwrap() + } + + #[tokio::test] + async fn load_manifest_list_errors_when_encrypted_but_no_manager_configured() { + let mut metadata = encryption_test_metadata(); + + let manifest_list_path = format!( + "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro", + env!("CARGO_MANIFEST_DIR"), + ); + let snapshot = metadata.snapshots.get_mut(&1).unwrap(); + let mut patched = snapshot.as_ref().clone(); + patched.manifest_list = manifest_list_path; + *snapshot = Arc::new(patched); + + let snapshot_ref = metadata.current_snapshot().unwrap().clone(); + let metadata_ref = TableMetadataRef::new(metadata); + + let err = ManifestListReader::new(snapshot_ref, FileIO::new_with_fs(), metadata_ref, None) + .load() + .await + .unwrap_err(); + assert_eq!(err.kind(), crate::ErrorKind::PreconditionFailed); + } + + #[tokio::test] + async fn load_manifest_list_decrypts_roundtrip() { + let mut metadata = encryption_test_metadata(); + + let manifest_list_path = format!( + "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro", + env!("CARGO_MANIFEST_DIR"), + ); + let snapshot = metadata.snapshots.get_mut(&1).unwrap(); + let mut patched = snapshot.as_ref().clone(); + patched.manifest_list = manifest_list_path; + *snapshot = Arc::new(patched); + + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key_bytes( + "master-1", + SensitiveBytes::new([ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, + 0x0e, 0x0f, + ]), + ) + .unwrap(); + + let mgr = EncryptionManager::builder() + .kms_client(Arc::new(kms) as Arc) + .table_key_id("master-1") + .encryption_keys(metadata.encryption_keys.clone()) + .build(); + + let snapshot_ref = metadata.current_snapshot().unwrap().clone(); + let metadata_ref = TableMetadataRef::new(metadata); + + let manifest_list = ManifestListReader::new( + snapshot_ref, + FileIO::new_with_fs(), + metadata_ref, + Some(Arc::new(mgr)), + ) + .load() + .await + .unwrap(); + assert_eq!(manifest_list.entries().len(), 0); + } +} diff --git a/crates/iceberg/src/spec/manifest_list/writer.rs b/crates/iceberg/src/spec/manifest_list/writer.rs new file mode 100644 index 0000000000..a52dae7dbf --- /dev/null +++ b/crates/iceberg/src/spec/manifest_list/writer.rs @@ -0,0 +1,683 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use apache_avro::Writer; +use bytes::Bytes; + +use super::_const_schema::{ + MANIFEST_LIST_AVRO_SCHEMA_V1, MANIFEST_LIST_AVRO_SCHEMA_V2, MANIFEST_LIST_AVRO_SCHEMA_V3, +}; +use super::_serde::{ManifestFileV1, ManifestFileV2, ManifestFileV3}; +use super::{FormatVersion, ManifestContentType, ManifestFile, UNASSIGNED_SEQUENCE_NUMBER}; +use crate::avro::new_writer; +use crate::error::{Result, invalid_data}; +use crate::io::{FileMetadata, FileWrite}; +use crate::{Error, ErrorKind}; + +/// A manifest list writer. +pub struct ManifestListWriter { + format_version: FormatVersion, + writer: Box, + avro_writer: Writer<'static, Vec>, + sequence_number: i64, + snapshot_id: i64, + next_row_id: Option, +} + +impl std::fmt::Debug for ManifestListWriter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ManifestListWriter") + .field("format_version", &self.format_version) + .field("avro_writer", &self.avro_writer.schema()) + .finish_non_exhaustive() + } +} + +impl ManifestListWriter { + /// Get the next row ID that will be assigned to the next data manifest added. + pub fn next_row_id(&self) -> Option { + self.next_row_id + } + + /// Construct a v1 [`ManifestListWriter`] that writes to a provided [`FileWrite`]. + pub fn v1( + writer: Box, + snapshot_id: i64, + parent_snapshot_id: Option, + ) -> Self { + let mut metadata = HashMap::from_iter([ + ("snapshot-id".to_string(), snapshot_id.to_string()), + ("format-version".to_string(), "1".to_string()), + ]); + if let Some(parent_snapshot_id) = parent_snapshot_id { + metadata.insert( + "parent-snapshot-id".to_string(), + parent_snapshot_id.to_string(), + ); + } + Self::new(FormatVersion::V1, writer, metadata, 0, snapshot_id, None) + } + + /// Construct a v2 [`ManifestListWriter`] that writes to a provided [`FileWrite`]. + pub fn v2( + writer: Box, + snapshot_id: i64, + parent_snapshot_id: Option, + sequence_number: i64, + ) -> Self { + let mut metadata = HashMap::from_iter([ + ("snapshot-id".to_string(), snapshot_id.to_string()), + ("sequence-number".to_string(), sequence_number.to_string()), + ("format-version".to_string(), "2".to_string()), + ]); + metadata.insert( + "parent-snapshot-id".to_string(), + parent_snapshot_id + .map(|v| v.to_string()) + .unwrap_or("null".to_string()), + ); + Self::new( + FormatVersion::V2, + writer, + metadata, + sequence_number, + snapshot_id, + None, + ) + } + + /// Construct a v3 [`ManifestListWriter`] that writes to a provided [`FileWrite`]. + pub fn v3( + writer: Box, + snapshot_id: i64, + parent_snapshot_id: Option, + sequence_number: i64, + first_row_id: Option, // Always None for delete manifests + ) -> Self { + let mut metadata = HashMap::from_iter([ + ("snapshot-id".to_string(), snapshot_id.to_string()), + ("sequence-number".to_string(), sequence_number.to_string()), + ("format-version".to_string(), "3".to_string()), + ]); + metadata.insert( + "parent-snapshot-id".to_string(), + parent_snapshot_id + .map(|v| v.to_string()) + .unwrap_or("null".to_string()), + ); + metadata.insert( + "first-row-id".to_string(), + first_row_id + .map(|v| v.to_string()) + .unwrap_or("null".to_string()), + ); + Self::new( + FormatVersion::V3, + writer, + metadata, + sequence_number, + snapshot_id, + first_row_id, + ) + } + + fn new( + format_version: FormatVersion, + writer: Box, + metadata: HashMap, + sequence_number: i64, + snapshot_id: i64, + first_row_id: Option, + ) -> Self { + let avro_schema = match format_version { + FormatVersion::V1 => &MANIFEST_LIST_AVRO_SCHEMA_V1, + FormatVersion::V2 => &MANIFEST_LIST_AVRO_SCHEMA_V2, + FormatVersion::V3 => &MANIFEST_LIST_AVRO_SCHEMA_V3, + }; + let mut avro_writer = new_writer(avro_schema, Vec::new()).expect( + "Manifest list Avro schemas should resolve because they refer to no named types.", + ); + for (key, value) in metadata { + avro_writer + .add_user_metadata(key, value) + .expect("Avro metadata should be added to the writer before the first record."); + } + Self { + format_version, + writer, + avro_writer, + sequence_number, + snapshot_id, + next_row_id: first_row_id, + } + } + + /// Append manifests to be written. + /// + /// If V3 Manifests are added and the `first_row_id` of any data manifest is unassigned, + /// it will be assigned based on the `next_row_id` of the writer, and the `next_row_id` of the writer will be updated accordingly. + /// If `first_row_id` is already assigned, it will be validated against the `next_row_id` of the writer. + pub fn add_manifests(&mut self, manifests: impl Iterator) -> Result<()> { + match self.format_version { + FormatVersion::V1 => { + for manifest in manifests { + let manifests: ManifestFileV1 = manifest.try_into()?; + self.avro_writer.append_ser(manifests)?; + } + } + FormatVersion::V2 | FormatVersion::V3 => { + for mut manifest in manifests { + self.assign_sequence_numbers(&mut manifest)?; + + if self.format_version == FormatVersion::V2 { + let manifest_entry: ManifestFileV2 = manifest.try_into()?; + self.avro_writer.append_ser(manifest_entry)?; + } else if self.format_version == FormatVersion::V3 { + self.assign_first_row_id(&mut manifest)?; + let manifest_entry: ManifestFileV3 = manifest.try_into()?; + self.avro_writer.append_ser(manifest_entry)?; + } + } + } + } + Ok(()) + } + + /// Write the manifest list and return its stored size. + pub async fn close(mut self) -> Result { + let data = self.avro_writer.into_inner()?; + self.writer.write(Bytes::from(data)).await?; + self.writer.close().await + } + + /// Assign sequence numbers to manifest if they are unassigned + fn assign_sequence_numbers(&self, manifest: &mut ManifestFile) -> Result<()> { + if manifest.sequence_number == UNASSIGNED_SEQUENCE_NUMBER { + if manifest.added_snapshot_id != self.snapshot_id { + return Err(invalid_data!( + "Found unassigned sequence number for a manifest from snapshot {}.", + manifest.added_snapshot_id + )); + } + manifest.sequence_number = self.sequence_number; + } + + if manifest.min_sequence_number == UNASSIGNED_SEQUENCE_NUMBER { + if manifest.added_snapshot_id != self.snapshot_id { + return Err(invalid_data!( + "Found unassigned sequence number for a manifest from snapshot {}.", + manifest.added_snapshot_id + )); + } + manifest.min_sequence_number = self.sequence_number; + } + + Ok(()) + } + + /// Returns number of newly assigned first-row-ids, if any. + fn assign_first_row_id(&mut self, manifest: &mut ManifestFile) -> Result<()> { + match manifest.content { + ManifestContentType::Data => { + match (self.next_row_id, manifest.first_row_id) { + (Some(_), Some(_)) => { + // Case: Manifest with already assigned first row ID. + // No need to increase next_row_id, as this manifest is already assigned. + } + (None, Some(manifest_first_row_id)) => { + // Case: Assigned first row ID for data manifest, but the writer does not have a next-row-id assigned. + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Found invalid first-row-id assignment for Manifest {}. Writer does not have a next-row-id assigned, but the manifest has first-row-id assigned to {}.", + manifest.manifest_path, manifest_first_row_id, + ), + )); + } + (Some(writer_next_row_id), None) => { + // Case: Unassigned first row ID for data manifest. This is either a new + // manifest, or a manifest from a pre-v3 snapshot. We need to assign one. + let (existing_rows_count, added_rows_count) = + require_row_counts_in_manifest(manifest)?; + manifest.first_row_id = Some(writer_next_row_id); + + self.next_row_id = writer_next_row_id + .checked_add(existing_rows_count) + .and_then(|sum| sum.checked_add(added_rows_count)) + .ok_or_else(|| { + invalid_data!( + "Row ID overflow when computing next row ID for Manifest {}. Next Row ID: {writer_next_row_id}, Existing Rows Count: {existing_rows_count}, Added Rows Count: {added_rows_count}", + manifest.manifest_path + ) + }).map(Some)?; + } + (None, None) => { + // Case: Table without row lineage. No action needed. + } + } + } + ManifestContentType::Deletes => { + // Deletes never have a first-row-id assigned. + manifest.first_row_id = None; + } + }; + + Ok(()) + } +} + +fn require_row_counts_in_manifest(manifest: &ManifestFile) -> Result<(u64, u64)> { + let existing_rows_count = manifest.existing_rows_count.ok_or_else(|| { + invalid_data!( + "Cannot include a Manifest without existing-rows-count to a table with row lineage enabled. Manifest path: {}", + manifest.manifest_path, + ) + })?; + let added_rows_count = manifest.added_rows_count.ok_or_else(|| { + invalid_data!( + "Cannot include a Manifest without added-rows-count to a table with row lineage enabled. Manifest path: {}", + manifest.manifest_path, + ) + })?; + Ok((existing_rows_count, added_rows_count)) +} + +#[cfg(test)] +mod test { + use std::fs; + use std::path::Path; + use std::sync::Arc; + + use tempfile::TempDir; + + use super::ManifestListWriter; + use crate::avro::assert_deflate_codec; + use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; + use crate::encryption::{EncryptedInputFile, EncryptionManager}; + use crate::io::{FileIO, FileWrite}; + use crate::spec::{ + Datum, FieldSummary, ManifestContentType, ManifestFile, ManifestList, + UNASSIGNED_SEQUENCE_NUMBER, + }; + + #[tokio::test] + async fn test_manifest_list_writer_v1() { + let expected_manifest_list = ManifestList { + entries: vec![ManifestFile { + manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(), + manifest_length: 5806, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: 0, + min_sequence_number: 0, + added_snapshot_id: 1646658105718557341, + added_files_count: Some(3), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}], + ), + key_metadata: None, + first_row_id: None, + }] + }; + + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("manifest_list_v1.avro"); + let io = FileIO::new_with_fs(); + let file_writer = file_writer(&path, io).await; + + let mut writer = ManifestListWriter::v1(file_writer, 1646658105718557341, Some(0)); + writer + .add_manifests(expected_manifest_list.entries.clone().into_iter()) + .unwrap(); + writer.close().await.unwrap(); + + let bs = fs::read(path).unwrap(); + assert_deflate_codec(&bs); + + let manifest_list = + ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V1).unwrap(); + assert_eq!(manifest_list, expected_manifest_list); + + temp_dir.close().unwrap(); + } + + #[tokio::test] + async fn test_manifest_list_writer_v2() { + let snapshot_id = 377075049360453639; + let seq_num = 1; + let mut expected_manifest_list = ManifestList { + entries: vec![ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + added_snapshot_id: snapshot_id, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: None, + }] + }; + + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("manifest_list_v2.avro"); + let io = FileIO::new_with_fs(); + let file_writer = file_writer(&path, io).await; + + let mut writer = ManifestListWriter::v2(file_writer, snapshot_id, Some(0), seq_num); + writer + .add_manifests(expected_manifest_list.entries.clone().into_iter()) + .unwrap(); + writer.close().await.unwrap(); + + let bs = fs::read(path).unwrap(); + assert_deflate_codec(&bs); + let manifest_list = + ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V2).unwrap(); + expected_manifest_list.entries[0].sequence_number = seq_num; + expected_manifest_list.entries[0].min_sequence_number = seq_num; + assert_eq!(manifest_list, expected_manifest_list); + + temp_dir.close().unwrap(); + } + + #[tokio::test] + async fn test_manifest_list_writer_v3() { + let snapshot_id = 377075049360453639; + let seq_num = 1; + let mut expected_manifest_list = ManifestList { + entries: vec![ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + added_snapshot_id: snapshot_id, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: Some(10), + }] + }; + + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("manifest_list_v2.avro"); + let io = FileIO::new_with_fs(); + let file_writer = file_writer(&path, io).await; + + let mut writer = + ManifestListWriter::v3(file_writer, snapshot_id, Some(0), seq_num, Some(10)); + writer + .add_manifests(expected_manifest_list.entries.clone().into_iter()) + .unwrap(); + writer.close().await.unwrap(); + + let bs = fs::read(path).unwrap(); + assert_deflate_codec(&bs); + let manifest_list = + ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V3).unwrap(); + expected_manifest_list.entries[0].sequence_number = seq_num; + expected_manifest_list.entries[0].min_sequence_number = seq_num; + expected_manifest_list.entries[0].first_row_id = Some(10); + assert_eq!(manifest_list, expected_manifest_list); + + temp_dir.close().unwrap(); + } + + #[tokio::test] + async fn test_manifest_list_writer_v1_as_v2() { + let expected_manifest_list = ManifestList { + entries: vec![ManifestFile { + manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(), + manifest_length: 5806, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: 0, + min_sequence_number: 0, + added_snapshot_id: 1646658105718557341, + added_files_count: Some(3), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: None, + }] + }; + + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("manifest_list_v1.avro"); + let io = FileIO::new_with_fs(); + let file_writer = file_writer(&path, io).await; + + let mut writer = ManifestListWriter::v1(file_writer, 1646658105718557341, Some(0)); + writer + .add_manifests(expected_manifest_list.entries.clone().into_iter()) + .unwrap(); + writer.close().await.unwrap(); + + let bs = fs::read(path).unwrap(); + + let manifest_list = + ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V2).unwrap(); + assert_eq!(manifest_list, expected_manifest_list); + + temp_dir.close().unwrap(); + } + + #[tokio::test] + async fn test_manifest_list_writer_v1_as_v3() { + let expected_manifest_list = ManifestList { + entries: vec![ManifestFile { + manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(), + manifest_length: 5806, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: 0, + min_sequence_number: 0, + added_snapshot_id: 1646658105718557341, + added_files_count: Some(3), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: None, + }] + }; + + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("manifest_list_v1.avro"); + let io = FileIO::new_with_fs(); + let file_writer = file_writer(&path, io).await; + + let mut writer = ManifestListWriter::v1(file_writer, 1646658105718557341, Some(0)); + writer + .add_manifests(expected_manifest_list.entries.clone().into_iter()) + .unwrap(); + writer.close().await.unwrap(); + + let bs = fs::read(path).unwrap(); + + let manifest_list = + ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V3).unwrap(); + assert_eq!(manifest_list, expected_manifest_list); + + temp_dir.close().unwrap(); + } + + #[tokio::test] + async fn test_manifest_list_writer_v2_as_v3() { + let snapshot_id = 377075049360453639; + let seq_num = 1; + let mut expected_manifest_list = ManifestList { + entries: vec![ManifestFile { + manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(), + manifest_length: 6926, + partition_spec_id: 1, + content: ManifestContentType::Data, + sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + added_snapshot_id: snapshot_id, + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(3), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some( + vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}] + ), + key_metadata: None, + first_row_id: None, + }] + }; + + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("manifest_list_v2.avro"); + let io = FileIO::new_with_fs(); + let file_writer = file_writer(&path, io).await; + + let mut writer = ManifestListWriter::v2(file_writer, snapshot_id, Some(0), seq_num); + writer + .add_manifests(expected_manifest_list.entries.clone().into_iter()) + .unwrap(); + writer.close().await.unwrap(); + + let bs = fs::read(path).unwrap(); + + let manifest_list = + ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V3).unwrap(); + expected_manifest_list.entries[0].sequence_number = seq_num; + expected_manifest_list.entries[0].min_sequence_number = seq_num; + assert_eq!(manifest_list, expected_manifest_list); + + temp_dir.close().unwrap(); + } + + #[tokio::test] + async fn test_manifest_list_writer_v3_encrypted_round_trip() { + let (mgr, file_io) = fresh_encryption_manager_and_io(); + let path = "memory:///manifest_list_v3_encrypted.avro"; + + let encrypted_output = mgr.encrypt(file_io.new_output(path).unwrap()); + + let snapshot_id = 9_000_000_000_000_001i64; + let seq_num = 7i64; + let mut expected = ManifestList { + entries: vec![ManifestFile { + manifest_path: "memory:///encrypted/v3_m0.avro".to_string(), + manifest_length: 1234, + partition_spec_id: 0, + content: ManifestContentType::Data, + sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER, + added_snapshot_id: snapshot_id, + added_files_count: Some(2), + existing_files_count: Some(0), + deleted_files_count: Some(0), + added_rows_count: Some(10), + existing_rows_count: Some(0), + deleted_rows_count: Some(0), + partitions: Some(vec![FieldSummary { + contains_null: false, + contains_nan: Some(false), + lower_bound: Some(Datum::long(1).to_bytes().unwrap()), + upper_bound: Some(Datum::long(1).to_bytes().unwrap()), + }]), + key_metadata: None, + first_row_id: None, + }], + }; + + let file_writer = encrypted_output.writer().await.unwrap(); + let mut writer = ManifestListWriter::v3(file_writer, snapshot_id, Some(0), seq_num, None); + writer + .add_manifests(expected.entries.clone().into_iter()) + .unwrap(); + let file_metadata = writer.close().await.unwrap(); + + let raw_bytes = file_io.new_input(path).unwrap().read().await.unwrap(); + assert!( + ManifestList::parse_with_version(&raw_bytes, crate::spec::FormatVersion::V3).is_err(), + "raw bytes should be ciphertext, not parseable as Avro" + ); + + let key_metadata = encrypted_output.key_metadata_with_saved_file_metadata(&file_metadata); + assert_eq!(key_metadata.file_length(), Some(raw_bytes.len() as u64)); + let plaintext = EncryptedInputFile::new(file_io.new_input(path).unwrap(), key_metadata) + .read() + .await + .unwrap(); + let manifest_list = + ManifestList::parse_with_version(&plaintext, crate::spec::FormatVersion::V3).unwrap(); + + expected.entries[0].sequence_number = seq_num; + expected.entries[0].min_sequence_number = seq_num; + assert_eq!(manifest_list, expected); + } + + fn fresh_encryption_manager_and_io() -> (EncryptionManager, FileIO) { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + let mgr = EncryptionManager::builder() + .kms_client(Arc::new(kms) as Arc) + .table_key_id("master-1") + .build(); + (mgr, FileIO::new_with_memory()) + } + + async fn file_writer(path: &Path, io: FileIO) -> Box { + io.new_output(path.to_str().unwrap()) + .unwrap() + .writer() + .await + .unwrap() + } +} diff --git a/crates/iceberg/src/spec/mod.rs b/crates/iceberg/src/spec/mod.rs index 90dafcc110..7bbb37c797 100644 --- a/crates/iceberg/src/spec/mod.rs +++ b/crates/iceberg/src/spec/mod.rs @@ -19,6 +19,7 @@ mod datatypes; mod encrypted_key; +pub mod geospatial; mod manifest; mod manifest_list; mod name_mapping; @@ -30,6 +31,7 @@ mod sort; mod statistic_file; mod table_metadata; mod table_metadata_builder; +mod table_properties; mod transform; mod values; mod view_metadata; @@ -38,6 +40,7 @@ mod view_version; pub use datatypes::*; pub use encrypted_key::*; +pub use geospatial::{GeographyType, GeometryType}; pub use manifest::*; pub use manifest_list::*; pub use name_mapping::*; @@ -48,7 +51,10 @@ pub use snapshot_summary::*; pub use sort::*; pub use statistic_file::*; pub use table_metadata::*; +pub(crate) use table_metadata_builder::FIRST_FIELD_ID; +pub use table_properties::*; pub use transform::*; +pub(crate) use values::decimal_utils; pub use values::*; pub use view_metadata::*; pub use view_version::*; diff --git a/crates/iceberg/src/spec/name_mapping/mod.rs b/crates/iceberg/src/spec/name_mapping/mod.rs index db9e44c290..95b8642fa6 100644 --- a/crates/iceberg/src/spec/name_mapping/mod.rs +++ b/crates/iceberg/src/spec/name_mapping/mod.rs @@ -17,13 +17,14 @@ //! Iceberg name mapping. +use std::str::FromStr; use std::sync::Arc; use serde::{Deserialize, Serialize}; use serde_with::{DefaultOnNull, serde_as}; -/// Property name for name mapping. -pub const DEFAULT_SCHEMA_NAME_MAPPING: &str = "schema.name-mapping.default"; +use crate::error::invalid_data; +use crate::{Error, Result}; /// Iceberg fallback field name to ID mapping. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] @@ -44,6 +45,17 @@ impl NameMapping { } } +impl FromStr for NameMapping { + type Err = Error; + + /// Parses a [`NameMapping`] from its JSON representation. + fn from_str(value: &str) -> Result { + serde_json::from_str(value).map_err(|error| { + invalid_data!("Failed to parse value as a NameMapping").with_source(error) + }) + } +} + /// Maps field names to IDs. #[serde_as] #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] diff --git a/crates/iceberg/src/spec/partition.rs b/crates/iceberg/src/spec/partition.rs index fff9b62d78..bd6ed35429 100644 --- a/crates/iceberg/src/spec/partition.rs +++ b/crates/iceberg/src/spec/partition.rs @@ -25,9 +25,10 @@ use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; use super::transform::Transform; -use super::{NestedField, Schema, SchemaRef, StructType}; +use super::{NestedField, PrimitiveType, Schema, SchemaRef, StructType}; +use crate::Result; +use crate::error::invalid_data; use crate::spec::Struct; -use crate::{Error, ErrorKind, Result}; pub(crate) const UNPARTITIONED_LAST_ASSIGNED_ID: i32 = 999; pub(crate) const DEFAULT_PARTITION_SPEC_ID: i32 = 0; @@ -103,8 +104,32 @@ impl PartitionSpec { } /// Returns the partition type of this partition spec. + /// If a source column is absent, preserves fixed transform result types and uses + /// unknown for result types that depend on the source type. pub fn partition_type(&self, schema: &Schema) -> Result { - PartitionSpecBuilder::partition_type(&self.fields, schema) + let mut struct_fields = Vec::with_capacity(self.fields.len()); + for partition_field in &self.fields { + let res_type = match schema.field_by_id(partition_field.source_id) { + Some(field) => partition_field.transform.result_type(&field.field_type)?, + // Historical specs may reference dropped source columns. Retain every + // field's position and any result type that is independent of its source. + None => match partition_field.transform { + Transform::Bucket(_) | Transform::Year | Transform::Month | Transform::Hour => { + PrimitiveType::Int.into() + } + Transform::Day => PrimitiveType::Date.into(), + Transform::Unknown => PrimitiveType::String.into(), + Transform::Identity | Transform::Truncate(_) | Transform::Void => { + PrimitiveType::Unknown.into() + } + }, + }; + struct_fields.push( + NestedField::optional(partition_field.field_id, &partition_field.name, res_type) + .into(), + ); + } + Ok(StructType::new(struct_fields)) } /// Convert to unbound partition spec @@ -155,7 +180,9 @@ impl PartitionSpec { true } - pub(crate) fn partition_to_path(&self, data: &Struct, schema: SchemaRef) -> String { + /// Returns partition path string containing partition type and partition + /// value as key-value pairs. + pub fn partition_to_path(&self, data: &Struct, schema: SchemaRef) -> String { let partition_type = self.partition_type(&schema).unwrap(); let field_types = partition_type.fields(); @@ -164,13 +191,14 @@ impl PartitionSpec { .enumerate() .map(|(i, field)| { let value = data[i].as_ref(); - format!( - "{}={}", - field.name, - field - .transform - .to_human_string(&field_types[i].field_type, value) - ) + form_urlencoded::Serializer::new(String::new()) + .append_pair( + &field.name, + &field + .transform + .to_human_string(&field_types[i].field_type, value), + ) + .finish() }) .join("/") } @@ -194,6 +222,15 @@ impl PartitionKey { Self { spec, schema, data } } + /// Creates a new partition key from another partition key, with a new data field. + pub fn copy_with_data(&self, data: Struct) -> Self { + Self { + spec: self.spec.clone(), + schema: self.schema.clone(), + data, + } + } + /// Generates a partition path based on the partition values. pub fn to_path(&self) -> String { self.spec.partition_to_path(&self.data, self.schema.clone()) @@ -207,24 +244,177 @@ impl PartitionKey { Some(pk) => pk.spec.is_unpartitioned(), } } + + /// Returns the associated [`PartitionSpec`]. + pub fn spec(&self) -> &PartitionSpec { + &self.spec + } + + /// Returns the associated [`SchemaRef`]. + pub fn schema(&self) -> &SchemaRef { + &self.schema + } + + /// Returns the associated [`Struct`]. + pub fn data(&self) -> &Struct { + &self.data + } } /// Reference to [`UnboundPartitionSpec`]. pub type UnboundPartitionSpecRef = Arc; /// Unbound partition field can be built without a schema and later bound to a schema. +/// +/// The fields are private so that an instance is always well formed: in particular +/// `source_ids` holds at least one id, which [`UnboundPartitionField::builder`] checks when the +/// field is built. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, TypedBuilder)] -#[serde(rename_all = "kebab-case")] +#[serde( + try_from = "self::_serde::UnboundPartitionFieldSerde", + into = "self::_serde::UnboundPartitionFieldSerde" +)] +#[builder(build_method(into = Result))] pub struct UnboundPartitionField { - /// A source column id from the table’s schema - pub source_id: i32, + /// The source column ids from the table’s schema. A single-argument transform reads one + /// id; a v3 multi-argument transform reads several. + source_ids: Vec, /// A partition field id that is used to identify a partition field and is unique within a partition spec. /// In v2 table metadata, it is unique across all partition specs. #[builder(default, setter(strip_option(fallback = field_id_opt)))] - pub field_id: Option, + field_id: Option, /// A partition name. - pub name: String, + #[builder(setter(into))] + name: String, /// A transform that is applied to the source column to produce a partition value. - pub transform: Transform, + transform: Transform, +} + +impl UnboundPartitionField { + /// The single source column id this field reads. + /// + /// Returns an error for a multi-argument field, which reads several columns and therefore + /// has no single source id. Use [`Self::source_ids`] to handle both shapes. + pub fn source_id(&self) -> Result { + match self.source_ids.as_slice() { + [source_id] => Ok(*source_id), + source_ids => Err(invalid_data!( + "Partition field '{}' reads {} source columns and has no single source id", + self.name, + source_ids.len() + )), + } + } + + /// The source column ids this field reads, in order. Never empty. + pub fn source_ids(&self) -> &[i32] { + &self.source_ids + } + + /// The partition field id, when one was assigned. + pub fn field_id(&self) -> Option { + self.field_id + } + + /// The partition name. + pub fn name(&self) -> &str { + &self.name + } + + /// The transform applied to the source columns to produce a partition value. + pub fn transform(&self) -> Transform { + self.transform + } + + /// Return this field with the given partition field id assigned. + pub fn with_field_id(self, field_id: i32) -> Self { + Self { + field_id: Some(field_id), + ..self + } + } + + fn validate(&self) -> Result<()> { + if self.source_ids.is_empty() { + return Err(invalid_data!("Empty source-ids is not allowed")); + } + Ok(()) + } +} + +impl From for Result { + fn from(field: UnboundPartitionField) -> Self { + field.validate()?; + Ok(field) + } +} + +mod _serde { + use serde::{Deserialize, Serialize}; + + use super::UnboundPartitionField; + use crate::Error; + use crate::error::invalid_data; + use crate::spec::Transform; + + /// Per the spec a single-argument field carries `source-id` and a multi-argument field + /// carries `source-ids`. Either spelling is read, but not both at once; the one that + /// matches the field is written. + #[derive(Serialize, Deserialize)] + #[serde(rename_all = "kebab-case")] + pub(super) struct UnboundPartitionFieldSerde { + #[serde(default, skip_serializing_if = "Option::is_none")] + source_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + source_ids: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + field_id: Option, + name: String, + transform: Transform, + } + + impl TryFrom for UnboundPartitionField { + type Error = Error; + + fn try_from(value: UnboundPartitionFieldSerde) -> Result { + let source_ids = match (value.source_id, value.source_ids) { + (Some(source_id), None) => vec![source_id], + (None, Some(source_ids)) => source_ids, + (Some(_), Some(_)) => { + return Err(invalid_data!( + "source-id and source-ids are mutually exclusive" + )); + } + (None, None) => { + return Err(invalid_data!( + "Either `source-id` or `source-ids` must be present" + )); + } + }; + + UnboundPartitionField::builder() + .source_ids(source_ids) + .field_id_opt(value.field_id) + .name(value.name) + .transform(value.transform) + .build() + } + } + + impl From for UnboundPartitionFieldSerde { + fn from(value: UnboundPartitionField) -> Self { + let (source_id, source_ids) = match value.source_ids.as_slice() { + [source_id] => (Some(*source_id), None), + _ => (None, Some(value.source_ids)), + }; + Self { + source_id, + source_ids, + field_id: value.field_id, + name: value.name, + transform: value.transform, + } + } + } } /// Unbound partition spec can be built without a schema and later bound to a schema. @@ -234,6 +424,7 @@ pub struct UnboundPartitionField { #[serde(rename_all = "kebab-case")] pub struct UnboundPartitionSpec { /// Identifier for PartitionSpec + #[serde(skip_serializing_if = "Option::is_none")] pub(crate) spec_id: Option, /// Details of the partition spec pub(crate) fields: Vec, @@ -246,6 +437,9 @@ impl UnboundPartitionSpec { } /// Bind this unbound partition spec to a schema. + /// + /// Unlike [`PartitionSpecBuilder::add_unbound_field`], this accepts redundant fields, such + /// as two time granularities of one column, because spec evolution produces them. pub fn bind(self, schema: impl Into) -> Result { PartitionSpecBuilder::new_from_unbound(self, schema)?.build() } @@ -287,7 +481,7 @@ fn has_sequential_ids(field_ids: impl Iterator) -> bool { impl From for UnboundPartitionField { fn from(field: PartitionField) -> Self { UnboundPartitionField { - source_id: field.source_id, + source_ids: vec![field.source_id], field_id: Some(field.field_id), name: field.name, transform: field.transform, @@ -327,19 +521,14 @@ impl UnboundPartitionSpecBuilder { } /// Add a new partition field to the partition spec from an unbound partition field. - pub fn add_partition_field( - self, - source_id: i32, - target_name: impl ToString, - transformation: Transform, - ) -> Result { - let field = UnboundPartitionField { - source_id, - field_id: None, - name: target_name.to_string(), - transform: transformation, - }; - self.add_partition_field_internal(field) + pub fn add_partition_field(mut self, field: UnboundPartitionField) -> Result { + self.check_name_set_and_unique(&field.name)?; + self.check_for_redundant_partitions(&field.source_ids, &field.transform)?; + if let Some(partition_field_id) = field.field_id { + self.check_partition_id_unique(partition_field_id)?; + } + self.fields.push(field); + Ok(self) } /// Add multiple partition fields to the partition spec. @@ -349,21 +538,11 @@ impl UnboundPartitionSpecBuilder { ) -> Result { let mut builder = self; for field in fields { - builder = builder.add_partition_field_internal(field)?; + builder = builder.add_partition_field(field)?; } Ok(builder) } - fn add_partition_field_internal(mut self, field: UnboundPartitionField) -> Result { - self.check_name_set_and_unique(&field.name)?; - self.check_for_redundant_partitions(field.source_id, &field.transform)?; - if let Some(partition_field_id) = field.field_id { - self.check_partition_id_unique(partition_field_id)?; - } - self.fields.push(field); - Ok(self) - } - /// Build the unbound partition spec. pub fn build(self) -> UnboundPartitionSpec { UnboundPartitionSpec { @@ -402,7 +581,7 @@ impl PartitionSpecBuilder { Self::new(schema).with_spec_id(unbound.spec_id.unwrap_or(DEFAULT_PARTITION_SPEC_ID)); for field in unbound.fields { - builder = builder.add_unbound_field(field)?; + builder = builder.add_field_allowing_redundancy(field)?; } Ok(builder) } @@ -434,17 +613,14 @@ impl PartitionSpecBuilder { .schema .field_by_name(source_name.as_ref()) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot find source column with name: {} in schema", - source_name.as_ref() - ), + invalid_data!( + "Cannot find source column with name: {} in schema", + source_name.as_ref() ) })? .id; let field = UnboundPartitionField { - source_id, + source_ids: vec![source_id], field_id: None, name: target_name.into(), transform, @@ -457,9 +633,20 @@ impl PartitionSpecBuilder { /// /// If partition field id is set, it is used as the field id. /// Otherwise, a new `field_id` is assigned. - pub fn add_unbound_field(mut self, field: UnboundPartitionField) -> Result { + /// + /// Rejects a field that is redundant with an existing one, such as a second time + /// granularity of the same column. [`UnboundPartitionSpec::bind`] accepts such fields. + pub fn add_unbound_field(self, field: UnboundPartitionField) -> Result { + self.check_name_set_and_unique(&field.name)?; + self.check_for_redundant_partitions(&field.source_ids, &field.transform)?; + self.add_field_allowing_redundancy(field) + } + + /// Same as [`Self::add_unbound_field`] without the redundancy check, as Java's + /// `PartitionSpec.Builder.add` does. Used to bind persisted specs, where evolution can leave + /// several time granularities or void fields for one column. + fn add_field_allowing_redundancy(mut self, field: UnboundPartitionField) -> Result { self.check_name_set_and_unique(&field.name)?; - self.check_for_redundant_partitions(field.source_id, &field.transform)?; Self::check_name_does_not_collide_with_schema(&field, &self.schema)?; Self::check_transform_compatibility(&field, &self.schema)?; if let Some(partition_field_id) = field.field_id { @@ -505,12 +692,8 @@ impl PartitionSpecBuilder { .collect::>(); fn _check_add_1(prev: i32) -> Result { - prev.checked_add(1).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot assign more partition ids. Overflow.", - ) - }) + prev.checked_add(1) + .ok_or_else(|| invalid_data!("Cannot assign more partition ids. Overflow.")) } let mut bound_fields = Vec::with_capacity(fields.len()); @@ -527,7 +710,7 @@ impl PartitionSpecBuilder { }; bound_fields.push(PartitionField { - source_id: field.source_id, + source_id: field.source_id()?, field_id: partition_field_id, name: field.name, transform: field.transform, @@ -537,61 +720,31 @@ impl PartitionSpecBuilder { Ok(bound_fields) } - /// Returns the partition type of this partition spec. - fn partition_type(fields: &Vec, schema: &Schema) -> Result { - let mut struct_fields = Vec::with_capacity(fields.len()); - for partition_field in fields { - let field = schema - .field_by_id(partition_field.source_id) - .ok_or_else(|| { - Error::new( - // This should never occur as check_transform_compatibility - // already ensures that the source field exists in the schema - ErrorKind::Unexpected, - format!( - "No column with source column id {} in schema {:?}", - partition_field.source_id, schema - ), - ) - })?; - let res_type = partition_field.transform.result_type(&field.field_type)?; - let field = - NestedField::optional(partition_field.field_id, &partition_field.name, res_type) - .into(); - struct_fields.push(field); - } - Ok(StructType::new(struct_fields)) - } - /// Ensure that the partition name is unique among columns in the schema. /// Duplicate names are allowed if: /// 1. The column is sourced from the column with the same name. - /// 2. AND the transformation is identity + /// 2. AND [`Transform::may_share_name_with_source`] holds for the transform. fn check_name_does_not_collide_with_schema( field: &UnboundPartitionField, schema: &Schema, ) -> Result<()> { match schema.field_by_name(field.name.as_str()) { Some(schema_collision) => { - if field.transform == Transform::Identity { - if schema_collision.id == field.source_id { + if field.transform.may_share_name_with_source() { + if schema_collision.id == field.source_id()? { Ok(()) } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot create identity partition sourced from different field in schema. Field name '{}' has id `{}` in schema but partition source id is `{}`", - field.name, schema_collision.id, field.source_id - ), + Err(invalid_data!( + "Cannot create identity or void partition sourced from different field in schema. Field name '{}' has id `{}` in schema but partition source id is `{}`", + field.name, + schema_collision.id, + field.source_id()? )) } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot create partition with name: '{}' that conflicts with schema field and is not an identity transform.", - field.name - ), + Err(invalid_data!( + "Cannot create partition with name: '{}' that conflicts with schema field and is not an identity or void transform.", + field.name )) } } @@ -602,24 +755,18 @@ impl PartitionSpecBuilder { /// Ensure that the transformation of the field is compatible with type of the field /// in the schema. Implicitly also checks if the source field exists in the schema. fn check_transform_compatibility(field: &UnboundPartitionField, schema: &Schema) -> Result<()> { - let schema_field = schema.field_by_id(field.source_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot find partition source field with id `{}` in schema", - field.source_id - ), - ) + // A multi-argument field is rejected here on purpose: a bound `PartitionField` has + // no place for more than one source id yet. + let source_id = field.source_id()?; + let schema_field = schema.field_by_id(source_id).ok_or_else(|| { + invalid_data!("Cannot find partition source field with id `{source_id}` in schema") })?; if field.transform != Transform::Void { if !schema_field.field_type.is_primitive() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot partition by non-primitive source field: '{}'.", - schema_field.field_type - ), + return Err(invalid_data!( + "Cannot partition by non-primitive source field: '{}'.", + schema_field.field_type )); } @@ -628,13 +775,10 @@ impl PartitionSpecBuilder { .result_type(&schema_field.field_type) .is_err() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid source type: '{}' for transform: '{}'.", - schema_field.field_type, - field.transform.dedup_name() - ), + return Err(invalid_data!( + "Invalid source type: '{}' for transform: '{}'.", + schema_field.field_type, + field.transform.dedup_name() )); } } @@ -648,36 +792,33 @@ trait CorePartitionSpecValidator { /// Ensure that the partition name is unique among the partition fields and is not empty. fn check_name_set_and_unique(&self, name: &str) -> Result<()> { if name.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot use empty partition name", - )); + return Err(invalid_data!("Cannot use empty partition name")); } if self.fields().iter().any(|f| f.name == name) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot use partition name more than once: {}", name), + return Err(invalid_data!( + "Cannot use partition name more than once: {name}" )); } Ok(()) } /// For a single source-column transformations must be unique. - fn check_for_redundant_partitions(&self, source_id: i32, transform: &Transform) -> Result<()> { + fn check_for_redundant_partitions( + &self, + source_ids: &[i32], + transform: &Transform, + ) -> Result<()> { let collision = self.fields().iter().find(|f| { - f.source_id == source_id && f.transform.dedup_name() == transform.dedup_name() + f.source_ids == source_ids && f.transform.dedup_name() == transform.dedup_name() }); if let Some(collision) = collision { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add redundant partition with source id `{}` and transform `{}`. A partition with the same source id and transform already exists with name `{}`", - source_id, - transform.dedup_name(), - collision.name - ), + Err(invalid_data!( + "Cannot add redundant partition with source ids `{:?}` and transform `{}`. A partition with the same source ids and transform already exists with name `{}`", + source_ids, + transform.dedup_name(), + collision.name )) } else { Ok(()) @@ -687,12 +828,8 @@ trait CorePartitionSpecValidator { /// Check field / partition_id unique within the partition spec if set fn check_partition_id_unique(&self, field_id: i32) -> Result<()> { if self.fields().iter().any(|f| f.field_id == Some(field_id)) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot use field id more than once in one PartitionSpec: {}", - field_id - ), + return Err(invalid_data!( + "Cannot use field id more than once in one PartitionSpec: {field_id}" )); } @@ -717,6 +854,7 @@ impl CorePartitionSpecValidator for UnboundPartitionSpecBuilder { #[cfg(test)] mod tests { use super::*; + use crate::ErrorKind; use crate::spec::{Literal, PrimitiveType, Type}; #[test] @@ -764,14 +902,8 @@ mod tests { fn test_is_unpartitioned() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -787,15 +919,17 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("id".to_string()) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), UnboundPartitionField::builder() - .source_id(2) + .source_ids(vec![2]) .name("name_string".to_string()) .transform(Transform::Void) - .build(), + .build() + .unwrap(), ]) .unwrap() .with_spec_id(1) @@ -810,15 +944,17 @@ mod tests { .with_spec_id(1) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(1) + .source_ids(vec![1]) .name("id_void".to_string()) .transform(Transform::Void) - .build(), + .build() + .unwrap(), UnboundPartitionField::builder() - .source_id(2) + .source_ids(vec![2]) .name("name_void".to_string()) .transform(Transform::Void) - .build(), + .build() + .unwrap(), ]) .unwrap() .build() @@ -856,17 +992,17 @@ mod tests { let partition_spec: UnboundPartitionSpec = serde_json::from_str(spec).unwrap(); assert_eq!(Some(1), partition_spec.spec_id); - assert_eq!(4, partition_spec.fields[0].source_id); + assert_eq!(4, partition_spec.fields[0].source_id().unwrap()); assert_eq!(Some(1000), partition_spec.fields[0].field_id); assert_eq!("ts_day", partition_spec.fields[0].name); assert_eq!(Transform::Day, partition_spec.fields[0].transform); - assert_eq!(1, partition_spec.fields[1].source_id); + assert_eq!(1, partition_spec.fields[1].source_id().unwrap()); assert_eq!(Some(1001), partition_spec.fields[1].field_id); assert_eq!("id_bucket", partition_spec.fields[1].name); assert_eq!(Transform::Bucket(16), partition_spec.fields[1].transform); - assert_eq!(2, partition_spec.fields[2].source_id); + assert_eq!(2, partition_spec.fields[2].source_id().unwrap()); assert_eq!(Some(1002), partition_spec.fields[2].field_id); assert_eq!("id_truncate", partition_spec.fields[2].name); assert_eq!(Transform::Truncate(4), partition_spec.fields[2].transform); @@ -883,24 +1019,57 @@ mod tests { let partition_spec: UnboundPartitionSpec = serde_json::from_str(spec).unwrap(); assert_eq!(None, partition_spec.spec_id); - assert_eq!(4, partition_spec.fields[0].source_id); + assert_eq!(4, partition_spec.fields[0].source_id().unwrap()); assert_eq!(None, partition_spec.fields[0].field_id); assert_eq!("ts_day", partition_spec.fields[0].name); assert_eq!(Transform::Day, partition_spec.fields[0].transform); } + #[test] + fn test_unbound_partition_spec_serialization_skips_none_fields() { + let spec = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .name("ts_day") + .transform(Transform::Day) + .build() + .unwrap(), + ) + .unwrap() + .build(); + + let value = serde_json::to_value(&spec).unwrap(); + let object = value.as_object().unwrap(); + assert!(!object.contains_key("spec-id")); + let field = object["fields"][0].as_object().unwrap(); + assert!(!field.contains_key("field-id")); + + let value = serde_json::to_value(spec.with_spec_id(1)).unwrap(); + let object = value.as_object().unwrap(); + assert_eq!(Some(&serde_json::json!(1)), object.get("spec-id")); + + // Explicit nulls must still deserialize to `None` for backwards + // compatibility. + let spec: UnboundPartitionSpec = serde_json::from_str( + r#"{ + "spec-id": null, + "fields": [ + {"source-id": 4, "name": "ts_day", "transform": "day", "field-id": null} + ] + }"#, + ) + .unwrap(); + assert_eq!(None, spec.spec_id); + assert_eq!(None, spec.fields[0].field_id); + } + #[test] fn test_new_unpartition() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -942,38 +1111,13 @@ mod tests { let partition_spec: PartitionSpec = serde_json::from_str(spec).unwrap(); let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "ts", Type::Primitive(PrimitiveType::Timestamp)).into(), + NestedField::required(4, "ts_day", Type::Primitive(PrimitiveType::Timestamp)) .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), - NestedField::required( - 3, - "ts", - Type::Primitive(crate::spec::PrimitiveType::Timestamp), - ) - .into(), - NestedField::required( - 4, - "ts_day", - Type::Primitive(crate::spec::PrimitiveType::Timestamp), - ) - .into(), - NestedField::required( - 5, - "id_bucket", - Type::Primitive(crate::spec::PrimitiveType::Int), - ) - .into(), - NestedField::required( - 6, - "id_truncate", - Type::Primitive(crate::spec::PrimitiveType::Int), - ) - .into(), + NestedField::required(5, "id_bucket", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(6, "id_truncate", Type::Primitive(PrimitiveType::Int)).into(), ]) .build() .unwrap(); @@ -985,7 +1129,7 @@ mod tests { NestedField::optional( partition_spec.fields[0].field_id, &partition_spec.fields[0].name, - Type::Primitive(crate::spec::PrimitiveType::Date) + Type::Primitive(PrimitiveType::Date) ) ); assert_eq!( @@ -993,7 +1137,7 @@ mod tests { NestedField::optional( partition_spec.fields[1].field_id, &partition_spec.fields[1].name, - Type::Primitive(crate::spec::PrimitiveType::Int) + Type::Primitive(PrimitiveType::Int) ) ); assert_eq!( @@ -1001,7 +1145,7 @@ mod tests { NestedField::optional( partition_spec.fields[2].field_id, &partition_spec.fields[2].name, - Type::Primitive(crate::spec::PrimitiveType::String) + Type::Primitive(PrimitiveType::String) ) ); } @@ -1018,38 +1162,13 @@ mod tests { let partition_spec: PartitionSpec = serde_json::from_str(spec).unwrap(); let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "ts", Type::Primitive(PrimitiveType::Timestamp)).into(), + NestedField::required(4, "ts_day", Type::Primitive(PrimitiveType::Timestamp)) .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), - NestedField::required( - 3, - "ts", - Type::Primitive(crate::spec::PrimitiveType::Timestamp), - ) - .into(), - NestedField::required( - 4, - "ts_day", - Type::Primitive(crate::spec::PrimitiveType::Timestamp), - ) - .into(), - NestedField::required( - 5, - "id_bucket", - Type::Primitive(crate::spec::PrimitiveType::Int), - ) - .into(), - NestedField::required( - 6, - "id_truncate", - Type::Primitive(crate::spec::PrimitiveType::Int), - ) - .into(), + NestedField::required(5, "id_bucket", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(6, "id_truncate", Type::Primitive(PrimitiveType::Int)).into(), ]) .build() .unwrap(); @@ -1059,7 +1178,7 @@ mod tests { } #[test] - fn test_partition_error() { + fn test_partition_type_with_dropped_source_column() { let spec = r#" { "spec-id": 1, @@ -1085,27 +1204,101 @@ mod tests { let partition_spec: PartitionSpec = serde_json::from_str(spec).unwrap(); let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + assert_eq!( + partition_spec.partition_type(&schema).unwrap(), + StructType::new(vec![ + NestedField::optional(1000, "ts_day", PrimitiveType::Date).into(), + NestedField::optional(1001, "id_bucket", PrimitiveType::Int).into(), + NestedField::optional(1002, "id_truncate", PrimitiveType::String).into(), + ]) + ); + + // Dropping all sources must retain every partition field in its original position. + let empty_schema = Schema::builder().build().unwrap(); + assert_eq!( + partition_spec.partition_type(&empty_schema).unwrap(), + StructType::new(vec![ + NestedField::optional(1000, "ts_day", PrimitiveType::Date).into(), + NestedField::optional(1001, "id_bucket", PrimitiveType::Int).into(), + NestedField::optional(1002, "id_truncate", PrimitiveType::Unknown).into(), + ]) + ); + + // Missing sources must not suppress validation of transforms on remaining sources. + let incompatible_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", PrimitiveType::Boolean).into(), ]) .build() .unwrap(); + let err = partition_spec + .partition_type(&incompatible_schema) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + "boolean is not a valid input type of bucket transform" + ); + } - assert!(partition_spec.partition_type(&schema).is_err()); + #[test] + fn test_partition_type_without_source_type() { + let schema = Schema::builder().build().unwrap(); + for (transform, expected_type) in [ + (Transform::Identity, PrimitiveType::Unknown), + (Transform::Truncate(4), PrimitiveType::Unknown), + (Transform::Void, PrimitiveType::Unknown), + (Transform::Bucket(16), PrimitiveType::Int), + (Transform::Year, PrimitiveType::Int), + (Transform::Month, PrimitiveType::Int), + (Transform::Day, PrimitiveType::Date), + (Transform::Hour, PrimitiveType::Int), + (Transform::Unknown, PrimitiveType::String), + ] { + let spec = PartitionSpec { + spec_id: 0, + fields: vec![PartitionField { + source_id: 1, + field_id: 1000, + name: "partition".to_string(), + transform, + }], + }; + assert_eq!( + spec.partition_type(&schema).unwrap(), + StructType::new(vec![ + NestedField::optional(1000, "partition", expected_type).into(), + ]) + ); + } } #[test] fn test_builder_disallow_duplicate_names() { UnboundPartitionSpec::builder() - .add_partition_field(1, "ts_day".to_string(), Transform::Day) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("ts_day") + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap() - .add_partition_field(2, "ts_day".to_string(), Transform::Day) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("ts_day") + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap_err(); } @@ -1113,27 +1306,21 @@ mod tests { fn test_builder_disallow_duplicate_field_ids() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); PartitionSpec::builder(schema.clone()) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(1000), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1000), name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1145,34 +1332,23 @@ mod tests { fn test_builder_auto_assign_field_ids() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), - NestedField::required( - 3, - "ts", - Type::Primitive(crate::spec::PrimitiveType::Timestamp), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "ts", Type::Primitive(PrimitiveType::Timestamp)).into(), ]) .build() .unwrap(); let spec = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], name: "id".to_string(), transform: Transform::Identity, field_id: Some(1012), }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], name: "name_void".to_string(), transform: Transform::Void, field_id: None, @@ -1180,7 +1356,7 @@ mod tests { .unwrap() // Should keep its ID even if its lower .add_unbound_field(UnboundPartitionField { - source_id: 3, + source_ids: vec![3], name: "year".to_string(), transform: Transform::Year, field_id: Some(1), @@ -1198,14 +1374,8 @@ mod tests { fn test_builder_valid_schema() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -1244,8 +1414,7 @@ mod tests { fn test_collision_with_schema_name() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), ]) .build() .unwrap(); @@ -1258,7 +1427,7 @@ mod tests { let err = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id".to_string(), transform: Transform::Bucket(16), @@ -1271,14 +1440,8 @@ mod tests { fn test_builder_collision_is_ok_for_identity_transforms() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "number", - Type::Primitive(crate::spec::PrimitiveType::Int), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "number", Type::Primitive(PrimitiveType::Int)).into(), ]) .build() .unwrap(); @@ -1291,7 +1454,7 @@ mod tests { PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id".to_string(), transform: Transform::Identity, @@ -1304,7 +1467,7 @@ mod tests { PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "id".to_string(), transform: Transform::Identity, @@ -1316,20 +1479,9 @@ mod tests { fn test_builder_all_source_ids_must_exist() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), - NestedField::required( - 3, - "ts", - Type::Primitive(crate::spec::PrimitiveType::Timestamp), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "ts", Type::Primitive(PrimitiveType::Timestamp)).into(), ]) .build() .unwrap(); @@ -1339,13 +1491,13 @@ mod tests { .with_spec_id(1) .add_unbound_fields(vec![ UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), }, UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "name".to_string(), transform: Transform::Identity, @@ -1360,13 +1512,13 @@ mod tests { .with_spec_id(1) .add_unbound_fields(vec![ UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), }, UnboundPartitionField { - source_id: 4, + source_ids: vec![4], field_id: None, name: "name".to_string(), transform: Transform::Identity, @@ -1375,27 +1527,204 @@ mod tests { .unwrap_err(); } + #[test] + fn test_builder_disallows_variant_source() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "v", Type::Variant(crate::spec::VariantType)).into(), + ]) + .build() + .unwrap(); + + let err = PartitionSpec::builder(schema) + .with_spec_id(1) + .add_unbound_fields(vec![UnboundPartitionField { + source_ids: vec![2], + field_id: None, + name: "v_part".to_string(), + transform: Transform::Identity, + }]) + .expect_err("variant must not be allowed as a partition source"); + + assert_eq!( + err.message(), + "Cannot partition by non-primitive source field: 'variant'." + ); + } + #[test] fn test_builder_disallows_redundant() { let err = UnboundPartitionSpec::builder() .with_spec_id(1) - .add_partition_field(1, "id_bucket[16]".to_string(), Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket[16]") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .add_partition_field( - 1, - "id_bucket_with_other_name".to_string(), - Transform::Bucket(16), + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id_bucket_with_other_name") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), ) .unwrap_err(); assert!(err.message().contains("redundant partition")); } + fn ts_schema() -> Schema { + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "ts", Type::Primitive(PrimitiveType::Timestamptz)).into(), + NestedField::required(2, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap() + } + + fn ts_field(field_id: i32, name: &str, transform: Transform) -> UnboundPartitionField { + UnboundPartitionField { + source_ids: vec![1], + field_id: Some(field_id), + name: name.to_string(), + transform, + } + } + + #[test] + fn test_add_unbound_field_rejects_second_time_granularity() { + let err = PartitionSpec::builder(ts_schema()) + .add_unbound_field(ts_field(1000, "ts_day", Transform::Day)) + .unwrap() + .add_unbound_field(ts_field(1001, "ts_hour", Transform::Hour)) + .unwrap_err(); + assert!(err.message().contains("redundant partition")); + } + + #[test] + fn test_bind_accepts_evolved_spec_fields() { + let fields = vec![ + ts_field(1000, "ts_day", Transform::Day), + ts_field(1001, "ts_hour", Transform::Hour), + ts_field(1002, "ts", Transform::Void), + ts_field(1003, "ts_null", Transform::Void), + ]; + let unbound = UnboundPartitionSpec { + spec_id: Some(1), + fields: fields.clone(), + }; + + let spec = unbound.bind(ts_schema()).unwrap(); + + let expected = PartitionSpec { + spec_id: 1, + fields: fields + .into_iter() + .map(|f| PartitionField { + source_id: 1, + field_id: f.field_id.unwrap(), + name: f.name, + transform: f.transform, + }) + .collect(), + }; + assert_eq!(spec, expected); + } + + #[test] + fn test_bind_rejects_void_field_named_after_other_column() { + let unbound = UnboundPartitionSpec { + spec_id: Some(1), + fields: vec![ts_field(1000, "id", Transform::Void)], + }; + + let err = unbound.bind(ts_schema()).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("Field name 'id'")); + } + + #[test] + fn test_bind_accepts_identical_fields_with_different_names() { + let bucket_field = |field_id: i32, name: &str| UnboundPartitionField { + source_ids: vec![2], + field_id: Some(field_id), + name: name.to_string(), + transform: Transform::Bucket(16), + }; + let unbound = UnboundPartitionSpec { + spec_id: Some(1), + fields: vec![ + bucket_field(1000, "id_bucket"), + bucket_field(1001, "id_bucket_2"), + ], + }; + + let spec = unbound.bind(ts_schema()).unwrap(); + + let expected = PartitionSpec { + spec_id: 1, + fields: vec![ + PartitionField { + source_id: 2, + field_id: 1000, + name: "id_bucket".to_string(), + transform: Transform::Bucket(16), + }, + PartitionField { + source_id: 2, + field_id: 1001, + name: "id_bucket_2".to_string(), + transform: Transform::Bucket(16), + }, + ], + }; + assert_eq!(spec, expected); + } + + #[test] + fn test_bind_rejects_repeated_partition_name() { + let unbound = UnboundPartitionSpec { + spec_id: Some(1), + fields: vec![ + ts_field(1000, "ts_part", Transform::Day), + ts_field(1001, "ts_part", Transform::Hour), + ], + }; + + let err = unbound.bind(ts_schema()).unwrap_err(); + + assert!( + err.message() + .contains("Cannot use partition name more than once") + ); + } + + #[test] + fn test_add_unbound_field_checks_name_before_redundancy() { + let err = PartitionSpec::builder(ts_schema()) + .add_unbound_field(ts_field(1000, "ts_part", Transform::Day)) + .unwrap() + .add_unbound_field(ts_field(1001, "ts_part", Transform::Hour)) + .unwrap_err(); + + assert!( + err.message() + .contains("Cannot use partition name more than once") + ); + } + #[test] fn test_builder_incompatible_transforms_disallowed() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), ]) .build() .unwrap(); @@ -1403,7 +1732,7 @@ mod tests { PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_year".to_string(), transform: Transform::Year, @@ -1416,7 +1745,7 @@ mod tests { let spec = UnboundPartitionSpec::builder() .with_spec_id(1) .add_partition_fields(vec![UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket[16]".to_string(), transform: Transform::Bucket(16), @@ -1427,7 +1756,7 @@ mod tests { assert_eq!(spec, UnboundPartitionSpec { spec_id: Some(1), fields: vec![UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket[16]".to_string(), transform: Transform::Bucket(16), @@ -1439,14 +1768,8 @@ mod tests { fn test_is_compatible_with() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -1454,7 +1777,7 @@ mod tests { let partition_spec_1 = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1466,7 +1789,7 @@ mod tests { let partition_spec_2 = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1482,8 +1805,7 @@ mod tests { fn test_not_compatible_with_transform_different() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), ]) .build() .unwrap(); @@ -1491,7 +1813,7 @@ mod tests { let partition_spec_1 = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1503,7 +1825,7 @@ mod tests { let partition_spec_2 = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(32), @@ -1519,14 +1841,8 @@ mod tests { fn test_not_compatible_with_source_id_different() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -1534,7 +1850,7 @@ mod tests { let partition_spec_1 = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1546,7 +1862,7 @@ mod tests { let partition_spec_2 = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1562,14 +1878,8 @@ mod tests { fn test_not_compatible_with_order_different() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -1577,14 +1887,14 @@ mod tests { let partition_spec_1 = PartitionSpec::builder(schema.clone()) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "name".to_string(), transform: Transform::Identity, @@ -1596,14 +1906,14 @@ mod tests { let partition_spec_2 = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: None, name: "name".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: None, name: "id_bucket".to_string(), transform: Transform::Bucket(16), @@ -1629,14 +1939,8 @@ mod tests { fn test_highest_field_id() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -1644,14 +1948,14 @@ mod tests { let spec = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(1001), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1000), name: "name".to_string(), transform: Transform::Identity, @@ -1667,14 +1971,8 @@ mod tests { fn test_has_sequential_ids() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -1682,14 +1980,14 @@ mod tests { let spec = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(1000), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1001), name: "name".to_string(), transform: Transform::Identity, @@ -1707,14 +2005,8 @@ mod tests { fn test_sequential_ids_must_start_at_1000() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -1722,14 +2014,14 @@ mod tests { let spec = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(999), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1000), name: "name".to_string(), transform: Transform::Identity, @@ -1747,14 +2039,8 @@ mod tests { fn test_sequential_ids_must_have_no_gaps() { let schema = Schema::builder() .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(crate::spec::PrimitiveType::Int)) - .into(), - NestedField::required( - 2, - "name", - Type::Primitive(crate::spec::PrimitiveType::String), - ) - .into(), + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -1762,14 +2048,14 @@ mod tests { let spec = PartitionSpec::builder(schema) .with_spec_id(1) .add_unbound_field(UnboundPartitionField { - source_id: 1, + source_ids: vec![1], field_id: Some(1000), name: "id".to_string(), transform: Transform::Identity, }) .unwrap() .add_unbound_field(UnboundPartitionField { - source_id: 2, + source_ids: vec![2], field_id: Some(1002), name: "name".to_string(), transform: Transform::Identity, @@ -1783,12 +2069,77 @@ mod tests { assert!(!spec.has_sequential_ids()); } + #[test] + fn test_partition_to_path_with_dropped_source_column() { + for (source_type, transform, value, expected) in [ + ( + PrimitiveType::Int, + Transform::Identity, + Literal::int(1), + "p=1", + ), + ( + PrimitiveType::Timestamp, + Transform::Identity, + Literal::long(1), + "p=1", + ), + ( + PrimitiveType::Int, + Transform::Truncate(10), + Literal::int(10), + "p=10", + ), + ( + PrimitiveType::Decimal { + precision: 9, + scale: 2, + }, + Transform::Identity, + Literal::decimal(123), + "p=123", + ), + ( + PrimitiveType::Uuid, + Transform::Identity, + Literal::uuid_from_str("f79c3e09-677c-4bbd-a479-3f349cb785e7").unwrap(), + "p=f79c3e09-677c-4bbd-a479-3f349cb785e7", + ), + ] { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "kept", PrimitiveType::Int).into(), + NestedField::required(2, "dropped", source_type).into(), + ]) + .build() + .unwrap(); + let spec = PartitionSpec::builder(schema) + .add_partition_field("dropped", "p", transform) + .unwrap() + .build() + .unwrap(); + let current_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "kept", PrimitiveType::Int).into(), + ]) + .build() + .unwrap(); + assert_eq!( + spec.partition_to_path(&Struct::from_iter([Some(value)]), current_schema.into()), + expected + ); + } + } + #[test] fn test_partition_to_path() { let schema = Schema::builder() .with_fields(vec![ NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "timestamp", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + NestedField::required(4, "empty", Type::Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap(); @@ -1798,14 +2149,209 @@ mod tests { .unwrap() .add_partition_field("name", "name", Transform::Identity) .unwrap() + .add_partition_field("timestamp", "ts_hour", Transform::Hour) + .unwrap() + .add_partition_field("empty", "empty_void", Transform::Void) + .unwrap() + .build() + .unwrap(); + + let data = Struct::from_iter([ + Some(Literal::int(42)), + Some(Literal::string("alice")), + Some(Literal::int(1000)), + Some(Literal::string("empty")), + ]); + + assert_eq!( + spec.partition_to_path(&data, schema.into()), + "id=42/name=alice/ts_hour=1970-02-11-16/empty_void=null" + ); + } + + #[test] + fn test_partition_to_path_float_matches_java() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "d", Type::Primitive(PrimitiveType::Double)).into(), + NestedField::required(2, "f", Type::Primitive(PrimitiveType::Float)).into(), + ]) + .build() + .unwrap(); + + let spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("d", "d", Transform::Identity) + .unwrap() + .add_partition_field("f", "f", Transform::Identity) + .unwrap() + .build() + .unwrap(); + + let data = Struct::from_iter([Some(Literal::double(1.0)), Some(Literal::float(1.0e7_f32))]); + + assert_eq!( + spec.partition_to_path(&data, schema.into()), + "d=1.0/f=1.0E7" + ); + } + + #[test] + fn test_partition_to_path_escaped_strings() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "\"esc\"#1", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + let spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("\"esc\"#1", "\"esc\"#1", Transform::Identity) + .unwrap() .build() .unwrap(); - let data = Struct::from_iter([Some(Literal::int(42)), Some(Literal::string("alice"))]); + let data = Struct::from_iter([ + Some(Literal::string("a/b/c/d")), + Some(Literal::string("val#1")), + ]); assert_eq!( spec.partition_to_path(&data, schema.into()), - "id=42/name=alice" + "%22esc%22%231=a%2Fb%2Fc%2Fd" + ); + } + + #[test] + fn test_partition_to_path_escaped_field_name() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "\"esc\"#1", Type::Primitive(PrimitiveType::String)) + .into(), + NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + let spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("data", "data", Transform::Identity) + .unwrap() + .add_partition_field("data", "data_truc_10", Transform::Truncate(10)) + .unwrap() + .build() + .unwrap(); + + let data = Struct::from_iter([ + Some(Literal::string("a/b/c/d")), + Some(Literal::string("a/b/c/d")), + ]); + + assert_eq!( + spec.partition_to_path(&data, schema.into()), + "data=a%2Fb%2Fc%2Fd/data_truc_10=a%2Fb%2Fc%2Fd" + ); + } + + #[test] + fn test_unbound_partition_field_reads_source_ids_only() { + let field: UnboundPartitionField = serde_json::from_str( + r#"{"source-ids": [1, 2], "name": "m", "transform": "bucket[4]"}"#, + ) + .unwrap(); + + assert_eq!([1, 2], field.source_ids()); + // a multi-argument field has no single source id + assert!( + field + .source_id() + .unwrap_err() + .to_string() + .contains("has no single source id") + ); + + let serialized = serde_json::to_value(&field).unwrap(); + assert_eq!( + Some(&serde_json::json!([1, 2])), + serialized.get("source-ids") + ); + assert!(serialized.get("source-id").is_none()); + } + + #[test] + fn test_unbound_partition_field_single_source_id_round_trip() { + let field: UnboundPartitionField = + serde_json::from_str(r#"{"source-id": 1, "name": "m", "transform": "identity"}"#) + .unwrap(); + + assert_eq!(1, field.source_id().unwrap()); + assert_eq!([1], field.source_ids()); + + // a single-argument field keeps writing source-id + let serialized = serde_json::to_value(&field).unwrap(); + assert_eq!(Some(&serde_json::json!(1)), serialized.get("source-id")); + assert!(serialized.get("source-ids").is_none()); + } + + #[test] + fn test_unbound_partition_field_rejects_malformed_source_ids() { + for (input, expected) in [ + ( + r#"{"source-ids": [], "name": "m", "transform": "identity"}"#, + "Empty source-ids is not allowed", + ), + ( + r#"{"name": "m", "transform": "identity"}"#, + "Either `source-id` or `source-ids` must be present", + ), + ( + r#"{"source-id": 1, "source-ids": [1], "name": "m", "transform": "identity"}"#, + "mutually exclusive", + ), + ] { + let err = serde_json::from_str::(input).unwrap_err(); + assert!( + err.to_string().contains(expected), + "unexpected error for {input}: {err}" + ); + } + } + + #[test] + fn test_binding_a_multi_argument_field_fails_loudly() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "b", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let spec: UnboundPartitionSpec = serde_json::from_str( + r#"{"spec-id": 1, "fields": [{"source-ids": [1, 2], "field-id": 1000, "name": "m", "transform": "bucket[4]"}]}"#, + ) + .unwrap(); + + // A bound PartitionField still carries a single source id, so binding has to refuse + // rather than quietly keep the first one. + let err = spec.bind(schema).unwrap_err(); + assert!( + err.to_string().contains("has no single source id"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_unbound_partition_field_builder_rejects_empty_source_ids() { + let err = UnboundPartitionField::builder() + .source_ids(vec![]) + .name("m") + .transform(Transform::Identity) + .build() + .unwrap_err(); + assert!( + err.to_string().contains("Empty source-ids is not allowed"), + "unexpected error: {err}" ); } } diff --git a/crates/iceberg/src/spec/schema/assign_fresh_ids.rs b/crates/iceberg/src/spec/schema/assign_fresh_ids.rs new file mode 100644 index 0000000000..94f3030681 --- /dev/null +++ b/crates/iceberg/src/spec/schema/assign_fresh_ids.rs @@ -0,0 +1,520 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::utils::try_insert_field; +use super::*; +use crate::error::invalid_data; + +/// Reassigns `schema`'s field ids, reusing ids from `base` for fields whose full name is unchanged +/// and drawing fresh ids from `last_column_id + 1` upwards for everything else. +/// +/// `last_column_id` must be every id the table has ever assigned, not merely `base`'s highest: pass +/// `table_metadata.last_column_id()`, which also reserves the ids of columns already dropped from +/// `base`. A reused id does not consume a fresh one, so a too-low `last_column_id` fails in one of +/// two ways. Falling below `base`'s own ids is rejected here. Landing on the id of a column already +/// dropped from `base` cannot be detected from `base` alone: nothing here or in `build()` notices, +/// and a new column silently inherits a retired id. Passing a schema's `highest_field_id()` instead +/// of the table's `last_column_id()` is the usual way to hit the latter. +/// +/// The returned `schema_id` is carried over unchanged and is not authoritative; it is arbitrated by +/// [`TableMetadataBuilder::add_schema`](crate::spec::TableMetadataBuilder::add_schema). +pub(crate) fn assign_fresh_ids( + schema: Schema, + base: &Schema, + last_column_id: i32, +) -> Result { + if last_column_id < base.highest_field_id() { + return Err(invalid_data!( + "last_column_id ({last_column_id}) is below base.highest_field_id() ({}); pass table_metadata.last_column_id()", + base.highest_field_id() + )); + } + + let start_from = last_column_id + .checked_add(1) + .ok_or_else(|| invalid_data!("Field ID overflowed, cannot add more fields"))?; + + let Schema { + r#struct, + schema_id, + identifier_field_ids, + alias_to_id, + id_to_name, + .. + } = schema; + let mut assigner = AssignFreshIds::new(id_to_name, base, start_from); + let fields = assigner.assign_fields(r#struct.fields().to_vec())?; + let identifier_field_ids = assigner.apply_to_identifier_fields(identifier_field_ids)?; + let alias_to_id = assigner.apply_to_aliases(alias_to_id)?; + + Schema::builder() + .with_schema_id(schema_id) + .with_fields(fields) + .with_identifier_field_ids(identifier_field_ids) + .with_alias(alias_to_id) + .build() +} + +struct AssignFreshIds { + next_field_id: i32, + target_names: HashMap, + base_ids: HashMap, + old_to_new_id: HashMap, +} + +impl AssignFreshIds { + fn new(target_names: HashMap, base: &Schema, start_from: i32) -> Self { + Self { + next_field_id: start_from, + target_names, + base_ids: base + .field_id_to_name_map() + .iter() + .map(|(id, name)| (name.clone(), *id)) + .collect(), + old_to_new_id: HashMap::new(), + } + } + + /// Returns `base`'s id when the field's full name is unchanged, otherwise consumes a fresh id by + /// advancing `next_field_id`. + fn resolve_or_assign_id(&mut self, old_id: i32) -> Result { + if let Some(id) = self + .target_names + .get(&old_id) + .and_then(|name| self.base_ids.get(name)) + { + return Ok(*id); + } + + let id = self.next_field_id; + self.next_field_id = self + .next_field_id + .checked_add(1) + .ok_or_else(|| invalid_data!("Field ID overflowed, cannot add more fields"))?; + Ok(id) + } + + fn assign_fields(&mut self, fields: Vec) -> Result> { + let outer_fields = fields + .into_iter() + .map(|field| { + let new_id = self.resolve_or_assign_id(field.id)?; + try_insert_field(&mut self.old_to_new_id, field.id, new_id)?; + Ok(Arc::new(Arc::unwrap_or_clone(field).with_id(new_id))) + }) + .collect::>>()?; + + outer_fields + .into_iter() + .map(|field| { + if !field.field_type.is_nested() { + Ok(field) + } else { + let mut field = Arc::unwrap_or_clone(field); + *field.field_type = self.assign_type(*field.field_type)?; + Ok(Arc::new(field)) + } + }) + .collect() + } + + fn assign_type(&mut self, field_type: Type) -> Result { + match field_type { + Type::Primitive(primitive) => Ok(Type::Primitive(primitive)), + Type::Struct(r#struct) => Ok(Type::Struct(StructType::new( + self.assign_fields(r#struct.fields().to_vec())?, + ))), + Type::List(list) => { + let new_id = self.resolve_or_assign_id(list.element_field.id)?; + try_insert_field(&mut self.old_to_new_id, list.element_field.id, new_id)?; + let mut element_field = Arc::unwrap_or_clone(list.element_field); + element_field.id = new_id; + *element_field.field_type = self.assign_type(*element_field.field_type)?; + Ok(Type::List(ListType { + element_field: Arc::new(element_field), + })) + } + Type::Map(map) => { + // Key and value ids are resolved before recursing into either, matching Java and + // intentionally unlike `ReassignFieldIds`, which recurses the key first. + let new_key_id = self.resolve_or_assign_id(map.key_field.id)?; + let new_value_id = self.resolve_or_assign_id(map.value_field.id)?; + try_insert_field(&mut self.old_to_new_id, map.key_field.id, new_key_id)?; + try_insert_field(&mut self.old_to_new_id, map.value_field.id, new_value_id)?; + + let mut key_field = Arc::unwrap_or_clone(map.key_field); + key_field.id = new_key_id; + *key_field.field_type = self.assign_type(*key_field.field_type)?; + + let mut value_field = Arc::unwrap_or_clone(map.value_field); + value_field.id = new_value_id; + *value_field.field_type = self.assign_type(*value_field.field_type)?; + + Ok(Type::Map(MapType { + key_field: Arc::new(key_field), + value_field: Arc::new(value_field), + })) + } + Type::Variant(variant) => Ok(Type::Variant(variant)), + } + } + + fn apply_to_identifier_fields(&self, field_ids: HashSet) -> Result> { + field_ids + .into_iter() + .map(|id| { + self.old_to_new_id + .get(&id) + .copied() + .ok_or_else(|| invalid_data!("identifier field id {id} not found")) + }) + .collect() + } + + fn apply_to_aliases(&self, aliases: BiHashMap) -> Result> { + aliases + .into_iter() + .map(|(name, id)| { + self.old_to_new_id + .get(&id) + .copied() + .ok_or_else(|| invalid_data!("Field with id {id} for alias {name} not found")) + .map(|new_id| (name, new_id)) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spec::{Literal, VariantType}; + + fn empty_schema() -> Schema { + Schema::builder().build().unwrap() + } + + #[test] + fn test_assign_fresh_ids() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(0, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(1, "c", Type::Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::int(23)) + .with_write_default(Literal::int(34)) + .into(), + NestedField::required(2, "B", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let expected = Schema::builder() + .with_fields(vec![ + NestedField::required(11, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(12, "c", Type::Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::int(23)) + .with_write_default(Literal::int(34)) + .into(), + NestedField::required(13, "B", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let assigned = assign_fresh_ids(schema, &empty_schema(), 10).unwrap(); + + assert_eq!(assigned.as_struct(), expected.as_struct()); + } + + #[test] + fn test_assign_fresh_ids_with_type() { + for test_type in [ + Type::Primitive(PrimitiveType::Boolean), + Type::Primitive(PrimitiveType::Int), + Type::Primitive(PrimitiveType::Long), + Type::Primitive(PrimitiveType::Float), + Type::Primitive(PrimitiveType::Double), + Type::Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 2, + }), + Type::Primitive(PrimitiveType::Date), + Type::Primitive(PrimitiveType::Time), + Type::Primitive(PrimitiveType::Timestamp), + Type::Primitive(PrimitiveType::Timestamptz), + Type::Primitive(PrimitiveType::TimestampNs), + Type::Primitive(PrimitiveType::TimestamptzNs), + Type::Primitive(PrimitiveType::String), + Type::Primitive(PrimitiveType::Uuid), + Type::Primitive(PrimitiveType::Fixed(16)), + Type::Primitive(PrimitiveType::Binary), + Type::Variant(VariantType), + ] { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(0, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(1, "data", test_type.clone()).into(), + ]) + .build() + .unwrap(); + let expected = Schema::builder() + .with_fields(vec![ + NestedField::required(11, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(12, "data", test_type.clone()).into(), + ]) + .build() + .unwrap(); + + let assigned = assign_fresh_ids(schema, &empty_schema(), 10).unwrap(); + + assert_eq!( + assigned.as_struct(), + expected.as_struct(), + "failed for type: {test_type:?}" + ); + } + } + + #[test] + fn test_assign_fresh_ids_reuses_full_names_and_assigns_new_ids_in_correct_order() { + let base = Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "nested", + Type::Struct(StructType::new(vec![ + NestedField::optional(2, "a", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ) + .into(), + NestedField::optional( + 3, + "items", + Type::List(ListType::new( + NestedField::list_element(4, Type::Primitive(PrimitiveType::String), false) + .into(), + )), + ) + .into(), + NestedField::optional( + 5, + "properties", + Type::Map(MapType::optional( + 6, + Type::Primitive(PrimitiveType::String), + 7, + Type::Primitive(PrimitiveType::Long), + )), + ) + .into(), + NestedField::required(8, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(9, "dropped", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let replacement = Schema::builder() + .with_schema_id(1) + .with_identifier_field_ids([18]) + .with_alias(BiHashMap::from_iter([("a_alias".to_string(), 12)])) + .with_fields(vec![ + NestedField::required( + 10, + "nested", + Type::Struct(StructType::new(vec![ + NestedField::optional(11, "b", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(12, "a", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ) + .into(), + NestedField::optional( + 13, + "items", + Type::List(ListType::new( + NestedField::list_element( + 14, + Type::Primitive(PrimitiveType::String), + false, + ) + .into(), + )), + ) + .into(), + NestedField::optional( + 15, + "properties", + Type::Map(MapType::optional( + 16, + Type::Primitive(PrimitiveType::String), + 17, + Type::Primitive(PrimitiveType::Long), + )), + ) + .into(), + NestedField::required(18, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(19, "z", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let assigned = assign_fresh_ids(replacement, &base, 9).unwrap(); + + assert_eq!(assigned.field_by_name("nested").unwrap().id, 1); + assert_eq!(assigned.field_by_name("nested.a").unwrap().id, 2); + assert_eq!(assigned.field_by_name("items").unwrap().id, 3); + assert_eq!(assigned.field_by_name("items.element").unwrap().id, 4); + assert_eq!(assigned.field_by_name("properties").unwrap().id, 5); + assert_eq!(assigned.field_by_name("properties.key").unwrap().id, 6); + assert_eq!(assigned.field_by_name("properties.value").unwrap().id, 7); + assert_eq!(assigned.field_by_name("x").unwrap().id, 8); + assert_eq!(assigned.field_by_name("z").unwrap().id, 10); + assert_eq!(assigned.field_by_name("nested.b").unwrap().id, 11); + assert_eq!( + assigned.identifier_field_ids().collect::>(), + HashSet::from([8]) + ); + assert_eq!(assigned.field_by_alias("a_alias").unwrap().id, 2); + assert_eq!(assigned.highest_field_id(), 11); + } + + /// The table once had ids 1..=5; only 1 and 3 survive in `base`, so 4 and 5 are retired and the + /// correct `last_column_id` is 5, not `base.highest_field_id()`. + fn schemas_with_dropped_column_ids() -> (Schema, Schema) { + let base = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(3, "b", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let replacement = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "fresh", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + (base, replacement) + } + + #[test] + fn test_assign_fresh_ids_skips_dropped_column_ids_when_seeded_correctly() { + let (base, replacement) = schemas_with_dropped_column_ids(); + + let assigned = assign_fresh_ids(replacement, &base, 5).unwrap(); + + assert_eq!(assigned.field_by_name("a").unwrap().id, 1); + assert_eq!(assigned.field_by_name("fresh").unwrap().id, 6); + } + + #[test] + fn test_assign_fresh_ids_seeded_too_low_reuses_dropped_column_id() { + let (base, replacement) = schemas_with_dropped_column_ids(); + + // Passing base's highest id as `last_column_id` clears the guard and `build()` sees no + // duplicates, so the retired id 4 is reused silently. `base` alone cannot reveal that 4 is + // retired, so this stays the caller's contract. + let assigned = assign_fresh_ids(replacement, &base, base.highest_field_id()).unwrap(); + + assert_eq!(assigned.field_by_name("fresh").unwrap().id, 4); + } + + #[test] + fn test_assign_fresh_ids_rejects_last_column_id_below_base_highest_field_id() { + let (base, replacement) = schemas_with_dropped_column_ids(); + + let err = assign_fresh_ids(replacement, &base, base.highest_field_id() - 1).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("last_column_id (2) is below base.highest_field_id() (3)") + ); + } + + #[test] + fn test_assign_fresh_ids_rejects_alias_for_dropped_field() { + // `build()` validates `identifier_field_ids` but not `alias_to_id`, so an alias left + // pointing at a field the replacement drops reaches `apply_to_aliases`. + let base = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "dropped", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let replacement = Schema::builder() + .with_alias(BiHashMap::from_iter([("dropped_alias".to_string(), 2)])) + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let err = assign_fresh_ids(replacement, &base, 2).unwrap_err(); + + assert!( + err.message() + .contains("Field with id 2 for alias dropped_alias not found") + ); + } + + #[test] + fn test_assign_fresh_ids_assigns_map_ids_before_nested_types() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required( + 1, + "map", + Type::Map(MapType::required( + 2, + Type::Struct(StructType::new(vec![ + NestedField::required( + 3, + "key_nested", + Type::Primitive(PrimitiveType::Int), + ) + .into(), + ])), + 4, + Type::Struct(StructType::new(vec![ + NestedField::required( + 5, + "value_nested", + Type::Primitive(PrimitiveType::Int), + ) + .into(), + ])), + )), + ) + .into(), + NestedField::required(6, "tail", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let assigned = assign_fresh_ids(schema, &empty_schema(), 10).unwrap(); + + assert_eq!(assigned.field_by_name("map").unwrap().id, 11); + assert_eq!(assigned.field_by_name("tail").unwrap().id, 12); + assert_eq!(assigned.field_by_name("map.key").unwrap().id, 13); + assert_eq!(assigned.field_by_name("map.value").unwrap().id, 14); + assert_eq!(assigned.field_by_name("map.key.key_nested").unwrap().id, 15); + assert_eq!( + assigned.field_by_name("map.value.value_nested").unwrap().id, + 16 + ); + } +} diff --git a/crates/iceberg/src/spec/schema/id_reassigner.rs b/crates/iceberg/src/spec/schema/id_reassigner.rs index 0443831853..dafaeaa641 100644 --- a/crates/iceberg/src/spec/schema/id_reassigner.rs +++ b/crates/iceberg/src/spec/schema/id_reassigner.rs @@ -17,6 +17,7 @@ use super::utils::try_insert_field; use super::*; +use crate::error::invalid_data; pub struct ReassignFieldIds { next_field_id: i32, @@ -52,7 +53,7 @@ impl ReassignFieldIds { outer_fields .into_iter() .map(|field| { - if field.field_type.is_primitive() { + if !field.field_type.is_nested() { Ok(field) } else { let mut new_field = Arc::unwrap_or_clone(field); @@ -102,16 +103,15 @@ impl ReassignFieldIds { value_field: Arc::new(value_field), })) } + Type::Variant(v) => Ok(Type::Variant(v)), } } fn increase_next_field_id(&mut self) -> Result<()> { - self.next_field_id = self.next_field_id.checked_add(1).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Field ID overflowed, cannot add more fields", - ) - })?; + self.next_field_id = self + .next_field_id + .checked_add(1) + .ok_or_else(|| invalid_data!("Field ID overflowed, cannot add more fields"))?; Ok(()) } @@ -119,12 +119,10 @@ impl ReassignFieldIds { field_ids .into_iter() .map(|id| { - self.old_to_new_id.get(&id).copied().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Identifier Field ID {} not found", id), - ) - }) + self.old_to_new_id + .get(&id) + .copied() + .ok_or_else(|| invalid_data!("identifier field id {id} not found")) }) .collect() } @@ -139,12 +137,7 @@ impl ReassignFieldIds { self.old_to_new_id .get(&id) .copied() - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Field with id {} for alias {} not found", id, name), - ) - }) + .ok_or_else(|| invalid_data!("Field with id {id} for alias {name} not found")) .map(|new_id| (name, new_id)) }) .collect() @@ -192,6 +185,37 @@ mod tests { assert_eq!(reassigned_schema.highest_field_id(), 2); } + #[test] + fn test_reassign_ids_variant() { + use crate::spec::VariantType; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(5, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "data", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(); + + let reassigned = schema + .into_builder() + .with_reassigned_field_ids(0) + .build() + .unwrap(); + + // Variant has no sub-fields, so it survives reassignment unchanged; only the + // top-level field ids shift (id → 0, data → 1). + let expected = Schema::builder() + .with_fields(vec![ + NestedField::required(0, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(1, "data", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(); + + pretty_assertions::assert_eq!(expected, reassigned); + } + #[test] fn test_reassigned_ids_nested() { let schema = table_schema_nested(); diff --git a/crates/iceberg/src/spec/schema/index.rs b/crates/iceberg/src/spec/schema/index.rs index d4e77ab2aa..eb2f990a36 100644 --- a/crates/iceberg/src/spec/schema/index.rs +++ b/crates/iceberg/src/spec/schema/index.rs @@ -17,6 +17,8 @@ use super::utils::try_insert_field; use super::*; +use crate::error::invalid_data; +use crate::spec::VariantType; /// Creates a field id to field map. pub fn index_by_id(r#struct: &StructType) -> Result> { @@ -53,6 +55,10 @@ pub fn index_by_id(r#struct: &StructType) -> Result fn primitive(&mut self, _: &PrimitiveType) -> Result { Ok(()) } + + fn variant(&mut self, _v: &VariantType) -> Result { + Ok(()) + } } let mut index = IndexById(HashMap::new()); @@ -145,6 +151,10 @@ pub fn index_parents(r#struct: &StructType) -> Result> { fn primitive(&mut self, _p: &PrimitiveType) -> Result { Ok(()) } + + fn variant(&mut self, _v: &VariantType) -> Result { + Ok(()) + } } let mut index = IndexByParent { @@ -174,11 +184,8 @@ impl IndexByName { .chain(vec![name]) .join("."); if let Some(existing_field_id) = self.name_to_id.get(full_name.as_str()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}" - ), + return Err(invalid_data!( + "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}" )); } else { self.name_to_id.insert(full_name, field_id); @@ -293,6 +300,10 @@ impl SchemaVisitor for IndexByName { fn primitive(&mut self, _p: &PrimitiveType) -> Result { Ok(()) } + + fn variant(&mut self, _v: &VariantType) -> Result { + Ok(()) + } } #[cfg(test)] @@ -315,4 +326,40 @@ mod tests { assert_eq!(result.get(&16).unwrap(), &15); assert_eq!(result.get(&17).unwrap(), &15); } + + #[test] + fn test_index_variant_by_id_and_name() { + // A variant is indexed like a leaf: by id and by full (dotted) name, both at the + // top level and nested inside a struct. Mirrors Java's TestTypeUtil index-by-id / + // index-name-by-id coverage for variant. + let s = StructType::new(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "v", Type::Variant(VariantType)).into(), + NestedField::required( + 3, + "nested", + Type::Struct(StructType::new(vec![ + NestedField::optional(4, "inner_v", Type::Variant(VariantType)).into(), + ])), + ) + .into(), + ]); + + let by_id = index_by_id(&s).unwrap(); + // All four fields (id, v, nested, nested.inner_v) are indexed exactly once. + assert_eq!(by_id.len(), 4); + assert!(by_id[&2].field_type.is_variant()); + assert!(by_id[&4].field_type.is_variant()); + + let (name_to_id, id_to_name) = { + let mut index = IndexByName::default(); + visit_struct(&s, &mut index).unwrap(); + index.indexes() + }; + assert_eq!(id_to_name.len(), 4); + assert_eq!(id_to_name[&2], "v"); + assert_eq!(id_to_name[&4], "nested.inner_v"); + assert_eq!(name_to_id["v"], 2); + assert_eq!(name_to_id["nested.inner_v"], 4); + } } diff --git a/crates/iceberg/src/spec/schema/mod.rs b/crates/iceberg/src/spec/schema/mod.rs index 6554629987..87e5535a60 100644 --- a/crates/iceberg/src/spec/schema/mod.rs +++ b/crates/iceberg/src/spec/schema/mod.rs @@ -25,6 +25,7 @@ mod utils; mod visitor; pub use self::visitor::*; pub(super) mod _serde; +mod assign_fresh_ids; mod id_reassigner; mod index; mod prune_columns; @@ -33,12 +34,14 @@ use itertools::{Itertools, zip_eq}; use serde::{Deserialize, Serialize}; use self::_serde::SchemaEnum; +pub(crate) use self::assign_fresh_ids::assign_fresh_ids; use self::id_reassigner::ReassignFieldIds; use self::index::{IndexByName, index_by_id, index_parents}; pub use self::prune_columns::prune_columns; use super::NestedField; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::expr::accessor::StructAccessor; +use crate::spec::FormatVersion; use crate::spec::datatypes::{ LIST_FIELD_NAME, ListType, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedFieldRef, PrimitiveType, StructType, Type, @@ -51,6 +54,11 @@ pub type SchemaId = i32; pub type SchemaRef = Arc; /// Default schema id. pub const DEFAULT_SCHEMA_ID: SchemaId = 0; +/// Delimiter for schema name, which denotes a nested struct. +pub const SCHEMA_NAME_DELIMITER: &str = "."; +/// Minimum format version that allows non-null field default values. +/// Mirrors Java's `Schema.DEFAULT_VALUES_MIN_FORMAT_VERSION`. +pub(crate) const DEFAULT_VALUES_MIN_FORMAT_VERSION: FormatVersion = FormatVersion::V3; /// Defines schema in iceberg. #[derive(Debug, Serialize, Deserialize, Clone)] @@ -65,7 +73,8 @@ pub struct Schema { id_to_field: HashMap, name_to_id: HashMap, - lowercase_name_to_id: HashMap, + // `None` marks a lower-cased name that resolves to more than one field. + lowercase_name_to_id: HashMap>, id_to_name: HashMap, field_id_to_accessor: HashMap>, @@ -102,8 +111,8 @@ impl SchemaBuilder { /// Reassignment starts from the field-id specified in `start_from` (inclusive). /// /// All specified aliases and identifier fields will be updated to the new field-ids. - pub(crate) fn with_reassigned_field_ids(mut self, start_from: u32) -> Self { - self.reassign_field_ids_from = Some(start_from.try_into().unwrap_or(i32::MAX)); + pub(crate) fn with_reassigned_field_ids(mut self, start_from: i32) -> Self { + self.reassign_field_ids_from = Some(start_from); self } @@ -144,10 +153,13 @@ impl SchemaBuilder { index.indexes() }; - let lowercase_name_to_id = name_to_id - .iter() - .map(|(k, v)| (k.to_lowercase(), *v)) - .collect(); + let mut lowercase_name_to_id = HashMap::new(); + for (name, field_id) in &name_to_id { + lowercase_name_to_id + .entry(name.to_lowercase()) + .and_modify(|existing| *existing = None) + .or_insert(Some(*field_id)); + } let highest_field_id = id_to_field.keys().max().cloned().unwrap_or(0); @@ -252,11 +264,8 @@ impl SchemaBuilder { let id_to_parent = index_parents(r#struct)?; for identifier_field_id in identifier_field_ids { let field = id_to_field.get(&identifier_field_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add identifier field {identifier_field_id}: field does not exist" - ), + invalid_data!( + "Cannot add identifier field {identifier_field_id}: field does not exist" ) })?; ensure_data_valid!( @@ -271,12 +280,9 @@ impl SchemaBuilder { field.name ); } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add field {} as an identifier field: not a primitive type field", - field.name - ), + return Err(invalid_data!( + "Cannot add field {} as an identifier field: not a primitive type field", + field.name )); } @@ -342,13 +348,47 @@ impl Schema { .and_then(|id| self.field_by_id(*id)) } - /// Get field by field name, but in case-insensitive way. + /// Get field by field name in a case-insensitive way. /// /// Both full name and short name could work here. + /// Returns `None` if the name is missing or matches multiple fields case-insensitively. + /// Use [`Self::field_by_name_case_insensitive_checked`] to distinguish those cases. pub fn field_by_name_case_insensitive(&self, field_name: &str) -> Option<&NestedFieldRef> { self.lowercase_name_to_id .get(&field_name.to_lowercase()) - .and_then(|id| self.field_by_id(*id)) + .copied() + .flatten() + .and_then(|id| self.field_by_id(id)) + } + + /// Get field by field name in a case-insensitive way, reporting ambiguous names. + /// + /// Both full name and short name could work here. Returns `Ok(None)` if the name is + /// missing, or an error if it matches multiple fields case-insensitively. A collision + /// on another name does not prevent this name from resolving. This differs from Iceberg + /// Java, which rejects all case-insensitive lookups after a collision, and PyIceberg, + /// which silently keeps one of the colliding fields. + pub fn field_by_name_case_insensitive_checked( + &self, + field_name: &str, + ) -> Result> { + let lowercase_name = field_name.to_lowercase(); + match self.lowercase_name_to_id.get(&lowercase_name) { + Some(Some(id)) => Ok(self.field_by_id(*id)), + Some(None) => { + let mut colliders: Vec<&str> = self + .name_to_id + .keys() + .filter(|name| name.to_lowercase() == lowercase_name) + .map(String::as_str) + .collect(); + colliders.sort_unstable(); + Err(invalid_data!( + "Multiple fields match {field_name} case-insensitively: {colliders:?}" + )) + } + None => Ok(None), + } } /// Get field by alias. @@ -358,25 +398,25 @@ impl Schema { .and_then(|id| self.field_by_id(*id)) } - /// Returns [`highest_field_id`]. + /// Returns the highest field ID assigned in this schema. #[inline] pub fn highest_field_id(&self) -> i32 { self.highest_field_id } - /// Returns [`schema_id`]. + /// Returns the Schema's ID. #[inline] pub fn schema_id(&self) -> SchemaId { self.schema_id } - /// Returns [`r#struct`]. + /// Returns the type for the struct. #[inline] pub fn as_struct(&self) -> &StructType { &self.r#struct } - /// Returns [`identifier_field_ids`]. + /// Returns the IDs of the fields. #[inline] pub fn identifier_field_ids(&self) -> impl ExactSizeIterator + '_ { self.identifier_field_ids.iter().copied() @@ -419,13 +459,96 @@ impl Schema { pub fn field_id_to_fields(&self) -> &HashMap { &self.id_to_field } + + /// Minimum [`FormatVersion`] required to represent all *types* in this schema. + /// + /// Walks the whole schema — every field, nested ones included — and returns the + /// highest per-type floor, so this is O(fields) rather than a cheap field read. + /// Types only; for initial-default version floors see [`Schema::check_format_compatibility`]. + pub fn calc_min_compatible_format(&self) -> FormatVersion { + // `id_to_field` is flattened, so the max over all fields covers nested ones too. + self.id_to_field + .values() + .map(|f| f.field_type.min_format_version()) + .max() + .unwrap_or(FormatVersion::V1) + } + + /// Returns an error listing every field incompatible with `format_version`. + /// Mirrors Java's `Schema.checkCompatibility()`. Two checks per field: + /// + /// - **Type** — minimum format version required to support that type, without + /// taking nested field types into account. + /// - **Initial default** — a non-null `initial_default` backfills pre-existing rows, + /// so it requires `DEFAULT_VALUES_MIN_FORMAT_VERSION`; `write_default` is not + /// checked, as it only affects newly written rows (read identically at any version). + pub fn check_format_compatibility(&self, format_version: FormatVersion) -> Result<()> { + // (field id, message); sorted by id below for a deterministic error. + let mut problems: Vec<(i32, String)> = Vec::new(); + + // `id_to_field` is flattened, so checking each field by its own type keeps the + // blame on the offending leaf, not its container (mirrors Java's `lazyIdToField`). + for field in self.id_to_field.values() { + let min_version = field.field_type.min_format_version(); + if format_version < min_version { + // Every id in `id_to_field` is also indexed in `id_to_name`; a miss means + // the schema's indexes are inconsistent (a bug), so surface it rather than + // guessing an unqualified name. + let name = self.name_by_field_id(field.id).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!( + "Field id {} is missing from the schema's name index", + field.id + ), + ) + })?; + problems.push((field.id, format!( + "Invalid type for {name}: {} is not supported until {min_version} but format version is {format_version}.", + field.field_type, + ))); + } + + if let Some(default) = &field.initial_default + && format_version < DEFAULT_VALUES_MIN_FORMAT_VERSION + { + let name = self.name_by_field_id(field.id).ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + format!( + "Field id {} is missing from the schema's name index", + field.id + ), + ) + })?; + problems.push((field.id, format!( + "Invalid initial default for {name}: non-null default ({default:?}) is not supported until {DEFAULT_VALUES_MIN_FORMAT_VERSION} but format version is {format_version}." + ))); + } + } + + if problems.is_empty() { + return Ok(()); + } + + // Stable sort by id: HashMap order is nondeterministic, and stability keeps a + // field's type problem before its default problem (matches Java's TreeMap order). + let message = problems + .into_iter() + .sorted_by_key(|(id, _)| *id) + .map(|(_, msg)| msg) + .join("\n- "); + Err(invalid_data!( + "Invalid schema for {format_version}:\n- {message}" + )) + } } impl Display for Schema { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { writeln!(f, "table {{")?; for field in self.as_struct().fields() { - writeln!(f, " {}", field)?; + writeln!(f, " {field}")?; } writeln!(f, "}}") } @@ -437,20 +560,221 @@ mod tests { use bimap::BiHashMap; - use crate::spec::datatypes::Type::{List, Map, Primitive, Struct}; + use crate::spec::datatypes::Type::{List, Map, Primitive, Struct, Variant}; use crate::spec::datatypes::{ - ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType, Type, + ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType, }; use crate::spec::schema::Schema; use crate::spec::values::Map as MapValue; use crate::spec::{Datum, Literal}; + #[test] + fn test_check_format_compatibility() { + use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType}; + + fn schema_with(fields: Vec) -> Schema { + Schema::builder().with_fields(fields).build().unwrap() + } + + // Variant type requires v3. + let variant = schema_with(vec![ + NestedField::optional(1, "v", Variant(VariantType)).into(), + ]); + assert!( + variant + .check_format_compatibility(FormatVersion::V2) + .is_err() + ); + assert!( + variant + .check_format_compatibility(FormatVersion::V3) + .is_ok() + ); + + // A non-null initial default requires v3, even for a v1-compatible type. + let with_default = schema_with(vec![ + NestedField::optional(1, "a", Primitive(PrimitiveType::Int)) + .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(1))) + .into(), + ]); + let err = with_default + .check_format_compatibility(FormatVersion::V2) + .unwrap_err(); + assert!( + err.message().contains("Invalid initial default for a"), + "{err}" + ); + assert!( + with_default + .check_format_compatibility(FormatVersion::V3) + .is_ok() + ); + + // No default is fine at any version (an absent/null default never trips). + let no_default = schema_with(vec![ + NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(), + ]); + assert!( + no_default + .check_format_compatibility(FormatVersion::V1) + .is_ok() + ); + + // Recursion: a default on a field nested inside a struct is caught. + let nested = schema_with(vec![ + NestedField::required( + 1, + "s", + Struct(StructType::new(vec![ + NestedField::optional(2, "inner", Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(7))) + .into(), + ])), + ) + .into(), + ]); + let err = nested + .check_format_compatibility(FormatVersion::V2) + .unwrap_err(); + assert!(err.message().contains("inner"), "{err}"); + + // A v3-only type nested in a container blames the leaf, never the container. + let nested_variant = schema_with(vec![ + NestedField::required( + 1, + "container", + Struct(StructType::new(vec![ + NestedField::optional(2, "v", Variant(VariantType)).into(), + ])), + ) + .into(), + ]); + let err = nested_variant + .check_format_compatibility(FormatVersion::V2) + .unwrap_err(); + assert!(err.message().contains("container.v"), "{err}"); + assert!( + !err.message().contains("Invalid type for container:"), + "container must not be blamed: {err}" + ); + } + + #[test] + fn test_calc_min_compatible_format() { + use crate::spec::{FormatVersion, GeometryType, VariantType}; + + fn schema_with(fields: Vec) -> Schema { + Schema::builder().with_fields(fields).build().unwrap() + } + + // All v1-compatible types → V1. + let v1 = schema_with(vec![ + NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "b", Primitive(PrimitiveType::String)).into(), + ]); + assert_eq!(v1.calc_min_compatible_format(), FormatVersion::V1); + + // A top-level variant → V3. + let variant = schema_with(vec![ + NestedField::optional(1, "v", Variant(VariantType)).into(), + ]); + assert_eq!(variant.calc_min_compatible_format(), FormatVersion::V3); + + // Unknown is a v3-only primitive type. + let unknown = schema_with(vec![ + NestedField::optional(1, "u", Primitive(PrimitiveType::Unknown)).into(), + ]); + assert_eq!(unknown.calc_min_compatible_format(), FormatVersion::V3); + assert!( + unknown + .check_format_compatibility(FormatVersion::V2) + .is_err() + ); + + // Geospatial types are v3-only. + let geometry = schema_with(vec![ + NestedField::optional( + 1, + "geom", + Primitive(PrimitiveType::Geometry(GeometryType::default())), + ) + .into(), + ]); + assert_eq!(geometry.calc_min_compatible_format(), FormatVersion::V3); + assert!( + geometry + .check_format_compatibility(FormatVersion::V2) + .is_err() + ); + + // A v3-only type nested inside a list inside a struct → V3 (flattened fields). + let nested = schema_with(vec![ + NestedField::required( + 1, + "s", + Struct(StructType::new(vec![ + NestedField::optional( + 2, + "l", + List(ListType::new( + NestedField::required( + 3, + "element", + Primitive(PrimitiveType::TimestampNs), + ) + .into(), + )), + ) + .into(), + ])), + ) + .into(), + ]); + assert_eq!(nested.calc_min_compatible_format(), FormatVersion::V3); + } + + #[test] + fn test_check_format_compatibility_message_order() { + use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType}; + + // Fields declared out of id order; field 2 fails both checks (ts_ns type + // AND a non-null initial default). The error must list problems by field id, + // with the type problem before the initial-default problem for field 2. + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(3, "c", Variant(VariantType)).into(), + NestedField::optional(2, "b", Primitive(PrimitiveType::TimestampNs)) + .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(0))) + .into(), + NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let message = schema + .check_format_compatibility(FormatVersion::V2) + .unwrap_err() + .message() + .to_string(); + + let lines: Vec<&str> = message.lines().skip(1).collect(); + assert_eq!( + lines, + vec![ + "- Invalid type for b: timestamp_ns is not supported until v3 but format version is v2.", + "- Invalid initial default for b: non-null default (Primitive(Long(0))) is not supported until v3 but format version is v2.", + "- Invalid type for c: variant is not supported until v3 but format version is v2.", + ], + "{message}" + ); + } + #[test] fn test_construct_schema() { let field1: NestedFieldRef = - NestedField::required(1, "f1", Type::Primitive(PrimitiveType::Boolean)).into(); + NestedField::required(1, "f1", Primitive(PrimitiveType::Boolean)).into(); let field2: NestedFieldRef = - NestedField::optional(2, "f2", Type::Primitive(PrimitiveType::Int)).into(); + NestedField::optional(2, "f2", Primitive(PrimitiveType::Int)).into(); let schema = Schema::builder() .with_fields(vec![field1.clone()]) @@ -471,9 +795,9 @@ mod tests { .with_schema_id(1) .with_identifier_field_ids(vec![2]) .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), + NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(), ]) .build() .unwrap(); @@ -510,16 +834,16 @@ mod tests { .with_schema_id(1) .with_identifier_field_ids(vec![2]) .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), + NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(), NestedField::required( 4, "qux", - Type::List(ListType { + List(ListType { element_field: NestedField::list_element( 5, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), true, ) .into(), @@ -529,23 +853,23 @@ mod tests { NestedField::required( 6, "quux", - Type::Map(MapType { + Map(MapType { key_field: NestedField::map_key_element( 7, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), ) .into(), value_field: NestedField::map_value_element( 8, - Type::Map(MapType { + Map(MapType { key_field: NestedField::map_key_element( 9, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), ) .into(), value_field: NestedField::map_value_element( 10, - Type::Primitive(PrimitiveType::Int), + Primitive(PrimitiveType::Int), true, ) .into(), @@ -559,20 +883,20 @@ mod tests { NestedField::required( 11, "location", - Type::List(ListType { + List(ListType { element_field: NestedField::list_element( 12, - Type::Struct(StructType::new(vec![ + Struct(StructType::new(vec![ NestedField::optional( 13, "latitude", - Type::Primitive(PrimitiveType::Float), + Primitive(PrimitiveType::Float), ) .into(), NestedField::optional( 14, "longitude", - Type::Primitive(PrimitiveType::Float), + Primitive(PrimitiveType::Float), ) .into(), ])), @@ -585,11 +909,9 @@ mod tests { NestedField::optional( 15, "person", - Type::Struct(StructType::new(vec![ - NestedField::optional(16, "name", Type::Primitive(PrimitiveType::String)) - .into(), - NestedField::required(17, "age", Type::Primitive(PrimitiveType::Int)) - .into(), + Struct(StructType::new(vec![ + NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(), + NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(), ])), ) .into(), @@ -707,6 +1029,40 @@ table { } } + #[test] + fn test_schema_field_by_name_case_insensitive_checked() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "id", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "ID", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "data", Primitive(PrimitiveType::String)).into(), + ]) + .build() + .expect("case-insensitive name ambiguity is valid in a schema"); + + assert_eq!( + schema + .field_by_name_case_insensitive_checked("DATA") + .unwrap() + .map(|field| field.id), + Some(3) + ); + assert!( + schema + .field_by_name_case_insensitive_checked("missing") + .unwrap() + .is_none() + ); + + let error = schema + .field_by_name_case_insensitive_checked("Id") + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::DataInvalid); + assert!(error.message().contains("case-insensitively"), "{error}"); + assert!(error.message().contains("[\"ID\", \"id\"]"), "{error}"); + assert!(schema.field_by_name_case_insensitive("Id").is_none()); + } + #[test] fn test_schema_find_column_name() { let expected_column_name = HashMap::from([ @@ -731,8 +1087,7 @@ table { assert_eq!( Some(name), schema.name_by_field_id(id), - "Column name for field id {} not match.", - id + "Column name for field id {id} not match." ); } } @@ -754,8 +1109,7 @@ table { assert_eq!( Some(name), schema.name_by_field_id(id), - "Column name for field id {} not match.", - id + "Column name for field id {id} not match." ); } } @@ -801,10 +1155,10 @@ table { NestedField::required( 4, "qux", - Type::List(ListType { + List(ListType { element_field: NestedField::list_element( 5, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), true, ) .into(), @@ -933,21 +1287,19 @@ table { NestedField::optional( 15, "person", - Type::Struct(StructType::new(vec![ - NestedField::optional(16, "name", Type::Primitive(PrimitiveType::String)) - .into(), - NestedField::required(17, "age", Type::Primitive(PrimitiveType::Int)) - .into(), + Struct(StructType::new(vec![ + NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(), + NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(), ])), ), ), ( 16, - NestedField::optional(16, "name", Type::Primitive(PrimitiveType::String)), + NestedField::optional(16, "name", Primitive(PrimitiveType::String)), ), ( 17, - NestedField::required(17, "age", Type::Primitive(PrimitiveType::Int)), + NestedField::required(17, "age", Primitive(PrimitiveType::Int)), ), ]); @@ -956,8 +1308,7 @@ table { assert_eq!( Some(&field), schema.field_by_id(id).map(|f| f.as_ref()), - "Field for {} not match.", - id + "Field for {id} not match." ); } } @@ -983,8 +1334,8 @@ table { )]))), Some(Literal::List(vec![Some(Literal::Struct( crate::spec::Struct::from_iter(vec![ - Some(Literal::float(52.509_09)), - Some(Literal::float(-1.885_249)), + Some(Literal::float(52.509_09_f32)), + Some(Literal::float(-1.885_249_f32)), ]), ))])), Some(Literal::Struct(crate::spec::Struct::from_iter(vec![ @@ -1057,9 +1408,9 @@ table { .with_identifier_field_ids(vec![5]) .with_alias(BiHashMap::from_iter(vec![("bar_alias".to_string(), 3)])) .with_fields(vec![ - NestedField::required(5, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), + NestedField::required(5, "foo", Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "bar", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(), ]) .build() .unwrap_err(); @@ -1092,12 +1443,12 @@ table { NestedField::required( 1, "Map", - Type::Map(MapType::new( - NestedField::map_key_element(2, Type::Primitive(PrimitiveType::String)) + Map(MapType::new( + NestedField::map_key_element(2, Primitive(PrimitiveType::String)) .into(), NestedField::map_value_element( 3, - Type::Primitive(PrimitiveType::Boolean), + Primitive(PrimitiveType::Boolean), true, ) .into(), @@ -1116,12 +1467,12 @@ table { NestedField::required( 1, "Map", - Type::Map(MapType::new( - NestedField::map_key_element(2, Type::Primitive(PrimitiveType::String)) + Map(MapType::new( + NestedField::map_key_element(2, Primitive(PrimitiveType::String)) .into(), NestedField::map_value_element( 3, - Type::Primitive(PrimitiveType::Boolean), + Primitive(PrimitiveType::Boolean), true, ) .into(), @@ -1142,13 +1493,9 @@ table { NestedField::required( 1, "List", - Type::List(ListType::new( - NestedField::list_element( - 2, - Type::Primitive(PrimitiveType::String), - true - ) - .into(), + List(ListType::new( + NestedField::list_element(2, Primitive(PrimitiveType::String), true) + .into(), )), ) .into() @@ -1166,15 +1513,10 @@ table { NestedField::optional( 1, "Struct", - Type::Struct(StructType::new(vec![ - NestedField::required( - 2, - "name", - Type::Primitive(PrimitiveType::String) - ) - .into(), - NestedField::optional(3, "age", Type::Primitive(PrimitiveType::Int)) + Struct(StructType::new(vec![ + NestedField::required(2, "name", Primitive(PrimitiveType::String)) .into(), + NestedField::optional(3, "age", Primitive(PrimitiveType::Int)).into(), ])), ) .into() @@ -1189,8 +1531,7 @@ table { .with_schema_id(1) .with_identifier_field_ids(vec![1]) .with_fields(vec![ - NestedField::required(1, "Float", Type::Primitive(PrimitiveType::Float),) - .into() + NestedField::required(1, "Float", Primitive(PrimitiveType::Float),).into() ]) .build() .is_err() @@ -1200,8 +1541,7 @@ table { .with_schema_id(1) .with_identifier_field_ids(vec![1]) .with_fields(vec![ - NestedField::required(1, "Double", Type::Primitive(PrimitiveType::Double),) - .into() + NestedField::required(1, "Double", Primitive(PrimitiveType::Double),).into() ]) .build() .is_err() @@ -1213,8 +1553,7 @@ table { .with_schema_id(1) .with_identifier_field_ids(vec![1]) .with_fields(vec![ - NestedField::required(1, "Required", Type::Primitive(PrimitiveType::String),) - .into() + NestedField::required(1, "Required", Primitive(PrimitiveType::String),).into() ]) .build() .is_ok() @@ -1224,11 +1563,159 @@ table { .with_schema_id(1) .with_identifier_field_ids(vec![1]) .with_fields(vec![ - NestedField::optional(1, "Optional", Type::Primitive(PrimitiveType::String),) - .into() + NestedField::optional(1, "Optional", Primitive(PrimitiveType::String),).into() ]) .build() .is_err() ); } + + #[test] + fn test_unknown_type_deserialization_rejects_non_null_default() { + let field_json = serde_json::json!({ + "id": 1, + "name": "empty", + "required": false, + "type": "unknown", + "initial-default": 1 + }); + + let error = serde_json::from_value::(field_json.clone()).unwrap_err(); + assert!( + error + .to_string() + .contains("Unknown type only supports null default values"), + "unexpected error: {error}" + ); + + let schema_json = serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [field_json] + }); + assert!(serde_json::from_value::(schema_json).is_err()); + } + + #[test] + fn test_unknown_type_deserialization_accepts_null_defaults() { + let schema_json = serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [ + { + "id": 1, + "name": "empty", + "required": false, + "type": "unknown", + "initial-default": null, + "write-default": null + } + ] + }); + + serde_json::from_value::(schema_json).unwrap(); + } + + #[test] + fn test_unknown_type_accepts_null_container_defaults() { + let cases = [ + ( + "struct", + Struct(StructType::new(vec![ + NestedField::optional(2, "empty", Primitive(PrimitiveType::Unknown)).into(), + ])), + Literal::Struct(crate::spec::Struct::from_iter([None])), + ), + ( + "list", + List(ListType::new( + NestedField::list_element(2, Primitive(PrimitiveType::Unknown), false).into(), + )), + Literal::List(vec![None]), + ), + ( + "map", + Map(MapType::optional( + 2, + Primitive(PrimitiveType::String), + 3, + Primitive(PrimitiveType::Unknown), + )), + Literal::Map(MapValue::from([(Literal::string("key"), None)])), + ), + ]; + + for (name, field_type, default) in cases { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, name, field_type) + .with_write_default(default) + .into(), + ]) + .build() + .unwrap(); + serde_json::to_value(schema).unwrap(); + } + } + + #[test] + fn test_unknown_type_deserialization_rejects_non_null_container_defaults() { + let cases = [ + ( + "struct", + serde_json::json!({ + "type": "struct", + "fields": [{ + "id": 2, + "name": "empty", + "required": false, + "type": "unknown" + }] + }), + serde_json::json!({"2": 1}), + ), + ( + "list", + serde_json::json!({ + "type": "list", + "element-id": 2, + "element-required": false, + "element": "unknown" + }), + serde_json::json!([1]), + ), + ( + "map", + serde_json::json!({ + "type": "map", + "key-id": 2, + "key": "string", + "value-id": 3, + "value-required": false, + "value": "unknown" + }), + serde_json::json!({"keys": ["key"], "values": [1]}), + ), + ]; + + for (name, field_type, default) in cases { + let schema_json = serde_json::json!({ + "type": "struct", + "schema-id": 1, + "fields": [{ + "id": 1, + "name": name, + "required": false, + "type": field_type, + "initial-default": default + }] + }); + + assert!( + serde_json::from_value::(schema_json).is_err(), + "non-null unknown default in {name} should be rejected" + ); + } + } } diff --git a/crates/iceberg/src/spec/schema/prune_columns.rs b/crates/iceberg/src/spec/schema/prune_columns.rs index 5a2f0b50fc..edfbb5fa1d 100644 --- a/crates/iceberg/src/spec/schema/prune_columns.rs +++ b/crates/iceberg/src/spec/schema/prune_columns.rs @@ -16,6 +16,8 @@ // under the License. use super::*; +use crate::error::invalid_data; +use crate::spec::VariantType; struct PruneColumn { selected: HashSet, @@ -110,19 +112,17 @@ impl SchemaVisitor for PruneColumn { if self.select_full_types { Ok(Some(*field.field_type.clone())) } else if field.field_type.is_struct() { - return Ok(Some(Type::Struct(PruneColumn::project_selected_struct( + Ok(Some(Type::Struct(PruneColumn::project_selected_struct( value, - )?))); + )?))) } else if !field.field_type.is_nested() { - return Ok(Some(*field.field_type.clone())); + Ok(Some(*field.field_type.clone())) } else { - return Err(Error::new( - ErrorKind::DataInvalid, + Err(invalid_data!( "Can't project list or map field directly when not selecting full type." - .to_string(), ) .with_context("field_id", field.id.to_string()) - .with_context("field_type", field.field_type.to_string())); + .with_context("field_type", field.field_type.to_string())) } } else { Ok(value) @@ -174,20 +174,18 @@ impl SchemaVisitor for PruneColumn { Ok(Some(Type::List(list.clone()))) } else if list.element_field.field_type.is_struct() { let projected_struct = PruneColumn::project_selected_struct(value).unwrap(); - return Ok(Some(Type::List(PruneColumn::project_list( + Ok(Some(Type::List(PruneColumn::project_list( list, Type::Struct(projected_struct), - )?))); + )?))) } else if list.element_field.field_type.is_primitive() { - return Ok(Some(Type::List(list.clone()))); + Ok(Some(Type::List(list.clone()))) } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot explicitly project List or Map types, List element {} of type {} was selected", - list.element_field.id, list.element_field.field_type - ), - )); + Err(invalid_data!( + "Cannot explicitly project List or Map types, List element {} of type {} was selected", + list.element_field.id, + list.element_field.field_type + )) } } else if let Some(result) = value { Ok(Some(Type::List(PruneColumn::project_list(list, result)?))) @@ -208,26 +206,24 @@ impl SchemaVisitor for PruneColumn { } else if map.value_field.field_type.is_struct() { let projected_struct = PruneColumn::project_selected_struct(Some(value.unwrap())).unwrap(); - return Ok(Some(Type::Map(PruneColumn::project_map( + Ok(Some(Type::Map(PruneColumn::project_map( map, Type::Struct(projected_struct), - )?))); + )?))) } else if map.value_field.field_type.is_primitive() { - return Ok(Some(Type::Map(map.clone()))); + Ok(Some(Type::Map(map.clone()))) } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot explicitly project List or Map types, Map value {} of type {} was selected", - map.value_field.id, map.value_field.field_type - ), - )); + Err(invalid_data!( + "Cannot explicitly project List or Map types, Map value {} of type {} was selected", + map.value_field.id, + map.value_field.field_type + )) } } else if let Some(value_result) = value { - return Ok(Some(Type::Map(PruneColumn::project_map( + Ok(Some(Type::Map(PruneColumn::project_map( map, value_result, - )?))); + )?))) } else if self.selected.contains(&map.key_field.id) { Ok(Some(Type::Map(map.clone()))) } else { @@ -238,6 +234,10 @@ impl SchemaVisitor for PruneColumn { fn primitive(&mut self, _p: &PrimitiveType) -> Result> { Ok(None) } + + fn variant(&mut self, _v: &VariantType) -> Result { + Ok(None) + } } #[cfg(test)] @@ -252,7 +252,7 @@ mod tests { let expected_type = Type::from( Schema::builder() .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap() @@ -271,7 +271,7 @@ mod tests { let expected_type = Type::from( Schema::builder() .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(), ]) .build() .unwrap() @@ -296,7 +296,7 @@ mod tests { Type::List(ListType { element_field: NestedField::list_element( 5, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), true, ) .into(), @@ -335,7 +335,7 @@ mod tests { Type::List(ListType { element_field: NestedField::list_element( 5, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), true, ) .into(), @@ -366,7 +366,7 @@ mod tests { Type::Map(MapType { key_field: NestedField::map_key_element( 7, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), ) .into(), value_field: NestedField::map_value_element( @@ -374,12 +374,12 @@ mod tests { Type::Map(MapType { key_field: NestedField::map_key_element( 9, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), ) .into(), value_field: NestedField::map_value_element( 10, - Type::Primitive(PrimitiveType::Int), + Primitive(PrimitiveType::Int), true, ) .into(), @@ -422,7 +422,7 @@ mod tests { Type::Map(MapType { key_field: NestedField::map_key_element( 7, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), ) .into(), value_field: NestedField::map_value_element( @@ -430,12 +430,12 @@ mod tests { Type::Map(MapType { key_field: NestedField::map_key_element( 9, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), ) .into(), value_field: NestedField::map_value_element( 10, - Type::Primitive(PrimitiveType::Int), + Primitive(PrimitiveType::Int), true, ) .into(), @@ -470,7 +470,7 @@ mod tests { Type::Map(MapType { key_field: NestedField::map_key_element( 7, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), ) .into(), value_field: NestedField::map_value_element( @@ -478,12 +478,12 @@ mod tests { Type::Map(MapType { key_field: NestedField::map_key_element( 9, - Type::Primitive(PrimitiveType::String), + Primitive(PrimitiveType::String), ) .into(), value_field: NestedField::map_value_element( 10, - Type::Primitive(PrimitiveType::Int), + Primitive(PrimitiveType::Int), true, ) .into(), @@ -516,12 +516,8 @@ mod tests { 15, "person", Type::Struct(StructType::new(vec![ - NestedField::optional( - 16, - "name", - Type::Primitive(PrimitiveType::String), - ) - .into(), + NestedField::optional(16, "name", Primitive(PrimitiveType::String)) + .into(), ])), ) .into(), @@ -547,12 +543,8 @@ mod tests { 15, "person", Type::Struct(StructType::new(vec![ - NestedField::optional( - 16, - "name", - Type::Primitive(PrimitiveType::String), - ) - .into(), + NestedField::optional(16, "name", Primitive(PrimitiveType::String)) + .into(), ])), ) .into(), @@ -628,11 +620,8 @@ mod tests { 6, "id_to_person", Type::Map(MapType { - key_field: NestedField::map_key_element( - 7, - Type::Primitive(PrimitiveType::Int), - ) - .into(), + key_field: NestedField::map_key_element(7, Primitive(PrimitiveType::Int)) + .into(), value_field: NestedField::map_value_element( 8, Type::Struct(StructType::new(vec![ @@ -659,7 +648,7 @@ mod tests { Type::Map(MapType { key_field: NestedField::map_key_element( 7, - Type::Primitive(PrimitiveType::Int), + Primitive(PrimitiveType::Int), ) .into(), value_field: NestedField::map_value_element( @@ -694,11 +683,8 @@ mod tests { 6, "id_to_person", Type::Map(MapType { - key_field: NestedField::map_key_element( - 7, - Type::Primitive(PrimitiveType::Int), - ) - .into(), + key_field: NestedField::map_key_element(7, Primitive(PrimitiveType::Int)) + .into(), value_field: NestedField::map_value_element( 8, Type::Struct(StructType::new(vec![ @@ -725,7 +711,7 @@ mod tests { Type::Map(MapType { key_field: NestedField::map_key_element( 7, - Type::Primitive(PrimitiveType::Int), + Primitive(PrimitiveType::Int), ) .into(), value_field: NestedField::map_value_element( @@ -760,4 +746,33 @@ mod tests { assert!(result.is_ok()); assert_eq!(result.unwrap(), Type::Struct(schema.as_struct().clone())); } + + #[test] + fn test_prune_columns_variant() { + // foo (String, id=1) + v (Variant, id=2). + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(), + NestedField::optional(2, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(); + + // A variant is a leaf (like a primitive): selecting it keeps it, the same way + // for select_full_types true and false. + let only_variant = Type::Struct(StructType::new(vec![ + NestedField::optional(2, "v", Type::Variant(VariantType)).into(), + ])); + for full in [false, true] { + let result = prune_columns(&schema, HashSet::from([2]), full).unwrap(); + assert_eq!(result, only_variant, "select_full_types={full}"); + } + + // Selecting a sibling prunes the variant out. + let only_foo = Type::Struct(StructType::new(vec![ + NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(), + ])); + let result = prune_columns(&schema, HashSet::from([1]), false).unwrap(); + assert_eq!(result, only_foo); + } } diff --git a/crates/iceberg/src/spec/schema/utils.rs b/crates/iceberg/src/spec/schema/utils.rs index 37e6833441..26f49b23da 100644 --- a/crates/iceberg/src/spec/schema/utils.rs +++ b/crates/iceberg/src/spec/schema/utils.rs @@ -17,18 +17,15 @@ use std::collections::HashMap; -use crate::{Error, ErrorKind, Result}; +use crate::Result; +use crate::error::invalid_data; pub fn try_insert_field(map: &mut HashMap, field_id: i32, value: V) -> Result<()> { map.insert(field_id, value).map_or_else( || Ok(()), |_| { - Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Found duplicate 'field.id' {}. Field ids must be unique.", - field_id - ), + Err(invalid_data!( + "Found duplicate 'field.id' {field_id}. Field ids must be unique." )) }, ) diff --git a/crates/iceberg/src/spec/schema/visitor.rs b/crates/iceberg/src/spec/schema/visitor.rs index 50f7c04caa..16b5fcdca9 100644 --- a/crates/iceberg/src/spec/schema/visitor.rs +++ b/crates/iceberg/src/spec/schema/visitor.rs @@ -16,6 +16,7 @@ // under the License. use super::*; +use crate::spec::VariantType; /// A post order schema visitor. /// @@ -69,6 +70,9 @@ pub trait SchemaVisitor { fn map(&mut self, map: &MapType, key_value: Self::T, value: Self::T) -> Result; /// Called when see a primitive type. fn primitive(&mut self, p: &PrimitiveType) -> Result; + + /// Called when see a variant type. + fn variant(&mut self, v: &VariantType) -> Result; } /// Visiting a type in post order. @@ -99,6 +103,7 @@ pub(crate) fn visit_type(r#type: &Type, visitor: &mut V) -> Re visitor.map(map, key_result, value_result) } Type::Struct(s) => visit_struct(s, visitor), + Type::Variant(v) => visitor.variant(v), } } @@ -185,6 +190,8 @@ pub trait SchemaWithPartnerVisitor

{ ) -> Result; /// Called when see a primitive type. fn primitive(&mut self, p: &PrimitiveType, partner: &P) -> Result; + /// Called when see a variant type. + fn variant(&mut self, v: &VariantType, partner: &P) -> Result; } /// Accessor used to get child partner from parent partner. @@ -242,6 +249,7 @@ pub(crate) fn visit_type_with_partner, A: Part visitor.map(map, partner, key_result, value_result) } Type::Struct(s) => visit_struct_with_partner(s, partner, visitor, accessor), + Type::Variant(v) => visitor.variant(v, partner), } } diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index 04a9e15b34..bbd53a0c32 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -21,29 +21,24 @@ use std::collections::HashMap; use std::sync::Arc; -use _serde::SnapshotV2; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use typed_builder::TypedBuilder; -use super::table_metadata::SnapshotLog; -use crate::error::{Result, timestamp_ms_to_utc}; -use crate::io::FileIO; -use crate::spec::{ManifestList, SchemaId, SchemaRef, TableMetadata}; -use crate::{Error, ErrorKind}; +use crate::error::{Result, invalid_data, timestamp_ms_to_utc}; +use crate::spec::{SchemaId, SchemaRef, TableMetadata}; /// The ref name of the main branch of the table. pub const MAIN_BRANCH: &str = "main"; -/// Placeholder for snapshot ID. The field with this value must be replaced with the actual snapshot ID before it is committed. -pub const UNASSIGNED_SNAPSHOT_ID: i64 = -1; /// Reference to [`Snapshot`]. pub type SnapshotRef = Arc; -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq, Clone)] #[serde(rename_all = "lowercase")] /// The operation field is used by some operations, like snapshot expiration, to skip processing certain snapshots. pub enum Operation { /// Only data files were added and no files were removed. + #[default] Append, /// Data and delete files were added and removed without changing table data; /// i.e., compaction, changing the data file format, or relocating data files. @@ -76,39 +71,52 @@ pub struct Summary { pub additional_properties: HashMap, } -impl Default for Operation { - fn default() -> Operation { - Self::Append - } +#[derive(Debug, PartialEq, Eq, Clone)] +/// Row range of a snapshot, contains first_row_id and added_rows_count. +pub struct SnapshotRowRange { + /// The first _row_id assigned to the first row in the first data file in the first manifest. + pub first_row_id: u64, + /// The upper bound of the number of rows with assigned row IDs + pub added_rows: u64, } -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, TypedBuilder)] -#[serde(from = "SnapshotV2", into = "SnapshotV2")] +#[derive(Debug, PartialEq, Eq, Clone, TypedBuilder)] #[builder(field_defaults(setter(prefix = "with_")))] /// A snapshot represents the state of a table at some time and is used to access the complete set of data files in the table. pub struct Snapshot { /// A unique long ID - snapshot_id: i64, + pub(crate) snapshot_id: i64, /// The snapshot ID of the snapshot’s parent. /// Omitted for any snapshot with no parent #[builder(default = None)] - parent_snapshot_id: Option, + pub(crate) parent_snapshot_id: Option, /// A monotonically increasing long that tracks the order of /// changes to a table. - sequence_number: i64, + pub(crate) sequence_number: i64, /// A timestamp when the snapshot was created, used for garbage /// collection and table inspection - timestamp_ms: i64, + pub(crate) timestamp_ms: i64, /// The location of a manifest list for this snapshot that /// tracks manifest files with additional metadata. /// Currently we only support manifest list file, and manifest files are not supported. #[builder(setter(into))] - manifest_list: String, + pub(crate) manifest_list: String, /// A string map that summarizes the snapshot changes, including operation. - summary: Summary, + pub(crate) summary: Summary, /// ID of the table’s current schema when the snapshot was created. #[builder(setter(strip_option(fallback = schema_id_opt)), default = None)] - schema_id: Option, + pub(crate) schema_id: Option, + /// Encryption Key ID + #[builder(default)] + pub(crate) encryption_key_id: Option, + /// Row range of this snapshot, required when the table version supports row lineage. + /// Specify as a tuple of (first_row_id, added_rows_count) + #[builder(default, setter(!strip_option, transform = |first_row_id: u64, added_rows: u64| Some(SnapshotRowRange { first_row_id, added_rows })))] + // This is specified as a struct instead of two separate fields to ensure that both fields are either set or not set. + // The java implementations uses two separate fields, then sets `added_row_counts` to Null if `first_row_id` is set to Null. + // It throws an error if `added_row_counts` is set but `first_row_id` is not set, or if either of the two is negative. + // We handle all cases infallible using the rust type system. + pub(crate) row_range: Option, } impl Snapshot { @@ -163,12 +171,7 @@ impl Snapshot { Ok(match self.schema_id() { Some(schema_id) => table_metadata .schema_by_id(schema_id) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Schema with id {} not found", schema_id), - ) - })? + .ok_or_else(|| invalid_data!("Schema with id {schema_id} not found"))? .clone(), None => table_metadata.current_schema().clone(), }) @@ -183,27 +186,35 @@ impl Snapshot { } } - /// Load manifest list. - pub async fn load_manifest_list( - &self, - file_io: &FileIO, - table_metadata: &TableMetadata, - ) -> Result { - let manifest_list_content = file_io.new_input(&self.manifest_list)?.read().await?; - ManifestList::parse_with_version( - &manifest_list_content, - // TODO: You don't really need the version since you could just project any Avro in - // the version that you'd like to get (probably always the latest) - table_metadata.format_version(), - ) - } - - #[allow(dead_code)] - pub(crate) fn log(&self) -> SnapshotLog { - SnapshotLog { - timestamp_ms: self.timestamp_ms, - snapshot_id: self.snapshot_id, - } + /// The row-id of the first newly added row in this snapshot. All rows added in this snapshot will + /// have a row-id assigned to them greater than this value. All rows with a row-id less than this + /// value were created in a snapshot that was added to the table (but not necessarily committed to + /// this branch) in the past. + /// + /// This field is optional but is required when the table version supports row lineage. + pub fn first_row_id(&self) -> Option { + self.row_range.as_ref().map(|r| r.first_row_id) + } + + /// The total number of newly added rows in this snapshot. It should be the summation of {@link + /// ManifestFile#ADDED_ROWS_COUNT} for every manifest added in this snapshot. + /// + /// This field is optional but is required when the table version supports row lineage. + pub fn added_rows_count(&self) -> Option { + self.row_range.as_ref().map(|r| r.added_rows) + } + + /// Returns the row range of this snapshot, if available. + /// This is a tuple containing (first_row_id, added_rows_count). + pub fn row_range(&self) -> Option<(u64, u64)> { + self.row_range + .as_ref() + .map(|r| (r.first_row_id, r.added_rows)) + } + + /// Get encryption key id, if available. + pub fn encryption_key_id(&self) -> Option<&str> { + self.encryption_key_id.as_deref() } } @@ -218,7 +229,30 @@ pub(super) mod _serde { use super::{Operation, Snapshot, Summary}; use crate::Error; + use crate::error::invalid_data; use crate::spec::SchemaId; + use crate::spec::snapshot::SnapshotRowRange; + + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] + #[serde(rename_all = "kebab-case")] + /// Defines the structure of a v3 snapshot for serialization/deserialization + pub(crate) struct SnapshotV3 { + pub snapshot_id: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_snapshot_id: Option, + pub sequence_number: i64, + pub timestamp_ms: i64, + pub manifest_list: String, + pub summary: Summary, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_row_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub added_rows: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub key_id: Option, + } #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] @@ -227,6 +261,7 @@ pub(super) mod _serde { pub snapshot_id: i64, #[serde(skip_serializing_if = "Option::is_none")] pub parent_snapshot_id: Option, + #[serde(default)] pub sequence_number: i64, pub timestamp_ms: i64, pub manifest_list: String, @@ -253,6 +288,52 @@ pub(super) mod _serde { pub schema_id: Option, } + impl From for Snapshot { + fn from(s: SnapshotV3) -> Self { + Snapshot { + snapshot_id: s.snapshot_id, + parent_snapshot_id: s.parent_snapshot_id, + sequence_number: s.sequence_number, + timestamp_ms: s.timestamp_ms, + manifest_list: s.manifest_list, + summary: s.summary, + schema_id: s.schema_id, + encryption_key_id: s.key_id, + row_range: match (s.first_row_id, s.added_rows) { + (Some(first_row_id), Some(added_rows)) => Some(SnapshotRowRange { + first_row_id, + added_rows, + }), + _ => None, + }, + } + } + } + + impl TryFrom for SnapshotV3 { + type Error = Error; + + fn try_from(s: Snapshot) -> Result { + let (first_row_id, added_rows) = match s.row_range { + Some(row_range) => (Some(row_range.first_row_id), Some(row_range.added_rows)), + None => (None, None), + }; + + Ok(SnapshotV3 { + snapshot_id: s.snapshot_id, + parent_snapshot_id: s.parent_snapshot_id, + sequence_number: s.sequence_number, + timestamp_ms: s.timestamp_ms, + manifest_list: s.manifest_list, + summary: s.summary, + schema_id: s.schema_id, + first_row_id, + added_rows, + key_id: s.encryption_key_id, + }) + } + } + impl From for Snapshot { fn from(v2: SnapshotV2) -> Self { Snapshot { @@ -263,6 +344,8 @@ pub(super) mod _serde { manifest_list: v2.manifest_list, summary: v2.summary, schema_id: v2.schema_id, + encryption_key_id: None, + row_range: None, } } } @@ -292,14 +375,24 @@ pub(super) mod _serde { timestamp_ms: v1.timestamp_ms, manifest_list: match (v1.manifest_list, v1.manifests) { (Some(file), None) => file, - (Some(_), Some(_)) => "Invalid v1 snapshot, when manifest list provided, manifest files should be omitted".to_string(), - (None, _) => "Unsupported v1 snapshot, only manifest list is supported".to_string() - }, + (Some(_), Some(_)) => { + return Err(invalid_data!( + "Invalid v1 snapshot, when manifest list provided, manifest files should be omitted" + )); + } + (None, _) => { + return Err(invalid_data!( + "Unsupported v1 snapshot, only manifest list is supported" + )); + } + }, summary: v1.summary.unwrap_or(Summary { operation: Operation::default(), additional_properties: HashMap::new(), }), schema_id: v1.schema_id, + encryption_key_id: None, + row_range: None, }) } } @@ -399,6 +492,7 @@ mod tests { use chrono::{TimeZone, Utc}; + use crate::spec::TableMetadata; use crate::spec::snapshot::_serde::SnapshotV1; use crate::spec::snapshot::{Operation, Snapshot, Summary}; @@ -486,6 +580,84 @@ mod tests { ); } + #[test] + fn test_v1_snapshot_with_manifest_list_and_manifests() { + { + let metadata = r#" + { + "format-version": 1, + "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", + "location": "s3://bucket/test/location", + "last-updated-ms": 1700000000000, + "last-column-id": 1, + "schema": { + "type": "struct", + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ] + }, + "partition-spec": [], + "properties": {}, + "current-snapshot-id": 111111111, + "snapshots": [ + { + "snapshot-id": 111111111, + "timestamp-ms": 1600000000000, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-123.avro", + "manifests": ["s3://bucket/metadata/manifest-1.avro"] + } + ] + } + "#; + + let result_both_manifest_list_and_manifest_set = + serde_json::from_str::(metadata); + assert!(result_both_manifest_list_and_manifest_set.is_err()); + assert_eq!( + result_both_manifest_list_and_manifest_set + .unwrap_err() + .to_string(), + "DataInvalid => Invalid v1 snapshot, when manifest list provided, manifest files should be omitted" + ) + } + + { + let metadata = r#" + { + "format-version": 1, + "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", + "location": "s3://bucket/test/location", + "last-updated-ms": 1700000000000, + "last-column-id": 1, + "schema": { + "type": "struct", + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ] + }, + "partition-spec": [], + "properties": {}, + "current-snapshot-id": 111111111, + "snapshots": [ + { + "snapshot-id": 111111111, + "timestamp-ms": 1600000000000, + "summary": {"operation": "append"}, + "manifests": ["s3://bucket/metadata/manifest-1.avro"] + } + ] + } + "#; + let result_missing_manifest_list = serde_json::from_str::(metadata); + assert!(result_missing_manifest_list.is_err()); + assert_eq!( + result_missing_manifest_list.unwrap_err().to_string(), + "DataInvalid => Unsupported v1 snapshot, only manifest list is supported" + ) + } + } + #[test] fn test_snapshot_v1_to_v2_with_missing_summary() { use crate::spec::snapshot::_serde::SnapshotV1; diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index 1b07ce3fa0..fdaf277043 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -20,6 +20,7 @@ use std::collections::HashMap; use itertools::Itertools; use super::{DataContentType, DataFile, PartitionSpecRef}; +use crate::error::invalid_data; use crate::spec::{ManifestContentType, ManifestFile, Operation, SchemaRef, Summary}; use crate::{Error, ErrorKind, Result}; @@ -47,6 +48,7 @@ const TOTAL_RECORDS: &str = "total-records"; const TOTAL_FILE_SIZE: &str = "total-files-size"; const CHANGED_PARTITION_COUNT_PROP: &str = "changed-partition-count"; const CHANGED_PARTITION_PREFIX: &str = "partitions."; +const ICEBERG_VERSION_PROP: &str = "iceberg-version"; /// `SnapshotSummaryCollector` collects and aggregates snapshot update metrics. /// It gathers metrics about added or removed data files and manifests, and tracks @@ -329,7 +331,14 @@ where T: PartialOrd + Default + ToString { } } -#[allow(dead_code)] +pub(crate) fn set_iceberg_version(properties: &mut HashMap) { + let version = env!("CARGO_PKG_VERSION"); + properties.insert( + ICEBERG_VERSION_PROP.to_string(), + format!("Apache Iceberg Rust {version}"), + ); +} + pub(crate) fn update_snapshot_summaries( summary: Summary, previous_summary: Option<&Summary>, @@ -340,20 +349,15 @@ pub(crate) fn update_snapshot_summaries( && summary.operation != Operation::Overwrite && summary.operation != Operation::Delete { - return Err(Error::new( - ErrorKind::DataInvalid, - "Operation is not supported.", - )); + return Err(invalid_data!("Operation is not supported.")); } let mut summary = match previous_summary { Some(prev_summary) if truncate_full_table && summary.operation == Operation::Overwrite => { - truncate_table_summary(summary, prev_summary) - .map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to truncate table summary.") - .with_source(err) - }) - .unwrap() + truncate_table_summary(summary, prev_summary).map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to truncate table summary.") + .with_source(err) + })? } _ => summary, }; @@ -408,23 +412,21 @@ pub(crate) fn update_snapshot_summaries( Ok(summary) } -#[allow(dead_code)] -fn get_prop(previous_summary: &Summary, prop: &str) -> Result { +fn get_prop(previous_summary: &Summary, prop: &str) -> Result { let value_str = previous_summary .additional_properties .get(prop) .map(String::as_str) .unwrap_or("0"); - value_str.parse::().map_err(|err| { + value_str.parse::().map_err(|err| { Error::new( ErrorKind::Unexpected, - "Failed to parse value from previous summary property.", + format!("Failed to parse summary property '{prop}' value '{value_str}' as u64."), ) .with_source(err) }) } -#[allow(dead_code)] fn truncate_table_summary(mut summary: Summary, previous_summary: &Summary) -> Result

{ for prop in [ TOTAL_DATA_FILES, @@ -481,7 +483,6 @@ fn truncate_table_summary(mut summary: Summary, previous_summary: &Summary) -> R Ok(summary) } -#[allow(dead_code)] fn update_totals( summary: &mut Summary, previous_summary: Option<&Summary>, @@ -489,28 +490,59 @@ fn update_totals( added_property: &str, removed_property: &str, ) { - let previous_total = previous_summary.map_or(0, |previous_summary| { - previous_summary - .additional_properties - .get(total_property) - .map_or(0, |value| value.parse::().unwrap()) - }); + let previous_total = match previous_summary { + None => 0, + Some(prev_summary) => match prev_summary.additional_properties.get(total_property) { + Some(value_str) => match value_str.parse::() { + Ok(v) => v, + Err(parse_err) => { + tracing::warn!( + "Property '{total_property}' could not be parsed in the previous snapshot summary: {parse_err}. \ + Skipping total computation.", + ); + return; + } + }, + None => { + tracing::debug!( + "Property '{total_property}' was not set in the previous snapshot summary. \ + Skipping total computation." + ); + return; + } + }, + }; - let mut new_total = previous_total; - if let Some(value) = summary - .additional_properties - .get(added_property) - .map(|value| value.parse::().unwrap()) - { - new_total += value; - } - if let Some(value) = summary - .additional_properties - .get(removed_property) - .map(|value| value.parse::().unwrap()) - { - new_total -= value; - } + // Parse the added/removed deltas, tolerating an unparsable value by skipping + // the total entirely rather than panicking. Computed metrics always overwrite + // user-supplied summary properties (see `SnapshotProducer::summary`), so a bad + // value should only ever come from a previous snapshot's summary; matching + // iceberg-java's `updateTotal`, we ignore it instead of failing the commit. + let parse_delta = |property: &str| -> Option { + match summary.additional_properties.get(property) { + None => Some(0), + Some(value) => match value.parse::() { + Ok(v) => Some(v), + Err(parse_err) => { + tracing::warn!( + "Property '{property}' could not be parsed when computing '{total_property}': {parse_err}. \ + Skipping total computation.", + ); + None + } + }, + } + }; + + let (Some(added), Some(removed)) = (parse_delta(added_property), parse_delta(removed_property)) + else { + return; + }; + + // Like Java, a total that would go negative is omitted rather than written as a false basis. + let Some(new_total) = previous_total.saturating_add(added).checked_sub(removed) else { + return; + }; summary .additional_properties .insert(total_property.to_string(), new_total.to_string()); @@ -715,6 +747,83 @@ mod tests { ); } + #[test] + fn test_update_snapshot_summaries_overwrite_truncate_handles_totals_above_i32_max() { + // A table can legitimately accumulate more than i32::MAX rows or files + // over its lifetime. Truncating such a table on overwrite must succeed + // and surface the previous totals into the deleted-* counters. + let big = (i32::MAX as u64 + 1).to_string(); // 2_147_483_648 + let prev_props: HashMap = [ + (TOTAL_DATA_FILES.to_string(), big.clone()), + (TOTAL_DELETE_FILES.to_string(), "0".to_string()), + (TOTAL_RECORDS.to_string(), big.clone()), + (TOTAL_FILE_SIZE.to_string(), "0".to_string()), + (TOTAL_POSITION_DELETES.to_string(), "0".to_string()), + (TOTAL_EQUALITY_DELETES.to_string(), "0".to_string()), + ] + .into_iter() + .collect(); + + let previous_summary = Summary { + operation: Operation::Overwrite, + additional_properties: prev_props, + }; + + let summary = Summary { + operation: Operation::Overwrite, + additional_properties: HashMap::new(), + }; + + let updated = update_snapshot_summaries(summary, Some(&previous_summary), true) + .expect("overwrite truncation should accept totals above i32::MAX"); + assert_eq!( + updated + .additional_properties + .get(DELETED_DATA_FILES) + .unwrap(), + &big + ); + assert_eq!( + updated.additional_properties.get(DELETED_RECORDS).unwrap(), + &big + ); + } + + #[test] + fn test_update_snapshot_summaries_overwrite_truncate_returns_err_on_malformed_total() { + // Non-numeric values in the previous summary (corruption, manual edits, + // a foreign implementation) must surface as a recoverable Err - not + // crash the process. + let prev_props: HashMap = [ + (TOTAL_DATA_FILES.to_string(), "not_a_number".to_string()), + (TOTAL_DELETE_FILES.to_string(), "0".to_string()), + (TOTAL_RECORDS.to_string(), "0".to_string()), + (TOTAL_FILE_SIZE.to_string(), "0".to_string()), + (TOTAL_POSITION_DELETES.to_string(), "0".to_string()), + (TOTAL_EQUALITY_DELETES.to_string(), "0".to_string()), + ] + .into_iter() + .collect(); + + let previous_summary = Summary { + operation: Operation::Overwrite, + additional_properties: prev_props, + }; + + let summary = Summary { + operation: Operation::Overwrite, + additional_properties: HashMap::new(), + }; + + let err = update_snapshot_summaries(summary, Some(&previous_summary), true) + .expect_err("malformed previous summary must produce an Err, not a panic"); + assert!( + err.message().contains("truncate table summary"), + "expected wrapped 'Failed to truncate table summary' context, got: {}", + err.message() + ); + } + #[test] fn test_snapshot_summary_collector_build() { let schema = Arc::new( @@ -731,10 +840,11 @@ mod tests { PartitionSpec::builder(schema.clone()) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(2) + .source_ids(vec![2]) .name("year".to_string()) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ]) .unwrap() .with_spec_id(1) @@ -767,7 +877,7 @@ mod tests { (3, Datum::string("x")), ]), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: Some(0), partition_spec_id: 0, @@ -799,7 +909,7 @@ mod tests { (3, Datum::string("x")), ]), key_metadata: None, - split_offsets: vec![4], + split_offsets: Some(vec![4]), equality_ids: None, sort_order_id: Some(0), partition_spec_id: 0, @@ -824,9 +934,9 @@ mod tests { assert!(props.contains_key(&partition_key)); let partition_summary = props.get(&partition_key).unwrap(); - assert!(partition_summary.contains(&format!("{}=200", ADDED_FILE_SIZE))); - assert!(partition_summary.contains(&format!("{}=1", ADDED_DATA_FILES))); - assert!(partition_summary.contains(&format!("{}=20", ADDED_RECORDS))); + assert!(partition_summary.contains(&format!("{ADDED_FILE_SIZE}=200"))); + assert!(partition_summary.contains(&format!("{ADDED_DATA_FILES}=1"))); + assert!(partition_summary.contains(&format!("{ADDED_RECORDS}=20"))); } #[test] @@ -850,6 +960,7 @@ mod tests { deleted_rows_count: Some(50), partitions: Some(Vec::new()), key_metadata: None, + first_row_id: None, }; collector @@ -880,10 +991,11 @@ mod tests { PartitionSpec::builder(schema.clone()) .add_unbound_fields(vec![ UnboundPartitionField::builder() - .source_id(2) + .source_ids(vec![2]) .name("year".to_string()) .transform(Transform::Identity) - .build(), + .build() + .unwrap(), ]) .unwrap() .with_spec_id(1) @@ -909,7 +1021,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -937,7 +1049,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -974,6 +1086,7 @@ mod tests { deleted_rows_count: Some(0), partitions: Some(Vec::new()), key_metadata: None, + first_row_id: None, }); summary_four.add_file( @@ -991,7 +1104,7 @@ mod tests { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), key_metadata: None, - split_offsets: vec![], + split_offsets: None, equality_ids: None, sort_order_id: None, partition_spec_id: 0, @@ -1015,4 +1128,181 @@ mod tests { .all(|(k, _)| !k.starts_with(CHANGED_PARTITION_PREFIX)) ); } + + #[test] + fn test_update_totals_skipped_when_previous_summary_missing_totals() { + let prev_props: HashMap = [(TOTAL_DATA_FILES, "8")] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let previous_summary = Summary { + operation: Operation::Overwrite, + additional_properties: prev_props, + }; + + let new_props: HashMap = [ + (ADDED_DATA_FILES, "4"), + (ADDED_DELETE_FILES, "2"), + (ADDED_RECORDS, "40"), + (ADDED_FILE_SIZE, "400"), + (ADDED_POSITION_DELETES, "5"), + (ADDED_EQUALITY_DELETES, "3"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let summary = Summary { + operation: Operation::Append, + additional_properties: new_props, + }; + + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + let props = &updated.additional_properties; + + assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "12"); + + for total_field in [ + TOTAL_DELETE_FILES, + TOTAL_RECORDS, + TOTAL_FILE_SIZE, + TOTAL_POSITION_DELETES, + TOTAL_EQUALITY_DELETES, + ] { + assert!( + !props.contains_key(total_field), + "{total_field} should not be set when previous summary lacks it", + ); + } + } + + #[test] + fn test_update_totals_tolerates_unparsable_added_value() { + // A non-integer added value (which can survive in a previous snapshot's + // summary) must not panic the commit. Matching iceberg-java's `updateTotal` + // try/catch, the affected total is skipped while other totals still compute. + let prev_props: HashMap = [(TOTAL_DATA_FILES, "8"), (TOTAL_RECORDS, "80")] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let previous_summary = Summary { + operation: Operation::Append, + additional_properties: prev_props, + }; + + let new_props: HashMap = + [(ADDED_DATA_FILES, "not-a-number"), (ADDED_RECORDS, "40")] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let summary = Summary { + operation: Operation::Append, + additional_properties: new_props, + }; + + // Must not panic. + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + let props = &updated.additional_properties; + + // The total whose added delta was unparsable is skipped... + assert!( + !props.contains_key(TOTAL_DATA_FILES), + "TOTAL_DATA_FILES should be skipped when its added value is unparsable", + ); + // ...while a sibling total with valid deltas still computes. + assert_eq!(props.get(TOTAL_RECORDS).unwrap(), "120"); + } + + #[test] + fn test_update_totals_computed_when_no_previous_summary() { + let new_props: HashMap = [ + (ADDED_DATA_FILES, "4"), + (ADDED_RECORDS, "40"), + (ADDED_FILE_SIZE, "400"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let summary = Summary { + operation: Operation::Append, + additional_properties: new_props, + }; + + let updated = update_snapshot_summaries(summary, None, false).unwrap(); + let props = &updated.additional_properties; + + assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "4"); + assert_eq!(props.get(TOTAL_RECORDS).unwrap(), "40"); + assert_eq!(props.get(TOTAL_FILE_SIZE).unwrap(), "400"); + } + + #[test] + fn test_update_totals_with_removes_only() { + let prev_props: HashMap = [ + (TOTAL_DATA_FILES, "10"), + (TOTAL_DELETE_FILES, "5"), + (TOTAL_RECORDS, "100"), + (TOTAL_FILE_SIZE, "1000"), + (TOTAL_POSITION_DELETES, "3"), + (TOTAL_EQUALITY_DELETES, "2"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let previous_summary = Summary { + operation: Operation::Overwrite, + additional_properties: prev_props, + }; + + let new_props: HashMap = [ + (DELETED_DATA_FILES, "2"), + (REMOVED_DELETE_FILES, "1"), + (DELETED_RECORDS, "20"), + (REMOVED_FILE_SIZE, "200"), + (REMOVED_POSITION_DELETES, "1"), + (REMOVED_EQUALITY_DELETES, "1"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let summary = Summary { + operation: Operation::Delete, + additional_properties: new_props, + }; + + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + let props = &updated.additional_properties; + + assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "8"); + assert_eq!(props.get(TOTAL_DELETE_FILES).unwrap(), "4"); + assert_eq!(props.get(TOTAL_RECORDS).unwrap(), "80"); + assert_eq!(props.get(TOTAL_FILE_SIZE).unwrap(), "800"); + assert_eq!(props.get(TOTAL_POSITION_DELETES).unwrap(), "2"); + assert_eq!(props.get(TOTAL_EQUALITY_DELETES).unwrap(), "1"); + } + + #[test] + fn test_update_totals_omits_total_when_removed_exceeds_previous_total() { + let previous_summary = Summary { + operation: Operation::Append, + additional_properties: HashMap::from([(TOTAL_DATA_FILES.to_string(), "2".to_string())]), + }; + let summary = Summary { + operation: Operation::Delete, + additional_properties: HashMap::from([( + DELETED_DATA_FILES.to_string(), + "3".to_string(), + )]), + }; + + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + + assert!(!updated.additional_properties.contains_key(TOTAL_DATA_FILES)); + } } diff --git a/crates/iceberg/src/spec/sort.rs b/crates/iceberg/src/spec/sort.rs index 379d44cc2d..c1c9fdbad0 100644 --- a/crates/iceberg/src/spec/sort.rs +++ b/crates/iceberg/src/spec/sort.rs @@ -27,7 +27,7 @@ use typed_builder::TypedBuilder; use super::transform::Transform; use crate::error::Result; -use crate::spec::Schema; +use crate::spec::{PrimitiveType, Schema, Type}; use crate::{Error, ErrorKind}; /// Reference to [`SortOrder`]. @@ -180,7 +180,7 @@ impl SortOrderBuilder { match schema.field_by_id(sort_field.source_id) { None => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::DataInvalid, format!("Cannot find source column for sort field: {sort_field}"), )); } @@ -189,20 +189,26 @@ impl SortOrderBuilder { if !source_type.is_primitive() { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::DataInvalid, format!("Cannot sort by non-primitive source field: {source_type}"), )); } - let field_transform = sort_field.transform; - if field_transform.result_type(source_type).is_err() { + if matches!( + source_type, + Type::Primitive(PrimitiveType::Geometry(_) | PrimitiveType::Geography(_)) + ) { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::DataInvalid, format!( - "Invalid source type {source_type} for transform {field_transform}" + "Cannot sort by geospatial source field: {}", + source_field.name ), )); } + + let field_transform = sort_field.transform; + field_transform.result_type(source_type)?; } } } @@ -214,7 +220,7 @@ impl SortOrderBuilder { #[cfg(test)] mod tests { use super::*; - use crate::spec::{ListType, NestedField, PrimitiveType, Type}; + use crate::spec::{GeometryType, ListType, NestedField, PrimitiveType, Type}; #[test] fn test_sort_field() { @@ -403,12 +409,12 @@ mod tests { ) .build(&schema); + let err = sort_order_builder_result.expect_err("Expected an Err value"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); assert_eq!( - sort_order_builder_result - .expect_err("Expected an Err value") - .message(), + err.message(), "Cannot find source column for sort field: SortField { source_id: 2, transform: identity, direction: ascending, null_order: first }" - ) + ); } #[test] @@ -444,12 +450,75 @@ mod tests { ) .build(&schema); + let err = sort_order_builder_result.expect_err("Expected an Err value"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); assert_eq!( - sort_order_builder_result - .expect_err("Expected an Err value") - .message(), + err.message(), "Cannot sort by non-primitive source field: list" - ) + ); + } + + #[test] + fn test_build_should_return_err_if_source_field_is_variant() { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::optional(1, "v", Type::Variant(crate::spec::VariantType)).into(), + ]) + .build() + .unwrap(); + + let sort_order_builder_result = SortOrder::builder() + .with_sort_field( + SortField::builder() + .source_id(1) + .direction(SortDirection::Ascending) + .null_order(NullOrder::First) + .transform(Transform::Identity) + .build(), + ) + .build(&schema); + + let err = sort_order_builder_result.expect_err("Expected an Err value"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + "Cannot sort by non-primitive source field: variant" + ); + } + + #[test] + fn test_build_should_reject_geospatial_source_fields() { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required( + 1, + "geom", + Type::Primitive(PrimitiveType::Geometry(GeometryType::default())), + ) + .into(), + ]) + .build() + .unwrap(); + + let err = SortOrder::builder() + .with_sort_field( + SortField::builder() + .source_id(1) + .direction(SortDirection::Ascending) + .null_order(NullOrder::First) + .transform(Transform::Void) + .build(), + ) + .build(&schema) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + "Cannot sort by geospatial source field: geom" + ); } #[test] @@ -473,12 +542,12 @@ mod tests { ) .build(&schema); + let err = sort_order_builder_result.expect_err("Expected an Err value"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); assert_eq!( - sort_order_builder_result - .expect_err("Expected an Err value") - .message(), - "Invalid source type int for transform year" - ) + err.message(), + "int is not a valid input type of year transform" + ); } #[test] diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index d7347d50ee..63199742f5 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -33,104 +33,31 @@ use uuid::Uuid; use super::snapshot::SnapshotReference; pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataBuilder}; use super::{ - DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, - SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, + DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, Schema, SchemaId, + SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, + TableProperties, Transform, }; -use crate::error::{Result, timestamp_ms_to_utc}; +use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation}; +use crate::compression::CompressionCodec; +use crate::error::{Result, invalid_data, timestamp_ms_to_utc}; use crate::io::FileIO; +use crate::partitioning::compute_unified_partition_type; +use crate::spec::EncryptedKey; use crate::{Error, ErrorKind}; static MAIN_BRANCH: &str = "main"; pub(crate) static ONE_MINUTE_MS: i64 = 60_000; +/// Sentinel value used by the Java implementation and older metadata files +/// to represent a missing/empty current snapshot ID. During deserialization, +/// this value is normalized to `None`. pub(crate) static EMPTY_SNAPSHOT_ID: i64 = -1; pub(crate) static INITIAL_SEQUENCE_NUMBER: i64 = 0; -/// Reserved table property for table format version. -/// -/// Iceberg will default a new table's format version to the latest stable and recommended -/// version. This reserved property keyword allows users to override the Iceberg format version of -/// the table metadata. -/// -/// If this table property exists when creating a table, the table will use the specified format -/// version. If a table updates this property, it will try to upgrade to the specified format -/// version. -pub const PROPERTY_FORMAT_VERSION: &str = "format-version"; -/// Reserved table property for table UUID. -pub const PROPERTY_UUID: &str = "uuid"; -/// Reserved table property for the total number of snapshots. -pub const PROPERTY_SNAPSHOT_COUNT: &str = "snapshot-count"; -/// Reserved table property for current snapshot summary. -pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str = "current-snapshot-summary"; -/// Reserved table property for current snapshot id. -pub const PROPERTY_CURRENT_SNAPSHOT_ID: &str = "current-snapshot-id"; -/// Reserved table property for current snapshot timestamp. -pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str = "current-snapshot-timestamp-ms"; -/// Reserved table property for the JSON representation of current schema. -pub const PROPERTY_CURRENT_SCHEMA: &str = "current-schema"; -/// Reserved table property for the JSON representation of current(default) partition spec. -pub const PROPERTY_DEFAULT_PARTITION_SPEC: &str = "default-partition-spec"; -/// Reserved table property for the JSON representation of current(default) sort order. -pub const PROPERTY_DEFAULT_SORT_ORDER: &str = "default-sort-order"; - -/// Property key for max number of previous versions to keep. -pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str = "write.metadata.previous-versions-max"; -/// Default value for max number of previous versions to keep. -pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100; - -/// Property key for max number of partitions to keep summary stats for. -pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str = "write.summary.partition-limit"; -/// Default value for the max number of partitions to keep summary stats for. -pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0; - -/// Reserved Iceberg table properties list. -/// -/// Reserved table properties are only used to control behaviors when creating or updating a -/// table. The value of these properties are not persisted as a part of the table metadata. -pub const RESERVED_PROPERTIES: [&str; 9] = [ - PROPERTY_FORMAT_VERSION, - PROPERTY_UUID, - PROPERTY_SNAPSHOT_COUNT, - PROPERTY_CURRENT_SNAPSHOT_ID, - PROPERTY_CURRENT_SNAPSHOT_SUMMARY, - PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP, - PROPERTY_CURRENT_SCHEMA, - PROPERTY_DEFAULT_PARTITION_SPEC, - PROPERTY_DEFAULT_SORT_ORDER, -]; - -/// Property key for number of commit retries. -pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries"; -/// Default value for number of commit retries. -pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; - -/// Property key for minimum wait time (ms) between retries. -pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms"; -/// Default value for minimum wait time (ms) between retries. -pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; - -/// Property key for maximum wait time (ms) between retries. -pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms"; -/// Default value for maximum wait time (ms) between retries. -pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute - -/// Property key for total maximum retry time (ms). -pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms"; -/// Default value for total maximum retry time (ms). -pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes - -/// Default file format for data files -pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = "write.format.default"; -/// Default file format for delete files -pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str = "write.delete.format.default"; -/// Default value for data file format -pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet"; - -/// Target file size for newly written files. -pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; -/// Default target file size -pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB - +/// Initial row id for row lineage for new v3 tables and older tables upgrading to v3. +pub const INITIAL_ROW_ID: u64 = 0; +/// Minimum format version that supports row lineage (v3). +pub const MIN_FORMAT_VERSION_ROW_LINEAGE: FormatVersion = FormatVersion::V3; /// Reference to [`TableMetadata`]. pub type TableMetadataRef = Arc; @@ -208,8 +135,10 @@ pub struct TableMetadata { pub(crate) statistics: HashMap, /// Mapping of snapshot ids to partition statistics files. pub(crate) partition_statistics: HashMap, - /// Encryption Keys - pub(crate) encryption_keys: HashMap, + /// Encryption Keys - map of key id to the actual key + pub(crate) encryption_keys: HashMap, + /// Next row id to be assigned for Row Lineage (v3) + pub(crate) next_row_id: u64, } impl TableMetadata { @@ -224,14 +153,6 @@ impl TableMetadata { TableMetadataBuilder::new_from_metadata(self, current_file_location) } - /// Check if a partition field name exists in any partition spec. - #[inline] - pub(crate) fn partition_name_exists(&self, name: &str) -> bool { - self.partition_specs - .values() - .any(|spec| spec.fields().iter().any(|pf| pf.name == name)) - } - /// Check if a field name exists in any schema. #[inline] pub(crate) fn name_exists_in_any_schema(&self, name: &str) -> bool { @@ -349,6 +270,19 @@ impl TableMetadata { &self.default_partition_type } + /// Returns the unified partition type across every partition spec in the table, resolved + /// against `schema`. + /// + /// Unlike [`Self::default_partition_type`], the result contains all partition fields ever + /// used by the table, so partition values stay readable across partition spec evolution. + /// See [`compute_unified_partition_type`] for the exact merge rules. + pub fn unified_partition_type(&self, schema: &Schema) -> Result { + compute_unified_partition_type( + self.partition_specs_iter().map(|spec| spec.as_ref()), + schema, + ) + } + #[inline] /// Returns spec id of the "current" partition spec. pub fn default_partition_spec_id(&self) -> i32 { @@ -400,7 +334,7 @@ impl TableMetadata { pub fn snapshot_for_ref(&self, ref_name: &str) -> Option<&SnapshotRef> { self.refs.get(ref_name).map(|r| { self.snapshot_by_id(r.snapshot_id) - .unwrap_or_else(|| panic!("Snapshot id of ref {} doesn't exist", ref_name)) + .unwrap_or_else(|| panic!("Snapshot id of ref {ref_name} doesn't exist")) }) } @@ -436,6 +370,35 @@ impl TableMetadata { &self.properties } + /// Returns the base location for metadata files (manifests, manifest lists). + /// + /// Honors the `write.metadata.path` table property when set, otherwise defaults + /// to the `metadata` subdirectory under the table location. + pub fn metadata_location(&self) -> Result { + Ok(self + .table_properties() + .write_metadata_path()? + .unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME))) + } + + /// Returns the metadata compression codec from table properties. + /// + /// Returns `CompressionCodec::None` if compression is disabled or not configured. + /// Returns `CompressionCodec::Gzip` if gzip compression is enabled. + /// + /// # Errors + /// + /// Returns an error if the compression codec property has an invalid value. + pub fn metadata_compression_codec(&self) -> Result { + self.table_properties().metadata_compression_codec() + } + + /// Returns a typed view that parses each table property when its getter is called. + #[inline] + pub fn table_properties(&self) -> TableProperties<'_> { + TableProperties::new(&self.properties) + } + /// Return location of statistics files. #[inline] pub fn statistics_iter(&self) -> impl ExactSizeIterator { @@ -466,41 +429,65 @@ impl TableMetadata { } fn construct_refs(&mut self) { - if let Some(current_snapshot_id) = self.current_snapshot_id { - if !self.refs.contains_key(MAIN_BRANCH) { - self.refs - .insert(MAIN_BRANCH.to_string(), SnapshotReference { - snapshot_id: current_snapshot_id, - retention: SnapshotRetention::Branch { - min_snapshots_to_keep: None, - max_snapshot_age_ms: None, - max_ref_age_ms: None, - }, - }); - } + if let Some(current_snapshot_id) = self.current_snapshot_id + && !self.refs.contains_key(MAIN_BRANCH) + { + self.refs + .insert(MAIN_BRANCH.to_string(), SnapshotReference { + snapshot_id: current_snapshot_id, + retention: SnapshotRetention::Branch { + min_snapshots_to_keep: None, + max_snapshot_age_ms: None, + max_ref_age_ms: None, + }, + }); } } /// Iterate over all encryption keys #[inline] - pub fn encryption_keys_iter(&self) -> impl ExactSizeIterator { - self.encryption_keys.iter() + pub fn encryption_keys_iter(&self) -> impl ExactSizeIterator { + self.encryption_keys.values() } /// Get the encryption key for a given key id #[inline] - pub fn encryption_key(&self, key_id: &str) -> Option<&String> { + pub fn encryption_key(&self, key_id: &str) -> Option<&EncryptedKey> { self.encryption_keys.get(key_id) } + /// Get the next row id to be assigned + #[inline] + pub fn next_row_id(&self) -> u64 { + self.next_row_id + } + /// Read table metadata from the given location. pub async fn read_from( file_io: &FileIO, metadata_location: impl AsRef, ) -> Result { + let metadata_location = metadata_location.as_ref(); let input_file = file_io.new_input(metadata_location)?; let metadata_content = input_file.read().await?; - let metadata = serde_json::from_slice::(&metadata_content)?; + + // Check if the file is compressed by looking for the gzip "magic number". + let metadata = if metadata_content.len() > 2 + && metadata_content[0] == 0x1F + && metadata_content[1] == 0x8B + { + let decompressed_data = CompressionCodec::gzip_default() + .decompress(metadata_content.to_vec()) + .map_err(|e| { + invalid_data!("Trying to read compressed metadata file") + .with_context("file_path", metadata_location) + .with_source(e) + })?; + serde_json::from_slice(&decompressed_data)? + } else { + serde_json::from_slice(&metadata_content)? + }; + Ok(metadata) } @@ -508,11 +495,35 @@ impl TableMetadata { pub async fn write_to( &self, file_io: &FileIO, - metadata_location: impl AsRef, + metadata_location: &MetadataLocation, ) -> Result<()> { + let json_data = serde_json::to_vec(self)?; + + // Check if compression codec from properties matches the one in metadata_location + let codec = self.table_properties().metadata_compression_codec()?; + + if codec != metadata_location.compression_codec() { + return Err(invalid_data!( + "Compression codec mismatch: metadata_location has {:?}, but table properties specify {:?}", + metadata_location.compression_codec(), + codec + )); + } + + // Apply compression based on codec + let data_to_write = match codec { + CompressionCodec::Gzip(_) => codec.compress(json_data)?, + CompressionCodec::None => json_data, + _ => { + return Err(invalid_data!( + "Unsupported metadata compression codec: {codec:?}" + )); + } + }; + file_io - .new_output(metadata_location)? - .write(serde_json::to_vec(self)?.into()) + .new_output(metadata_location.to_string())? + .write(data_to_write.into()) .await } @@ -533,13 +544,29 @@ impl TableMetadata { // Normalize location (remove trailing slash) self.location = self.location.trim_end_matches('/').to_string(); self.validate_snapshot_sequence_number()?; + self.validate_schema_format_compatibility()?; self.try_normalize_partition_spec()?; self.try_normalize_sort_order()?; Ok(self) } - /// If the default partition spec is not present in specs, add it + /// Validate active default-spec sources and add the spec if it is not present. fn try_normalize_partition_spec(&mut self) -> Result<()> { + for field in self.default_spec.fields() { + // Historical specs may reference dropped columns, but active default-spec + // fields need a source for new writes. Void fields do not read their source. + if field.transform != Transform::Void + && self.current_schema().field_by_id(field.source_id).is_none() + { + return Err(invalid_data!( + "Default partition spec {} references missing source field {} in current schema {}", + self.default_spec.spec_id(), + field.source_id, + self.current_schema_id + )); + } + } + if self .partition_spec_by_id(self.default_spec.spec_id()) .is_none() @@ -555,17 +582,27 @@ impl TableMetadata { /// If the default sort order is unsorted but the sort order is not present, add it fn try_normalize_sort_order(&mut self) -> Result<()> { + // Validate that sort order ID 0 (reserved for unsorted) has no fields + if let Some(sort_order) = self.sort_order_by_id(SortOrder::UNSORTED_ORDER_ID) + && !sort_order.fields.is_empty() + { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Sort order ID {} is reserved for unsorted order", + SortOrder::UNSORTED_ORDER_ID + ), + )); + } + if self.sort_order_by_id(self.default_sort_order_id).is_some() { return Ok(()); } if self.default_sort_order_id != SortOrder::UNSORTED_ORDER_ID { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "No sort order exists with the default sort order id {}.", - self.default_sort_order_id - ), + return Err(invalid_data!( + "No sort order exists with the default sort order id {}.", + self.default_sort_order_id )); } @@ -578,12 +615,9 @@ impl TableMetadata { /// Validate the current schema is set and exists. fn validate_current_schema(&self) -> Result<()> { if self.schema_by_id(self.current_schema_id).is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "No schema exists with the current schema id {}.", - self.current_schema_id - ), + return Err(invalid_data!( + "No schema exists with the current schema id {}.", + self.current_schema_id )); } Ok(()) @@ -595,12 +629,8 @@ impl TableMetadata { if current_snapshot_id == EMPTY_SNAPSHOT_ID { self.current_snapshot_id = None; } else if self.snapshot_by_id(current_snapshot_id).is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Snapshot for current snapshot id {} does not exist in the existing snapshots list", - current_snapshot_id - ), + return Err(invalid_data!( + "Snapshot for current snapshot id {current_snapshot_id} does not exist in the existing snapshots list" )); } } @@ -611,33 +641,26 @@ impl TableMetadata { fn validate_refs(&self) -> Result<()> { for (name, snapshot_ref) in self.refs.iter() { if self.snapshot_by_id(snapshot_ref.snapshot_id).is_none() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Snapshot for reference {name} does not exist in the existing snapshots list" - ), + return Err(invalid_data!( + "Snapshot for reference {name} does not exist in the existing snapshots list" )); } } let main_ref = self.refs.get(MAIN_BRANCH); if self.current_snapshot_id.is_some() { - if let Some(main_ref) = main_ref { - if main_ref.snapshot_id != self.current_snapshot_id.unwrap_or_default() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Current snapshot id does not match main branch ({:?} != {:?})", - self.current_snapshot_id.unwrap_or_default(), - main_ref.snapshot_id - ), - )); - } + if let Some(main_ref) = main_ref + && main_ref.snapshot_id != self.current_snapshot_id.unwrap_or_default() + { + return Err(invalid_data!( + "Current snapshot id does not match main branch ({:?} != {:?})", + self.current_snapshot_id.unwrap_or_default(), + main_ref.snapshot_id + )); } } else if main_ref.is_some() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Current snapshot is not set, but main branch exists", + return Err(invalid_data!( + "Current snapshot is not set, but main branch exists" )); } @@ -647,31 +670,24 @@ impl TableMetadata { /// Validate that for V1 Metadata the last_sequence_number is 0 fn validate_snapshot_sequence_number(&self) -> Result<()> { if self.format_version < FormatVersion::V2 && self.last_sequence_number != 0 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Last sequence number must be 0 in v1. Found {}", - self.last_sequence_number - ), + return Err(invalid_data!( + "Last sequence number must be 0 in v1. Found {}", + self.last_sequence_number )); } - if self.format_version >= FormatVersion::V2 { - if let Some(snapshot) = self + if self.format_version >= FormatVersion::V2 + && let Some(snapshot) = self .snapshots .values() .find(|snapshot| snapshot.sequence_number() > self.last_sequence_number) - { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid snapshot with id {} and sequence number {} greater than last sequence number {}", - snapshot.snapshot_id(), - snapshot.sequence_number(), - self.last_sequence_number - ), - )); - } + { + return Err(invalid_data!( + "Invalid snapshot with id {} and sequence number {} greater than last sequence number {}", + snapshot.snapshot_id(), + snapshot.sequence_number(), + self.last_sequence_number + )); } Ok(()) @@ -684,10 +700,7 @@ impl TableMetadata { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if curr.timestamp_ms - prev.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - "Expected sorted snapshot log entries", - )); + return Err(invalid_data!("Expected sorted snapshot log entries")); } } @@ -695,12 +708,10 @@ impl TableMetadata { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if self.last_updated_ms - last.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid update timestamp {}: before last snapshot log entry at {}", - self.last_updated_ms, last.timestamp_ms - ), + return Err(invalid_data!( + "Invalid update timestamp {}: before last snapshot log entry at {}", + self.last_updated_ms, + last.timestamp_ms )); } } @@ -713,10 +724,7 @@ impl TableMetadata { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if curr.timestamp_ms - prev.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - "Expected sorted metadata log entries", - )); + return Err(invalid_data!("Expected sorted metadata log entries")); } } @@ -724,18 +732,23 @@ impl TableMetadata { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if self.last_updated_ms - last.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid update timestamp {}: before last metadata log entry at {}", - self.last_updated_ms, last.timestamp_ms - ), + return Err(invalid_data!( + "Invalid update timestamp {}: before last metadata log entry at {}", + self.last_updated_ms, + last.timestamp_ms )); } } Ok(()) } + + /// Validates that every type used in the current schema is supported by the + /// table's format version. Delegates to [`Schema::check_format_compatibility`]. + fn validate_schema_format_compatibility(&self) -> Result<()> { + self.current_schema() + .check_format_compatibility(self.format_version) + } } pub(super) mod _serde { @@ -755,20 +768,23 @@ pub(super) mod _serde { use uuid::Uuid; use super::{ - DEFAULT_PARTITION_SPEC_ID, FormatVersion, MAIN_BRANCH, MetadataLog, SnapshotLog, - TableMetadata, + DEFAULT_PARTITION_SPEC_ID, EMPTY_SNAPSHOT_ID, FormatVersion, MAIN_BRANCH, MetadataLog, + SnapshotLog, TableMetadata, }; + use crate::error::invalid_data; use crate::spec::schema::_serde::{SchemaV1, SchemaV2}; - use crate::spec::snapshot::_serde::{SnapshotV1, SnapshotV2}; + use crate::spec::snapshot::_serde::{SnapshotV1, SnapshotV2, SnapshotV3}; use crate::spec::{ - PartitionField, PartitionSpec, PartitionSpecRef, PartitionStatisticsFile, Schema, - SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, SortOrder, StatisticsFile, + EncryptedKey, INITIAL_ROW_ID, PartitionField, PartitionSpec, PartitionSpecRef, + PartitionStatisticsFile, Schema, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, + SortOrder, StatisticsFile, UnboundPartitionSpec, }; use crate::{Error, ErrorKind}; #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(untagged)] pub(super) enum TableMetadataEnum { + V3(TableMetadataV3), V2(TableMetadataV2), V1(TableMetadataV1), } @@ -776,8 +792,21 @@ pub(super) mod _serde { #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] /// Defines the structure of a v2 table metadata for serialization/deserialization - pub(super) struct TableMetadataV2 { - pub format_version: VersionNumber<2>, + pub(super) struct TableMetadataV3 { + pub format_version: VersionNumber<3>, + #[serde(flatten)] + pub shared: TableMetadataV2V3Shared, + pub next_row_id: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub encryption_keys: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub snapshots: Option>, + } + + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] + #[serde(rename_all = "kebab-case")] + /// Defines the structure of a v2 table metadata for serialization/deserialization + pub(super) struct TableMetadataV2V3Shared { pub table_uuid: Uuid, pub location: String, pub last_sequence_number: i64, @@ -793,8 +822,6 @@ pub(super) mod _serde { #[serde(skip_serializing_if = "Option::is_none")] pub current_snapshot_id: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub snapshots: Option>, - #[serde(skip_serializing_if = "Option::is_none")] pub snapshot_log: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub metadata_log: Option>, @@ -808,6 +835,17 @@ pub(super) mod _serde { pub partition_statistics: Vec, } + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] + #[serde(rename_all = "kebab-case")] + /// Defines the structure of a v2 table metadata for serialization/deserialization + pub(super) struct TableMetadataV2 { + pub format_version: VersionNumber<2>, + #[serde(flatten)] + pub shared: TableMetadataV2V3Shared, + #[serde(skip_serializing_if = "Option::is_none")] + pub snapshots: Option>, + } + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] /// Defines the structure of a v1 table metadata for serialization/deserialization @@ -888,6 +926,7 @@ pub(super) mod _serde { type Error = Error; fn try_from(value: TableMetadataEnum) -> Result { match value { + TableMetadataEnum::V3(value) => value.try_into(), TableMetadataEnum::V2(value) => value.try_into(), TableMetadataEnum::V1(value) => value.try_into(), } @@ -898,16 +937,24 @@ pub(super) mod _serde { type Error = Error; fn try_from(value: TableMetadata) -> Result { Ok(match value.format_version { + FormatVersion::V3 => TableMetadataEnum::V3(value.try_into()?), FormatVersion::V2 => TableMetadataEnum::V2(value.into()), FormatVersion::V1 => TableMetadataEnum::V1(value.try_into()?), }) } } - impl TryFrom for TableMetadata { + impl TryFrom for TableMetadata { type Error = Error; - fn try_from(value: TableMetadataV2) -> Result { - let current_snapshot_id = if let &Some(-1) = &value.current_snapshot_id { + fn try_from(value: TableMetadataV3) -> Result { + let TableMetadataV3 { + format_version: _, + shared: value, + next_row_id, + encryption_keys, + snapshots, + } = value; + let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) { None } else { value.current_snapshot_id @@ -922,12 +969,9 @@ pub(super) mod _serde { let current_schema: &SchemaRef = schemas.get(&value.current_schema_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "No schema exists with the current schema id {}.", - value.current_schema_id - ), + invalid_data!( + "No schema exists with the current schema id {}.", + value.current_schema_id ) })?; let partition_specs = HashMap::from_iter( @@ -944,12 +988,112 @@ pub(super) mod _serde { (DEFAULT_PARTITION_SPEC_ID == default_spec_id) .then(PartitionSpec::unpartition_spec) }) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Default partition spec {default_spec_id} not found"), + .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))? + .into(); + let default_partition_type = default_spec.partition_type(current_schema)?; + + let mut metadata = TableMetadata { + format_version: FormatVersion::V3, + table_uuid: value.table_uuid, + location: value.location, + last_sequence_number: value.last_sequence_number, + last_updated_ms: value.last_updated_ms, + last_column_id: value.last_column_id, + current_schema_id: value.current_schema_id, + schemas, + partition_specs, + default_partition_type, + default_spec, + last_partition_id: value.last_partition_id, + properties: value.properties.unwrap_or_default(), + current_snapshot_id, + snapshots: snapshots + .map(|snapshots| { + HashMap::from_iter( + snapshots + .into_iter() + .map(|x| (x.snapshot_id, Arc::new(x.into()))), + ) + }) + .unwrap_or_default(), + snapshot_log: value.snapshot_log.unwrap_or_default(), + metadata_log: value.metadata_log.unwrap_or_default(), + sort_orders: HashMap::from_iter( + value + .sort_orders + .into_iter() + .map(|x| (x.order_id, Arc::new(x))), + ), + default_sort_order_id: value.default_sort_order_id, + refs: value.refs.unwrap_or_else(|| { + if let Some(snapshot_id) = current_snapshot_id { + HashMap::from_iter(vec![(MAIN_BRANCH.to_string(), SnapshotReference { + snapshot_id, + retention: SnapshotRetention::Branch { + min_snapshots_to_keep: None, + max_snapshot_age_ms: None, + max_ref_age_ms: None, + }, + })]) + } else { + HashMap::new() + } + }), + statistics: index_statistics(value.statistics), + partition_statistics: index_partition_statistics(value.partition_statistics), + encryption_keys: encryption_keys + .map(|keys| { + HashMap::from_iter(keys.into_iter().map(|key| (key.key_id.clone(), key))) + }) + .unwrap_or_default(), + next_row_id, + }; + + metadata.borrow_mut().try_normalize()?; + Ok(metadata) + } + } + + impl TryFrom for TableMetadata { + type Error = Error; + fn try_from(value: TableMetadataV2) -> Result { + let snapshots = value.snapshots; + let value = value.shared; + let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) { + None + } else { + value.current_snapshot_id + }; + let schemas = HashMap::from_iter( + value + .schemas + .into_iter() + .map(|schema| Ok((schema.schema_id, Arc::new(schema.try_into()?)))) + .collect::, Error>>()?, + ); + + let current_schema: &SchemaRef = + schemas.get(&value.current_schema_id).ok_or_else(|| { + invalid_data!( + "No schema exists with the current schema id {}.", + value.current_schema_id ) - })? + })?; + let partition_specs = HashMap::from_iter( + value + .partition_specs + .into_iter() + .map(|x| (x.spec_id(), Arc::new(x))), + ); + let default_spec_id = value.default_spec_id; + let default_spec: PartitionSpecRef = partition_specs + .get(&value.default_spec_id) + .map(|spec| (**spec).clone()) + .or_else(|| { + (DEFAULT_PARTITION_SPEC_ID == default_spec_id) + .then(PartitionSpec::unpartition_spec) + }) + .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))? .into(); let default_partition_type = default_spec.partition_type(current_schema)?; @@ -968,8 +1112,7 @@ pub(super) mod _serde { last_partition_id: value.last_partition_id, properties: value.properties.unwrap_or_default(), current_snapshot_id, - snapshots: value - .snapshots + snapshots: snapshots .map(|snapshots| { HashMap::from_iter( snapshots @@ -1004,6 +1147,7 @@ pub(super) mod _serde { statistics: index_statistics(value.statistics), partition_statistics: index_partition_statistics(value.partition_statistics), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, }; metadata.borrow_mut().try_normalize()?; @@ -1014,7 +1158,7 @@ pub(super) mod _serde { impl TryFrom for TableMetadata { type Error = Error; fn try_from(value: TableMetadataV1) -> Result { - let current_snapshot_id = if let &Some(-1) = &value.current_snapshot_id { + let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) { None } else { value.current_snapshot_id @@ -1039,12 +1183,8 @@ pub(super) mod _serde { let schema = schema_map .get(&schema_id) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "No schema exists with the current schema id {}.", - schema_id - ), + invalid_data!( + "No schema exists with the current schema id {schema_id}." ) })? .clone(); @@ -1058,9 +1198,8 @@ pub(super) mod _serde { (schema_map, schema_id, schema_arc) } else { // Option 3: No valid schema configuration found - return Err(Error::new( - ErrorKind::DataInvalid, - "No valid schema configuration found in table metadata", + return Err(invalid_data!( + "No valid schema configuration found in table metadata" )); }; @@ -1073,10 +1212,14 @@ pub(super) mod _serde { .collect::>() } else if let Some(partition_spec) = value.partition_spec { // Option 2: Fall back to 'partition_spec' - let spec = PartitionSpec::builder(current_schema.clone()) - .with_spec_id(DEFAULT_PARTITION_SPEC_ID) - .add_unbound_fields(partition_spec.into_iter().map(|f| f.into_unbound()))? - .build()?; + let spec = UnboundPartitionSpec { + spec_id: Some(DEFAULT_PARTITION_SPEC_ID), + fields: partition_spec + .into_iter() + .map(|f| f.into_unbound()) + .collect(), + } + .bind(current_schema.clone())?; HashMap::from_iter(vec![(DEFAULT_PARTITION_SPEC_ID, Arc::new(spec))]) } else { @@ -1097,12 +1240,7 @@ pub(super) mod _serde { let default_spec: PartitionSpecRef = partition_specs .get(&default_spec_id) .map(|x| Arc::unwrap_or_clone(x.clone())) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Default partition spec {default_spec_id} not found"), - ) - })? + .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))? .into(); let default_partition_type = default_spec.partition_type(¤t_schema)?; @@ -1161,6 +1299,7 @@ pub(super) mod _serde { statistics: index_statistics(value.statistics), partition_statistics: index_partition_statistics(value.partition_statistics), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, // v1 has no row lineage }; metadata.borrow_mut().try_normalize()?; @@ -1168,10 +1307,63 @@ pub(super) mod _serde { } } + impl TryFrom for TableMetadataV3 { + type Error = Error; + + fn try_from(mut v: TableMetadata) -> Result { + let next_row_id = v.next_row_id; + let encryption_keys = std::mem::take(&mut v.encryption_keys); + let snapshots = std::mem::take(&mut v.snapshots); + let shared = v.into(); + + Ok(TableMetadataV3 { + format_version: VersionNumber::<3>, + shared, + next_row_id, + encryption_keys: if encryption_keys.is_empty() { + None + } else { + Some(encryption_keys.into_values().collect()) + }, + snapshots: if snapshots.is_empty() { + None + } else { + Some( + snapshots + .into_values() + .map(|s| SnapshotV3::try_from(Arc::unwrap_or_clone(s))) + .collect::>()?, + ) + }, + }) + } + } + impl From for TableMetadataV2 { - fn from(v: TableMetadata) -> Self { + fn from(mut v: TableMetadata) -> Self { + let snapshots = std::mem::take(&mut v.snapshots); + let shared = v.into(); + TableMetadataV2 { format_version: VersionNumber::<2>, + shared, + snapshots: if snapshots.is_empty() { + None + } else { + Some( + snapshots + .into_values() + .map(|s| SnapshotV2::from(Arc::unwrap_or_clone(s))) + .collect(), + ) + }, + } + } + } + + impl From for TableMetadataV2V3Shared { + fn from(v: TableMetadata) -> Self { + TableMetadataV2V3Shared { table_uuid: v.table_uuid, location: v.location, last_sequence_number: v.last_sequence_number, @@ -1200,20 +1392,6 @@ pub(super) mod _serde { Some(v.properties) }, current_snapshot_id: v.current_snapshot_id, - snapshots: if v.snapshots.is_empty() { - None - } else { - Some( - v.snapshots - .into_values() - .map(|x| { - Arc::try_unwrap(x) - .unwrap_or_else(|snapshot| snapshot.as_ref().clone()) - .into() - }) - .collect(), - ) - }, snapshot_log: if v.snapshot_log.is_empty() { None } else { @@ -1343,6 +1521,8 @@ pub enum FormatVersion { V1 = 1u8, /// Iceberg spec version 2 V2 = 2u8, + /// Iceberg spec version 3 + V3 = 3u8, } impl PartialOrd for FormatVersion { @@ -1362,6 +1542,7 @@ impl Display for FormatVersion { match self { FormatVersion::V1 => write!(f, "v1"), FormatVersion::V2 => write!(f, "v2"), + FormatVersion::V3 => write!(f, "v3"), } } } @@ -1387,7 +1568,7 @@ pub struct SnapshotLog { } impl SnapshotLog { - /// Returns the last updated timestamp as a DateTime with millisecond precision + /// Returns the last updated timestamp as a [`DateTime`] with millisecond precision pub fn timestamp(self) -> Result> { timestamp_ms_to_utc(self.timestamp_ms) } @@ -1406,19 +1587,23 @@ mod tests { use std::sync::Arc; use anyhow::Result; + use base64::Engine as _; use pretty_assertions::assert_eq; use tempfile::TempDir; use uuid::Uuid; use super::{FormatVersion, MetadataLog, SnapshotLog, TableMetadataBuilder}; - use crate::TableCreation; - use crate::io::FileIOBuilder; + use crate::catalog::MetadataLocation; + use crate::compression::CompressionCodec; + use crate::io::FileIO; use crate::spec::table_metadata::TableMetadata; use crate::spec::{ - BlobMetadata, NestedField, NullOrder, Operation, PartitionSpec, PartitionStatisticsFile, - PrimitiveType, Schema, Snapshot, SnapshotReference, SnapshotRetention, SortDirection, - SortField, SortOrder, StatisticsFile, Summary, Transform, Type, UnboundPartitionField, + BlobMetadata, EncryptedKey, INITIAL_ROW_ID, Literal, NestedField, NullOrder, Operation, + PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType, Schema, Snapshot, + SnapshotReference, SnapshotRetention, SortDirection, SortField, SortOrder, StatisticsFile, + Summary, TableProperties, Transform, Type, UnboundPartitionField, UnboundPartitionSpec, }; + use crate::{ErrorKind, TableCreation}; fn check_table_metadata_serde(json: &str, expected_type: TableMetadata) { let desered_type: TableMetadata = serde_json::from_str(json).unwrap(); @@ -1431,12 +1616,22 @@ mod tests { } fn get_test_table_metadata(file_name: &str) -> TableMetadata { - let path = format!("testdata/table_metadata/{}", file_name); + let path = format!("testdata/table_metadata/{file_name}"); let metadata: String = fs::read_to_string(path).unwrap(); serde_json::from_str(&metadata).unwrap() } + /// Loads a test table metadata and relocates it to `location`, so that derived + /// metadata paths point at a writable (e.g. temp) directory. + fn get_test_table_metadata_at(file_name: &str, location: &str) -> TableMetadata { + TableMetadataBuilder::new_from_metadata(get_test_table_metadata(file_name), None) + .set_location(location.to_string()) + .build() + .unwrap() + .metadata + } + #[test] fn test_table_data_v2() { let data = r#" @@ -1522,12 +1717,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "ts_day".to_string(), - transform: Transform::Day, - source_id: 4, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("ts_day".to_string()) + .transform(Transform::Day) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -1563,6 +1761,186 @@ mod tests { statistics: HashMap::new(), partition_statistics: HashMap::new(), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, + }; + + let expected_json_value = serde_json::to_value(&expected).unwrap(); + check_table_metadata_serde(data, expected); + + let json_value = serde_json::from_str::(data).unwrap(); + assert_eq!(json_value, expected_json_value); + } + + #[test] + fn test_table_data_v3() { + let data = r#" + { + "format-version" : 3, + "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94", + "location": "s3://b/wh/data.db/table", + "last-sequence-number" : 1, + "last-updated-ms": 1515100955770, + "last-column-id": 1, + "next-row-id": 5, + "schemas": [ + { + "schema-id" : 1, + "type" : "struct", + "fields" :[ + { + "id": 4, + "name": "ts", + "required": true, + "type": "timestamp" + } + ] + } + ], + "current-schema-id" : 1, + "partition-specs": [ + { + "spec-id": 0, + "fields": [ + { + "source-id": 4, + "field-id": 1000, + "name": "ts_day", + "transform": "day" + } + ] + } + ], + "default-spec-id": 0, + "last-partition-id": 1000, + "properties": { + "commit.retry.num-retries": "1" + }, + "metadata-log": [ + { + "metadata-file": "s3://bucket/.../v1.json", + "timestamp-ms": 1515100 + } + ], + "refs": {}, + "snapshots" : [ { + "snapshot-id" : 1, + "timestamp-ms" : 1662532818843, + "sequence-number" : 0, + "first-row-id" : 0, + "added-rows" : 4, + "key-id" : "key1", + "summary" : { + "operation" : "append" + }, + "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro", + "schema-id" : 0 + } + ], + "encryption-keys": [ + { + "key-id": "key1", + "encrypted-by-id": "KMS", + "encrypted-key-metadata": "c29tZS1lbmNyeXB0aW9uLWtleQ==", + "properties": { + "p1": "v1" + } + } + ], + "sort-orders": [ + { + "order-id": 0, + "fields": [] + } + ], + "default-sort-order-id": 0 + } + "#; + + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![Arc::new(NestedField::required( + 4, + "ts", + Type::Primitive(PrimitiveType::Timestamp), + ))]) + .build() + .unwrap(); + + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![4]) + .field_id(1000) + .name("ts_day".to_string()) + .transform(Transform::Day) + .build() + .unwrap(), + ) + .unwrap() + .build() + .unwrap(); + + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_timestamp_ms(1662532818843) + .with_sequence_number(0) + .with_row_range(0, 4) + .with_encryption_key_id(Some("key1".to_string())) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_manifest_list("/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro".to_string()) + .with_schema_id(0) + .build(); + + let encryption_key = EncryptedKey::builder() + .key_id("key1".to_string()) + .encrypted_by_id("KMS".to_string()) + .encrypted_key_metadata( + base64::prelude::BASE64_STANDARD + .decode("c29tZS1lbmNyeXB0aW9uLWtleQ==") + .unwrap(), + ) + .properties(HashMap::from_iter(vec![( + "p1".to_string(), + "v1".to_string(), + )])) + .build(); + + let default_partition_type = partition_spec.partition_type(&schema).unwrap(); + let expected = TableMetadata { + format_version: FormatVersion::V3, + table_uuid: Uuid::parse_str("fb072c92-a02b-11e9-ae9c-1bb7bc9eca94").unwrap(), + location: "s3://b/wh/data.db/table".to_string(), + last_updated_ms: 1515100955770, + last_column_id: 1, + schemas: HashMap::from_iter(vec![(1, Arc::new(schema))]), + current_schema_id: 1, + partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]), + default_partition_type, + default_spec: partition_spec.into(), + last_partition_id: 1000, + default_sort_order_id: 0, + sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]), + snapshots: HashMap::from_iter(vec![(1, snapshot.into())]), + current_snapshot_id: None, + last_sequence_number: 1, + properties: HashMap::from_iter(vec![( + "commit.retry.num-retries".to_string(), + "1".to_string(), + )]), + snapshot_log: Vec::new(), + metadata_log: vec![MetadataLog { + metadata_file: "s3://bucket/.../v1.json".to_string(), + timestamp_ms: 1515100, + }], + refs: HashMap::new(), + statistics: HashMap::new(), + partition_statistics: HashMap::new(), + encryption_keys: HashMap::from_iter(vec![("key1".to_string(), encryption_key)]), + next_row_id: 5, }; let expected_json_value = serde_json::to_value(&expected).unwrap(); @@ -1739,6 +2117,7 @@ mod tests { statistics: HashMap::new(), partition_statistics: HashMap::new(), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, }; check_table_metadata_serde(data, expected); @@ -1837,6 +2216,7 @@ mod tests { statistics: HashMap::new(), partition_statistics: HashMap::new(), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, }; let expected_json_value = serde_json::to_value(&expected).unwrap(); @@ -2203,7 +2583,6 @@ mod tests { "#; let err = serde_json::from_str::(data).unwrap_err(); - println!("{}", err); assert!(err.to_string().contains( "Invalid snapshot with id 3055729675574597004 and sequence number 4 greater than last sequence number 1" )); @@ -2372,6 +2751,7 @@ mod tests { }, })]), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, }; check_table_metadata_serde(data, expected); @@ -2507,6 +2887,7 @@ mod tests { }, })]), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, }; check_table_metadata_serde(data, expected); @@ -2535,6 +2916,98 @@ mod tests { Ok(()) } + #[test] + fn test_table_metadata_v3_valid_minimal() { + let metadata_str = + fs::read_to_string("testdata/table_metadata/TableMetadataV3ValidMinimal.json").unwrap(); + + let table_metadata = serde_json::from_str::(&metadata_str).unwrap(); + assert_eq!(table_metadata.format_version, FormatVersion::V3); + + let schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + Arc::new( + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)) + .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(1))) + .with_write_default(Literal::Primitive(PrimitiveLiteral::Long(1))), + ), + Arc::new( + NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)) + .with_doc("comment"), + ), + Arc::new(NestedField::required( + 3, + "z", + Type::Primitive(PrimitiveType::Long), + )), + ]) + .build() + .unwrap(); + + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(0) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1000) + .name("x".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build() + .unwrap(); + + let sort_order = SortOrder::builder() + .with_order_id(3) + .with_sort_field(SortField { + source_id: 2, + transform: Transform::Identity, + direction: SortDirection::Ascending, + null_order: NullOrder::First, + }) + .with_sort_field(SortField { + source_id: 3, + transform: Transform::Bucket(4), + direction: SortDirection::Descending, + null_order: NullOrder::Last, + }) + .build_unbound() + .unwrap(); + + let default_partition_type = partition_spec.partition_type(&schema).unwrap(); + let expected = TableMetadata { + format_version: FormatVersion::V3, + table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(), + location: "s3://bucket/test/location".to_string(), + last_updated_ms: 1602638573590, + last_column_id: 3, + schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]), + current_schema_id: 0, + partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]), + default_spec: Arc::new(partition_spec), + default_partition_type, + last_partition_id: 1000, + default_sort_order_id: 3, + sort_orders: HashMap::from_iter(vec![(3, sort_order.into())]), + snapshots: HashMap::default(), + current_snapshot_id: None, + last_sequence_number: 34, + properties: HashMap::new(), + snapshot_log: Vec::new(), + metadata_log: Vec::new(), + refs: HashMap::new(), + statistics: HashMap::new(), + partition_statistics: HashMap::new(), + encryption_keys: HashMap::new(), + next_row_id: 0, // V3 specific field from the JSON + }; + + check_table_metadata_serde(&metadata_str, expected); + } + #[test] fn test_table_metadata_v2_file_valid() { let metadata = @@ -2574,12 +3047,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema2.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "x".to_string(), - transform: Transform::Identity, - source_id: 1, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1000) + .name("x".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -2669,6 +3145,7 @@ mod tests { statistics: HashMap::new(), partition_statistics: HashMap::new(), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, }; check_table_metadata_serde(&metadata, expected); @@ -2702,12 +3179,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "x".to_string(), - transform: Transform::Identity, - source_id: 1, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1000) + .name("x".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -2754,6 +3234,7 @@ mod tests { statistics: HashMap::new(), partition_statistics: HashMap::new(), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, }; check_table_metadata_serde(&metadata, expected); @@ -2787,12 +3268,15 @@ mod tests { let partition_spec = PartitionSpec::builder(schema.clone()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "x".to_string(), - transform: Transform::Identity, - source_id: 1, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1000) + .name("x".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -2823,11 +3307,51 @@ mod tests { statistics: HashMap::new(), partition_statistics: HashMap::new(), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, }; check_table_metadata_serde(&metadata, expected); } + #[test] + fn test_table_metadata_v1_legacy_partition_spec_with_void_fields() { + let mut metadata: serde_json::Value = serde_json::from_str( + &fs::read_to_string("testdata/table_metadata/TableMetadataV1Valid.json").unwrap(), + ) + .unwrap(); + // What a format version 1 spec keeps after identity(x) was removed twice. + metadata["partition-spec"] = serde_json::json!([ + {"name": "x", "transform": "void", "source-id": 1, "field-id": 1000}, + {"name": "x_null", "transform": "void", "source-id": 1, "field-id": 1001}, + ]); + + let metadata: TableMetadata = serde_json::from_value(metadata).unwrap(); + + let fields = metadata.default_partition_spec().fields(); + assert_eq!( + fields + .iter() + .map(|f| (f.field_id, f.name.as_str(), f.transform)) + .collect::>(), + vec![ + (1000, "x", Transform::Void), + (1001, "x_null", Transform::Void) + ] + ); + } + + #[test] + fn test_empty_snapshot_id_is_normalized_to_none() { + let metadata = + fs::read_to_string("testdata/table_metadata/TableMetadataV1Valid.json").unwrap(); + let deserialized: TableMetadata = serde_json::from_str(&metadata).unwrap(); + assert_eq!( + deserialized.current_snapshot_id(), + None, + "current_snapshot_id of -1 should be deserialized as None" + ); + } + #[test] fn test_table_metadata_v1_compat() { let metadata = @@ -2890,8 +3414,7 @@ mod tests { let error_message = desered.unwrap_err().to_string(); assert!( error_message.contains("No valid schema configuration found"), - "Expected error about no valid schema configuration, got: {}", - error_message + "Expected error about no valid schema configuration, got: {error_message}" ); } @@ -3018,6 +3541,70 @@ mod tests { assert_eq!(FormatVersion::V2, FormatVersion::V2); } + #[test] + fn test_deserialize_default_partition_spec_with_dropped_source() { + for version in [1, 2, 3] { + for transform in ["identity", "truncate[4]", "bucket[16]"] { + let mut metadata = serde_json::json!({ + "format-version": version, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590_i64, + "last-column-id": 2, + "current-schema-id": 1, + "schemas": [ + { + "schema-id": 0, + "type": "struct", + "fields": [ + {"id": 1, "name": "x", "required": false, "type": "long"}, + {"id": 2, "name": "y", "required": false, "type": "long"} + ] + }, + { + "schema-id": 1, + "type": "struct", + "fields": [ + {"id": 2, "name": "y", "required": false, "type": "long"} + ] + } + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": [{ + "source-id": 1, "field-id": 1000, "name": "x_partition", + "transform": transform + }]}], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + "next-row-id": 0 + }); + + // Retaining the source in history does not make it writable using + // the current schema and default spec. + let err = serde_json::from_value::(metadata.clone()).unwrap_err(); + assert!(err.to_string().contains( + "Default partition spec 0 references missing source field 1 in current schema 1" + )); + + // The same spec is allowed once it becomes historical. + metadata["partition-specs"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({"spec-id": 1, "fields": []})); + metadata["default-spec-id"] = serde_json::json!(1); + serde_json::from_value::(metadata.clone()).unwrap(); + + // A voided field does not require its dropped source, even in the default spec. + metadata["default-spec-id"] = serde_json::json!(0); + metadata["partition-specs"][0]["fields"][0]["transform"] = + serde_json::json!("void"); + serde_json::from_value::(metadata).unwrap(); + } + } + } + #[test] fn test_default_partition_spec() { let default_spec_id = 1234; @@ -3112,13 +3699,16 @@ mod tests { let temp_path = temp_dir.path().to_str().unwrap(); // Create a FileIO instance - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); // Use an existing test metadata from the test files - let original_metadata: TableMetadata = get_test_table_metadata("TableMetadataV2Valid.json"); + let original_metadata: TableMetadata = + get_test_table_metadata_at("TableMetadataV2Valid.json", temp_path); // Define the metadata location - let metadata_location = format!("{}/metadata.json", temp_path); + let metadata_location = + MetadataLocation::try_new_with_metadata(&original_metadata).unwrap(); + let metadata_location_str = metadata_location.to_string(); // Write the metadata original_metadata @@ -3127,10 +3717,10 @@ mod tests { .unwrap(); // Verify the file exists - assert!(fs::metadata(&metadata_location).is_ok()); + assert!(fs::metadata(&metadata_location_str).is_ok()); // Read the metadata back - let read_metadata = TableMetadata::read_from(&file_io, &metadata_location) + let read_metadata = TableMetadata::read_from(&file_io, &metadata_location_str) .await .unwrap(); @@ -3139,91 +3729,97 @@ mod tests { } #[tokio::test] - async fn test_table_metadata_read_nonexistent_file() { - // Create a FileIO instance - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + async fn test_table_metadata_read_compressed() { + let temp_dir = TempDir::new().unwrap(); + let metadata_location = temp_dir.path().join("v1.gz.metadata.json"); - // Try to read a non-existent file - let result = TableMetadata::read_from(&file_io, "/nonexistent/path/metadata.json").await; + let original_metadata: TableMetadata = get_test_table_metadata("TableMetadataV2Valid.json"); + let json = serde_json::to_string(&original_metadata).unwrap(); - // Verify it returns an error - assert!(result.is_err()); - } + let compressed = CompressionCodec::gzip_default() + .compress(json.into_bytes()) + .expect("failed to compress metadata"); + fs::write(&metadata_location, &compressed).expect("failed to write metadata"); - #[test] - fn test_partition_name_exists() { - let schema = Schema::builder() - .with_fields(vec![ - NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "partition_col", Type::Primitive(PrimitiveType::Int)) - .into(), - ]) - .build() - .unwrap(); - - let spec1 = PartitionSpec::builder(schema.clone()) - .with_spec_id(1) - .add_partition_field("data", "data_partition", Transform::Identity) - .unwrap() - .build() + // Read the metadata back + let file_io = FileIO::new_with_fs(); + let metadata_location = metadata_location.to_str().unwrap(); + let read_metadata = TableMetadata::read_from(&file_io, metadata_location) + .await .unwrap(); - let spec2 = PartitionSpec::builder(schema.clone()) - .with_spec_id(2) - .add_partition_field("partition_col", "partition_bucket", Transform::Bucket(16)) - .unwrap() - .build() - .unwrap(); + // Verify the metadata matches + assert_eq!(read_metadata, original_metadata); + } - // Build metadata with these specs - let metadata = TableMetadataBuilder::new( - schema, - spec1.clone().into_unbound(), - SortOrder::unsorted_order(), - "s3://test/location".to_string(), - FormatVersion::V2, - HashMap::new(), - ) - .unwrap() - .add_partition_spec(spec2.into_unbound()) - .unwrap() - .build() - .unwrap() - .metadata; + #[tokio::test] + async fn test_table_metadata_read_nonexistent_file() { + // Create a FileIO instance + let file_io = FileIO::new_with_fs(); - assert!(metadata.partition_name_exists("data_partition")); - assert!(metadata.partition_name_exists("partition_bucket")); + // Try to read a non-existent file + let result = TableMetadata::read_from(&file_io, "/nonexistent/path/metadata.json").await; - assert!(!metadata.partition_name_exists("nonexistent_field")); - assert!(!metadata.partition_name_exists("data")); // schema field name, not partition field name - assert!(!metadata.partition_name_exists("")); + // Verify it returns an error + assert!(result.is_err()); } - #[test] - fn test_partition_name_exists_empty_specs() { - // Create metadata with no partition specs (unpartitioned table) - let schema = Schema::builder() - .with_fields(vec![ - NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() + #[tokio::test] + async fn test_table_metadata_write_with_gzip_compression() { + let temp_dir = TempDir::new().unwrap(); + let temp_path = temp_dir.path().to_str().unwrap(); + let file_io = FileIO::new_with_fs(); + + // Get a test metadata and add gzip compression property + let original_metadata: TableMetadata = + get_test_table_metadata_at("TableMetadataV2Valid.json", temp_path); + + // Modify properties to enable gzip compression (using mixed case to test case-insensitive matching) + let mut props = original_metadata.properties.clone(); + props.insert( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "GziP".to_string(), + ); + // Use builder to create new metadata with updated properties + let compressed_metadata = + TableMetadataBuilder::new_from_metadata(original_metadata.clone(), None) + .assign_uuid(original_metadata.table_uuid) + .set_properties(props.clone()) + .unwrap() + .build() + .unwrap() + .metadata; + + // Create MetadataLocation with compression codec from metadata + let metadata_location = + MetadataLocation::try_new_with_metadata(&compressed_metadata).unwrap(); + let metadata_location_str = metadata_location.to_string(); + + // Verify the location has the .gz extension + assert!(metadata_location_str.contains(".gz.metadata.json")); + + // Write the metadata with compression + compressed_metadata + .write_to(&file_io, &metadata_location) + .await .unwrap(); - let metadata = TableMetadataBuilder::new( - schema, - PartitionSpec::unpartition_spec().into_unbound(), - SortOrder::unsorted_order(), - "s3://test/location".to_string(), - FormatVersion::V2, - HashMap::new(), - ) - .unwrap() - .build() - .unwrap() - .metadata; + // Verify the compressed file exists + assert!(std::path::Path::new(&metadata_location_str).exists()); + + // Read the raw file and check it's gzip compressed + let raw_content = fs::read(&metadata_location_str).unwrap(); + assert!(raw_content.len() > 2); + assert_eq!(raw_content[0], 0x1F); // gzip magic number + assert_eq!(raw_content[1], 0x8B); // gzip magic number + + // Read the metadata back using the compressed location + let read_metadata = TableMetadata::read_from(&file_io, &metadata_location_str) + .await + .unwrap(); - assert!(!metadata.partition_name_exists("any_field")); - assert!(!metadata.partition_name_exists("data")); + // Verify the complete round-trip: read metadata should match what we wrote + assert_eq!(read_metadata, compressed_metadata); } #[test] @@ -3354,12 +3950,547 @@ mod tests { .unwrap() .metadata; - assert!(!final_metadata.partition_name_exists("nonexistent_partition")); // unpartitioned table - assert!(final_metadata.name_exists_in_any_schema("id")); // exists in both schemas assert!(final_metadata.name_exists_in_any_schema("name")); // exists in both schemas assert!(final_metadata.name_exists_in_any_schema("deprecated_field")); // exists in both schemas assert!(final_metadata.name_exists_in_any_schema("new_field")); // only in current schema assert!(!final_metadata.name_exists_in_any_schema("never_existed")); } + + #[test] + fn test_invalid_sort_order_id_zero_with_fields() { + let metadata = r#" + { + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 111, + "last-updated-ms": 1600000000000, + "last-column-id": 3, + "current-schema-id": 1, + "schemas": [ + { + "type": "struct", + "schema-id": 1, + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"}, + {"id": 2, "name": "y", "required": true, "type": "long"} + ] + } + ], + "default-spec-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [ + { + "order-id": 0, + "fields": [ + { + "transform": "identity", + "source-id": 1, + "direction": "asc", + "null-order": "nulls-first" + } + ] + } + ], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [] + } + "#; + + let result: Result = serde_json::from_str(metadata); + + // Should fail because sort order ID 0 is reserved for unsorted order and cannot have fields + assert!( + result.is_err(), + "Parsing should fail for sort order ID 0 with fields" + ); + } + + #[test] + fn test_table_properties_with_defaults() { + use crate::spec::TableProperties; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let metadata = TableMetadataBuilder::new( + schema, + PartitionSpec::unpartition_spec().into_unbound(), + SortOrder::unsorted_order(), + "s3://test/location".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + let props = metadata.table_properties(); + + assert_eq!( + props.commit_num_retries().unwrap(), + TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT + ); + assert_eq!( + props.write_target_file_size_bytes().unwrap(), + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT + ); + } + + #[test] + fn test_table_properties_with_custom_values() { + use crate::spec::TableProperties; + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let properties = HashMap::from([ + ( + TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), + "10".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "1024".to_string(), + ), + ]); + + let metadata = TableMetadataBuilder::new( + schema, + PartitionSpec::unpartition_spec().into_unbound(), + SortOrder::unsorted_order(), + "s3://test/location".to_string(), + FormatVersion::V2, + properties, + ) + .unwrap() + .build() + .unwrap() + .metadata; + + let props = metadata.table_properties(); + + assert_eq!(props.commit_num_retries().unwrap(), 10); + assert_eq!(props.write_target_file_size_bytes().unwrap(), 1024); + } + + #[test] + fn test_deserialize_metadata_defers_invalid_table_property_errors() { + let invalid_retries = "not_a_number"; + let invalid_codec = "unknown"; + let target_file_size = "1024"; + + for file_name in [ + "TableMetadataV1Valid.json", + "TableMetadataV2ValidMinimal.json", + "TableMetadataV3ValidMinimal.json", + ] { + let path = format!("testdata/table_metadata/{file_name}"); + let mut json: serde_json::Value = + serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap(); + json["properties"] = serde_json::json!({ + (TableProperties::PROPERTY_COMMIT_NUM_RETRIES): invalid_retries, + (TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC): invalid_codec, + (TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES): target_file_size, + }); + + let metadata: TableMetadata = serde_json::from_value(json).unwrap(); + assert_eq!( + metadata + .properties() + .get(TableProperties::PROPERTY_COMMIT_NUM_RETRIES) + .map(String::as_str), + Some(invalid_retries) + ); + + let table_properties = metadata.table_properties(); + let error = table_properties.commit_num_retries().unwrap_err(); + assert!( + error + .message() + .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES) + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + 1024 + ); + let error = table_properties.metadata_compression_codec().unwrap_err(); + assert!( + format!("{error}").contains(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) + ); + + let serialized = serde_json::to_value(metadata).unwrap(); + assert_eq!( + serialized["properties"][TableProperties::PROPERTY_COMMIT_NUM_RETRIES], + invalid_retries + ); + assert_eq!( + serialized["properties"][TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC], + invalid_codec + ); + } + } + + #[test] + fn test_table_properties_with_invalid_value() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let properties = HashMap::from([ + ( + TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), + "not_a_number".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "1024".to_string(), + ), + ]); + + let metadata = TableMetadataBuilder::new( + schema, + PartitionSpec::unpartition_spec().into_unbound(), + SortOrder::unsorted_order(), + "s3://test/location".to_string(), + FormatVersion::V2, + properties, + ) + .unwrap() + .build() + .unwrap() + .metadata; + + let table_properties = metadata.table_properties(); + let err = table_properties.commit_num_retries().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES) + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + 1024 + ); + } + + #[test] + fn test_v2_to_v3_upgrade_preserves_existing_snapshots_without_row_lineage() { + // Create a v2 table metadata + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let v2_metadata = TableMetadataBuilder::new( + schema, + PartitionSpec::unpartition_spec().into_unbound(), + SortOrder::unsorted_order(), + "s3://bucket/test/location".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + // Add a v2 snapshot + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_timestamp_ms(v2_metadata.last_updated_ms + 1) + .with_sequence_number(1) + .with_schema_id(0) + .with_manifest_list("s3://bucket/test/metadata/snap-1.avro") + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::from([( + "added-data-files".to_string(), + "1".to_string(), + )]), + }) + .build(); + + let v2_with_snapshot = v2_metadata + .into_builder(Some("s3://bucket/test/metadata/v00001.json".to_string())) + .add_snapshot(snapshot) + .unwrap() + .set_ref("main", SnapshotReference { + snapshot_id: 1, + retention: SnapshotRetention::Branch { + min_snapshots_to_keep: None, + max_snapshot_age_ms: None, + max_ref_age_ms: None, + }, + }) + .unwrap() + .build() + .unwrap() + .metadata; + + // Verify v2 serialization works fine + let v2_json = serde_json::to_string(&v2_with_snapshot); + assert!(v2_json.is_ok(), "v2 serialization should work"); + + // Upgrade to v3 + let v3_metadata = v2_with_snapshot + .into_builder(Some("s3://bucket/test/metadata/v00002.json".to_string())) + .upgrade_format_version(FormatVersion::V3) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(v3_metadata.format_version, FormatVersion::V3); + assert_eq!(v3_metadata.next_row_id, INITIAL_ROW_ID); + assert_eq!(v3_metadata.snapshots.len(), 1); + + // Verify the snapshot has no row_range + let snapshot = v3_metadata.snapshots.values().next().unwrap(); + assert!( + snapshot.row_range().is_none(), + "Snapshot should have no row_range after upgrade" + ); + + // Try to serialize v3 metadata - this should now work + let v3_json = serde_json::to_string(&v3_metadata); + assert!( + v3_json.is_ok(), + "v3 serialization should work for upgraded tables" + ); + + // Verify we can deserialize it back + let deserialized: TableMetadata = serde_json::from_str(&v3_json.unwrap()).unwrap(); + assert_eq!(deserialized.format_version, FormatVersion::V3); + assert_eq!(deserialized.snapshots.len(), 1); + + // Verify the deserialized snapshot still has no row_range + let deserialized_snapshot = deserialized.snapshots.values().next().unwrap(); + assert!( + deserialized_snapshot.row_range().is_none(), + "Deserialized snapshot should have no row_range" + ); + } + + #[test] + fn test_v3_snapshot_with_row_lineage_serialization() { + // Create a v3 table metadata + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let v3_metadata = TableMetadataBuilder::new( + schema, + PartitionSpec::unpartition_spec().into_unbound(), + SortOrder::unsorted_order(), + "s3://bucket/test/location".to_string(), + FormatVersion::V3, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + // Add a v3 snapshot with row lineage + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_timestamp_ms(v3_metadata.last_updated_ms + 1) + .with_sequence_number(1) + .with_schema_id(0) + .with_manifest_list("s3://bucket/test/metadata/snap-1.avro") + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::from([( + "added-data-files".to_string(), + "1".to_string(), + )]), + }) + .with_row_range(100, 50) // first_row_id=100, added_rows=50 + .build(); + + let v3_with_snapshot = v3_metadata + .into_builder(Some("s3://bucket/test/metadata/v00001.json".to_string())) + .add_snapshot(snapshot) + .unwrap() + .set_ref("main", SnapshotReference { + snapshot_id: 1, + retention: SnapshotRetention::Branch { + min_snapshots_to_keep: None, + max_snapshot_age_ms: None, + max_ref_age_ms: None, + }, + }) + .unwrap() + .build() + .unwrap() + .metadata; + + // Verify the snapshot has row_range + let snapshot = v3_with_snapshot.snapshots.values().next().unwrap(); + assert!( + snapshot.row_range().is_some(), + "Snapshot should have row_range" + ); + let (first_row_id, added_rows) = snapshot.row_range().unwrap(); + assert_eq!(first_row_id, 100); + assert_eq!(added_rows, 50); + + // Serialize v3 metadata - this should work + let v3_json = serde_json::to_string(&v3_with_snapshot); + assert!( + v3_json.is_ok(), + "v3 serialization should work for snapshots with row lineage" + ); + + // Verify we can deserialize it back + let deserialized: TableMetadata = serde_json::from_str(&v3_json.unwrap()).unwrap(); + assert_eq!(deserialized.format_version, FormatVersion::V3); + assert_eq!(deserialized.snapshots.len(), 1); + + // Verify the deserialized snapshot has the correct row_range + let deserialized_snapshot = deserialized.snapshots.values().next().unwrap(); + assert!( + deserialized_snapshot.row_range().is_some(), + "Deserialized snapshot should have row_range" + ); + let (deserialized_first_row_id, deserialized_added_rows) = + deserialized_snapshot.row_range().unwrap(); + assert_eq!(deserialized_first_row_id, 100); + assert_eq!(deserialized_added_rows, 50); + } + + #[test] + fn test_metadata_location_default() { + // Verify metadata files go to `/metadata` when `write.metadata.path` is not set + let metadata = get_test_table_metadata("TableMetadataV2Valid.json"); + assert_eq!(metadata.location(), "s3://bucket/test/location"); + assert_eq!( + metadata.metadata_location().unwrap(), + "s3://bucket/test/location/metadata" + ); + } + + #[test] + fn test_metadata_location_honors_write_metadata_path() { + let metadata = get_test_table_metadata("TableMetadataV2Valid.json") + .into_builder(None) + .set_properties(HashMap::from([( + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "s3://other-bucket/custom-meta".to_string(), + )])) + .unwrap() + .build() + .unwrap() + .metadata; + assert_eq!( + metadata.metadata_location().unwrap(), + "s3://other-bucket/custom-meta" + ); + } + + #[test] + fn test_metadata_location_trims_trailing_slash() { + // A configured path with a trailing slash must not yield a doubled separator + let metadata = get_test_table_metadata("TableMetadataV2Valid.json") + .into_builder(None) + .set_properties(HashMap::from([( + TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(), + "s3://other-bucket/custom-meta/".to_string(), + )])) + .unwrap() + .build() + .unwrap() + .metadata; + assert_eq!( + metadata.metadata_location().unwrap(), + "s3://other-bucket/custom-meta" + ); + } + + #[test] + fn test_unified_partition_type_spans_all_specs() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let metadata = TableMetadataBuilder::new( + schema.clone(), + UnboundPartitionSpec::builder() + .with_spec_id(0) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("y") + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build(), + SortOrder::unsorted_order(), + "s3://bucket/table".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata + .into_builder(None) + .add_partition_spec( + UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![3]) + .name("z") + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + // The default spec only knows about `y`, but `_partition` must expose both. + assert_eq!(metadata.default_partition_type().fields().len(), 1); + + let unified = metadata.unified_partition_type(&schema).unwrap(); + let names: Vec<&str> = unified + .fields() + .iter() + .map(|field| field.name.as_str()) + .collect(); + assert_eq!(names, vec!["y", "z"]); + } } diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs index 068f2002f9..82c0a521c6 100644 --- a/crates/iceberg/src/spec/table_metadata_builder.rs +++ b/crates/iceberg/src/spec/table_metadata_builder.rs @@ -20,18 +20,19 @@ use std::sync::Arc; use uuid::Uuid; +use super::schema::assign_fresh_ids; use super::{ DEFAULT_PARTITION_SPEC_ID, DEFAULT_SCHEMA_ID, FormatVersion, MAIN_BRANCH, MetadataLog, - ONE_MINUTE_MS, PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX, - PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT, PartitionSpec, PartitionSpecBuilder, - PartitionStatisticsFile, RESERVED_PROPERTIES, Schema, SchemaRef, Snapshot, SnapshotLog, - SnapshotReference, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, - TableMetadata, UNPARTITIONED_LAST_ASSIGNED_ID, UnboundPartitionSpec, + ONE_MINUTE_MS, PartitionField, PartitionSpec, PartitionSpecBuilder, PartitionStatisticsFile, + Schema, SchemaRef, Snapshot, SnapshotLog, SnapshotReference, SnapshotRetention, SortOrder, + SortOrderRef, StatisticsFile, StructType, TableMetadata, TableProperties, Transform, + UNPARTITIONED_LAST_ASSIGNED_ID, UnboundPartitionField, UnboundPartitionSpec, }; -use crate::error::{Error, ErrorKind, Result}; +use crate::error::{Error, ErrorKind, Result, invalid_data}; +use crate::spec::{EncryptedKey, INITIAL_ROW_ID, MIN_FORMAT_VERSION_ROW_LINEAGE}; use crate::{TableCreation, TableUpdate}; -const FIRST_FIELD_ID: u32 = 1; +pub(crate) const FIRST_FIELD_ID: i32 = 1; /// Manipulating table metadata. /// @@ -121,6 +122,7 @@ impl TableMetadataBuilder { statistics: HashMap::new(), partition_statistics: HashMap::new(), encryption_keys: HashMap::new(), + next_row_id: INITIAL_ROW_ID, }, last_updated_ms: None, changes: vec![], @@ -171,14 +173,11 @@ impl TableMetadataBuilder { partition_spec, sort_order, properties, + format_version, } = table_creation; - let location = location.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Can't create table without location", - ) - })?; + let location = + location.ok_or_else(|| invalid_data!("Can't create table without location"))?; let partition_spec = partition_spec.unwrap_or(UnboundPartitionSpec { spec_id: None, fields: vec![], @@ -189,11 +188,87 @@ impl TableMetadataBuilder { partition_spec, sort_order.unwrap_or(SortOrder::unsorted_order()), location, - FormatVersion::V2, + format_version, properties, ) } + /// Replaces the table's schema, partition spec and sort order, leaving it with no + /// current snapshot. Backs `REPLACE TABLE`. + /// + /// The table keeps its uuid, properties and snapshots, so earlier snapshots stay readable. + /// Only the `main` branch is removed; other branches and tags are kept. + /// + /// Columns are matched to the current schema by full name and keep their field ids, even + /// when their type changes; new columns get ids above `last_column_id`, never a dropped + /// column's id. + /// + /// Partition fields reuse the highest id an equivalent field has in any of the table's specs, + /// and other fields get ids above `last_partition_id`. + /// + /// `location` moves the table, `None` leaves it in place. `format_version` upgrades the table, + /// `None` keeps its version. `properties` are applied on top of the existing ones, so an + /// existing `write.parquet.compression-codec` is kept. + /// + /// The caller adds the snapshot holding the new contents, if any, before building. + /// + /// # Errors + /// - The table is format version 1, which is not supported. + /// - `format_version` would downgrade the table. + /// - The partition spec or sort order names a column missing from `schema`. + /// - `schema` takes the name of a partition field in one of the table's specs without being + /// that field's source column. + /// - `properties` contains a reserved property. + // TODO: Use for table replacement in https://github.com/apache/iceberg-rust/issues/3056. + #[allow(dead_code)] + pub(crate) fn build_replacement( + self, + schema: Schema, + spec: impl Into, + sort_order: SortOrder, + location: Option, + format_version: Option, + properties: HashMap, + ) -> Result { + if self.metadata.format_version == FormatVersion::V1 { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Table replacement is not supported for format version 1", + )); + } + + let format_version = format_version.unwrap_or(self.metadata.format_version); + let spec = spec.into(); + + let previous_id_to_name = schema.field_id_to_name_map().clone(); + let fresh_schema = assign_fresh_ids( + schema, + self.get_current_schema()?, + self.metadata.last_column_id, + )?; + + let (fresh_spec, fresh_sort_order) = + Self::rebind_to_schema(&previous_id_to_name, &fresh_schema, spec, sort_order)?; + let fresh_spec = self.reassign_partition_field_ids(fresh_spec, &fresh_schema)?; + + let mut builder = self.upgrade_format_version(format_version)?; + if builder.metadata.refs.contains_key(MAIN_BRANCH) { + builder = builder.remove_ref(MAIN_BRANCH); + } + + let builder = builder + .add_current_schema(fresh_schema)? + .add_default_partition_spec(fresh_spec)? + .add_default_sort_order(fresh_sort_order)?; + + let builder = match location { + Some(location) => builder.set_location(location), + None => builder, + }; + + builder.set_properties(properties) + } + /// Changes uuid of table metadata. pub fn assign_uuid(mut self, uuid: Uuid) -> Self { if self.metadata.table_uuid != uuid { @@ -210,12 +285,10 @@ impl TableMetadataBuilder { /// - Cannot downgrade to older format versions. pub fn upgrade_format_version(mut self, format_version: FormatVersion) -> Result { if format_version < self.metadata.format_version { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot downgrade FormatVersion from {} to {}", - self.metadata.format_version, format_version - ), + return Err(invalid_data!( + "Cannot downgrade FormatVersion from {} to {}", + self.metadata.format_version, + format_version )); } @@ -229,6 +302,13 @@ impl TableMetadataBuilder { self.changes .push(TableUpdate::UpgradeFormatVersion { format_version }); } + FormatVersion::V3 => { + self.metadata.format_version = format_version; + // Set next-row-id to 0 when upgrading to v3 as per Iceberg spec + self.metadata.next_row_id = INITIAL_ROW_ID; + self.changes + .push(TableUpdate::UpgradeFormatVersion { format_version }); + } } } @@ -247,17 +327,14 @@ impl TableMetadataBuilder { // List of specified properties that are RESERVED and should not be persisted. let reserved_properties = properties .keys() - .filter(|key| RESERVED_PROPERTIES.contains(&key.as_str())) + .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str())) .map(ToString::to_string) .collect::>(); if !reserved_properties.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Table properties should not contain reserved properties, but got: [{}]", - reserved_properties.join(", ") - ), + return Err(invalid_data!( + "Table properties should not contain reserved properties, but got: [{}]", + reserved_properties.join(", ") )); } @@ -285,17 +362,14 @@ impl TableMetadataBuilder { // disallow removal of reserved properties let reserved_properties = properties .iter() - .filter(|key| RESERVED_PROPERTIES.contains(&key.as_str())) + .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str())) .map(ToString::to_string) .collect::>(); if !reserved_properties.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Table properties to remove contain reserved properties: [{}]", - reserved_properties.join(", ") - ), + return Err(invalid_data!( + "Table properties to remove contain reserved properties: [{}]", + reserved_properties.join(", ") )); } @@ -330,15 +404,18 @@ impl TableMetadataBuilder { /// # Errors /// - Snapshot id already exists. /// - For format version > 1: the sequence number of the snapshot is lower than the highest sequence number specified so far. + /// - For format version >= 3: the first-row-id of the snapshot is lower than the next-row-id of the table. + /// - For format version >= 3: added-rows is null or first-row-id is null. + /// - For format version >= 3: next-row-id would overflow when adding added-rows. pub fn add_snapshot(mut self, snapshot: Snapshot) -> Result { if self .metadata .snapshots .contains_key(&snapshot.snapshot_id()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Snapshot already exists for: '{}'", snapshot.snapshot_id()), + return Err(invalid_data!( + "Snapshot already exists for: '{}'", + snapshot.snapshot_id() )); } @@ -346,13 +423,10 @@ impl TableMetadataBuilder { && snapshot.sequence_number() <= self.metadata.last_sequence_number && snapshot.parent_snapshot_id().is_some() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add snapshot with sequence number {} older than last sequence number {}", - snapshot.sequence_number(), - self.metadata.last_sequence_number - ), + return Err(invalid_data!( + "Cannot add snapshot with sequence number {} older than last sequence number {}", + snapshot.sequence_number(), + self.metadata.last_sequence_number )); } @@ -360,13 +434,10 @@ impl TableMetadataBuilder { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if snapshot.timestamp_ms() - last.timestamp_ms < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid snapshot timestamp {}: before last snapshot timestamp {}", - snapshot.timestamp_ms(), - last.timestamp_ms - ), + return Err(invalid_data!( + "Invalid snapshot timestamp {}: before last snapshot timestamp {}", + snapshot.timestamp_ms(), + last.timestamp_ms )); } } @@ -376,16 +447,43 @@ impl TableMetadataBuilder { .unwrap_or_default() .max(self.metadata.last_updated_ms); if snapshot.timestamp_ms() - max_last_updated < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid snapshot timestamp {}: before last updated timestamp {}", - snapshot.timestamp_ms(), - max_last_updated - ), + return Err(invalid_data!( + "Invalid snapshot timestamp {}: before last updated timestamp {}", + snapshot.timestamp_ms(), + max_last_updated )); } + let mut added_rows = None; + if self.metadata.format_version >= MIN_FORMAT_VERSION_ROW_LINEAGE { + if let Some((first_row_id, added_rows_count)) = snapshot.row_range() { + if first_row_id < self.metadata.next_row_id { + return Err(invalid_data!( + "Cannot add a snapshot, first-row-id is behind table next-row-id: {first_row_id} < {}", + self.metadata.next_row_id + )); + } + + added_rows = Some(added_rows_count); + } else { + return Err(invalid_data!( + "Cannot add a snapshot: first-row-id is null. first-row-id must be set for format version >= {MIN_FORMAT_VERSION_ROW_LINEAGE}", + )); + } + } + + if let Some(added_rows) = added_rows { + self.metadata.next_row_id = self + .metadata + .next_row_id + .checked_add(added_rows) + .ok_or_else(|| { + invalid_data!( + "Cannot add snapshot: next-row-id overflowed when adding added-rows" + ) + })?; + } + // Mutation happens in next line - must be infallible from here self.changes.push(TableUpdate::AddSnapshot { snapshot: snapshot.clone(), @@ -410,9 +508,8 @@ impl TableMetadataBuilder { let reference = if let Some(mut reference) = reference { if !reference.is_branch() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot append snapshot to non-branch reference '{branch}'",), + return Err(invalid_data!( + "Cannot append snapshot to non-branch reference '{branch}'", )); } @@ -434,18 +531,17 @@ impl TableMetadataBuilder { /// Remove snapshots by its ids from the table metadata. /// Does nothing if a snapshot id is not present. - /// Keeps as changes only the snapshots that were actually removed. + /// Records removed snapshot IDs once each, in the order they first appear + /// in `snapshot_ids`. + /// Refs pointing to a removed snapshot are removed as well, see [`Self::remove_ref`]. pub fn remove_snapshots(mut self, snapshot_ids: &[i64]) -> Self { let mut removed_snapshots = Vec::with_capacity(snapshot_ids.len()); - self.metadata.snapshots.retain(|k, _| { - if snapshot_ids.contains(k) { - removed_snapshots.push(*k); - false - } else { - true + for &snapshot_id in snapshot_ids { + if self.metadata.snapshots.remove(&snapshot_id).is_some() { + removed_snapshots.push(snapshot_id); } - }); + } if !removed_snapshots.is_empty() { self.changes.push(TableUpdate::RemoveSnapshots { @@ -454,9 +550,18 @@ impl TableMetadataBuilder { } // Remove refs that are no longer valid - self.metadata + let mut dangling_refs: Vec = self + .metadata .refs - .retain(|_, v| self.metadata.snapshots.contains_key(&v.snapshot_id)); + .iter() + .filter(|(_, v)| !self.metadata.snapshots.contains_key(&v.snapshot_id)) + .map(|(name, _)| name.clone()) + .collect(); + dangling_refs.sort(); + + for ref_name in dangling_refs { + self = self.remove_ref(&ref_name); + } self } @@ -476,12 +581,9 @@ impl TableMetadataBuilder { } let Some(snapshot) = self.metadata.snapshots.get(&reference.snapshot_id) else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot set '{ref_name}' to unknown snapshot: '{}'", - reference.snapshot_id - ), + return Err(invalid_data!( + "Cannot set '{ref_name}' to unknown snapshot: '{}'", + reference.snapshot_id )); }; @@ -521,11 +623,10 @@ impl TableMetadataBuilder { /// Remove a reference /// - /// If `ref_name='main'` the current snapshot id is set to -1. + /// If `ref_name='main'` the current snapshot id is set to `None`. pub fn remove_ref(mut self, ref_name: &str) -> Self { if ref_name == MAIN_BRANCH { self.metadata.current_snapshot_id = None; - self.metadata.snapshot_log.clear(); } if self.metadata.refs.remove(ref_name).is_some() || ref_name == MAIN_BRANCH { @@ -590,9 +691,6 @@ impl TableMetadataBuilder { /// Important: Use this method with caution. The builder does not check /// if the added schema is compatible with the current schema. pub fn add_schema(mut self, schema: Schema) -> Result { - // Validate that new schema fields don't conflict with existing partition field names - self.validate_schema_field_names(&schema)?; - let new_schema_id = self.reuse_or_create_new_schema_id(&schema); let schema_found = self.metadata.schemas.contains_key(&new_schema_id); @@ -636,12 +734,12 @@ impl TableMetadataBuilder { /// Errors: /// - provided `schema_id` is -1 but no schema has been added via `add_schema`. /// - No schema with the provided `schema_id` exists. + /// - A field of the schema takes the name of a partition field it is not the source of. pub fn set_current_schema(mut self, mut schema_id: i32) -> Result { if schema_id == Self::LAST_ADDED { schema_id = self.last_added_schema_id.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot set current schema to last added schema: no schema has been added.", + invalid_data!( + "Cannot set current schema to last added schema: no schema has been added." ) })?; }; @@ -651,15 +749,10 @@ impl TableMetadataBuilder { return Ok(self); } - let _schema = self.metadata.schemas.get(&schema_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot set current schema to unknown schema with id: '{}'", - schema_id - ), - ) + let schema = self.metadata.schemas.get(&schema_id).ok_or_else(|| { + invalid_data!("Cannot set current schema to unknown schema with id: '{schema_id}'") })?; + self.validate_schema_field_names(schema)?; // Old partition specs and sort-orders should be preserved even if they are not compatible with the new schema, // so that older metadata can still be interpreted. @@ -686,32 +779,37 @@ impl TableMetadataBuilder { .set_current_schema(Self::LAST_ADDED) } - /// Validate schema field names against partition field names across all historical schemas. + /// Validate schema field names against the partition field names of every spec. /// - /// Due to Iceberg's multi-version property, this check ignores existing schema fields - /// that match partition names (schema evolution allows re-adding previously removed fields). - /// Only NEW field names that conflict with partition names are rejected. + /// A partition field whose source column is absent from `schema` is skipped: its spec is + /// historical, kept only so old metadata can be interpreted, so the name is free to reuse. + /// Otherwise the name may only be taken by the source column itself, which is the identity + /// partition case that [`Self::validate_partition_field_names`] allows in the other direction. /// /// # Errors - /// - Schema field name conflicts with partition field name but doesn't exist in any historical schema. + /// - A schema field takes the name of a partition field it is not the source of. fn validate_schema_field_names(&self, schema: &Schema) -> Result<()> { - if self.metadata.schemas.is_empty() { - return Ok(()); - } - - for field_name in schema.field_id_to_name_map().values() { - let has_partition_conflict = self.metadata.partition_name_exists(field_name); - let is_new_field = !self.metadata.name_exists_in_any_schema(field_name); + for (spec_id, spec) in &self.metadata.partition_specs { + for partition_field in spec.fields() { + if schema.field_by_id(partition_field.source_id).is_none() { + continue; + } - if has_partition_conflict && is_new_field { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add schema field '{}' because it conflicts with existing partition field name. \ - Schema evolution cannot introduce field names that match existing partition field names.", - field_name - ), - )); + let Some(schema_field) = schema.field_by_name(&partition_field.name) else { + continue; + }; + + if schema_field.id != partition_field.source_id { + return Err(invalid_data!( + "Cannot set current schema {}: field '{}' with id {} conflicts with a \ + partition field of that name in spec {} sourced from field id {}", + schema.schema_id(), + partition_field.name, + schema_field.id, + spec_id, + partition_field.source_id + )); + } } } @@ -721,12 +819,13 @@ impl TableMetadataBuilder { /// Validate partition field names against schema field names across all historical schemas. /// /// Due to Iceberg's multi-version property, partition fields can share names with schema fields - /// if they meet specific requirements (identity transform + matching source field ID). + /// if they meet specific requirements (identity or void transform + matching source field ID). /// This validation enforces those rules across all historical schema versions. /// /// # Errors - /// - Partition field name conflicts with schema field name but doesn't use identity transform. - /// - Partition field uses identity transform but references wrong source field ID. + /// - Partition field name conflicts with schema field name but doesn't use identity or void + /// transform. + /// - Partition field uses identity or void transform but references wrong source field ID. fn validate_partition_field_names(&self, unbound_spec: &UnboundPartitionSpec) -> Result<()> { if self.metadata.schemas.is_empty() { return Ok(()); @@ -736,7 +835,7 @@ impl TableMetadataBuilder { for partition_field in unbound_spec.fields() { let exists_in_any_schema = self .metadata - .name_exists_in_any_schema(&partition_field.name); + .name_exists_in_any_schema(partition_field.name()); // Skip if partition field name doesn't conflict with any schema field if !exists_in_any_schema { @@ -744,29 +843,23 @@ impl TableMetadataBuilder { } // If name exists in schemas, validate against current schema rules - if let Some(schema_field) = current_schema.field_by_name(&partition_field.name) { - let is_identity_transform = - partition_field.transform == crate::spec::Transform::Identity; - let has_matching_source_id = schema_field.id == partition_field.source_id; + if let Some(schema_field) = current_schema.field_by_name(partition_field.name()) { + let has_matching_source_id = schema_field.id == partition_field.source_id()?; - if !is_identity_transform { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot create partition with name '{}' that conflicts with schema field and is not an identity transform.", - partition_field.name - ), + if !partition_field.transform().may_share_name_with_source() { + return Err(invalid_data!( + "Cannot create partition with name '{}' that conflicts with schema field and is not an identity or void transform.", + partition_field.name() )); } if !has_matching_source_id { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot create identity partition sourced from different field in schema. \ + return Err(invalid_data!( + "Cannot create identity or void partition sourced from different field in schema. \ Field name '{}' has id `{}` in schema but partition source id is `{}`", - partition_field.name, schema_field.id, partition_field.source_id - ), + partition_field.name(), + schema_field.id, + partition_field.source_id()? )); } } @@ -791,6 +884,9 @@ impl TableMetadataBuilder { // Check if partition field names conflict with schema field names across all schemas self.validate_partition_field_names(&unbound_spec)?; + // Reuse field IDs for equivalent fields from existing partition specs + let unbound_spec = self.reuse_partition_field_ids(unbound_spec)?; + let spec = PartitionSpecBuilder::new_from_unbound(unbound_spec.clone(), schema)? .with_last_assigned_field_id(self.metadata.last_partition_id) .build()?; @@ -811,9 +907,8 @@ impl TableMetadataBuilder { } if self.metadata.format_version <= FormatVersion::V1 && !spec.has_sequential_ids() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot add partition spec with non-sequential field ids to format version 1 table", + return Err(invalid_data!( + "Cannot add partition spec with non-sequential field ids to format version 1 table" )); } @@ -833,6 +928,60 @@ impl TableMetadataBuilder { Ok(self) } + /// Reuse partition field IDs for equivalent fields from existing partition specs. + /// + /// According to the Iceberg spec, partition field IDs must be reused if an existing + /// partition spec contains an equivalent field (same source_id and transform). + fn reuse_partition_field_ids( + &self, + unbound_spec: UnboundPartitionSpec, + ) -> Result { + let equivalent_field_ids = Self::highest_equivalent_field_ids( + self.metadata + .partition_specs + .values() + .flat_map(|spec| spec.fields()), + ); + + // Create new fields with reused field IDs where possible + let fields = unbound_spec + .fields + .into_iter() + .map(|field| { + if field.field_id().is_none() + && let Some(&existing_field_id) = + equivalent_field_ids.get(&(field.source_ids().to_vec(), field.transform())) + { + field.with_field_id(existing_field_id) + } else { + field + } + }) + .collect(); + + Ok(UnboundPartitionSpec { + spec_id: unbound_spec.spec_id, + fields, + }) + } + + /// Map each (source_ids, transform) pair to the highest field id assigned to it. + /// + /// Taking the highest keeps the result independent of the order `fields` arrive in, which for + /// the specs of a `TableMetadata` is an arbitrary `HashMap` iteration order. + fn highest_equivalent_field_ids<'a>( + fields: impl Iterator, + ) -> HashMap<(Vec, Transform), i32> { + let mut equivalent_field_ids = HashMap::new(); + for field in fields { + equivalent_field_ids + .entry((vec![field.source_id], field.transform)) + .and_modify(|id: &mut i32| *id = (*id).max(field.field_id)) + .or_insert(field.field_id); + } + equivalent_field_ids + } + /// Set the default partition spec. /// /// # Errors @@ -841,9 +990,8 @@ impl TableMetadataBuilder { pub fn set_default_partition_spec(mut self, mut spec_id: i32) -> Result { if spec_id == Self::LAST_ADDED { spec_id = self.last_added_spec_id.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot set default partition spec to last added spec: no spec has been added.", + invalid_data!( + "Cannot set default partition spec to last added spec: no spec has been added." ) })?; } @@ -853,9 +1001,8 @@ impl TableMetadataBuilder { } if !self.metadata.partition_specs.contains_key(&spec_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Cannot set default partition spec to unknown spec with id: '{spec_id}'",), + return Err(invalid_data!( + "Cannot set default partition spec to unknown spec with id: '{spec_id}'", )); } @@ -864,11 +1011,8 @@ impl TableMetadataBuilder { .partition_specs .get(&spec_id) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot set default partition spec to unknown spec with id: '{spec_id}'", - ), + invalid_data!( + "Cannot set default partition spec to unknown spec with id: '{spec_id}'", ) })? .clone(); @@ -902,10 +1046,7 @@ impl TableMetadataBuilder { /// - Cannot remove the default partition spec. pub fn remove_partition_specs(mut self, spec_ids: &[i32]) -> Result { if spec_ids.contains(&self.metadata.default_spec.spec_id()) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot remove default partition spec", - )); + return Err(invalid_data!("Cannot remove default partition spec")); } let mut removed_specs = Vec::with_capacity(spec_ids.len()); @@ -955,11 +1096,8 @@ impl TableMetadataBuilder { .with_fields(sort_order.fields) .build(&schema) .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Sort order to add is incompatible with current schema: {e}"), - ) - .with_source(e) + invalid_data!("Sort order to add is incompatible with current schema: {e}") + .with_source(e) })?; self.last_added_order_id = Some(new_order_id); @@ -979,9 +1117,8 @@ impl TableMetadataBuilder { pub fn set_default_sort_order(mut self, mut sort_order_id: i64) -> Result { if sort_order_id == Self::LAST_ADDED as i64 { sort_order_id = self.last_added_order_id.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot set default sort order to last added order: no order has been added.", + invalid_data!( + "Cannot set default sort order to last added order: no order has been added." ) })?; } @@ -991,11 +1128,8 @@ impl TableMetadataBuilder { } if !self.metadata.sort_orders.contains_key(&sort_order_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot set default sort order to unknown order with id: '{sort_order_id}'" - ), + return Err(invalid_data!( + "Cannot set default sort order to unknown order with id: '{sort_order_id}'" )); } @@ -1019,12 +1153,39 @@ impl TableMetadataBuilder { .set_default_sort_order(Self::LAST_ADDED as i64) } + /// Add an encryption key to the table metadata. + pub fn add_encryption_key(mut self, key: EncryptedKey) -> Self { + let key_id = key.key_id().to_string(); + if self.metadata.encryption_keys.contains_key(&key_id) { + // already exists + return self; + } + + self.metadata.encryption_keys.insert(key_id, key.clone()); + self.changes.push(TableUpdate::AddEncryptionKey { + encryption_key: key, + }); + self + } + + /// Remove an encryption key from the table metadata. + pub fn remove_encryption_key(mut self, key_id: &str) -> Self { + if self.metadata.encryption_keys.remove(key_id).is_some() { + self.changes.push(TableUpdate::RemoveEncryptionKey { + key_id: key_id.to_string(), + }); + } + self + } + /// Build the table metadata. pub fn build(mut self) -> Result { self.metadata.last_updated_ms = self .last_updated_ms .unwrap_or_else(|| chrono::Utc::now().timestamp_millis()); + self.check_format_compatibility()?; + // Check compatibility of the current schema to the default partition spec and sort order. // We use the `get_xxx` methods from the builder to avoid using the panicking // `TableMetadata.default_partition_spec` etc. methods. @@ -1043,12 +1204,13 @@ impl TableMetadataBuilder { .build(&schema)?; self.update_snapshot_log()?; - self.metadata.try_normalize()?; + // Validate the metadata log with the entry for the previous file, as every reader does. if let Some(hist_entry) = self.previous_history_entry.take() { self.metadata.metadata_log.push(hist_entry); } let expired_metadata_logs = self.expire_metadata_log(); + self.metadata.try_normalize()?; Ok(TableMetadataBuildResult { metadata: self.metadata, @@ -1061,9 +1223,9 @@ impl TableMetadataBuilder { let max_size = self .metadata .properties - .get(PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX) + .get(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX) .and_then(|v| v.parse::().ok()) - .unwrap_or(PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT) + .unwrap_or(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT) .max(1); if self.metadata.metadata_log.len() > max_size { @@ -1107,9 +1269,8 @@ impl TableMetadataBuilder { if let Some(current_snapshot_id) = self.metadata.current_snapshot_id { let last_id = new_snapshot_log.last().map(|entry| entry.snapshot_id); if last_id != Some(current_snapshot_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot set invalid snapshot log: latest entry is not the current snapshot", + return Err(invalid_data!( + "Cannot set invalid snapshot log: latest entry is not the current snapshot" )); } }; @@ -1162,6 +1323,38 @@ impl TableMetadataBuilder { .collect() } + /// Gives each field of `spec` the highest id an equivalent field has in the table's specs, or + /// the next id above `last_partition_id`. + fn reassign_partition_field_ids( + &self, + spec: PartitionSpec, + schema: &Schema, + ) -> Result { + let fields = spec + .fields() + .iter() + .map(|field| { + UnboundPartitionField::builder() + .source_ids(vec![field.source_id]) + .name(field.name.clone()) + .transform(field.transform) + .build() + }) + .collect::>>()?; + let spec = self.reuse_partition_field_ids( + UnboundPartitionSpec::builder() + .add_partition_fields(fields)? + .build(), + )?; + + Ok( + PartitionSpecBuilder::new_from_unbound(spec, schema.clone())? + .with_last_assigned_field_id(self.metadata.last_partition_id) + .build()? + .into_unbound(), + ) + } + fn reassign_ids( schema: Schema, spec: UnboundPartitionSpec, @@ -1175,20 +1368,36 @@ impl TableMetadataBuilder { .with_reassigned_field_ids(FIRST_FIELD_ID) .build()?; + let (fresh_spec, fresh_sort_order) = + Self::rebind_to_schema(&previous_id_to_name, &fresh_schema, spec, sort_order)?; + + Ok((fresh_schema, fresh_spec, fresh_sort_order)) + } + + /// Rebuilds `spec` and `sort_order` against `fresh_schema`, looking up each source column by + /// the name `previous_id_to_name` gives for the field id they were built against. + fn rebind_to_schema( + previous_id_to_name: &HashMap, + fresh_schema: &Schema, + spec: UnboundPartitionSpec, + sort_order: SortOrder, + ) -> Result<(PartitionSpec, SortOrder)> { // Re-build partition spec with new ids let mut fresh_spec = PartitionSpecBuilder::new(fresh_schema.clone()); for field in spec.fields() { - let source_field_name = previous_id_to_name.get(&field.source_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot find source column with id {} for partition column {} in schema.", - field.source_id, field.name - ), + let source_id = field.source_id()?; + let source_field_name = previous_id_to_name.get(&source_id).ok_or_else(|| { + invalid_data!( + "Cannot find source column with id {} for partition column {} in schema.", + source_id, + field.name() ) })?; - fresh_spec = - fresh_spec.add_partition_field(source_field_name, &field.name, field.transform)?; + fresh_spec = fresh_spec.add_partition_field( + source_field_name, + field.name(), + field.transform(), + )?; } let fresh_spec = fresh_spec.build()?; @@ -1196,12 +1405,9 @@ impl TableMetadataBuilder { let mut fresh_order = SortOrder::builder(); for mut field in sort_order.fields { let source_field_name = previous_id_to_name.get(&field.source_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot find source column with id {} for sort column in schema.", - field.source_id - ), + invalid_data!( + "Cannot find source column with id {} for sort column in schema.", + field.source_id ) })?; let new_field_id = fresh_schema @@ -1210,17 +1416,43 @@ impl TableMetadataBuilder { Error::new( ErrorKind::Unexpected, format!( - "Cannot find source column with name {} for sort column in re-assigned schema.", - source_field_name + "Cannot find source column with name {source_field_name} for sort column in re-assigned schema." ), ) })?.id; field.source_id = new_field_id; fresh_order.with_sort_field(field); } - let fresh_sort_order = fresh_order.build(&fresh_schema)?; + let fresh_sort_order = fresh_order.build(fresh_schema)?; - Ok((fresh_schema, fresh_spec, fresh_sort_order)) + Ok((fresh_spec, fresh_sort_order)) + } + + fn check_format_compatibility(&self) -> Result<()> { + let format_version = self.metadata.format_version; + if format_version < FormatVersion::V3 { + let encryption_properties = [ + TableProperties::PROPERTY_ENCRYPTION_KEY_ID, + TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, + ] + .into_iter() + .filter(|key| self.metadata.properties.contains_key(*key)) + .collect::>(); + if !encryption_properties.is_empty() { + return Err(invalid_data!( + "Invalid properties for {format_version}: [{}]", + encryption_properties.join(", ") + )); + } + + if !self.metadata.encryption_keys.is_empty() { + return Err(invalid_data!( + "Encryption keys are not supported for {format_version} tables" + )); + } + } + + Ok(()) } fn reuse_or_create_new_schema_id(&self, new_schema: &Schema) -> i32 { @@ -1245,12 +1477,9 @@ impl TableMetadataBuilder { .schemas .get(&self.metadata.current_schema_id) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Current schema with id '{}' not found in table metadata.", - self.metadata.current_schema_id - ), + invalid_data!( + "Current schema with id '{}' not found in table metadata.", + self.metadata.current_schema_id ) }) } @@ -1261,12 +1490,9 @@ impl TableMetadataBuilder { .get(&self.metadata.default_sort_order_id) .cloned() .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Default sort order with id '{}' not found in table metadata.", - self.metadata.default_sort_order_id - ), + invalid_data!( + "Default sort order with id '{}' not found in table metadata.", + self.metadata.default_sort_order_id ) }) } @@ -1315,10 +1541,7 @@ impl TableMetadataBuilder { /// Does nothing if a schema id is not present. Active schemas should not be removed. pub fn remove_schemas(mut self, schema_id_to_remove: &[i32]) -> Result { if schema_id_to_remove.contains(&self.metadata.current_schema_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot remove current schema", - )); + return Err(invalid_data!("Cannot remove current schema")); } if schema_id_to_remove.is_empty() { @@ -1357,13 +1580,15 @@ mod tests { use super::*; use crate::TableIdent; - use crate::io::FileIOBuilder; + use crate::io::FileIO; use crate::spec::{ BlobMetadata, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema, - SnapshotRetention, SortDirection, SortField, StructType, Summary, Transform, Type, - UnboundPartitionField, + SnapshotRetention, SortDirection, SortField, StructType, Summary, TableProperties, + Transform, Type, UnboundPartitionField, }; use crate::table::Table; + use crate::test_utils::scan::TableTestFixture; + use crate::test_utils::test_runtime; const TEST_LOCATION: &str = "s3://bucket/test/location"; const LAST_ASSIGNED_COLUMN_ID: i32 = 3; @@ -1396,7 +1621,14 @@ mod tests { fn partition_spec() -> UnboundPartitionSpec { UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(2, "y", Transform::Identity) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("y") + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() } @@ -1630,12 +1862,15 @@ mod tests { // partition_spec() has None set for field-id spec: PartitionSpec::builder(schema()) .with_spec_id(0) - .add_unbound_field(UnboundPartitionField { - name: "y".to_string(), - transform: Transform::Identity, - source_id: 2, - field_id: Some(1000) - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("y".to_string()) + .transform(Transform::Identity) + .build() + .unwrap() + ) .unwrap() .build() .unwrap() @@ -1703,20 +1938,21 @@ mod tests { let added_spec = UnboundPartitionSpec::builder() .with_spec_id(10) .add_partition_fields(vec![ - UnboundPartitionField { - // The previous field - has field_id set - name: "y".to_string(), - transform: Transform::Identity, - source_id: 2, - field_id: Some(1000), - }, - UnboundPartitionField { - // A new field without field id - should still be without field id in changes - name: "z".to_string(), - transform: Transform::Identity, - source_id: 3, - field_id: None, - }, + // The previous field - has field_id set + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("y".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + // A new field without field id - should still be without field id in changes + UnboundPartitionField::builder() + .source_ids(vec![3]) + .name("z".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), ]) .unwrap() .build(); @@ -1731,19 +1967,25 @@ mod tests { let expected_change = added_spec.with_spec_id(1); let expected_spec = PartitionSpec::builder(schema()) .with_spec_id(1) - .add_unbound_field(UnboundPartitionField { - name: "y".to_string(), - transform: Transform::Identity, - source_id: 2, - field_id: Some(1000), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("y".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() - .add_unbound_field(UnboundPartitionField { - name: "z".to_string(), - transform: Transform::Identity, - source_id: 3, - field_id: Some(1001), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![3]) + .field_id(1001) + .name("z".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -1781,7 +2023,14 @@ mod tests { let schema = builder.get_current_schema().unwrap().clone(); let added_spec = UnboundPartitionSpec::builder() .with_spec_id(10) - .add_partition_field(1, "y_bucket[2]", Transform::Bucket(2)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("y_bucket[2]") + .transform(Transform::Bucket(2)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -1795,12 +2044,15 @@ mod tests { let expected_spec = PartitionSpec::builder(schema) .with_spec_id(1) - .add_unbound_field(UnboundPartitionField { - name: "y_bucket[2]".to_string(), - transform: Transform::Bucket(2), - source_id: 1, - field_id: Some(1001), - }) + .add_unbound_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(1001) + .name("y_bucket[2]".to_string()) + .transform(Transform::Bucket(2)) + .build() + .unwrap(), + ) .unwrap() .build() .unwrap(); @@ -1985,36 +2237,162 @@ mod tests { assert_eq!(build_result.metadata.metadata_log.len(), 0); } + const KEY_ID: (&str, &str) = (TableProperties::PROPERTY_ENCRYPTION_KEY_ID, "master-1"); + const KEY_LENGTH: (&str, &str) = (TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, "16"); + + fn string_map(entries: &[(&str, &str)]) -> HashMap { + entries + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect() + } + + fn build_error(builder: TableMetadataBuilder) -> (ErrorKind, String) { + let err = builder.build().unwrap_err(); + (err.kind(), err.message().to_string()) + } + #[test] - fn test_no_metadata_log_entry_for_no_previous_location() { - // Used for first commit after stage-creation of tables - let metadata = builder_without_changes(FormatVersion::V2) - .build() - .unwrap() - .metadata; - assert_eq!(metadata.metadata_log.len(), 1); + fn test_build_rejects_encryption_properties_before_v3() { + for version in [FormatVersion::V1, FormatVersion::V2] { + for (properties, names) in [ + (vec![KEY_ID], "encryption.key-id"), + (vec![KEY_LENGTH], "encryption.data-key-length"), + ( + vec![KEY_ID, KEY_LENGTH], + "encryption.key-id, encryption.data-key-length", + ), + ] { + let builder = builder_without_changes(version) + .set_properties(string_map(&properties)) + .unwrap(); + assert_eq!( + build_error(builder), + ( + ErrorKind::DataInvalid, + format!("Invalid properties for {version}: [{names}]") + ) + ); + } + } + } - let build_result = metadata - .into_builder(None) - .set_properties(HashMap::from_iter(vec![( - "foo".to_string(), - "bar".to_string(), - )])) + #[test] + fn test_build_rejects_encryption_keys_before_v3() { + let key = EncryptedKey::builder() + .key_id("key-1") + .encrypted_key_metadata(vec![1, 2, 3, 4]) + .build(); + + for version in [FormatVersion::V1, FormatVersion::V2] { + let builder = builder_without_changes(version).add_encryption_key(key.clone()); + assert_eq!( + build_error(builder), + ( + ErrorKind::DataInvalid, + format!("Encryption keys are not supported for {version} tables") + ) + ); + } + } + + #[test] + fn test_build_accepts_encryption_properties_on_v3() { + let properties = string_map(&[KEY_ID, KEY_LENGTH]); + + let metadata = builder_without_changes(FormatVersion::V3) + .set_properties(properties.clone()) .unwrap() .build() - .unwrap(); + .unwrap() + .metadata; - assert_eq!(build_result.metadata.metadata_log.len(), 1); + assert_eq!(metadata.properties, properties); } #[test] - fn test_from_metadata_generates_metadata_log() { - let metadata_path = "s3://bucket/test/location/metadata/metadata1.json"; - let builder = TableMetadataBuilder::new( - schema(), - partition_spec(), - sort_order(), - TEST_LOCATION.to_string(), + fn test_pre_v3_encryption_property_blocks_updates_until_removed() { + let legacy = || { + let mut builder = builder_without_changes(FormatVersion::V2); + builder.metadata.properties = string_map(&[KEY_ID]); + builder + }; + + let other_update = legacy() + .set_properties(string_map(&[("key", "value")])) + .unwrap(); + assert_eq!( + build_error(other_update), + ( + ErrorKind::DataInvalid, + "Invalid properties for v2: [encryption.key-id]".to_string() + ) + ); + + let repaired = legacy() + .remove_properties(&[KEY_ID.0.to_string()]) + .unwrap() + .build() + .unwrap(); + assert_eq!(repaired.metadata.properties, HashMap::new()); + } + + #[test] + fn test_table_properties_view_reflects_metadata_updates() { + let property = TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(); + let metadata = builder_without_changes(FormatVersion::V2) + .set_properties(HashMap::from([(property.clone(), "7".to_string())])) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(metadata.table_properties().commit_num_retries().unwrap(), 7); + + let metadata = metadata + .into_builder(None) + .remove_properties(&[property]) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!( + metadata.table_properties().commit_num_retries().unwrap(), + TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT + ); + } + + #[test] + fn test_no_metadata_log_entry_for_no_previous_location() { + // Used for first commit after stage-creation of tables + let metadata = builder_without_changes(FormatVersion::V2) + .build() + .unwrap() + .metadata; + assert_eq!(metadata.metadata_log.len(), 1); + + let build_result = metadata + .into_builder(None) + .set_properties(HashMap::from_iter(vec![( + "foo".to_string(), + "bar".to_string(), + )])) + .unwrap() + .build() + .unwrap(); + + assert_eq!(build_result.metadata.metadata_log.len(), 1); + } + + #[test] + fn test_from_metadata_generates_metadata_log() { + let metadata_path = "s3://bucket/test/location/metadata/metadata1.json"; + let builder = TableMetadataBuilder::new( + schema(), + partition_spec(), + sort_order(), + TEST_LOCATION.to_string(), FormatVersion::V2, HashMap::new(), ) @@ -2170,6 +2548,73 @@ mod tests { assert_eq!(result.metadata.current_snapshot().unwrap().snapshot_id(), 2); } + #[test] + fn test_remove_main_ref_keeps_snapshot_log() { + let builder = builder_without_changes(FormatVersion::V2); + + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_timestamp_ms(builder.metadata.last_updated_ms + 1) + .with_sequence_number(0) + .with_schema_id(0) + .with_manifest_list("/snap-1.avro") + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::from_iter(vec![ + ( + "spark.app.id".to_string(), + "local-1662532784305".to_string(), + ), + ("added-data-files".to_string(), "4".to_string()), + ("added-records".to_string(), "4".to_string()), + ("added-files-size".to_string(), "6001".to_string()), + ]), + }) + .build(); + + let result = builder + .add_snapshot(snapshot.clone()) + .unwrap() + .set_ref(MAIN_BRANCH, SnapshotReference { + snapshot_id: 1, + retention: SnapshotRetention::Branch { + min_snapshots_to_keep: Some(10), + max_snapshot_age_ms: None, + max_ref_age_ms: None, + }, + }) + .unwrap() + .build() + .unwrap(); + + // Verify snapshot log was created + assert_eq!(result.metadata.snapshot_log.len(), 1); + assert_eq!(result.metadata.snapshot_log[0].snapshot_id, 1); + assert_eq!(result.metadata.current_snapshot_id, Some(1)); + + // Remove the main ref + let result_after_remove = result + .metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata2.json".to_string(), + )) + .remove_ref(MAIN_BRANCH) + .build() + .unwrap(); + + // Verify snapshot log is kept even after removing main ref + assert_eq!(result_after_remove.metadata.snapshot_log.len(), 1); + assert_eq!(result_after_remove.metadata.snapshot_log[0].snapshot_id, 1); + assert_eq!(result_after_remove.metadata.current_snapshot_id, None); + assert_eq!(result_after_remove.changes.len(), 1); + assert_eq!( + result_after_remove.changes[0], + TableUpdate::RemoveSnapshotRef { + ref_name: MAIN_BRANCH.to_string() + } + ); + } + #[test] fn test_set_branch_snapshot_creates_branch_if_not_exists() { let builder = builder_without_changes(FormatVersion::V2); @@ -2215,6 +2660,147 @@ mod tests { ]); } + /// Builds metadata with one snapshot per id and the given refs, as committed metadata + /// without pending changes. + fn metadata_with_snapshots_and_refs( + snapshot_ids: &[i64], + refs: &[(&str, SnapshotReference)], + ) -> TableMetadata { + let mut builder = builder_without_changes(FormatVersion::V2); + let timestamp_ms = builder.metadata.last_updated_ms + 1; + for (i, snapshot_id) in snapshot_ids.iter().enumerate() { + let snapshot = Snapshot::builder() + .with_snapshot_id(*snapshot_id) + .with_timestamp_ms(timestamp_ms + i as i64) + .with_sequence_number(i as i64 + 1) + .with_schema_id(0) + .with_manifest_list(format!("/snap-{snapshot_id}.avro")) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .build(); + builder = builder.add_snapshot(snapshot).unwrap(); + } + for (ref_name, reference) in refs { + builder = builder.set_ref(ref_name, reference.clone()).unwrap(); + } + builder.build().unwrap().metadata + } + + fn branch_ref(snapshot_id: i64) -> SnapshotReference { + SnapshotReference { + snapshot_id, + retention: SnapshotRetention::Branch { + min_snapshots_to_keep: None, + max_snapshot_age_ms: None, + max_ref_age_ms: None, + }, + } + } + + fn tag_ref(snapshot_id: i64) -> SnapshotReference { + SnapshotReference { + snapshot_id, + retention: SnapshotRetention::Tag { + max_ref_age_ms: None, + }, + } + } + + #[test] + fn test_remove_snapshots_records_removal_of_refs_to_removed_snapshots() { + let metadata = metadata_with_snapshots_and_refs(&[1, 2], &[ + (MAIN_BRANCH, branch_ref(2)), + ("branch-on-1", branch_ref(1)), + ("tag-on-1", tag_ref(1)), + ("branch-on-2", branch_ref(2)), + ]); + + let result = metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata2.json".to_string(), + )) + .remove_snapshots(&[1]) + .build() + .unwrap(); + + assert_eq!(result.changes, vec![ + TableUpdate::RemoveSnapshots { + snapshot_ids: vec![1] + }, + TableUpdate::RemoveSnapshotRef { + ref_name: "branch-on-1".to_string() + }, + TableUpdate::RemoveSnapshotRef { + ref_name: "tag-on-1".to_string() + }, + ]); + assert_eq!( + result.metadata.refs, + HashMap::from([ + (MAIN_BRANCH.to_string(), branch_ref(2)), + ("branch-on-2".to_string(), branch_ref(2)), + ]) + ); + assert_eq!(result.metadata.current_snapshot_id, Some(2)); + } + + #[test] + fn test_remove_snapshots_clears_current_snapshot_when_main_snapshot_removed() { + let metadata = metadata_with_snapshots_and_refs(&[1, 2], &[ + (MAIN_BRANCH, branch_ref(2)), + ("branch-on-1", branch_ref(1)), + ]); + + let result = metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata2.json".to_string(), + )) + .remove_snapshots(&[2]) + .build() + .unwrap(); + + assert_eq!(result.changes, vec![ + TableUpdate::RemoveSnapshots { + snapshot_ids: vec![2] + }, + TableUpdate::RemoveSnapshotRef { + ref_name: MAIN_BRANCH.to_string() + }, + ]); + assert_eq!(result.metadata.current_snapshot_id, None); + assert_eq!( + result.metadata.refs, + HashMap::from([("branch-on-1".to_string(), branch_ref(1))]) + ); + assert!(result.metadata.snapshot_log.is_empty()); + } + + #[test] + fn test_remove_snapshots_keeps_refs_to_retained_snapshots() { + let metadata = metadata_with_snapshots_and_refs(&[1, 2, 3], &[ + (MAIN_BRANCH, branch_ref(3)), + ("branch-on-2", branch_ref(2)), + ("tag-on-3", tag_ref(3)), + ]); + let refs_before = metadata.refs.clone(); + + let result = metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata2.json".to_string(), + )) + .remove_snapshots(&[1]) + .build() + .unwrap(); + + assert_eq!(result.changes, vec![TableUpdate::RemoveSnapshots { + snapshot_ids: vec![1] + }]); + assert_eq!(result.metadata.refs, refs_before); + assert_eq!(result.metadata.current_snapshot_id, Some(3)); + } + #[test] fn test_cannot_add_duplicate_snapshot_id() { let builder = builder_without_changes(FormatVersion::V2); @@ -2243,6 +2829,40 @@ mod tests { builder.add_snapshot(snapshot).unwrap_err(); } + #[test] + fn test_remove_snapshots_records_existing_ids_once() { + let fixture = TableTestFixture::new_with_deep_history(); + let metadata = fixture.table.metadata().clone(); + let first_id = metadata.snapshot_log[0].snapshot_id; + let third_id = metadata.snapshot_log[2].snapshot_id; + let mut expected_metadata = metadata.clone(); + expected_metadata.snapshots = [ + 3055729675574597004, + 3057729675574597004, + 3059729675574597004, + ] + .into_iter() + .map(|id| (id, metadata.snapshot_by_id(id).unwrap().clone())) + .collect(); + expected_metadata.snapshot_log = metadata.snapshot_log[3..].to_vec(); + let expected = TableMetadataBuildResult { + metadata: expected_metadata, + changes: vec![TableUpdate::RemoveSnapshots { + snapshot_ids: vec![third_id, first_id], + }], + expired_metadata_logs: vec![], + }; + let mut builder = metadata.into_builder(None); + builder.last_updated_ms = Some(expected.metadata.last_updated_ms); + + let result = builder + .remove_snapshots(&[third_id, 0, first_id, third_id, 0, first_id]) + .build() + .unwrap(); + + assert_eq!(result, expected); + } + #[test] fn test_add_incompatible_current_schema_fails() { let builder = builder_without_changes(FormatVersion::V2); @@ -2272,18 +2892,20 @@ mod tests { let added_spec = UnboundPartitionSpec::builder() .with_spec_id(10) .add_partition_fields(vec![ - UnboundPartitionField { - name: "y".to_string(), - transform: Transform::Identity, - source_id: 2, - field_id: Some(1000), - }, - UnboundPartitionField { - name: "z".to_string(), - transform: Transform::Identity, - source_id: 3, - field_id: Some(1002), - }, + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(1000) + .name("y".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), + UnboundPartitionField::builder() + .source_ids(vec![3]) + .field_id(1002) + .name("z".to_string()) + .transform(Transform::Identity) + .build() + .unwrap(), ]) .unwrap() .build(); @@ -2295,48 +2917,196 @@ mod tests { } #[test] - fn test_expire_metadata_log() { - let builder = builder_without_changes(FormatVersion::V2); + fn test_add_partition_spec_accepts_void_fields_of_removed_fields() { + let builder = builder_without_changes(FormatVersion::V1); + let void_field = |field_id: i32, name: &str| { + UnboundPartitionField::builder() + .source_ids(vec![2]) + .field_id(field_id) + .name(name) + .transform(Transform::Void) + .build() + .unwrap() + }; + let added_spec = UnboundPartitionSpec { + spec_id: None, + fields: vec![void_field(1000, "y"), void_field(1001, "y_null")], + }; + let metadata = builder - .set_properties(HashMap::from_iter(vec![( - PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), - "2".to_string(), - )])) + .add_default_partition_spec(added_spec.clone()) + .unwrap() + .build() .unwrap() + .metadata; + + let expected = added_spec.with_spec_id(1).bind(schema()).unwrap(); + assert_eq!(metadata.default_partition_spec().as_ref(), &expected); + assert_eq!(metadata.last_partition_id, 1001); + } + + #[test] + fn test_add_partition_spec_accepts_two_time_granularities_of_one_column() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "ts", Type::Primitive(PrimitiveType::Timestamptz)).into(), + ]) .build() .unwrap(); - assert_eq!(metadata.metadata.metadata_log.len(), 1); - assert_eq!(metadata.expired_metadata_logs.len(), 0); + let time_field = |field_id: i32, name: &str, transform: Transform| { + UnboundPartitionField::builder() + .source_ids(vec![1]) + .field_id(field_id) + .name(name) + .transform(transform) + .build() + .unwrap() + }; + let added_spec = UnboundPartitionSpec { + spec_id: None, + fields: vec![ + time_field(1000, "ts_day", Transform::Day), + time_field(1001, "ts_hour", Transform::Hour), + ], + }; + let metadata = TableMetadataBuilder::new( + schema.clone(), + PartitionSpec::unpartition_spec(), + SortOrder::unsorted_order(), + TEST_LOCATION.to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; let metadata = metadata - .metadata - .into_builder(Some("path2".to_string())) - .set_properties(HashMap::from_iter(vec![( - "change_nr".to_string(), - "1".to_string(), - )])) + .into_builder(None) + .add_default_partition_spec(added_spec.clone()) .unwrap() .build() - .unwrap(); - - assert_eq!(metadata.metadata.metadata_log.len(), 2); - assert_eq!(metadata.expired_metadata_logs.len(), 0); - + .unwrap() + .metadata; + // Every later commit re-validates the default spec. let metadata = metadata - .metadata - .into_builder(Some("path2".to_string())) - .set_properties(HashMap::from_iter(vec![( - "change_nr".to_string(), - "2".to_string(), - )])) + .into_builder(None) + .set_properties(HashMap::from([("key".to_string(), "value".to_string())])) .unwrap() .build() - .unwrap(); - assert_eq!(metadata.metadata.metadata_log.len(), 2); - assert_eq!(metadata.expired_metadata_logs.len(), 1); + .unwrap() + .metadata; + + let expected = added_spec.with_spec_id(1).bind(schema).unwrap(); + assert_eq!(metadata.default_partition_spec().as_ref(), &expected); } - #[test] + /// Adds a branch snapshot from a writer whose clock is `skew_ms` ahead, then changes a + /// property, which takes the current time. Returns the snapshot time and the second build. + fn build_after_fast_branch_snapshot(skew_ms: i64) -> (i64, Result) { + let builder = builder_without_changes(FormatVersion::V2); + let fast = builder.metadata.last_updated_ms + skew_ms; + let snapshot = Snapshot::builder() + .with_snapshot_id(2) + .with_timestamp_ms(fast) + .with_sequence_number(1) + .with_schema_id(0) + .with_manifest_list("/snap-2.avro") + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .build(); + let on_branch = builder + .set_branch_snapshot(snapshot, "b") + .unwrap() + .build() + .unwrap() + .metadata; + assert_eq!(on_branch.last_updated_ms, fast); + + let result = on_branch + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata2.json".to_string(), + )) + .set_properties(HashMap::from([("k".to_string(), "v".to_string())])) + .unwrap() + .build(); + (fast, result) + } + + #[test] + fn test_metadata_change_after_a_fast_branch_snapshot_is_refused_in_build() { + // The change is two hours before the entry for the previous file. No reader could load + // the result, so the build fails. + let (fast, result) = build_after_fast_branch_snapshot(2 * 60 * 60_000); + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .ends_with(&format!(": before last metadata log entry at {fast}")), + "{}", + err.message() + ); + } + + #[test] + fn test_metadata_change_after_a_slightly_fast_branch_snapshot_builds() { + // 30 seconds of clock skew is within the one-minute tolerance. + let (fast, result) = build_after_fast_branch_snapshot(30_000); + assert_eq!( + result.unwrap().metadata.metadata_log.last(), + Some(&MetadataLog { + metadata_file: "s3://bucket/test/location/metadata/metadata2.json".to_string(), + timestamp_ms: fast, + }) + ); + } + + #[test] + fn test_expire_metadata_log() { + let builder = builder_without_changes(FormatVersion::V2); + let metadata = builder + .set_properties(HashMap::from_iter(vec![( + TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(), + "2".to_string(), + )])) + .unwrap() + .build() + .unwrap(); + assert_eq!(metadata.metadata.metadata_log.len(), 1); + assert_eq!(metadata.expired_metadata_logs.len(), 0); + + let metadata = metadata + .metadata + .into_builder(Some("path2".to_string())) + .set_properties(HashMap::from_iter(vec![( + "change_nr".to_string(), + "1".to_string(), + )])) + .unwrap() + .build() + .unwrap(); + + assert_eq!(metadata.metadata.metadata_log.len(), 2); + assert_eq!(metadata.expired_metadata_logs.len(), 0); + + let metadata = metadata + .metadata + .into_builder(Some("path2".to_string())) + .set_properties(HashMap::from_iter(vec![( + "change_nr".to_string(), + "2".to_string(), + )])) + .unwrap() + .build() + .unwrap(); + assert_eq!(metadata.metadata.metadata_log.len(), 2); + assert_eq!(metadata.expired_metadata_logs.len(), 1); + } + + #[test] fn test_v2_sequence_number_cannot_decrease() { let builder = builder_without_changes(FormatVersion::V2); @@ -2533,9 +3303,10 @@ mod tests { let table = Table::builder() .metadata(resp) - .metadata_location("s3://bucket/test/location/metadata/v1.json".to_string()) + .metadata_location("s3://bucket/test/location/metadata/v1.json") .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .file_io(FileIOBuilder::new("memory").build().unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) .build() .unwrap(); @@ -2564,9 +3335,10 @@ mod tests { let table = Table::builder() .metadata(resp) - .metadata_location("s3://bucket/test/location/metadata/v1.json".to_string()) + .metadata_location("s3://bucket/test/location/metadata/v1.json") .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .file_io(FileIOBuilder::new("memory").build().unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) .build() .unwrap(); @@ -2606,7 +3378,14 @@ mod tests { let partition_spec_with_bucket = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "bucket_data", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("bucket_data") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2650,8 +3429,8 @@ mod tests { assert!(result.is_err()); let error = result.unwrap_err(); let error_message = error.message(); - assert!(error_message.contains("Cannot add schema field 'bucket_data' because it conflicts with existing partition field name")); - assert!(error_message.contains("Schema evolution cannot introduce field names that match existing partition field names")); + assert!(error_message.contains("Cannot set current schema")); + assert!(error_message.contains("sourced from field id 1")); } #[test] @@ -2665,7 +3444,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "partition_col", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("partition_col") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2714,7 +3500,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "data_bucket", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("data_bucket") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2737,7 +3530,14 @@ mod tests { let conflicting_partition_spec = UnboundPartitionSpec::builder() .with_spec_id(1) - .add_partition_field(1, "existing_field", Transform::Bucket(8)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("existing_field") + .transform(Transform::Bucket(8)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2750,7 +3550,7 @@ mod tests { assert!(error_message.contains( "Cannot create partition with name 'existing_field' that conflicts with schema field" )); - assert!(error_message.contains("and is not an identity transform")); + assert!(error_message.contains("and is not an identity or void transform")); } #[test] @@ -2767,7 +3567,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "bucket_data", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("bucket_data") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2842,11 +3649,12 @@ mod tests { let builder2 = metadata.into_builder(Some("test_location".to_string())); let result2 = builder2.add_current_schema(conflicting_schema); - // This should fail because "bucket_data" conflicts with partition field name - // and doesn't exist in any historical schema + // This should fail because "bucket_data" takes the name of a partition field whose + // source column `data` is still in the schema assert!(result2.is_err()); let error = result2.unwrap_err(); - assert!(error.message().contains("Cannot add schema field 'bucket_data' because it conflicts with existing partition field name")); + assert!(error.message().contains("Cannot set current schema")); + assert!(error.message().contains("sourced from field id 1")); } #[test] @@ -2863,7 +3671,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(2, "partition_data", Transform::Identity) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("partition_data") + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2900,6 +3715,230 @@ mod tests { assert!(result.is_ok()); } + /// A v2 table of `data: string` partitioned by `bucket(data, 16)` as `bucket_data`. + fn bucket_partitioned_metadata() -> TableMetadata { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + let spec = UnboundPartitionSpec::builder() + .with_spec_id(0) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("bucket_data") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) + .unwrap() + .build(); + + TableMetadataBuilder::new( + schema, + spec, + SortOrder::unsorted_order(), + TEST_LOCATION.to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata + } + + /// Replaces the default spec with an unpartitioned one. Spec 0 is retained but superseded. + fn unpartition(metadata: TableMetadata, file: &str) -> TableMetadata { + let metadata = metadata + .into_builder(Some(file.to_string())) + .add_default_partition_spec(UnboundPartitionSpec::builder().build()) + .unwrap() + .build() + .unwrap() + .metadata; + assert_ne!(metadata.default_partition_spec().spec_id(), 0); + assert!(metadata.partition_specs.contains_key(&0)); + metadata + } + + #[test] + fn test_schema_evolution_allows_a_superseded_partition_field_name_once_its_source_is_dropped() { + // `data`, the source of spec 0's `bucket_data`, is gone, so spec 0 can only be historical + // and its names are free. Iceberg Java skips its name check for exactly this case. + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let metadata = unpartition( + bucket_partitioned_metadata(), + "s3://bucket/t/metadata/m1.json", + ) + .into_builder(Some("s3://bucket/t/metadata/m2.json".to_string())) + .add_current_schema(schema.clone()) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(metadata.current_schema().as_struct(), schema.as_struct()); + assert!(metadata.current_schema().field_by_name("data").is_none()); + // Spec 0 is kept so the snapshots written under it stay interpretable. + assert_eq!( + metadata.partition_spec_by_id(0).unwrap().fields()[0].name, + "bucket_data" + ); + } + + #[test] + fn test_schema_evolution_rejects_a_partition_field_name_while_its_source_remains() { + // `data` is still here, so `bucket_data` would be ambiguous. Iceberg Java rejects this + // whether or not spec 0 is still the default, because it rebuilds every spec. + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let error = bucket_partitioned_metadata() + .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string())) + .add_current_schema(schema.clone()) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!( + error.message().contains("sourced from field id 1"), + "{error}" + ); + + let error = unpartition( + bucket_partitioned_metadata(), + "s3://bucket/t/metadata/m1.json", + ) + .into_builder(Some("s3://bucket/t/metadata/m2.json".to_string())) + .add_current_schema(schema) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!( + error.message().contains("sourced from field id 1"), + "{error}" + ); + } + + #[test] + fn test_schema_evolution_partition_name_check_is_order_independent() { + // Dropping the source column and dropping the partitioning in one set of changes must + // behave the same either way round. + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let unpartitioned = UnboundPartitionSpec::builder().build(); + + let schema_first = bucket_partitioned_metadata() + .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string())) + .add_current_schema(schema.clone()) + .and_then(|b| b.add_default_partition_spec(unpartitioned.clone())) + .and_then(|b| b.build()); + let spec_first = bucket_partitioned_metadata() + .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string())) + .add_default_partition_spec(unpartitioned) + .and_then(|b| b.add_current_schema(schema)) + .and_then(|b| b.build()); + + assert!(schema_first.is_ok(), "{:?}", schema_first.err()); + assert!(spec_first.is_ok(), "{:?}", spec_first.err()); + } + + #[test] + fn test_set_current_schema_rejects_a_partition_field_name_from_a_spec_added_after_the_schema() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + let conflicting_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let spec = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("bucket_data") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) + .unwrap() + .build(); + + let error = TableMetadataBuilder::new( + schema, + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + TEST_LOCATION.to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .add_schema(conflicting_schema) + .and_then(|b| b.add_partition_spec(spec)) + .and_then(|b| b.set_current_schema(TableMetadataBuilder::LAST_ADDED)) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!( + error.message().contains("sourced from field id 1"), + "{error}" + ); + } + + #[test] + fn test_schema_with_a_conflicting_partition_field_name_can_be_added_but_not_made_current() { + let conflicting_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + + let metadata = bucket_partitioned_metadata() + .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string())) + .add_schema(conflicting_schema.clone()) + .unwrap() + .build() + .unwrap() + .metadata; + assert_eq!(metadata.current_schema_id(), 0); + assert_eq!( + metadata.schema_by_id(1).unwrap().as_struct(), + conflicting_schema.as_struct() + ); + + let error = metadata + .into_builder(Some("s3://bucket/t/metadata/m2.json".to_string())) + .set_current_schema(1) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!( + error.message().contains("sourced from field id 1"), + "{error}" + ); + } + #[test] fn test_schema_evolution_prevents_new_field_conflicting_with_partition_field() { // Create initial schema WITHOUT the conflicting field @@ -2912,7 +3951,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "bucket_data", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("bucket_data") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2942,11 +3988,12 @@ mod tests { let builder = metadata.into_builder(Some("test_location".to_string())); let result = builder.add_current_schema(conflicting_schema); - // This should fail because "bucket_data" conflicts with partition field name - // and doesn't exist in any historical schema + // This should fail because "bucket_data" takes the name of a partition field whose + // source column `data` is still in the schema assert!(result.is_err()); let error = result.unwrap_err(); - assert!(error.message().contains("Cannot add schema field 'bucket_data' because it conflicts with existing partition field name")); + assert!(error.message().contains("Cannot set current schema")); + assert!(error.message().contains("sourced from field id 1")); } #[test] @@ -2962,7 +4009,14 @@ mod tests { let partition_spec = UnboundPartitionSpec::builder() .with_spec_id(0) - .add_partition_field(1, "data_bucket", Transform::Bucket(16)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("data_bucket") + .transform(Transform::Bucket(16)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2986,7 +4040,14 @@ mod tests { // Try to add a partition spec with a field name that does NOT conflict with existing schema fields let non_conflicting_partition_spec = UnboundPartitionSpec::builder() .with_spec_id(1) - .add_partition_field(2, "new_partition_field", Transform::Bucket(8)) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("new_partition_field") + .transform(Transform::Bucket(8)) + .build() + .unwrap(), + ) .unwrap() .build(); @@ -2994,4 +4055,1394 @@ mod tests { assert!(result.is_ok()); } + + #[test] + fn test_row_lineage_addition() { + let new_rows = 30; + let base = builder_without_changes(FormatVersion::V3) + .build() + .unwrap() + .metadata; + let add_rows = Snapshot::builder() + .with_snapshot_id(0) + .with_timestamp_ms(base.last_updated_ms + 1) + .with_sequence_number(0) + .with_schema_id(0) + .with_manifest_list("foo") + .with_parent_snapshot_id(None) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_row_range(base.next_row_id(), new_rows) + .build(); + + let first_addition = base + .into_builder(None) + .add_snapshot(add_rows.clone()) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(first_addition.next_row_id(), new_rows); + + let add_more_rows = Snapshot::builder() + .with_snapshot_id(1) + .with_timestamp_ms(first_addition.last_updated_ms + 1) + .with_sequence_number(1) + .with_schema_id(0) + .with_manifest_list("foo") + .with_parent_snapshot_id(Some(0)) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_row_range(first_addition.next_row_id(), new_rows) + .build(); + + let second_addition = first_addition + .into_builder(None) + .add_snapshot(add_more_rows) + .unwrap() + .build() + .unwrap() + .metadata; + assert_eq!(second_addition.next_row_id(), new_rows * 2); + } + + #[test] + fn test_row_lineage_invalid_snapshot() { + let new_rows = 30; + let base = builder_without_changes(FormatVersion::V3) + .build() + .unwrap() + .metadata; + + // add rows to check TableMetadata validation; Snapshot rejects negative next-row-id + let add_rows = Snapshot::builder() + .with_snapshot_id(0) + .with_timestamp_ms(base.last_updated_ms + 1) + .with_sequence_number(0) + .with_schema_id(0) + .with_manifest_list("foo") + .with_parent_snapshot_id(None) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_row_range(base.next_row_id(), new_rows) + .build(); + + let added = base + .into_builder(None) + .add_snapshot(add_rows) + .unwrap() + .build() + .unwrap() + .metadata; + + let invalid_new_rows = Snapshot::builder() + .with_snapshot_id(1) + .with_timestamp_ms(added.last_updated_ms + 1) + .with_sequence_number(1) + .with_schema_id(0) + .with_manifest_list("foo") + .with_parent_snapshot_id(Some(0)) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + // first_row_id is behind table next_row_id + .with_row_range(added.next_row_id() - 1, 10) + .build(); + + let err = added + .into_builder(None) + .add_snapshot(invalid_new_rows) + .unwrap_err(); + assert!( + err.to_string().contains( + "Cannot add a snapshot, first-row-id is behind table next-row-id: 29 < 30" + ) + ); + } + + #[test] + fn test_row_lineage_append_branch() { + // Appends to a branch should still change last-row-id even if not on main, these changes + // should also affect commits to main + + let branch = "some_branch"; + + // Start with V3 metadata to support row lineage + let base = builder_without_changes(FormatVersion::V3) + .build() + .unwrap() + .metadata; + + // Initial next_row_id should be 0 + assert_eq!(base.next_row_id(), 0); + + // Write to Branch - append 30 rows + let branch_snapshot_1 = Snapshot::builder() + .with_snapshot_id(1) + .with_timestamp_ms(base.last_updated_ms + 1) + .with_sequence_number(0) + .with_schema_id(0) + .with_manifest_list("foo") + .with_parent_snapshot_id(None) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_row_range(base.next_row_id(), 30) + .build(); + + let table_after_branch_1 = base + .into_builder(None) + .set_branch_snapshot(branch_snapshot_1.clone(), branch) + .unwrap() + .build() + .unwrap() + .metadata; + + // Current snapshot should be null (no main branch snapshot yet) + assert!(table_after_branch_1.current_snapshot().is_none()); + + // Branch snapshot should have first_row_id = 0 + let branch_ref = table_after_branch_1.refs.get(branch).unwrap(); + let branch_snap_1 = table_after_branch_1 + .snapshots + .get(&branch_ref.snapshot_id) + .unwrap(); + assert_eq!(branch_snap_1.first_row_id(), Some(0)); + + // Next row id should be 30 + assert_eq!(table_after_branch_1.next_row_id(), 30); + + // Write to Main - append 28 rows + let main_snapshot = Snapshot::builder() + .with_snapshot_id(2) + .with_timestamp_ms(table_after_branch_1.last_updated_ms + 1) + .with_sequence_number(1) + .with_schema_id(0) + .with_manifest_list("bar") + .with_parent_snapshot_id(None) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_row_range(table_after_branch_1.next_row_id(), 28) + .build(); + + let table_after_main = table_after_branch_1 + .into_builder(None) + .add_snapshot(main_snapshot.clone()) + .unwrap() + .set_ref(MAIN_BRANCH, SnapshotReference { + snapshot_id: main_snapshot.snapshot_id(), + retention: SnapshotRetention::Branch { + min_snapshots_to_keep: None, + max_snapshot_age_ms: None, + max_ref_age_ms: None, + }, + }) + .unwrap() + .build() + .unwrap() + .metadata; + + // Main snapshot should have first_row_id = 30 + let current_snapshot = table_after_main.current_snapshot().unwrap(); + assert_eq!(current_snapshot.first_row_id(), Some(30)); + + // Next row id should be 58 (30 + 28) + assert_eq!(table_after_main.next_row_id(), 58); + + // Write again to branch - append 21 rows + let branch_snapshot_2 = Snapshot::builder() + .with_snapshot_id(3) + .with_timestamp_ms(table_after_main.last_updated_ms + 1) + .with_sequence_number(2) + .with_schema_id(0) + .with_manifest_list("baz") + .with_parent_snapshot_id(Some(branch_snapshot_1.snapshot_id())) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .with_row_range(table_after_main.next_row_id(), 21) + .build(); + + let table_after_branch_2 = table_after_main + .into_builder(None) + .set_branch_snapshot(branch_snapshot_2.clone(), branch) + .unwrap() + .build() + .unwrap() + .metadata; + + // Branch snapshot should have first_row_id = 58 (30 + 28) + let branch_ref_2 = table_after_branch_2.refs.get(branch).unwrap(); + let branch_snap_2 = table_after_branch_2 + .snapshots + .get(&branch_ref_2.snapshot_id) + .unwrap(); + assert_eq!(branch_snap_2.first_row_id(), Some(58)); + + // Next row id should be 79 (30 + 28 + 21) + assert_eq!(table_after_branch_2.next_row_id(), 79); + } + + #[test] + fn test_encryption_keys() { + let builder = builder_without_changes(FormatVersion::V3); + + // Create test encryption keys + let encryption_key_1 = EncryptedKey::builder() + .key_id("key-1") + .encrypted_key_metadata(vec![1, 2, 3, 4]) + .encrypted_by_id("encryption-service-1") + .properties(HashMap::from_iter(vec![( + "algorithm".to_string(), + "AES-256".to_string(), + )])) + .build(); + + let encryption_key_2 = EncryptedKey::builder() + .key_id("key-2") + .encrypted_key_metadata(vec![5, 6, 7, 8]) + .encrypted_by_id("encryption-service-2") + .properties(HashMap::new()) + .build(); + + // Add first encryption key + let build_result = builder + .add_encryption_key(encryption_key_1.clone()) + .build() + .unwrap(); + + assert_eq!(build_result.changes.len(), 1); + assert_eq!(build_result.metadata.encryption_keys.len(), 1); + assert_eq!( + build_result.metadata.encryption_key("key-1"), + Some(&encryption_key_1) + ); + assert_eq!(build_result.changes[0], TableUpdate::AddEncryptionKey { + encryption_key: encryption_key_1.clone() + }); + + // Add second encryption key + let build_result = build_result + .metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata1.json".to_string(), + )) + .add_encryption_key(encryption_key_2.clone()) + .build() + .unwrap(); + + assert_eq!(build_result.changes.len(), 1); + assert_eq!(build_result.metadata.encryption_keys.len(), 2); + assert_eq!( + build_result.metadata.encryption_key("key-1"), + Some(&encryption_key_1) + ); + assert_eq!( + build_result.metadata.encryption_key("key-2"), + Some(&encryption_key_2) + ); + assert_eq!(build_result.changes[0], TableUpdate::AddEncryptionKey { + encryption_key: encryption_key_2.clone() + }); + + // Try to add duplicate key - should not create a change + let build_result = build_result + .metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata2.json".to_string(), + )) + .add_encryption_key(encryption_key_1.clone()) + .build() + .unwrap(); + + assert_eq!(build_result.changes.len(), 0); + assert_eq!(build_result.metadata.encryption_keys.len(), 2); + + // Remove first encryption key + let build_result = build_result + .metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata3.json".to_string(), + )) + .remove_encryption_key("key-1") + .build() + .unwrap(); + + assert_eq!(build_result.changes.len(), 1); + assert_eq!(build_result.metadata.encryption_keys.len(), 1); + assert_eq!(build_result.metadata.encryption_key("key-1"), None); + assert_eq!( + build_result.metadata.encryption_key("key-2"), + Some(&encryption_key_2) + ); + assert_eq!(build_result.changes[0], TableUpdate::RemoveEncryptionKey { + key_id: "key-1".to_string() + }); + + // Try to remove non-existent key - should not create a change + let build_result = build_result + .metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata4.json".to_string(), + )) + .remove_encryption_key("non-existent-key") + .build() + .unwrap(); + + assert_eq!(build_result.changes.len(), 0); + assert_eq!(build_result.metadata.encryption_keys.len(), 1); + + // Test encryption_keys_iter() + let keys = build_result + .metadata + .encryption_keys_iter() + .collect::>(); + assert_eq!(keys.len(), 1); + assert_eq!(keys[0], &encryption_key_2); + + // Remove last encryption key + let build_result = build_result + .metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata5.json".to_string(), + )) + .remove_encryption_key("key-2") + .build() + .unwrap(); + + assert_eq!(build_result.changes.len(), 1); + assert_eq!(build_result.metadata.encryption_keys.len(), 0); + assert_eq!(build_result.metadata.encryption_key("key-2"), None); + assert_eq!(build_result.changes[0], TableUpdate::RemoveEncryptionKey { + key_id: "key-2".to_string() + }); + + // Verify empty encryption_keys_iter() + let keys = build_result.metadata.encryption_keys_iter(); + assert_eq!(keys.len(), 0); + } + + #[test] + fn test_partition_field_id_reuse_across_specs() { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "timestamp", Type::Primitive(PrimitiveType::Timestamp)) + .into(), + ]) + .build() + .unwrap(); + + // Create initial table with spec 0: identity(id) -> field_id = 1000 + let initial_spec = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id") + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build(); + + let mut metadata = TableMetadataBuilder::new( + schema, + initial_spec, + SortOrder::unsorted_order(), + "s3://bucket/table".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + // Add spec 1: bucket(data) -> field_id = 1001 + let spec1 = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("data_bucket") + .transform(Transform::Bucket(10)) + .build() + .unwrap(), + ) + .unwrap() + .build(); + let builder = metadata.into_builder(Some("s3://bucket/table/metadata/v1.json".to_string())); + let result = builder.add_partition_spec(spec1).unwrap().build().unwrap(); + metadata = result.metadata; + + // Add spec 2: identity(id) + bucket(data) + year(timestamp) + // Should reuse field_id 1000 for identity(id) and 1001 for bucket(data) + let spec2 = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("id") + .transform(Transform::Identity) + .build() + .unwrap(), + ) // Should reuse 1000 + .unwrap() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("data_bucket") + .transform(Transform::Bucket(10)) + .build() + .unwrap(), + ) // Should reuse 1001 + .unwrap() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![3]) + .name("year") + .transform(Transform::Year) + .build() + .unwrap(), + ) // Should get new 1002 + .unwrap() + .build(); + let builder = metadata.into_builder(Some("s3://bucket/table/metadata/v2.json".to_string())); + let result = builder.add_partition_spec(spec2).unwrap().build().unwrap(); + + // Verify field ID reuse: spec 2 should reuse IDs from specs 0 and 1, assign new ID for new field + let spec2 = result.metadata.partition_spec_by_id(2).unwrap(); + let field_ids: Vec = spec2.fields().iter().map(|f| f.field_id).collect(); + assert_eq!(field_ids, vec![1000, 1001, 1002]); // Reused 1000, 1001; new 1002 + } + + fn unbound_field(source_id: i32, name: &str, transform: Transform) -> UnboundPartitionField { + UnboundPartitionField::builder() + .source_ids(vec![source_id]) + .name(name) + .transform(transform) + .build() + .unwrap() + } + + fn replacement_schema() -> Schema { + // `z` and `x` already exist in the table, `w` is new, `y` is dropped. + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "z", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(3, "w", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap() + } + + fn partition_replacement_builder(format_version: FormatVersion) -> TableMetadataBuilder { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + let spec = UnboundPartitionSpec::builder() + .add_partition_field(unbound_field(1, "x", Transform::Identity)) + .unwrap() + .add_partition_field(unbound_field(2, "y", Transform::Identity)) + .unwrap() + .build(); + + TableMetadataBuilder::new( + schema, + spec, + SortOrder::unsorted_order(), + TEST_LOCATION.to_string(), + format_version, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata1.json".to_string(), + )) + } + + fn populated_builder() -> TableMetadataBuilder { + let builder = builder_without_changes(FormatVersion::V2); + let snapshot = Snapshot::builder() + .with_snapshot_id(1) + .with_timestamp_ms(builder.metadata.last_updated_ms + 1) + .with_sequence_number(0) + .with_schema_id(0) + .with_manifest_list("/snap-1.avro") + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .build(); + + builder + .add_snapshot(snapshot) + .unwrap() + .set_ref(MAIN_BRANCH, SnapshotReference { + snapshot_id: 1, + retention: SnapshotRetention::branch(None, None, None), + }) + .unwrap() + .build() + .unwrap() + .metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata2.json".to_string(), + )) + } + + fn replaced_schema() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(4, "w", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap() + } + + #[test] + fn test_build_replacement_keeps_existing_field_ids_and_assigns_new_ones_above_last_column_id() { + let metadata = builder_without_changes(FormatVersion::V2) + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(metadata.current_schema().as_ref(), &replaced_schema()); + assert_eq!(metadata.last_column_id, 4); + } + + #[test] + fn test_build_replacement_does_not_reuse_the_id_of_a_dropped_column() { + let without_y = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + let replacement = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "w", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + let metadata = partition_replacement_builder(FormatVersion::V2) + .add_current_schema(without_y) + .unwrap() + .build_replacement( + replacement, + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + let expected = Schema::builder() + .with_schema_id(2) + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(3, "w", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + assert_eq!(metadata.current_schema().as_ref(), &expected); + assert_eq!(metadata.last_column_id, 3); + } + + #[test] + fn test_build_replacement_resolves_spec_and_sort_order_source_ids_against_the_replacement_schema() + { + let spec = UnboundPartitionSpec::builder() + .add_partition_field(unbound_field(3, "w", Transform::Identity)) + .unwrap() + .build(); + let sort_field = SortField { + source_id: 3, + transform: Transform::Identity, + direction: SortDirection::Ascending, + null_order: NullOrder::First, + }; + let sort_order = SortOrder::builder() + .with_sort_field(sort_field.clone()) + .build(&replacement_schema()) + .unwrap(); + + let metadata = builder_without_changes(FormatVersion::V2) + .build_replacement( + replacement_schema(), + spec, + sort_order, + None, + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + let expected_spec = PartitionSpec::builder(replaced_schema()) + .with_spec_id(1) + .add_unbound_field(unbound_field(4, "w", Transform::Identity).with_field_id(1001)) + .unwrap() + .build() + .unwrap(); + let expected_sort_order = SortOrder::builder() + .with_order_id(2) + .with_sort_field(SortField { + source_id: 4, + ..sort_field + }) + .build(&replaced_schema()) + .unwrap(); + assert_eq!(metadata.default_partition_spec().as_ref(), &expected_spec); + assert_eq!(metadata.default_sort_order().as_ref(), &expected_sort_order); + } + + #[test] + fn test_build_replacement_reassigns_partition_ids_for_v2() { + let updated_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "z", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + let updated_spec = UnboundPartitionSpec::builder() + .add_partition_field(unbound_field(2, "z_bucket", Transform::Bucket(8))) + .unwrap() + .add_partition_field(unbound_field(1, "x", Transform::Identity)) + .unwrap() + .build(); + + let result = partition_replacement_builder(FormatVersion::V2) + .build_replacement( + updated_schema, + updated_spec, + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap(); + let metadata = &result.metadata; + + let expected_spec = PartitionSpec::builder(metadata.current_schema().clone()) + .with_spec_id(1) + .add_unbound_fields([ + unbound_field(3, "z_bucket", Transform::Bucket(8)).with_field_id(1002), + unbound_field(1, "x", Transform::Identity).with_field_id(1000), + ]) + .unwrap() + .build() + .unwrap(); + assert_eq!(metadata.default_partition_spec().as_ref(), &expected_spec); + assert_eq!(metadata.last_partition_id, 1002); + assert!(result.changes.contains(&TableUpdate::AddSpec { + spec: expected_spec.into_unbound() + })); + } + + #[test] + fn test_build_replacement_rejects_a_new_partition_field_when_no_partition_id_is_left() { + let mut builder = partition_replacement_builder(FormatVersion::V2); + builder.metadata.last_partition_id = i32::MAX; + let updated_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "z", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + let updated_spec = UnboundPartitionSpec::builder() + .add_partition_field(unbound_field(2, "z_bucket", Transform::Bucket(8))) + .unwrap() + .build(); + + let err = builder + .build_replacement( + updated_schema, + updated_spec, + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap_err(); + + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::DataInvalid, + "Cannot assign more partition ids. Overflow." + ) + ); + } + + #[test] + fn test_build_replacement_keeps_the_table_and_its_snapshots_but_not_its_current_one() { + let base = populated_builder(); + let previous = base.metadata.clone(); + + let metadata = base + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(metadata.table_uuid, previous.table_uuid); + assert_eq!(metadata.location, previous.location); + assert_eq!(metadata.current_snapshot_id, None); + assert_eq!(metadata.snapshots, previous.snapshots); + assert_eq!(metadata.snapshot_log, previous.snapshot_log); + assert_eq!(metadata.schema_by_id(0), previous.schema_by_id(0)); + } + + #[test] + fn test_build_replacement_then_adding_a_snapshot_makes_it_current_on_main() { + let builder = populated_builder() + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap(); + let snapshot = Snapshot::builder() + .with_snapshot_id(2) + .with_timestamp_ms(builder.metadata.last_updated_ms + 1) + .with_sequence_number(builder.metadata.next_sequence_number()) + .with_schema_id(builder.metadata.current_schema_id) + .with_manifest_list("/snap-2.avro") + .with_summary(Summary { + operation: Operation::Overwrite, + additional_properties: HashMap::new(), + }) + .build(); + let main = SnapshotReference { + snapshot_id: 2, + retention: SnapshotRetention::branch(None, None, None), + }; + + let metadata = builder + .add_snapshot(snapshot.clone()) + .unwrap() + .set_ref(MAIN_BRANCH, main.clone()) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(metadata.current_snapshot(), Some(&Arc::new(snapshot))); + assert_eq!( + metadata.refs, + HashMap::from([(MAIN_BRANCH.to_string(), main)]) + ); + } + + #[test] + fn test_build_replacement_changes_describe_the_replacement() { + let changes = populated_builder() + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .changes; + + assert_eq!(changes, vec![ + TableUpdate::RemoveSnapshotRef { + ref_name: MAIN_BRANCH.to_string() + }, + TableUpdate::AddSchema { + schema: replaced_schema() + }, + TableUpdate::SetCurrentSchema { + schema_id: TableMetadataBuilder::LAST_ADDED + }, + TableUpdate::AddSpec { + spec: UnboundPartitionSpec::builder().with_spec_id(1).build() + }, + TableUpdate::SetDefaultSpec { + spec_id: TableMetadataBuilder::LAST_ADDED + }, + TableUpdate::AddSortOrder { + sort_order: SortOrder::unsorted_order() + }, + TableUpdate::SetDefaultSortOrder { + sort_order_id: TableMetadataBuilder::LAST_ADDED as i64 + }, + ]); + } + + #[test] + fn test_build_replacement_moves_the_table_when_given_a_location() { + let metadata = builder_without_changes(FormatVersion::V2) + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + Some("s3://bucket/somewhere/else".to_string()), + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(metadata.location, "s3://bucket/somewhere/else"); + } + + #[test] + fn test_build_replacement_applies_format_version_and_properties() { + let result = builder_without_changes(FormatVersion::V2) + .set_properties(HashMap::from_iter([( + "existing".to_string(), + "value".to_string(), + )])) + .unwrap() + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + Some(FormatVersion::V3), + HashMap::from_iter([("new".to_string(), "value".to_string())]), + ) + .unwrap() + .build() + .unwrap(); + + assert_eq!(result.metadata.format_version, FormatVersion::V3); + assert_eq!( + result.metadata.properties, + HashMap::from_iter([ + ("existing".to_string(), "value".to_string()), + ("new".to_string(), "value".to_string()), + ]) + ); + assert!(result.changes.contains(&TableUpdate::UpgradeFormatVersion { + format_version: FormatVersion::V3 + })); + assert!(result.changes.contains(&TableUpdate::SetProperties { + updates: HashMap::from_iter([("new".to_string(), "value".to_string())]) + })); + } + + #[test] + fn test_build_replacement_with_everything_unchanged_reuses_the_existing_ids() { + let base = builder_without_changes(FormatVersion::V2) + .build() + .unwrap() + .metadata; + let schema = base.current_schema().as_ref().clone(); + let spec = base + .default_partition_spec() + .as_ref() + .clone() + .into_unbound(); + let sort_order = base.default_sort_order().as_ref().clone(); + + let result = base + .clone() + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata2.json".to_string(), + )) + .build_replacement( + schema.clone(), + spec.clone(), + sort_order.clone(), + None, + Some(FormatVersion::V3), + HashMap::new(), + ) + .unwrap() + .build() + .unwrap(); + + let metadata = &result.metadata; + assert_eq!(metadata.schemas, base.schemas); + assert_eq!(metadata.current_schema_id, base.current_schema_id); + assert_eq!(metadata.last_column_id, base.last_column_id); + assert_eq!(metadata.partition_specs, base.partition_specs); + assert_eq!(metadata.default_spec, base.default_spec); + assert_eq!(metadata.last_partition_id, base.last_partition_id); + assert_eq!(metadata.sort_orders, base.sort_orders); + assert_eq!(metadata.default_sort_order_id, base.default_sort_order_id); + assert_eq!(result.changes, vec![ + TableUpdate::UpgradeFormatVersion { + format_version: FormatVersion::V3 + }, + TableUpdate::AddSchema { schema }, + TableUpdate::AddSpec { spec }, + TableUpdate::AddSortOrder { sort_order }, + ]); + } + + #[test] + fn test_build_replacement_rejects_v1() { + let err = builder_without_changes(FormatVersion::V1) + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap_err(); + + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::FeatureUnsupported, + "Table replacement is not supported for format version 1" + ) + ); + } + + #[test] + fn test_build_replacement_rejects_v1_with_format_upgrade() { + let err = builder_without_changes(FormatVersion::V1) + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + Some(FormatVersion::V2), + HashMap::new(), + ) + .unwrap_err(); + + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::FeatureUnsupported, + "Table replacement is not supported for format version 1" + ) + ); + } + + #[test] + fn test_build_replacement_rejects_reserved_properties() { + let err = builder_without_changes(FormatVersion::V2) + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::from_iter([( + TableProperties::PROPERTY_UUID.to_string(), + "ignored".to_string(), + )]), + ) + .unwrap_err(); + + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::DataInvalid, + "Table properties should not contain reserved properties, but got: [uuid]" + ) + ); + } + + #[test] + fn test_build_replacement_rejects_a_spec_on_a_missing_column() { + let err = builder_without_changes(FormatVersion::V2) + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder() + .add_partition_field(unbound_field(99, "nope", Transform::Identity)) + .unwrap() + .build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap_err(); + + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::DataInvalid, + "Cannot find source column with id 99 for partition column nope in schema." + ) + ); + } + + #[test] + fn test_build_replacement_rejects_a_format_version_downgrade() { + let err = builder_without_changes(FormatVersion::V3) + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + Some(FormatVersion::V2), + HashMap::new(), + ) + .unwrap_err(); + + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::DataInvalid, + "Cannot downgrade FormatVersion from v3 to v2" + ) + ); + } + + #[test] + fn test_build_replacement_rejects_a_sort_order_on_a_missing_column() { + let sort_order = SortOrder::builder() + .with_sort_field(SortField { + source_id: 99, + transform: Transform::Identity, + direction: SortDirection::Ascending, + null_order: NullOrder::First, + }) + .build_unbound() + .unwrap(); + + let err = builder_without_changes(FormatVersion::V2) + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + sort_order, + None, + None, + HashMap::new(), + ) + .unwrap_err(); + + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::DataInvalid, + "Cannot find source column with id 99 for sort column in schema." + ) + ); + } + + #[test] + fn test_build_replacement_rejects_a_column_named_after_a_partition_field() { + let builder = TableMetadataBuilder::new( + schema(), + UnboundPartitionSpec::builder() + .add_partition_field(unbound_field(1, "x_bucket", Transform::Bucket(8))) + .unwrap() + .build(), + SortOrder::unsorted_order(), + TEST_LOCATION.to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/metadata1.json".to_string(), + )); + let replacement = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "x_bucket", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let err = builder + .build_replacement( + replacement, + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap_err(); + + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::DataInvalid, + "Cannot set current schema 1: field 'x_bucket' with id 4 conflicts with a \ + partition field of that name in spec 0 sourced from field id 1" + ) + ); + } + + #[test] + fn test_build_replacement_keeps_refs_other_than_main() { + let audit = SnapshotReference { + snapshot_id: 1, + retention: SnapshotRetention::branch(None, None, None), + }; + let release = SnapshotReference { + snapshot_id: 1, + retention: SnapshotRetention::Tag { + max_ref_age_ms: None, + }, + }; + let builder = populated_builder() + .set_ref("audit", audit.clone()) + .unwrap() + .set_ref("release", release.clone()) + .unwrap(); + + let metadata = builder + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!( + metadata.refs, + HashMap::from_iter([ + ("audit".to_string(), audit), + ("release".to_string(), release), + ]) + ); + assert_eq!(metadata.current_snapshot_id, None); + } + + #[test] + fn test_build_replacement_keeps_the_id_of_a_column_whose_type_changes() { + let replacement = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + let metadata = builder_without_changes(FormatVersion::V2) + .build_replacement( + replacement, + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + let x = metadata.current_schema().field_by_name("x").unwrap(); + assert_eq!(x.id, 1); + assert_eq!(*x.field_type, Type::Primitive(PrimitiveType::String)); + } + + #[test] + fn test_build_replacement_keeps_the_existing_compression_codec() { + let codec = HashMap::from_iter([( + TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), + "gzip".to_string(), + )]); + + let metadata = builder_without_changes(FormatVersion::V2) + .set_properties(codec.clone()) + .unwrap() + .build_replacement( + replacement_schema(), + UnboundPartitionSpec::builder().build(), + SortOrder::unsorted_order(), + None, + None, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + + assert_eq!(metadata.properties, codec); + } + + #[test] + fn test_highest_equivalent_field_ids_is_independent_of_field_order() { + let field = |source_id, field_id, transform| PartitionField { + source_id, + field_id, + name: format!("f{field_id}"), + transform, + }; + let fields = [ + field(1, 1000, Transform::Identity), + field(2, 1003, Transform::Identity), + field(1, 1006, Transform::Identity), + field(1, 1005, Transform::Bucket(16)), + field(2, 1001, Transform::Identity), + field(1, 1002, Transform::Identity), + field(1, 1004, Transform::Bucket(16)), + ]; + let expected = HashMap::from([ + ((vec![1], Transform::Identity), 1006), + ((vec![2], Transform::Identity), 1003), + ((vec![1], Transform::Bucket(16)), 1005), + ]); + + assert_eq!( + TableMetadataBuilder::highest_equivalent_field_ids(fields.iter()), + expected + ); + assert_eq!( + TableMetadataBuilder::highest_equivalent_field_ids(fields.iter().rev()), + expected + ); + } + + #[test] + fn test_partition_field_id_reuse_picks_the_highest_of_several_candidates() { + // These specs give identity(x) four different field ids, which a writer that does not + // reuse ids can leave behind. `partition_specs` is a HashMap, so the id reused for a new + // spec must not depend on which of them is visited first. + let metadata: TableMetadata = serde_json::from_value(serde_json::json!({ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590i64, + "last-column-id": 2, + "current-schema-id": 0, + "schemas": [{ + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"}, + {"id": 2, "name": "y", "required": true, "type": "long"}, + ], + }], + "default-spec-id": 3, + "last-partition-id": 1006, + "partition-specs": [ + {"spec-id": 0, "fields": [ + {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}, + ]}, + {"spec-id": 1, "fields": [ + {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1002}, + ]}, + {"spec-id": 2, "fields": [ + {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1004}, + ]}, + {"spec-id": 3, "fields": [ + {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1006}, + ]}, + ], + "default-sort-order-id": 0, + "sort-orders": [{"order-id": 0, "fields": []}], + })) + .unwrap(); + + // Two fields, so this is not compatible with any existing spec and is added as a new + // one rather than resolving to an existing spec id. + let new_spec = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("x") + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("y") + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build(); + + let result = metadata + .into_builder(Some( + "s3://bucket/test/location/metadata/v1.json".to_string(), + )) + .add_partition_spec(new_spec) + .unwrap() + .build() + .unwrap(); + + let spec = result.metadata.partition_spec_by_id(4).unwrap(); + let fields: Vec<(i32, i32)> = spec + .fields() + .iter() + .map(|f| (f.source_id, f.field_id)) + .collect(); + assert_eq!(fields, vec![(1, 1006), (2, 1007)]); + assert_eq!(result.metadata.last_partition_id, 1007); + } } diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs new file mode 100644 index 0000000000..392c5cf0af --- /dev/null +++ b/crates/iceberg/src/spec/table_properties.rs @@ -0,0 +1,1236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::properties_view; + +use crate::compression::CompressionCodec; +use crate::encryption::AesKeySize; +use crate::error::{Result, invalid_data}; +use crate::spec::NameMapping; +use crate::util::location::strip_trailing_slash; + +fn parse_location_property(path: &str) -> Result { + if path.is_empty() { + return Err(invalid_data!("path must not be empty")); + } + + Ok(strip_trailing_slash(path).to_string()) +} + +fn parse_metadata_compression(value: &str) -> Result { + // Handle empty string as None + if value.is_empty() { + return Ok(CompressionCodec::None); + } + + // Lowercase the value for case-insensitive parsing + let lowercase_value = value.to_lowercase(); + + // Use serde to parse the codec (which has rename_all = "lowercase") + let codec: CompressionCodec = + serde_json::from_value(serde_json::Value::String(lowercase_value)).map_err(|_| { + invalid_data!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.", + CompressionCodec::None.name(), + CompressionCodec::gzip_default().name() + ) + })?; + + // Validate that only None and Gzip are used for metadata + match codec { + CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec), + _ => Err(invalid_data!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", + CompressionCodec::None.name(), + CompressionCodec::gzip_default().name() + )), + } +} + +/// Parse the Parquet data-file compression codec (`write.parquet.compression-codec`) +/// and fold in the compression level (`write.parquet.compression-level`) for the +/// codecs that accept one (`zstd`, `gzip`, `brotli`). +fn parse_parquet_compression( + properties: &HashMap, + codec_key: &str, + additional_keys: &[&str], + default: CompressionCodec, +) -> Result { + let level_key = additional_keys[0]; + let codec = properties + .get(codec_key) + .map(|value| { + serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| { + invalid_data!( + "Invalid Parquet compression codec: {value}. Supported codecs: \ + uncompressed, snappy, gzip, lzo, brotli, lz4, lz4_raw, zstd" + ) + }) + }) + .transpose()? + .unwrap_or(default); + + let level = properties + .get(level_key) + .map(|value| { + value + .parse::() + .map_err(|error| invalid_data!("Invalid value for {level_key}: {error}")) + }) + .transpose()?; + + Ok(match (codec, level) { + (CompressionCodec::Zstd(_), Some(level)) => CompressionCodec::Zstd(level), + (CompressionCodec::Gzip(_), Some(level)) => CompressionCodec::Gzip(level), + (CompressionCodec::Brotli(_), Some(level)) => CompressionCodec::Brotli(level), + (codec, _) => codec, + }) +} + +properties_view! { +/// Typed view over the properties of a table. +#[derive(Debug)] +pub struct TableProperties { + /// The number of times to retry a commit. + #[property( + key = Self::PROPERTY_COMMIT_NUM_RETRIES, + default = Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, + getter + )] + commit_num_retries: usize, + /// The minimum wait time between retries. + #[property( + key = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, + default = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, + getter + )] + commit_min_retry_wait_ms: u64, + /// The maximum wait time between retries. + #[property( + key = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, + default = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, + getter + )] + commit_max_retry_wait_ms: u64, + /// The total timeout for commit retries. + #[property( + key = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, + default = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, + getter + )] + commit_total_retry_timeout_ms: u64, + /// The default format for files. + #[property( + key = Self::PROPERTY_DEFAULT_FILE_FORMAT, + default = Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT, + getter + )] + write_format_default: String, + /// The target file size for files. + #[property( + key = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, + default = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, + getter + )] + write_target_file_size_bytes: usize, + /// Base directory for metadata files (manifests, manifest lists), with any + /// trailing slash trimmed. `None` if `write.metadata.path` is not set. + #[property( + key = Self::PROPERTY_WRITE_METADATA_PATH, + default = None, + parse_with = parse_location_property, + getter + )] + write_metadata_path: Option, + /// Compression codec for metadata files (JSON) + #[property( + key = Self::PROPERTY_METADATA_COMPRESSION_CODEC, + default = CompressionCodec::None, + parse_with = parse_metadata_compression, + getter + )] + metadata_compression_codec: CompressionCodec, + /// Whether to use `FanoutWriter` for partitioned tables. + #[property( + key = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, + default = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, + getter + )] + write_datafusion_fanout_enabled: bool, + /// Whether garbage collection is enabled on drop. + /// When `false`, data files will not be deleted when a table is dropped. + #[property( + key = Self::PROPERTY_GC_ENABLED, + default = Self::PROPERTY_GC_ENABLED_DEFAULT, + getter + )] + gc_enabled: bool, + /// Default maximum age of a snapshot to keep when expiring snapshots. + #[property( + key = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS, + default = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT, + getter + )] + max_snapshot_age_ms: i64, + /// Default minimum number of snapshots to keep per branch when expiring snapshots. + #[property( + key = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP, + default = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT, + getter + )] + min_snapshots_to_keep: usize, + /// Default maximum age of a snapshot reference to keep when expiring snapshots. + #[property( + key = Self::PROPERTY_MAX_REF_AGE_MS, + default = Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT, + getter + )] + max_ref_age_ms: i64, + /// Whether content-defined chunking is enabled. + /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`. + #[property( + key = Self::PROPERTY_PARQUET_CDC_ENABLED, + default = Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT, + getter + )] + cdc_enabled: bool, + /// Content-defined chunking minimum chunk size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE, + default = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, + getter + )] + cdc_min_chunk_size: usize, + /// Content-defined chunking maximum chunk size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE, + default = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, + getter + )] + cdc_max_chunk_size: usize, + /// Content-defined chunking normalization level (gearhash bit adjustment). + #[property( + key = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL, + default = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT, + getter + )] + cdc_norm_level: i32, + /// Parquet compression codec for data files, with the resolved compression + /// level folded in (from `write.parquet.compression-level`, or the codec's + /// default when unset). + #[property( + key = Self::PROPERTY_PARQUET_COMPRESSION_CODEC, + additional_keys = [Self::PROPERTY_PARQUET_COMPRESSION_LEVEL], + default = CompressionCodec::zstd_default(), + parse_properties_with = parse_parquet_compression, + getter + )] + parquet_compression_codec: CompressionCodec, + /// Approximate maximum Parquet row group size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES, + default = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT, + getter + )] + parquet_row_group_size_bytes: usize, + /// Approximate maximum Parquet data page size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES, + default = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT, + getter + )] + parquet_page_size_bytes: usize, + /// Maximum number of rows per Parquet data page. + #[property( + key = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT, + default = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT, + getter + )] + parquet_page_row_limit: usize, + /// Approximate maximum Parquet dictionary page size in bytes. + #[property( + key = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES, + default = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT, + getter + )] + parquet_dict_size_bytes: usize, + /// The master key id used to encrypt this table's manifest list and data + /// files. `None` if `encryption.key-id` is not set. + #[property( + key = Self::PROPERTY_ENCRYPTION_KEY_ID, + default = None, + getter + )] + encryption_key_id: Option, + /// The encryption data encryption key length in bytes. + #[property( + key = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, + default = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT, + getter + )] + encryption_data_key_length: usize, + /// Base directory for data files, with any trailing slash trimmed. + #[property( + key = Self::PROPERTY_WRITE_DATA_LOCATION, + default = None, + parse_with = parse_location_property, + getter + )] + write_data_location: Option, + /// Deprecated table property for data file write location, with any trailing slash trimmed. + /// + /// Property will be removed at a later date. + /// Superseded by [`TableProperties::write_data_location`]. + #[property( + key = Self::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION, + default = None, + parse_with = parse_location_property, + getter + )] + write_folder_storage_location: Option, + /// Deprecated table property for data file write location for object storage location generator, + /// with any trailing slash trimmed. + /// + /// Property will be removed at a later date. + /// Superseded by [`TableProperties::write_data_location`]. + #[property( + key = Self::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION, + default = None, + parse_with = parse_location_property, + getter + )] + write_object_storage_location: Option, + /// Whether new data files use the object storage location layout, which + /// injects hash entropy into file paths to spread object-store prefixes. + #[property( + key = Self::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED, + default = Self::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT, + getter + )] + write_object_storage_enabled: bool, + /// Whether partition values are included in object storage paths. + #[property( + key = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS, + default = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT, + getter + )] + write_object_storage_partitioned_paths: bool, + /// Whether to delete the oldest tracked metadata files after each commit. + #[property( + key = Self::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED, + default = Self::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT, + getter + )] + metadata_delete_after_commit_enabled: bool, + /// The table's default name mapping, used to assign field ids when reading data files + /// that carry no field id metadata. `None` if `schema.name-mapping.default` is not set. + #[property( + key = Self::PROPERTY_DEFAULT_NAME_MAPPING, + default = None, + getter + )] + default_name_mapping: Option, +} +} + +impl TableProperties<'_> { + /// Reserved table property for table format version. + /// + /// Iceberg will default a new table's format version to the latest stable and recommended + /// version. This reserved property keyword allows users to override the Iceberg format version of + /// the table metadata. + /// + /// If this table property exists when creating a table, the table will use the specified format + /// version. If a table updates this property, it will try to upgrade to the specified format + /// version. + pub const PROPERTY_FORMAT_VERSION: &'static str = "format-version"; + /// Reserved table property for table UUID. + pub const PROPERTY_UUID: &'static str = "uuid"; + /// Reserved table property for the total number of snapshots. + pub const PROPERTY_SNAPSHOT_COUNT: &'static str = "snapshot-count"; + /// Reserved table property for current snapshot summary. + pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &'static str = "current-snapshot-summary"; + /// Reserved table property for current snapshot id. + pub const PROPERTY_CURRENT_SNAPSHOT_ID: &'static str = "current-snapshot-id"; + /// Reserved table property for current snapshot timestamp. + pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &'static str = "current-snapshot-timestamp-ms"; + /// Reserved table property for the JSON representation of current schema. + pub const PROPERTY_CURRENT_SCHEMA: &'static str = "current-schema"; + /// Reserved table property for the JSON representation of current(default) partition spec. + pub const PROPERTY_DEFAULT_PARTITION_SPEC: &'static str = "default-partition-spec"; + /// Reserved table property for the JSON representation of current(default) sort order. + pub const PROPERTY_DEFAULT_SORT_ORDER: &'static str = "default-sort-order"; + + /// Property key for max number of previous versions to keep. + pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &'static str = + "write.metadata.previous-versions-max"; + /// Default value for max number of previous versions to keep. + pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100; + + /// Property key for deleting the previous metadata files that fall out of the metadata log + /// (bounded by [`TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX`]) after a commit. + pub const PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED: &'static str = + "write.metadata.delete-after-commit.enabled"; + /// Default value for [`TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED`]. + pub const PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT: bool = false; + + /// Property key for max number of partitions to keep summary stats for. + pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &'static str = + "write.summary.partition-limit"; + /// Default value for the max number of partitions to keep summary stats for. + pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0; + + /// Reserved Iceberg table properties list. + /// + /// Reserved table properties are only used to control behaviors when creating or updating a + /// table. The value of these properties are not persisted as a part of the table metadata. + pub const RESERVED_PROPERTIES: [&'static str; 9] = [ + Self::PROPERTY_FORMAT_VERSION, + Self::PROPERTY_UUID, + Self::PROPERTY_SNAPSHOT_COUNT, + Self::PROPERTY_CURRENT_SNAPSHOT_ID, + Self::PROPERTY_CURRENT_SNAPSHOT_SUMMARY, + Self::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP, + Self::PROPERTY_CURRENT_SCHEMA, + Self::PROPERTY_DEFAULT_PARTITION_SPEC, + Self::PROPERTY_DEFAULT_SORT_ORDER, + ]; + + /// Property key for number of commit retries. + pub const PROPERTY_COMMIT_NUM_RETRIES: &'static str = "commit.retry.num-retries"; + /// Default value for number of commit retries. + pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4; + + /// Property key for minimum wait time (ms) between retries. + pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &'static str = "commit.retry.min-wait-ms"; + /// Default value for minimum wait time (ms) between retries. + pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100; + + /// Property key for maximum wait time (ms) between retries. + pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &'static str = "commit.retry.max-wait-ms"; + /// Default value for maximum wait time (ms) between retries. + pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute + + /// Property key for total maximum retry time (ms). + pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &'static str = "commit.retry.total-timeout-ms"; + /// Default value for total maximum retry time (ms). + pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes + + /// Default file format for data files + pub const PROPERTY_DEFAULT_FILE_FORMAT: &'static str = "write.format.default"; + /// Default file format for delete files + pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str = "write.delete.format.default"; + /// Default value for data file format + pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str = "parquet"; + + /// Target file size for newly written files. + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &'static str = "write.target-file-size-bytes"; + /// Default target file size + pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB + + /// Base location for metadata files (manifests, manifest lists, table metadata). + /// When unset, metadata files default to the `metadata` directory under the table + /// location. + pub const PROPERTY_WRITE_METADATA_PATH: &'static str = "write.metadata.path"; + + /// Property key for the table's default name mapping, stored as a JSON + /// [`NameMapping`] document. + pub const PROPERTY_DEFAULT_NAME_MAPPING: &'static str = "schema.name-mapping.default"; + + /// Compression codec for metadata files (JSON) + pub const PROPERTY_METADATA_COMPRESSION_CODEC: &'static str = + "write.metadata.compression-codec"; + /// Default metadata compression codec - uncompressed + pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &'static str = "none"; + /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data). + /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient). + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &'static str = + "write.datafusion.fanout.enabled"; + /// Default value for fanout writer enabled + pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; + + /// Property key for enabling garbage collection on drop. + /// When set to `false`, data files will not be deleted when a table is dropped. + /// Defaults to `true`. + pub const PROPERTY_GC_ENABLED: &'static str = "gc.enabled"; + /// Default value for gc.enabled + pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true; + + /// Property key for the default maximum age of a snapshot to keep when expiring snapshots. + pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &'static str = "history.expire.max-snapshot-age-ms"; + /// Default value for history.expire.max-snapshot-age-ms (5 days). + pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000; + /// Property key for the default minimum number of snapshots to keep when expiring snapshots. + pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str = "history.expire.min-snapshots-to-keep"; + /// Default value for history.expire.min-snapshots-to-keep. + pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1; + /// Property key for the default maximum age of a snapshot reference to keep when expiring. + pub const PROPERTY_MAX_REF_AGE_MS: &'static str = "history.expire.max-ref-age-ms"; + /// Default value for history.expire.max-ref-age-ms (effectively never expire refs). + pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX; + + /// Enable content-defined chunking with parquet defaults (or per-property overrides). + pub const PROPERTY_PARQUET_CDC_ENABLED: &'static str = + "write.parquet.content-defined-chunking.enabled"; + /// Default value for content-defined chunking enabled. + pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false; + /// Minimum chunk size in bytes for content-defined chunking. + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &'static str = + "write.parquet.content-defined-chunking.min-chunk-size"; + /// Default matches `parquet::file::properties::DEFAULT_CDC_MIN_CHUNK_SIZE`. + pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024; + /// Maximum chunk size in bytes for content-defined chunking. + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str = + "write.parquet.content-defined-chunking.max-chunk-size"; + /// Default matches `parquet::file::properties::DEFAULT_CDC_MAX_CHUNK_SIZE`. + pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024; + /// Normalization level (gearhash bit adjustment) for content-defined chunking. + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &'static str = + "write.parquet.content-defined-chunking.norm-level"; + /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`. + pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0; + + /// Compression codec for Parquet data files (e.g. `zstd`, `gzip`, `snappy`, + /// `lz4`, `lz4_raw`, `brotli`, `lzo`, `uncompressed`). The codec name is + /// parsed into a [`CompressionCodec`] when properties are parsed; the level's + /// range is validated when the writer is built. + pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &'static str = "write.parquet.compression-codec"; + /// Default Parquet compression codec. + pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &'static str = "zstd"; + /// Compression level for Parquet data files, for codecs that take one + /// (`gzip`, `zstd`, `brotli`). When unset, the codec's default level is used. + pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &'static str = "write.parquet.compression-level"; + + /// Approximate maximum size of a Parquet row group in bytes. + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &'static str = + "write.parquet.row-group-size-bytes"; + /// Default Parquet row group size in bytes. + pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024; + + /// Approximate maximum size of a Parquet data page in bytes. + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &'static str = "write.parquet.page-size-bytes"; + /// Default Parquet page size in bytes. + pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024; + + /// Maximum number of rows per Parquet data page. + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &'static str = "write.parquet.page-row-limit"; + /// Default Parquet page row limit. + pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000; + + /// Approximate maximum size of the Parquet dictionary page in bytes. + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &'static str = "write.parquet.dict-size-bytes"; + /// Default Parquet dictionary page size in bytes. + pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024; + + /// Property key for the master key id used to encrypt the table's manifest + /// list and data files as defined in . + pub const PROPERTY_ENCRYPTION_KEY_ID: &'static str = "encryption.key-id"; + + /// Property key for the encryption data encryption key (DEK) length in bytes. + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &'static str = "encryption.data-key-length"; + /// Default value for the encryption DEK length (16 bytes = AES-128). + pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16; + /// Property key for the base directory for data files + pub const PROPERTY_WRITE_DATA_LOCATION: &'static str = "write.data.path"; + /// Property key for deprecated [`TableProperties::write_folder_storage_location`] + pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str = "write.folder-storage.path"; + /// Property key for deprecated object storage path, kept as a fallback for compatibility. + pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str = "write.object-storage.path"; + /// Property key for enabling the object storage location layout for new data files. + pub const PROPERTY_WRITE_OBJECT_STORAGE_ENABLED: &'static str = "write.object-storage.enabled"; + /// Default value for [`TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED`] + pub const PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT: bool = false; + /// Property key for controlling whether partition values are included in object storage paths. + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str = + "write.object-storage.partitioned-paths"; + /// Default value for [`TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS`] + pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true; + + /// The AES key size to use when generating data encryption keys, derived + /// from `encryption.data-key-length`. + pub fn data_encryption_key_size(&self) -> Result { + AesKeySize::from_key_length(self.encryption_data_key_length()?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ErrorKind; + use crate::compression::CompressionCodec; + + #[test] + fn test_table_properties_default() { + let props = HashMap::new(); + let table_properties = TableProperties::new(&props); + assert_eq!( + table_properties.commit_num_retries().unwrap(), + TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT + ); + assert_eq!( + table_properties.commit_min_retry_wait_ms().unwrap(), + TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT + ); + assert_eq!( + table_properties.commit_max_retry_wait_ms().unwrap(), + TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT + ); + assert_eq!( + table_properties.write_format_default().unwrap(), + TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string() + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT + ); + // Test compression defaults (none means CompressionCodec::None) + assert_eq!( + table_properties.metadata_compression_codec().unwrap(), + CompressionCodec::None + ); + assert_eq!( + table_properties.gc_enabled().unwrap(), + TableProperties::PROPERTY_GC_ENABLED_DEFAULT + ); + assert_eq!( + table_properties.max_snapshot_age_ms().unwrap(), + TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT + ); + assert_eq!( + table_properties.min_snapshots_to_keep().unwrap(), + TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT + ); + assert_eq!( + table_properties.max_ref_age_ms().unwrap(), + TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT + ); + } + + #[test] + fn test_table_properties_history_expire_overrides() { + let props = HashMap::from([ + ( + TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS.to_string(), + "1234".to_string(), + ), + ( + TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP.to_string(), + "7".to_string(), + ), + ( + TableProperties::PROPERTY_MAX_REF_AGE_MS.to_string(), + "5678".to_string(), + ), + ]); + let table_properties = TableProperties::new(&props); + assert_eq!(table_properties.max_snapshot_age_ms().unwrap(), 1234); + assert_eq!(table_properties.min_snapshots_to_keep().unwrap(), 7); + assert_eq!(table_properties.max_ref_age_ms().unwrap(), 5678); + } + + #[test] + fn test_table_properties_location_paths() { + // Test unset. + let raw_properties = HashMap::new(); + let table_properties = TableProperties::new(&raw_properties); + assert_eq!(table_properties.write_metadata_path().unwrap(), None); + assert_eq!(table_properties.write_data_location().unwrap(), None); + assert_eq!( + table_properties.write_folder_storage_location().unwrap(), + None + ); + assert_eq!( + table_properties.write_object_storage_location().unwrap(), + None + ); + + for key in [ + TableProperties::PROPERTY_WRITE_METADATA_PATH, + TableProperties::PROPERTY_WRITE_DATA_LOCATION, + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION, + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION, + ] { + // Test empty paths are invalid and retain the property key as error context. + let raw_properties = HashMap::from([(key.to_string(), String::new())]); + let table_properties = TableProperties::new(&raw_properties); + let error = match key { + TableProperties::PROPERTY_WRITE_METADATA_PATH => { + table_properties.write_metadata_path().unwrap_err() + } + TableProperties::PROPERTY_WRITE_DATA_LOCATION => { + table_properties.write_data_location().unwrap_err() + } + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => table_properties + .write_folder_storage_location() + .unwrap_err(), + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => table_properties + .write_object_storage_location() + .unwrap_err(), + _ => unreachable!(), + }; + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(format!("{error}").contains(key)); + + // Test all supported location properties share trailing-slash normalization. + let raw_properties = HashMap::from([( + key.to_string(), + "s3://other-bucket/custom-path/".to_string(), + )]); + let table_properties = TableProperties::new(&raw_properties); + let parsed = match key { + TableProperties::PROPERTY_WRITE_METADATA_PATH => { + table_properties.write_metadata_path().unwrap() + } + TableProperties::PROPERTY_WRITE_DATA_LOCATION => { + table_properties.write_data_location().unwrap() + } + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => { + table_properties.write_folder_storage_location().unwrap() + } + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => { + table_properties.write_object_storage_location().unwrap() + } + _ => unreachable!(), + }; + assert_eq!(parsed.as_deref(), Some("s3://other-bucket/custom-path")); + } + } + + #[test] + fn test_table_properties_compression() { + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "gzip".to_string(), + )]); + let table_properties = TableProperties::new(&props); + assert_eq!( + table_properties.metadata_compression_codec().unwrap(), + CompressionCodec::gzip_default() + ); + } + + #[test] + fn test_table_properties_compression_none() { + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "none".to_string(), + )]); + let table_properties = TableProperties::new(&props); + assert_eq!( + table_properties.metadata_compression_codec().unwrap(), + CompressionCodec::None + ); + } + + #[test] + fn test_table_properties_compression_case_insensitive() { + // Test uppercase + let props_upper = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "GZIP".to_string(), + )]); + let table_properties = TableProperties::new(&props_upper); + assert_eq!( + table_properties.metadata_compression_codec().unwrap(), + CompressionCodec::gzip_default() + ); + + // Test mixed case + let props_mixed = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "GzIp".to_string(), + )]); + let table_properties = TableProperties::new(&props_mixed); + assert_eq!( + table_properties.metadata_compression_codec().unwrap(), + CompressionCodec::gzip_default() + ); + + // Test "NONE" should also be case-insensitive + let props_none_upper = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "NONE".to_string(), + )]); + let table_properties = TableProperties::new(&props_none_upper); + assert_eq!( + table_properties.metadata_compression_codec().unwrap(), + CompressionCodec::None + ); + } + + #[test] + fn test_table_properties_valid() { + let props = HashMap::from([ + ( + TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), + "10".to_string(), + ), + ( + TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), + "20".to_string(), + ), + ( + TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(), + "avro".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "512".to_string(), + ), + ( + TableProperties::PROPERTY_GC_ENABLED.to_string(), + "false".to_string(), + ), + ]); + let table_properties = TableProperties::new(&props); + assert_eq!(table_properties.commit_num_retries().unwrap(), 10); + assert_eq!(table_properties.commit_max_retry_wait_ms().unwrap(), 20); + assert_eq!( + table_properties.write_format_default().unwrap(), + "avro".to_string() + ); + assert_eq!( + table_properties.write_target_file_size_bytes().unwrap(), + 512 + ); + assert!(!table_properties.gc_enabled().unwrap()); + } + + #[test] + fn test_table_properties_invalid() { + let invalid_retries = HashMap::from([( + TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(), + "abc".to_string(), + )]); + + let table_properties = TableProperties::new(&invalid_retries); + let error = table_properties.commit_num_retries().unwrap_err(); + assert!( + error.to_string().contains( + "Invalid value for commit.retry.num-retries: invalid digit found in string" + ) + ); + + let invalid_min_wait = HashMap::from([( + TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(), + "abc".to_string(), + )]); + let table_properties = TableProperties::new(&invalid_min_wait); + let error = table_properties.commit_min_retry_wait_ms().unwrap_err(); + assert!( + error.to_string().contains( + "Invalid value for commit.retry.min-wait-ms: invalid digit found in string" + ) + ); + + let invalid_max_wait = HashMap::from([( + TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(), + "abc".to_string(), + )]); + let table_properties = TableProperties::new(&invalid_max_wait); + let error = table_properties.commit_max_retry_wait_ms().unwrap_err(); + assert!( + error.to_string().contains( + "Invalid value for commit.retry.max-wait-ms: invalid digit found in string" + ) + ); + + let invalid_target_size = HashMap::from([( + TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), + "abc".to_string(), + )]); + let table_properties = TableProperties::new(&invalid_target_size); + let error = table_properties.write_target_file_size_bytes().unwrap_err(); + assert!(error.to_string().contains( + "Invalid value for write.target-file-size-bytes: invalid digit found in string" + )); + + let invalid_gc_enabled = HashMap::from([( + TableProperties::PROPERTY_GC_ENABLED.to_string(), + "notabool".to_string(), + )]); + let table_properties = TableProperties::new(&invalid_gc_enabled); + let error = table_properties.gc_enabled().unwrap_err(); + assert!(error.to_string().contains("Invalid value for gc.enabled")); + } + + #[test] + fn test_table_properties_compression_invalid_rejected() { + let invalid_codecs = ["lz4", "zstd", "snappy"]; + + for codec in invalid_codecs { + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + codec.to_string(), + )]); + let err = TableProperties::new(&props) + .metadata_compression_codec() + .unwrap_err(); + let err_msg = err.to_string(); + assert!( + err_msg.contains(&format!("Invalid metadata compression codec: {codec}")), + "Expected error message to contain codec '{codec}', got: {err_msg}" + ); + assert!( + err_msg.contains("Only 'none' and 'gzip' are supported"), + "Expected error message to contain supported codecs, got: {err_msg}" + ); + } + } + + #[test] + fn test_cdc_disabled_by_default() { + let props = HashMap::new(); + let tp = TableProperties::new(&props); + assert!(!tp.cdc_enabled().unwrap()); + } + + #[test] + fn test_cdc_enabled_via_flag() { + let props = HashMap::from([( + TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), + "true".to_string(), + )]); + let tp = TableProperties::new(&props); + assert!(tp.cdc_enabled().unwrap()); + assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024); + assert_eq!(tp.cdc_norm_level().unwrap(), 0); + } + + #[test] + fn test_cdc_size_props_alone_do_not_enable() { + let props = HashMap::from([( + TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), + "262144".to_string(), + )]); + let tp = TableProperties::new(&props); + assert!(!tp.cdc_enabled().unwrap()); + } + + #[test] + fn test_cdc_custom_values() { + let props = HashMap::from([ + ( + TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), + "true".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), + "200000".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(), + "900000".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), + "1".to_string(), + ), + ]); + let tp = TableProperties::new(&props); + assert!(tp.cdc_enabled().unwrap()); + assert_eq!(tp.cdc_min_chunk_size().unwrap(), 200000); + assert_eq!(tp.cdc_max_chunk_size().unwrap(), 900000); + assert_eq!(tp.cdc_norm_level().unwrap(), 1); + } + + #[test] + fn test_cdc_partial_override() { + let props = HashMap::from([ + ( + TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), + "true".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), + "2".to_string(), + ), + ]); + let tp = TableProperties::new(&props); + assert!(tp.cdc_enabled().unwrap()); + assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024); + assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024); + assert_eq!(tp.cdc_norm_level().unwrap(), 2); + } + + #[test] + fn test_cdc_negative_norm_level() { + let props = HashMap::from([ + ( + TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), + "true".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), + "-2".to_string(), + ), + ]); + let tp = TableProperties::new(&props); + assert_eq!(tp.cdc_norm_level().unwrap(), -2); + } + + #[test] + fn test_cdc_invalid_min_chunk_size() { + let props = HashMap::from([ + ( + TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), + "true".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), + "not_a_number".to_string(), + ), + ]); + let err = TableProperties::new(&props) + .cdc_min_chunk_size() + .unwrap_err(); + assert!( + err.to_string().contains( + "Invalid value for write.parquet.content-defined-chunking.min-chunk-size" + ) + ); + } + + #[test] + fn test_cdc_invalid_norm_level() { + let props = HashMap::from([ + ( + TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), + "true".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), + "not_a_number".to_string(), + ), + ]); + let err = TableProperties::new(&props).cdc_norm_level().unwrap_err(); + assert!( + err.to_string() + .contains("Invalid value for write.parquet.content-defined-chunking.norm-level") + ); + } + + #[test] + fn test_cdc_no_properties() { + let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]); + let tp = TableProperties::new(&props); + assert!(!tp.cdc_enabled().unwrap()); + } + + #[test] + fn test_parquet_sizing_defaults() { + let props = HashMap::new(); + let tp = TableProperties::new(&props); + // Default codec is zstd at its default level. + assert_eq!( + tp.parquet_compression_codec().unwrap(), + CompressionCodec::zstd_default() + ); + assert_eq!( + tp.parquet_row_group_size_bytes().unwrap(), + TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT + ); + assert_eq!( + tp.parquet_page_size_bytes().unwrap(), + TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT + ); + assert_eq!( + tp.parquet_page_row_limit().unwrap(), + TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT + ); + assert_eq!( + tp.parquet_dict_size_bytes().unwrap(), + TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT + ); + } + + #[test] + fn test_parquet_sizing_overrides() { + let props = HashMap::from([ + ( + TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), + "gzip".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(), + "4".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(), + "1048576".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES.to_string(), + "65536".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT.to_string(), + "5000".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES.to_string(), + "131072".to_string(), + ), + ]); + let tp = TableProperties::new(&props); + // Codec name and level are folded into a single CompressionCodec. + assert_eq!( + tp.parquet_compression_codec().unwrap(), + CompressionCodec::Gzip(4) + ); + assert_eq!(tp.parquet_row_group_size_bytes().unwrap(), 1048576); + assert_eq!(tp.parquet_page_size_bytes().unwrap(), 65536); + assert_eq!(tp.parquet_page_row_limit().unwrap(), 5000); + assert_eq!(tp.parquet_dict_size_bytes().unwrap(), 131072); + } + + #[test] + fn test_parquet_invalid_sizing_rejected() { + let props = HashMap::from([( + TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(), + "not_a_number".to_string(), + )]); + let err = TableProperties::new(&props) + .parquet_row_group_size_bytes() + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES) + ); + } + + #[test] + fn test_parquet_all_codecs_parse() { + // Every codec name parquet-java supports must parse (parity with Java's + // `CompressionCodecName.valueOf`). + for (name, expected) in [ + ("uncompressed", CompressionCodec::None), + ("snappy", CompressionCodec::Snappy), + ("gzip", CompressionCodec::gzip_default()), + ("lzo", CompressionCodec::Lzo), + ("brotli", CompressionCodec::brotli_default()), + ("lz4", CompressionCodec::Lz4), + ("lz4_raw", CompressionCodec::Lz4Raw), + ("zstd", CompressionCodec::zstd_default()), + ] { + let props = HashMap::from([( + TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), + name.to_string(), + )]); + let tp = TableProperties::new(&props); + assert_eq!( + tp.parquet_compression_codec().unwrap(), + expected, + "codec {name}" + ); + } + } + + #[test] + fn test_parquet_compression_level_ignored_for_levelless_codec() { + // A level set alongside a codec that carries none (e.g. snappy) is + // ignored rather than rejected, matching parquet-java. + let props = HashMap::from([ + ( + TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), + "snappy".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(), + "5".to_string(), + ), + ]); + let tp = TableProperties::new(&props); + assert_eq!( + tp.parquet_compression_codec().unwrap(), + CompressionCodec::Snappy + ); + } + + #[test] + fn test_parse_boolean_property_case_insensitive() { + let false_variants = ["False", "FALSE"]; + let true_variants = ["True", "TRUE"]; + + for f in false_variants { + let props = HashMap::from([( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), + f.to_string(), + )]); + let tp = TableProperties::new(&props); + assert!(!tp.write_object_storage_partitioned_paths().unwrap()); + } + + for t in true_variants { + let props = HashMap::from([( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), + t.to_string(), + )]); + let tp = TableProperties::new(&props); + assert!(tp.write_object_storage_partitioned_paths().unwrap()); + } + } + + #[test] + fn test_table_properties_default_name_mapping() { + // Test unset. + let properties = HashMap::new(); + assert!( + TableProperties::new(&properties) + .default_name_mapping() + .unwrap() + .is_none() + ); + + let properties = HashMap::from([( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(), + r#"[{"field-id":1,"names":["id","record_id"]}]"#.to_string(), + )]); + let mapping = TableProperties::new(&properties) + .default_name_mapping() + .unwrap() + .unwrap(); + assert_eq!(mapping.fields().len(), 1); + assert_eq!(mapping.fields()[0].field_id(), Some(1)); + assert_eq!(mapping.fields()[0].names(), &[ + "id".to_string(), + "record_id".to_string() + ]); + } + + #[test] + fn test_table_properties_malformed_name_mapping() { + let properties = HashMap::from([( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(), + "{ not valid json".to_string(), + )]); + let error = TableProperties::new(&properties) + .default_name_mapping() + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + // The property key must survive as error context. + assert!( + format!("{error}").contains(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING), + "{error}" + ); + } +} diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index 8b6d2225a9..f52202b78b 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -21,12 +21,14 @@ use std::cmp::Ordering; use std::fmt::{Display, Formatter}; use std::str::FromStr; +use chrono::Datelike; use fnv::FnvHashSet; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use super::values::temporal::date; use super::{Datum, PrimitiveLiteral}; use crate::ErrorKind; -use crate::error::{Error, Result}; +use crate::error::{Error, Result, invalid_data}; use crate::expr::{ BinaryExpression, BoundPredicate, BoundReference, Predicate, PredicateOperator, Reference, SetExpression, UnaryExpression, @@ -35,6 +37,9 @@ use crate::spec::Literal; use crate::spec::datatypes::{PrimitiveType, Type}; use crate::transform::{BoxedTransformFunction, create_transform_function}; +/// The year the Unix epoch falls in, which every temporal ordinal counts from. +const UNIX_EPOCH_YEAR: i32 = 1970; + /// Transform is used to transform predicates to partition predicates, /// in addition to transforming data values. /// @@ -47,7 +52,7 @@ use crate::transform::{BoxedTransformFunction, create_transform_function}; /// predicates and partition predicates. /// /// All transforms must return `null` for a `null` input value. -#[derive(Debug, PartialEq, Eq, Clone, Copy)] +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] pub enum Transform { /// Source value, unmodified /// @@ -136,38 +141,157 @@ pub enum Transform { impl Transform { /// Returns a human-readable String representation of a transformed value. + /// + /// The temporal transforms store an ordinal count since the Unix epoch, and + /// this method renders that count as a date so that partition paths and + /// snapshot summary keys match the Java reference implementation: + /// + /// | Transform | Format | Example | + /// |-----------|-----------------|-----------------| + /// | `Year` | `yyyy` | `2017` | + /// | `Month` | `yyyy-MM` | `2017-06` | + /// | `Day` | `yyyy-MM-dd` | `2017-06-15` | + /// | `Hour` | `yyyy-MM-dd-HH` | `2017-06-15-16` | + /// + /// `Void` renders as `null`, as does an absent value for any transform. + /// + /// # Example + /// + /// ``` + /// use iceberg::spec::{Literal, PrimitiveType, Transform, Type}; + /// + /// let int = Type::Primitive(PrimitiveType::Int); + /// let date = Type::Primitive(PrimitiveType::Date); + /// + /// // A stored value carries no logical type of its own. For transforms that do + /// // not format it themselves, the declared field type decides how it renders. + /// let stored = Literal::int(17332); + /// assert_eq!( + /// Transform::Identity.to_human_string(&int, Some(&stored)), + /// "17332" + /// ); + /// assert_eq!( + /// Transform::Identity.to_human_string(&date, Some(&stored)), + /// "2017-06-15" + /// ); + /// + /// // The temporal transforms format their own ordinal and ignore the declared + /// // type. All four ordinals below are the same instant, 2017-06-15T16:00:00Z, + /// // counted at four granularities. + /// assert_eq!( + /// Transform::Year.to_human_string(&int, Some(&Literal::int(47))), + /// "2017" + /// ); + /// assert_eq!( + /// Transform::Month.to_human_string(&int, Some(&Literal::int(569))), + /// "2017-06" + /// ); + /// assert_eq!( + /// Transform::Day.to_human_string(&int, Some(&Literal::int(17332))), + /// "2017-06-15" + /// ); + /// assert_eq!( + /// Transform::Hour.to_human_string(&int, Some(&Literal::int(415984))), + /// "2017-06-15-16" + /// ); + /// + /// // `Void` and an absent value render as `null`. + /// assert_eq!( + /// Transform::Void.to_human_string(&int, Some(&Literal::int(47))), + /// "null" + /// ); + /// assert_eq!(Transform::Year.to_human_string(&int, None), "null"); + /// ``` pub fn to_human_string(&self, field_type: &Type, value: Option<&Literal>) -> String { - if let Some(value) = value { - if let Some(value) = value.as_primitive_literal() { + let Some(value) = value.and_then(Literal::as_primitive_literal) else { + return "null".to_string(); + }; + + match (*self, value) { + (Self::Void, _) => "null".to_string(), + // The temporal transforms store an ordinal count since the Unix epoch, + // which the datum arm below cannot render: it would show the raw count + // for `Year`, `Month` and `Hour`, and would leave `Day` dependent on the + // field type happening to be `date`. Java overrides `toHumanString` on + // each of these four transforms and ignores the declared type, so do the + // same. Any other literal falls through to the datum. + (Self::Year, PrimitiveLiteral::Int(ordinal)) => Self::human_year(ordinal), + (Self::Month, PrimitiveLiteral::Int(ordinal)) => Self::human_month(ordinal), + (Self::Day, PrimitiveLiteral::Int(ordinal)) => Self::human_day(ordinal), + (Self::Hour, PrimitiveLiteral::Int(ordinal)) => Self::human_hour(ordinal), + (_, value) => { let field_type = field_type.as_primitive_type().unwrap(); - let datum = Datum::new(field_type.clone(), value); - match self { - Self::Identity => datum.to_human_string(), - Self::Void => "null".to_string(), - _ => { - todo!() - } - } - } else { - "null".to_string() + Datum::new(field_type.clone(), value).to_human_string() } - } else { - "null".to_string() } } + /// Formats a year ordinal, the number of years since 1970, as `yyyy`. + /// + /// Mirrors the output of `TransformUtil.humanYear`. + fn human_year(year_ordinal: i32) -> String { + format!("{:04}", UNIX_EPOCH_YEAR + year_ordinal) + } + + /// Formats a month ordinal, the number of months since 1970-01, as `yyyy-MM`. + /// + /// Mirrors the output of `TransformUtil.humanMonth`, which divides with + /// `Math.floorDiv` and `Math.floorMod` rather than `/` and `%`. Truncating + /// division rounds toward zero, which is the wrong direction before 1970: + /// ordinal -1 is 1969-12, but truncating yields 1970-01. `div_euclid` and + /// `rem_euclid` round toward negative infinity and so agree with Java. + fn human_month(month_ordinal: i32) -> String { + format!( + "{:04}-{:02}", + UNIX_EPOCH_YEAR + month_ordinal.div_euclid(12), + 1 + month_ordinal.rem_euclid(12) + ) + } + + /// Formats a day ordinal, the number of days since 1970-01-01, as `yyyy-MM-dd`. + /// + /// Mirrors the output of `TransformUtil.humanDay`. Like the Java `Days` + /// transform, whose signature is `toHumanString(Type alwaysDate, Integer + /// value)`, this ignores the declared field type rather than relying on it being + /// `date`. + fn human_day(day_ordinal: i32) -> String { + let date = date::days_to_date(day_ordinal); + format!("{:04}-{:02}-{:02}", date.year(), date.month(), date.day()) + } + + /// Formats an hour ordinal, the number of hours since 1970-01-01T00:00:00Z, + /// as `yyyy-MM-dd-HH`. + /// + /// Mirrors the output of `TransformUtil.humanHour`. `div_euclid` and `rem_euclid` + /// split the ordinal into whole days and the hour within the day so that hours + /// before 1970 round the way they do in Java. + fn human_hour(hour_ordinal: i32) -> String { + format!( + "{}-{:02}", + Self::human_day(hour_ordinal.div_euclid(24)), + hour_ordinal.rem_euclid(24) + ) + } + /// Get the return type of transform given the input type. /// Returns `None` if it can't be transformed. pub fn result_type(&self, input_type: &Type) -> Result { match self { Transform::Identity => { - if matches!(input_type, Type::Primitive(_)) { - Ok(input_type.clone()) - } else { + if matches!( + input_type, + Type::Primitive(PrimitiveType::Geometry(_) | PrimitiveType::Geography(_)) + ) { Err(Error::new( ErrorKind::DataInvalid, format!("{input_type} is not a valid input type of identity transform",), )) + } else if matches!(input_type, Type::Primitive(_)) { + Ok(input_type.clone()) + } else { + Err(invalid_data!( + "{input_type} is not a valid input type of identity transform", + )) } } Transform::Void => Ok(input_type.clone()), @@ -188,15 +312,13 @@ impl Transform { | PrimitiveType::Uuid | PrimitiveType::Fixed(_) | PrimitiveType::Binary => Ok(Type::Primitive(PrimitiveType::Int)), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of bucket transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of bucket transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of bucket transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of bucket transform", )) } } @@ -208,15 +330,13 @@ impl Transform { | PrimitiveType::String | PrimitiveType::Binary | PrimitiveType::Decimal { .. } => Ok(input_type.clone()), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of truncate transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of truncate transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of truncate transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of truncate transform", )) } } @@ -228,15 +348,13 @@ impl Transform { | PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs | PrimitiveType::Date => Ok(Type::Primitive(PrimitiveType::Int)), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )) } } @@ -248,15 +366,13 @@ impl Transform { | PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs | PrimitiveType::Date => Ok(Type::Primitive(PrimitiveType::Date)), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )) } } @@ -267,15 +383,13 @@ impl Transform { | PrimitiveType::Timestamptz | PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs => Ok(Type::Primitive(PrimitiveType::Int)), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + _ => Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )), } } else { - Err(Error::new( - ErrorKind::DataInvalid, - format!("{input_type} is not a valid input type of {self} transform",), + Err(invalid_data!( + "{input_type} is not a valid input type of {self} transform", )) } } @@ -290,6 +404,14 @@ impl Transform { ) } + /// Whether a partition field with this transform may be named after its source column. + /// + /// Void is allowed because removing an identity field from a format version 1 spec leaves + /// a void field that keeps the identity field's name. + pub(crate) fn may_share_name_with_source(&self) -> bool { + matches!(self, Transform::Identity | Transform::Void) + } + /// Return the unique transform name to check if similar transforms for the same source field /// are added multiple times in partition spec builder. pub fn dedup_name(&self) -> String { @@ -378,12 +500,9 @@ impl Transform { PrimitiveLiteral::String(s) => s.len(), PrimitiveLiteral::Binary(b) => b.len(), _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Expected a string or binary literal, got: {:?}", - expr.literal() - ), + return Err(invalid_data!( + "Expected a string or binary literal, got: {:?}", + expr.literal() )); } }; @@ -405,12 +524,9 @@ impl Transform { PrimitiveLiteral::String(s) => s.len(), PrimitiveLiteral::Binary(b) => b.len(), _ => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Expected a string or binary literal, got: {:?}", - expr.literal() - ), + return Err(invalid_data!( + "Expected a string or binary literal, got: {:?}", + expr.literal() )); } }; @@ -661,25 +777,43 @@ impl Transform { PredicateOperator::LessThan => match (datum.data_type(), datum.literal()) { (PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Some(Datum::int(v - 1)), (PrimitiveType::Long, PrimitiveLiteral::Long(v)) => Some(Datum::long(v - 1)), - (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => { - Some(Datum::decimal(v - 1)?) + (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(_)) => { + Some(Self::try_decrement_number(datum)?) } (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => Some(Datum::date(v - 1)), (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { Some(Datum::timestamp_micros(v - 1)) } + (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(v)) => { + Some(Datum::timestamptz_micros(v - 1)) + } + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(v)) => { + Some(Datum::timestamp_nanos(v - 1)) + } + (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(v)) => { + Some(Datum::timestamptz_nanos(v - 1)) + } _ => Some(datum.to_owned()), }, PredicateOperator::GreaterThan => match (datum.data_type(), datum.literal()) { (PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Some(Datum::int(v + 1)), (PrimitiveType::Long, PrimitiveLiteral::Long(v)) => Some(Datum::long(v + 1)), - (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => { - Some(Datum::decimal(v + 1)?) + (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(_)) => { + Some(Self::try_increment_number(datum)?) } (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => Some(Datum::date(v + 1)), (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { Some(Datum::timestamp_micros(v + 1)) } + (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(v)) => { + Some(Datum::timestamptz_micros(v + 1)) + } + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(v)) => { + Some(Datum::timestamp_nanos(v + 1)) + } + (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(v)) => { + Some(Datum::timestamptz_nanos(v + 1)) + } _ => Some(datum.to_owned()), }, PredicateOperator::Eq @@ -713,10 +847,10 @@ impl Transform { PredicateOperator::GreaterThan => Some(PredicateOperator::GreaterThanOrEq), PredicateOperator::StartsWith => match datum.literal() { PrimitiveLiteral::String(s) => { - if let Some(w) = width { - if s.len() == w as usize { - return Some(PredicateOperator::Eq); - }; + if let Some(w) = width + && s.len() == w as usize + { + return Some(PredicateOperator::Eq); }; Some(*op) } @@ -754,52 +888,56 @@ impl Transform { transformed: &Datum, ) -> Option { let should_adjust = match self { - Transform::Day => matches!(original.data_type(), PrimitiveType::Timestamp), + Transform::Day => matches!( + original.data_type(), + PrimitiveType::Timestamp + | PrimitiveType::Timestamptz + | PrimitiveType::TimestampNs + | PrimitiveType::TimestamptzNs + ), Transform::Year | Transform::Month => true, _ => false, }; - if should_adjust { - if let &PrimitiveLiteral::Int(v) = transformed.literal() { - match op { - PredicateOperator::LessThan - | PredicateOperator::LessThanOrEq - | PredicateOperator::In => { - if v < 0 { + if should_adjust && let &PrimitiveLiteral::Int(v) = transformed.literal() { + match op { + PredicateOperator::LessThan + | PredicateOperator::LessThanOrEq + | PredicateOperator::In => { + if v < 0 { + // # TODO + // An ugly hack to fix. Refine the increment and decrement logic later. + match self { + Transform::Day => { + return Some(AdjustedProjection::Single(Datum::date(v + 1))); + } + _ => { + return Some(AdjustedProjection::Single(Datum::int(v + 1))); + } + } + }; + } + PredicateOperator::Eq => { + if v < 0 { + let new_set = FnvHashSet::from_iter(vec![ + transformed.to_owned(), // # TODO // An ugly hack to fix. Refine the increment and decrement logic later. - match self { - Transform::Day => { - return Some(AdjustedProjection::Single(Datum::date(v + 1))); + { + match self { + Transform::Day => Datum::date(v + 1), + _ => Datum::int(v + 1), } - _ => { - return Some(AdjustedProjection::Single(Datum::int(v + 1))); - } - } - }; - } - PredicateOperator::Eq => { - if v < 0 { - let new_set = FnvHashSet::from_iter(vec![ - transformed.to_owned(), - // # TODO - // An ugly hack to fix. Refine the increment and decrement logic later. - { - match self { - Transform::Day => Datum::date(v + 1), - _ => Datum::int(v + 1), - } - }, - ]); - return Some(AdjustedProjection::Set(new_set)); - } - } - _ => { - return None; + }, + ]); + return Some(AdjustedProjection::Set(new_set)); } } - }; - } + _ => { + return None; + } + } + }; None } @@ -810,7 +948,9 @@ impl Transform { match (datum.data_type(), datum.literal()) { (PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Ok(Datum::int(v + 1)), (PrimitiveType::Long, PrimitiveLiteral::Long(v)) => Ok(Datum::long(v + 1)), - (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => Datum::decimal(v + 1), + (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => { + datum.with_decimal_mantissa(v + 1) + } (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => Ok(Datum::date(v + 1)), (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { Ok(Datum::timestamp_micros(v + 1)) @@ -846,7 +986,9 @@ impl Transform { match (datum.data_type(), datum.literal()) { (PrimitiveType::Int, PrimitiveLiteral::Int(v)) => Ok(Datum::int(v - 1)), (PrimitiveType::Long, PrimitiveLiteral::Long(v)) => Ok(Datum::long(v - 1)), - (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => Datum::decimal(v - 1), + (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(v)) => { + datum.with_decimal_mantissa(v - 1) + } (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => Ok(Datum::date(v - 1)), (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { Ok(Datum::timestamp_micros(v - 1)) @@ -894,9 +1036,8 @@ impl Transform { | &PrimitiveType::TimestampNs | &PrimitiveType::TimestamptzNs ) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Expected a numeric literal, got: {:?}", boundary), + return Err(invalid_data!( + "Expected a numeric literal, got: {boundary:?}" )); } @@ -1001,11 +1142,7 @@ impl FromStr for Transform { .trim_end_matches(']') .parse() .map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - format!("transform bucket type {v:?} is invalid"), - ) - .with_source(err) + invalid_data!("transform bucket type {v:?} is invalid").with_source(err) })?; Transform::Bucket(length) @@ -1018,20 +1155,13 @@ impl FromStr for Transform { .trim_end_matches(']') .parse() .map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - format!("transform truncate type {v:?} is invalid"), - ) - .with_source(err) + invalid_data!("transform truncate type {v:?} is invalid").with_source(err) })?; Transform::Truncate(width) } v => { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("transform {v:?} is invalid"), - )); + return Err(invalid_data!("transform {v:?} is invalid")); } }; @@ -1061,3 +1191,153 @@ enum AdjustedProjection { Single(Datum), Set(FnvHashSet), } + +#[cfg(test)] +mod tests { + use super::*; + + fn check_boundary(op: PredicateOperator, input: Datum, expected: Datum) { + let result = Transform::adjust_boundary(&op, &input).unwrap().unwrap(); + assert_eq!(result, expected); + } + + #[test] + fn test_adjust_boundary_timestamp_types() { + for (datum, dec, inc) in [ + ( + Datum::timestamptz_micros(1000), + Datum::timestamptz_micros(999), + Datum::timestamptz_micros(1001), + ), + ( + Datum::timestamp_nanos(5000), + Datum::timestamp_nanos(4999), + Datum::timestamp_nanos(5001), + ), + ( + Datum::timestamptz_nanos(5000), + Datum::timestamptz_nanos(4999), + Datum::timestamptz_nanos(5001), + ), + ] { + check_boundary(PredicateOperator::LessThan, datum.clone(), dec); + check_boundary(PredicateOperator::GreaterThan, datum.clone(), inc); + check_boundary( + PredicateOperator::LessThanOrEq, + datum.clone(), + datum.clone(), + ); + check_boundary(PredicateOperator::GreaterThanOrEq, datum.clone(), datum); + } + } + + /// Renders `ordinal` through the public API with the given declared type. + fn human(transform: Transform, primitive: PrimitiveType, ordinal: i32) -> String { + transform.to_human_string(&Type::Primitive(primitive), Some(&Literal::int(ordinal))) + } + + /// Renders `ordinal` for a transform whose result type is `int`. + fn human_int(transform: Transform, ordinal: i32) -> String { + human(transform, PrimitiveType::Int, ordinal) + } + + #[test] + fn test_to_human_string_year() { + assert_eq!(human_int(Transform::Year, -1970), "0000"); + assert_eq!(human_int(Transform::Year, -1), "1969"); + assert_eq!(human_int(Transform::Year, 0), "1970"); + assert_eq!(human_int(Transform::Year, 47), "2017"); + } + + #[test] + fn test_to_human_string_month() { + assert_eq!(human_int(Transform::Month, -1970 * 12), "0000-01"); + assert_eq!(human_int(Transform::Month, -13), "1968-12"); + assert_eq!(human_int(Transform::Month, -12), "1969-01"); + assert_eq!(human_int(Transform::Month, -1), "1969-12"); + assert_eq!(human_int(Transform::Month, 0), "1970-01"); + assert_eq!(human_int(Transform::Month, 11), "1970-12"); + assert_eq!(human_int(Transform::Month, 12), "1971-01"); + assert_eq!(human_int(Transform::Month, 569), "2017-06"); + } + + #[test] + fn test_to_human_string_day() { + assert_eq!(human_int(Transform::Day, -1), "1969-12-31"); + assert_eq!(human_int(Transform::Day, 0), "1970-01-01"); + assert_eq!(human_int(Transform::Day, 31), "1970-02-01"); + assert_eq!(human_int(Transform::Day, 17332), "2017-06-15"); + } + + #[test] + fn test_to_human_string_hour() { + assert_eq!(human_int(Transform::Hour, -24), "1969-12-31-00"); + assert_eq!(human_int(Transform::Hour, -1), "1969-12-31-23"); + assert_eq!(human_int(Transform::Hour, 0), "1970-01-01-00"); + assert_eq!(human_int(Transform::Hour, 23), "1970-01-01-23"); + assert_eq!(human_int(Transform::Hour, 24), "1970-01-02-00"); + assert_eq!(human_int(Transform::Hour, 1000), "1970-02-11-16"); + assert_eq!(human_int(Transform::Hour, 415984), "2017-06-15-16"); + } + + /// The temporal transforms ignore the declared field type, matching the Java + /// signatures `toHumanString(Type alwaysInt, ..)` and + /// `toHumanString(Type alwaysDate, ..)`. + #[test] + fn test_to_human_string_ignores_declared_type_for_temporal_transforms() { + assert_eq!(human(Transform::Year, PrimitiveType::Date, 47), "2017"); + assert_eq!(human(Transform::Month, PrimitiveType::Date, 569), "2017-06"); + assert_eq!( + human(Transform::Day, PrimitiveType::Int, 17332), + "2017-06-15" + ); + assert_eq!( + human(Transform::Hour, PrimitiveType::Date, 415984), + "2017-06-15-16" + ); + } + + /// Transforms with no temporal format keep deferring to the datum, which + /// renders according to the declared field type. + #[test] + fn test_to_human_string_defers_to_datum_for_other_transforms() { + assert_eq!( + human(Transform::Identity, PrimitiveType::Int, 17332), + "17332" + ); + assert_eq!( + human(Transform::Identity, PrimitiveType::Date, 17332), + "2017-06-15" + ); + assert_eq!(human(Transform::Bucket(16), PrimitiveType::Int, 5), "5"); + + // A literal that is not an `int` also defers to the datum rather than being + // reported as `null`, so the temporal arms add no second behaviour change. + assert_eq!( + Transform::Year.to_human_string( + &Type::Primitive(PrimitiveType::String), + Some(&Literal::string("unformatted")) + ), + "unformatted" + ); + } + + #[test] + fn test_to_human_string_null_cases() { + assert_eq!(human(Transform::Void, PrimitiveType::Int, 47), "null"); + assert_eq!(human(Transform::Void, PrimitiveType::Date, 17332), "null"); + for transform in [ + Transform::Year, + Transform::Month, + Transform::Day, + Transform::Hour, + Transform::Identity, + Transform::Void, + ] { + assert_eq!( + transform.to_human_string(&Type::Primitive(PrimitiveType::Int), None), + "null" + ); + } + } +} diff --git a/crates/iceberg/src/spec/values.rs b/crates/iceberg/src/spec/values.rs deleted file mode 100644 index 7f214810e9..0000000000 --- a/crates/iceberg/src/spec/values.rs +++ /dev/null @@ -1,3957 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -/*! - * Value in iceberg - */ - -use std::any::Any; -use std::cmp::Ordering; -use std::collections::HashMap; -use std::fmt::{Display, Formatter}; -use std::hash::Hash; -use std::ops::Index; -use std::str::FromStr; - -pub use _serde::RawLiteral; -use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; -use num_bigint::BigInt; -use ordered_float::{Float, OrderedFloat}; -use rust_decimal::Decimal; -use rust_decimal::prelude::ToPrimitive; -use serde::de::{ - MapAccess, {self}, -}; -use serde::ser::SerializeStruct; -use serde::{Deserialize, Serialize}; -use serde_bytes::ByteBuf; -use serde_json::{Map as JsonMap, Number, Value as JsonValue}; -use timestamp::nanoseconds_to_datetime; -use uuid::Uuid; - -use super::datatypes::{PrimitiveType, Type}; -use crate::error::Result; -use crate::spec::MAX_DECIMAL_PRECISION; -use crate::spec::values::date::{date_from_naive_date, days_to_date, unix_epoch}; -use crate::spec::values::time::microseconds_to_time; -use crate::spec::values::timestamp::microseconds_to_datetime; -use crate::spec::values::timestamptz::{microseconds_to_datetimetz, nanoseconds_to_datetimetz}; -use crate::{Error, ErrorKind, ensure_data_valid}; - -/// Maximum value for [`PrimitiveType::Time`] type in microseconds, e.g. 23 hours 59 minutes 59 seconds 999999 microseconds. -const MAX_TIME_VALUE: i64 = 24 * 60 * 60 * 1_000_000i64 - 1; - -const INT_MAX: i32 = 2147483647; -const INT_MIN: i32 = -2147483648; -const LONG_MAX: i64 = 9223372036854775807; -const LONG_MIN: i64 = -9223372036854775808; - -/// Values present in iceberg type -#[derive(Clone, Debug, PartialOrd, PartialEq, Hash, Eq)] -pub enum PrimitiveLiteral { - /// 0x00 for false, non-zero byte for true - Boolean(bool), - /// Stored as 4-byte little-endian - Int(i32), - /// Stored as 8-byte little-endian - Long(i64), - /// Stored as 4-byte little-endian - Float(OrderedFloat), - /// Stored as 8-byte little-endian - Double(OrderedFloat), - /// UTF-8 bytes (without length) - String(String), - /// Binary value (without length) - Binary(Vec), - /// Stored as 16-byte little-endian - Int128(i128), - /// Stored as 16-byte little-endian - UInt128(u128), - /// When a number is larger than it can hold - AboveMax, - /// When a number is smaller than it can hold - BelowMin, -} - -impl PrimitiveLiteral { - /// Returns true if the Literal represents a primitive type - /// that can be a NaN, and that it's value is NaN - pub fn is_nan(&self) -> bool { - match self { - PrimitiveLiteral::Double(val) => val.is_nan(), - PrimitiveLiteral::Float(val) => val.is_nan(), - _ => false, - } - } -} - -/// Literal associated with its type. The value and type pair is checked when construction, so the type and value is -/// guaranteed to be correct when used. -/// -/// By default, we decouple the type and value of a literal, so we can use avoid the cost of storing extra type info -/// for each literal. But associate type with literal can be useful in some cases, for example, in unbound expression. -#[derive(Clone, Debug, PartialEq, Hash, Eq)] -pub struct Datum { - r#type: PrimitiveType, - literal: PrimitiveLiteral, -} - -impl Serialize for Datum { - fn serialize( - &self, - serializer: S, - ) -> std::result::Result { - let mut struct_ser = serializer - .serialize_struct("Datum", 2) - .map_err(serde::ser::Error::custom)?; - struct_ser - .serialize_field("type", &self.r#type) - .map_err(serde::ser::Error::custom)?; - struct_ser - .serialize_field( - "literal", - &RawLiteral::try_from( - Literal::Primitive(self.literal.clone()), - &Type::Primitive(self.r#type.clone()), - ) - .map_err(serde::ser::Error::custom)?, - ) - .map_err(serde::ser::Error::custom)?; - struct_ser.end() - } -} - -impl<'de> Deserialize<'de> for Datum { - fn deserialize>( - deserializer: D, - ) -> std::result::Result { - #[derive(Deserialize)] - #[serde(field_identifier, rename_all = "lowercase")] - enum Field { - Type, - Literal, - } - - struct DatumVisitor; - - impl<'de> serde::de::Visitor<'de> for DatumVisitor { - type Value = Datum; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("struct Datum") - } - - fn visit_seq(self, mut seq: A) -> std::result::Result - where A: serde::de::SeqAccess<'de> { - let r#type = seq - .next_element::()? - .ok_or_else(|| serde::de::Error::invalid_length(0, &self))?; - let value = seq - .next_element::()? - .ok_or_else(|| serde::de::Error::invalid_length(1, &self))?; - let Literal::Primitive(primitive) = value - .try_into(&Type::Primitive(r#type.clone())) - .map_err(serde::de::Error::custom)? - .ok_or_else(|| serde::de::Error::custom("None value"))? - else { - return Err(serde::de::Error::custom("Invalid value")); - }; - - Ok(Datum::new(r#type, primitive)) - } - - fn visit_map(self, mut map: V) -> std::result::Result - where V: MapAccess<'de> { - let mut raw_primitive: Option = None; - let mut r#type: Option = None; - while let Some(key) = map.next_key()? { - match key { - Field::Type => { - if r#type.is_some() { - return Err(de::Error::duplicate_field("type")); - } - r#type = Some(map.next_value()?); - } - Field::Literal => { - if raw_primitive.is_some() { - return Err(de::Error::duplicate_field("literal")); - } - raw_primitive = Some(map.next_value()?); - } - } - } - let Some(r#type) = r#type else { - return Err(serde::de::Error::missing_field("type")); - }; - let Some(raw_primitive) = raw_primitive else { - return Err(serde::de::Error::missing_field("literal")); - }; - let Literal::Primitive(primitive) = raw_primitive - .try_into(&Type::Primitive(r#type.clone())) - .map_err(serde::de::Error::custom)? - .ok_or_else(|| serde::de::Error::custom("None value"))? - else { - return Err(serde::de::Error::custom("Invalid value")); - }; - Ok(Datum::new(r#type, primitive)) - } - } - const FIELDS: &[&str] = &["type", "literal"]; - deserializer.deserialize_struct("Datum", FIELDS, DatumVisitor) - } -} - -// Compare following iceberg float ordering rules: -// -NaN < -Infinity < -value < -0 < 0 < value < Infinity < NaN -fn iceberg_float_cmp(a: T, b: T) -> Option { - if a.is_nan() && b.is_nan() { - return match (a.is_sign_negative(), b.is_sign_negative()) { - (true, false) => Some(Ordering::Less), - (false, true) => Some(Ordering::Greater), - _ => Some(Ordering::Equal), - }; - } - - if a.is_nan() { - return Some(if a.is_sign_negative() { - Ordering::Less - } else { - Ordering::Greater - }); - } - - if b.is_nan() { - return Some(if b.is_sign_negative() { - Ordering::Greater - } else { - Ordering::Less - }); - } - - a.partial_cmp(&b) -} - -impl PartialOrd for Datum { - fn partial_cmp(&self, other: &Self) -> Option { - match (&self.literal, &other.literal, &self.r#type, &other.r#type) { - // generate the arm with same type and same literal - ( - PrimitiveLiteral::Boolean(val), - PrimitiveLiteral::Boolean(other_val), - PrimitiveType::Boolean, - PrimitiveType::Boolean, - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::Int(val), - PrimitiveLiteral::Int(other_val), - PrimitiveType::Int, - PrimitiveType::Int, - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::Long(val), - PrimitiveLiteral::Long(other_val), - PrimitiveType::Long, - PrimitiveType::Long, - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::Float(val), - PrimitiveLiteral::Float(other_val), - PrimitiveType::Float, - PrimitiveType::Float, - ) => iceberg_float_cmp(*val, *other_val), - ( - PrimitiveLiteral::Double(val), - PrimitiveLiteral::Double(other_val), - PrimitiveType::Double, - PrimitiveType::Double, - ) => iceberg_float_cmp(*val, *other_val), - ( - PrimitiveLiteral::Int(val), - PrimitiveLiteral::Int(other_val), - PrimitiveType::Date, - PrimitiveType::Date, - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::Long(val), - PrimitiveLiteral::Long(other_val), - PrimitiveType::Time, - PrimitiveType::Time, - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::Long(val), - PrimitiveLiteral::Long(other_val), - PrimitiveType::Timestamp, - PrimitiveType::Timestamp, - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::Long(val), - PrimitiveLiteral::Long(other_val), - PrimitiveType::Timestamptz, - PrimitiveType::Timestamptz, - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::String(val), - PrimitiveLiteral::String(other_val), - PrimitiveType::String, - PrimitiveType::String, - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::UInt128(val), - PrimitiveLiteral::UInt128(other_val), - PrimitiveType::Uuid, - PrimitiveType::Uuid, - ) => Uuid::from_u128(*val).partial_cmp(&Uuid::from_u128(*other_val)), - ( - PrimitiveLiteral::Binary(val), - PrimitiveLiteral::Binary(other_val), - PrimitiveType::Fixed(_), - PrimitiveType::Fixed(_), - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::Binary(val), - PrimitiveLiteral::Binary(other_val), - PrimitiveType::Binary, - PrimitiveType::Binary, - ) => val.partial_cmp(other_val), - ( - PrimitiveLiteral::Int128(val), - PrimitiveLiteral::Int128(other_val), - PrimitiveType::Decimal { - precision: _, - scale, - }, - PrimitiveType::Decimal { - precision: _, - scale: other_scale, - }, - ) => { - let val = Decimal::from_i128_with_scale(*val, *scale); - let other_val = Decimal::from_i128_with_scale(*other_val, *other_scale); - val.partial_cmp(&other_val) - } - _ => None, - } - } -} - -impl Display for Datum { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match (&self.r#type, &self.literal) { - (_, PrimitiveLiteral::Boolean(val)) => write!(f, "{}", val), - (PrimitiveType::Int, PrimitiveLiteral::Int(val)) => write!(f, "{}", val), - (PrimitiveType::Long, PrimitiveLiteral::Long(val)) => write!(f, "{}", val), - (_, PrimitiveLiteral::Float(val)) => write!(f, "{}", val), - (_, PrimitiveLiteral::Double(val)) => write!(f, "{}", val), - (PrimitiveType::Date, PrimitiveLiteral::Int(val)) => { - write!(f, "{}", days_to_date(*val)) - } - (PrimitiveType::Time, PrimitiveLiteral::Long(val)) => { - write!(f, "{}", microseconds_to_time(*val)) - } - (PrimitiveType::Timestamp, PrimitiveLiteral::Long(val)) => { - write!(f, "{}", microseconds_to_datetime(*val)) - } - (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(val)) => { - write!(f, "{}", microseconds_to_datetimetz(*val)) - } - (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(val)) => { - write!(f, "{}", nanoseconds_to_datetime(*val)) - } - (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(val)) => { - write!(f, "{}", nanoseconds_to_datetimetz(*val)) - } - (_, PrimitiveLiteral::String(val)) => write!(f, r#""{}""#, val), - (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(val)) => { - write!(f, "{}", Uuid::from_u128(*val)) - } - (_, PrimitiveLiteral::Binary(val)) => display_bytes(val, f), - ( - PrimitiveType::Decimal { - precision: _, - scale, - }, - PrimitiveLiteral::Int128(val), - ) => { - write!(f, "{}", Decimal::from_i128_with_scale(*val, *scale)) - } - (_, _) => { - unreachable!() - } - } - } -} - -fn display_bytes(bytes: &[u8], f: &mut Formatter<'_>) -> std::fmt::Result { - let mut s = String::with_capacity(bytes.len() * 2); - for b in bytes { - s.push_str(&format!("{:02X}", b)); - } - f.write_str(&s) -} - -impl From for Literal { - fn from(value: Datum) -> Self { - Literal::Primitive(value.literal) - } -} - -impl From for PrimitiveLiteral { - fn from(value: Datum) -> Self { - value.literal - } -} - -impl Datum { - /// Creates a `Datum` from a `PrimitiveType` and a `PrimitiveLiteral` - pub(crate) fn new(r#type: PrimitiveType, literal: PrimitiveLiteral) -> Self { - Datum { r#type, literal } - } - - /// Create iceberg value from bytes. - /// - /// See [this spec](https://iceberg.apache.org/spec/#binary-single-value-serialization) for reference. - pub fn try_from_bytes(bytes: &[u8], data_type: PrimitiveType) -> Result { - let literal = match data_type { - PrimitiveType::Boolean => { - if bytes.len() == 1 && bytes[0] == 0u8 { - PrimitiveLiteral::Boolean(false) - } else { - PrimitiveLiteral::Boolean(true) - } - } - PrimitiveType::Int => PrimitiveLiteral::Int(i32::from_le_bytes(bytes.try_into()?)), - PrimitiveType::Long => { - if bytes.len() == 4 { - // In the case of an evolved field - PrimitiveLiteral::Long(i32::from_le_bytes(bytes.try_into()?) as i64) - } else { - PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) - } - } - PrimitiveType::Float => { - PrimitiveLiteral::Float(OrderedFloat(f32::from_le_bytes(bytes.try_into()?))) - } - PrimitiveType::Double => { - if bytes.len() == 4 { - // In the case of an evolved field - PrimitiveLiteral::Double(OrderedFloat( - f32::from_le_bytes(bytes.try_into()?) as f64 - )) - } else { - PrimitiveLiteral::Double(OrderedFloat(f64::from_le_bytes(bytes.try_into()?))) - } - } - PrimitiveType::Date => PrimitiveLiteral::Int(i32::from_le_bytes(bytes.try_into()?)), - PrimitiveType::Time => PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)), - PrimitiveType::Timestamp => { - PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) - } - PrimitiveType::Timestamptz => { - PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) - } - PrimitiveType::TimestampNs => { - PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) - } - PrimitiveType::TimestamptzNs => { - PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) - } - PrimitiveType::String => { - PrimitiveLiteral::String(std::str::from_utf8(bytes)?.to_string()) - } - PrimitiveType::Uuid => { - PrimitiveLiteral::UInt128(u128::from_be_bytes(bytes.try_into()?)) - } - PrimitiveType::Fixed(_) => PrimitiveLiteral::Binary(Vec::from(bytes)), - PrimitiveType::Binary => PrimitiveLiteral::Binary(Vec::from(bytes)), - PrimitiveType::Decimal { .. } => { - let unscaled_value = BigInt::from_signed_bytes_be(bytes); - PrimitiveLiteral::Int128(unscaled_value.to_i128().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't convert bytes to i128: {:?}", bytes), - ) - })?) - } - }; - Ok(Datum::new(data_type, literal)) - } - - /// Convert the value to bytes - /// - /// See [this spec](https://iceberg.apache.org/spec/#binary-single-value-serialization) for reference. - pub fn to_bytes(&self) -> Result { - let buf = match &self.literal { - PrimitiveLiteral::Boolean(val) => { - if *val { - ByteBuf::from([1u8]) - } else { - ByteBuf::from([0u8]) - } - } - PrimitiveLiteral::Int(val) => ByteBuf::from(val.to_le_bytes()), - PrimitiveLiteral::Long(val) => ByteBuf::from(val.to_le_bytes()), - PrimitiveLiteral::Float(val) => ByteBuf::from(val.to_le_bytes()), - PrimitiveLiteral::Double(val) => ByteBuf::from(val.to_le_bytes()), - PrimitiveLiteral::String(val) => ByteBuf::from(val.as_bytes()), - PrimitiveLiteral::UInt128(val) => ByteBuf::from(val.to_be_bytes()), - PrimitiveLiteral::Binary(val) => ByteBuf::from(val.as_slice()), - PrimitiveLiteral::Int128(val) => { - let PrimitiveType::Decimal { precision, .. } = self.r#type else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "PrimitiveLiteral Int128 must be PrimitiveType Decimal but got {}", - &self.r#type - ), - )); - }; - - // It's required by iceberg spec that we must keep the minimum - // number of bytes for the value - let Ok(required_bytes) = Type::decimal_required_bytes(precision) else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "PrimitiveType Decimal must has valid precision but got {}", - precision - ), - )); - }; - - // The primitive literal is unscaled value. - let unscaled_value = BigInt::from(*val); - // Convert into two's-complement byte representation of the BigInt - // in big-endian byte order. - let mut bytes = unscaled_value.to_signed_bytes_be(); - // Truncate with required bytes to make sure. - bytes.truncate(required_bytes as usize); - - ByteBuf::from(bytes) - } - PrimitiveLiteral::AboveMax | PrimitiveLiteral::BelowMin => { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot convert AboveMax or BelowMin to bytes".to_string(), - )); - } - }; - - Ok(buf) - } - - /// Creates a boolean value. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; - /// let t = Datum::bool(true); - /// - /// assert_eq!(format!("{}", t), "true".to_string()); - /// assert_eq!( - /// Literal::from(t), - /// Literal::Primitive(PrimitiveLiteral::Boolean(true)) - /// ); - /// ``` - pub fn bool>(t: T) -> Self { - Self { - r#type: PrimitiveType::Boolean, - literal: PrimitiveLiteral::Boolean(t.into()), - } - } - - /// Creates a boolean value from string. - /// See [Parse bool from str](https://doc.rust-lang.org/stable/std/primitive.bool.html#impl-FromStr-for-bool) for reference. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; - /// let t = Datum::bool_from_str("false").unwrap(); - /// - /// assert_eq!(&format!("{}", t), "false"); - /// assert_eq!( - /// Literal::Primitive(PrimitiveLiteral::Boolean(false)), - /// t.into() - /// ); - /// ``` - pub fn bool_from_str>(s: S) -> Result { - let v = s.as_ref().parse::().map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse string to bool.").with_source(e) - })?; - Ok(Self::bool(v)) - } - - /// Creates an 32bit integer. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; - /// let t = Datum::int(23i8); - /// - /// assert_eq!(&format!("{}", t), "23"); - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Int(23)), t.into()); - /// ``` - pub fn int>(t: T) -> Self { - Self { - r#type: PrimitiveType::Int, - literal: PrimitiveLiteral::Int(t.into()), - } - } - - /// Creates an 64bit integer. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; - /// let t = Datum::long(24i8); - /// - /// assert_eq!(&format!("{t}"), "24"); - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Long(24)), t.into()); - /// ``` - pub fn long>(t: T) -> Self { - Self { - r#type: PrimitiveType::Long, - literal: PrimitiveLiteral::Long(t.into()), - } - } - - /// Creates an 32bit floating point number. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; - /// use ordered_float::OrderedFloat; - /// let t = Datum::float(32.1f32); - /// - /// assert_eq!(&format!("{t}"), "32.1"); - /// assert_eq!( - /// Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(32.1))), - /// t.into() - /// ); - /// ``` - pub fn float>(t: T) -> Self { - Self { - r#type: PrimitiveType::Float, - literal: PrimitiveLiteral::Float(OrderedFloat(t.into())), - } - } - - /// Creates an 64bit floating point number. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; - /// use ordered_float::OrderedFloat; - /// let t = Datum::double(32.1f64); - /// - /// assert_eq!(&format!("{t}"), "32.1"); - /// assert_eq!( - /// Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(32.1))), - /// t.into() - /// ); - /// ``` - pub fn double>(t: T) -> Self { - Self { - r#type: PrimitiveType::Double, - literal: PrimitiveLiteral::Double(OrderedFloat(t.into())), - } - } - - /// Creates date literal from number of days from unix epoch directly. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; - /// // 2 days after 1970-01-01 - /// let t = Datum::date(2); - /// - /// assert_eq!(&format!("{t}"), "1970-01-03"); - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Int(2)), t.into()); - /// ``` - pub fn date(days: i32) -> Self { - Self { - r#type: PrimitiveType::Date, - literal: PrimitiveLiteral::Int(days), - } - } - - /// Creates date literal in `%Y-%m-%d` format, assume in utc timezone. - /// - /// See [`NaiveDate::from_str`]. - /// - /// Example - /// ```rust - /// use iceberg::spec::{Datum, Literal}; - /// let t = Datum::date_from_str("1970-01-05").unwrap(); - /// - /// assert_eq!(&format!("{t}"), "1970-01-05"); - /// assert_eq!(Literal::date(4), t.into()); - /// ``` - pub fn date_from_str>(s: S) -> Result { - let t = s.as_ref().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse date from string: {}", s.as_ref()), - ) - .with_source(e) - })?; - - Ok(Self::date(date_from_naive_date(t))) - } - - /// Create date literal from calendar date (year, month and day). - /// - /// See [`NaiveDate::from_ymd_opt`]. - /// - /// Example: - /// - ///```rust - /// use iceberg::spec::{Datum, Literal}; - /// let t = Datum::date_from_ymd(1970, 1, 5).unwrap(); - /// - /// assert_eq!(&format!("{t}"), "1970-01-05"); - /// assert_eq!(Literal::date(4), t.into()); - /// ``` - pub fn date_from_ymd(year: i32, month: u32, day: u32) -> Result { - let t = NaiveDate::from_ymd_opt(year, month, day).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't create date from year: {year}, month: {month}, day: {day}"), - ) - })?; - - Ok(Self::date(date_from_naive_date(t))) - } - - /// Creates time literal in microseconds directly. - /// - /// It will return error when it's negative or too large to fit in 24 hours. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::{Datum, Literal}; - /// let micro_secs = { - /// 1 * 3600 * 1_000_000 + // 1 hour - /// 2 * 60 * 1_000_000 + // 2 minutes - /// 1 * 1_000_000 + // 1 second - /// 888999 // microseconds - /// }; - /// - /// let t = Datum::time_micros(micro_secs).unwrap(); - /// - /// assert_eq!(&format!("{t}"), "01:02:01.888999"); - /// assert_eq!(Literal::time(micro_secs), t.into()); - /// - /// let negative_value = -100; - /// assert!(Datum::time_micros(negative_value).is_err()); - /// - /// let too_large_value = 36 * 60 * 60 * 1_000_000; // Too large to fit in 24 hours. - /// assert!(Datum::time_micros(too_large_value).is_err()); - /// ``` - pub fn time_micros(value: i64) -> Result { - ensure_data_valid!( - (0..=MAX_TIME_VALUE).contains(&value), - "Invalid value for Time type: {}", - value - ); - - Ok(Self { - r#type: PrimitiveType::Time, - literal: PrimitiveLiteral::Long(value), - }) - } - - /// Creates time literal from [`chrono::NaiveTime`]. - fn time_from_naive_time(t: NaiveTime) -> Self { - let duration = t - unix_epoch().time(); - // It's safe to unwrap here since less than 24 hours will never overflow. - let micro_secs = duration.num_microseconds().unwrap(); - - Self { - r#type: PrimitiveType::Time, - literal: PrimitiveLiteral::Long(micro_secs), - } - } - - /// Creates time literal in microseconds in `%H:%M:%S:.f` format. - /// - /// See [`NaiveTime::from_str`] for details. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Datum, Literal}; - /// let t = Datum::time_from_str("01:02:01.888999777").unwrap(); - /// - /// assert_eq!(&format!("{t}"), "01:02:01.888999"); - /// ``` - pub fn time_from_str>(s: S) -> Result { - let t = s.as_ref().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse time from string: {}", s.as_ref()), - ) - .with_source(e) - })?; - - Ok(Self::time_from_naive_time(t)) - } - - /// Creates time literal from hour, minute, second, and microseconds. - /// - /// See [`NaiveTime::from_hms_micro_opt`]. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Datum, Literal}; - /// let t = Datum::time_from_hms_micro(22, 15, 33, 111).unwrap(); - /// - /// assert_eq!(&format!("{t}"), "22:15:33.000111"); - /// ``` - pub fn time_from_hms_micro(hour: u32, min: u32, sec: u32, micro: u32) -> Result { - let t = NaiveTime::from_hms_micro_opt(hour, min, sec, micro) - .ok_or_else(|| Error::new( - ErrorKind::DataInvalid, - format!("Can't create time from hour: {hour}, min: {min}, second: {sec}, microsecond: {micro}"), - ))?; - Ok(Self::time_from_naive_time(t)) - } - - /// Creates a timestamp from unix epoch in microseconds. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// let t = Datum::timestamp_micros(1000); - /// - /// assert_eq!(&format!("{t}"), "1970-01-01 00:00:00.001"); - /// ``` - pub fn timestamp_micros(value: i64) -> Self { - Self { - r#type: PrimitiveType::Timestamp, - literal: PrimitiveLiteral::Long(value), - } - } - - /// Creates a timestamp from unix epoch in nanoseconds. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// let t = Datum::timestamp_nanos(1000); - /// - /// assert_eq!(&format!("{t}"), "1970-01-01 00:00:00.000001"); - /// ``` - pub fn timestamp_nanos(value: i64) -> Self { - Self { - r#type: PrimitiveType::TimestampNs, - literal: PrimitiveLiteral::Long(value), - } - } - - /// Creates a timestamp from [`DateTime`]. - /// - /// Example: - /// - /// ```rust - /// use chrono::{NaiveDate, NaiveDateTime, TimeZone, Utc}; - /// use iceberg::spec::Datum; - /// let t = Datum::timestamp_from_datetime( - /// NaiveDate::from_ymd_opt(1992, 3, 1) - /// .unwrap() - /// .and_hms_micro_opt(1, 2, 3, 88) - /// .unwrap(), - /// ); - /// - /// assert_eq!(&format!("{t}"), "1992-03-01 01:02:03.000088"); - /// ``` - pub fn timestamp_from_datetime(dt: NaiveDateTime) -> Self { - Self::timestamp_micros(dt.and_utc().timestamp_micros()) - } - - /// Parse a timestamp in [`%Y-%m-%dT%H:%M:%S%.f`] format. - /// - /// See [`NaiveDateTime::from_str`]. - /// - /// Example: - /// - /// ```rust - /// use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; - /// use iceberg::spec::{Datum, Literal}; - /// let t = Datum::timestamp_from_str("1992-03-01T01:02:03.000088").unwrap(); - /// - /// assert_eq!(&format!("{t}"), "1992-03-01 01:02:03.000088"); - /// ``` - pub fn timestamp_from_str>(s: S) -> Result { - let dt = s.as_ref().parse::().map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse timestamp.").with_source(e) - })?; - - Ok(Self::timestamp_from_datetime(dt)) - } - - /// Creates a timestamp with timezone from unix epoch in microseconds. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// let t = Datum::timestamptz_micros(1000); - /// - /// assert_eq!(&format!("{t}"), "1970-01-01 00:00:00.001 UTC"); - /// ``` - pub fn timestamptz_micros(value: i64) -> Self { - Self { - r#type: PrimitiveType::Timestamptz, - literal: PrimitiveLiteral::Long(value), - } - } - - /// Creates a timestamp with timezone from unix epoch in nanoseconds. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// let t = Datum::timestamptz_nanos(1000); - /// - /// assert_eq!(&format!("{t}"), "1970-01-01 00:00:00.000001 UTC"); - /// ``` - pub fn timestamptz_nanos(value: i64) -> Self { - Self { - r#type: PrimitiveType::TimestamptzNs, - literal: PrimitiveLiteral::Long(value), - } - } - - /// Creates a timestamp with timezone from [`DateTime`]. - /// Example: - /// - /// ```rust - /// use chrono::{TimeZone, Utc}; - /// use iceberg::spec::Datum; - /// let t = Datum::timestamptz_from_datetime(Utc.timestamp_opt(1000, 0).unwrap()); - /// - /// assert_eq!(&format!("{t}"), "1970-01-01 00:16:40 UTC"); - /// ``` - pub fn timestamptz_from_datetime(dt: DateTime) -> Self { - Self::timestamptz_micros(dt.with_timezone(&Utc).timestamp_micros()) - } - - /// Parse timestamp with timezone in RFC3339 format. - /// - /// See [`DateTime::from_str`]. - /// - /// Example: - /// - /// ```rust - /// use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; - /// use iceberg::spec::{Datum, Literal}; - /// let t = Datum::timestamptz_from_str("1992-03-01T01:02:03.000088+08:00").unwrap(); - /// - /// assert_eq!(&format!("{t}"), "1992-02-29 17:02:03.000088 UTC"); - /// ``` - pub fn timestamptz_from_str>(s: S) -> Result { - let dt = DateTime::::from_str(s.as_ref()).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse datetime.").with_source(e) - })?; - - Ok(Self::timestamptz_from_datetime(dt)) - } - - /// Creates a string literal. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// let t = Datum::string("ss"); - /// - /// assert_eq!(&format!("{t}"), r#""ss""#); - /// ``` - pub fn string(s: S) -> Self { - Self { - r#type: PrimitiveType::String, - literal: PrimitiveLiteral::String(s.to_string()), - } - } - - /// Creates uuid literal. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// use uuid::uuid; - /// let t = Datum::uuid(uuid!("a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8")); - /// - /// assert_eq!(&format!("{t}"), "a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8"); - /// ``` - pub fn uuid(uuid: Uuid) -> Self { - Self { - r#type: PrimitiveType::Uuid, - literal: PrimitiveLiteral::UInt128(uuid.as_u128()), - } - } - - /// Creates uuid from str. See [`Uuid::parse_str`]. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// let t = Datum::uuid_from_str("a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8").unwrap(); - /// - /// assert_eq!(&format!("{t}"), "a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8"); - /// ``` - pub fn uuid_from_str>(s: S) -> Result { - let uuid = Uuid::parse_str(s.as_ref()).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse uuid from string: {}", s.as_ref()), - ) - .with_source(e) - })?; - Ok(Self::uuid(uuid)) - } - - /// Creates a fixed literal from bytes. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; - /// let t = Datum::fixed(vec![1u8, 2u8]); - /// - /// assert_eq!(&format!("{t}"), "0102"); - /// ``` - pub fn fixed>(input: I) -> Self { - let value: Vec = input.into_iter().collect(); - Self { - r#type: PrimitiveType::Fixed(value.len() as u64), - literal: PrimitiveLiteral::Binary(value), - } - } - - /// Creates a binary literal from bytes. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// let t = Datum::binary(vec![1u8, 100u8]); - /// - /// assert_eq!(&format!("{t}"), "0164"); - /// ``` - pub fn binary>(input: I) -> Self { - Self { - r#type: PrimitiveType::Binary, - literal: PrimitiveLiteral::Binary(input.into_iter().collect()), - } - } - - /// Creates decimal literal from string. See [`Decimal::from_str_exact`]. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// use itertools::assert_equal; - /// use rust_decimal::Decimal; - /// let t = Datum::decimal_from_str("123.45").unwrap(); - /// - /// assert_eq!(&format!("{t}"), "123.45"); - /// ``` - pub fn decimal_from_str>(s: S) -> Result { - let decimal = Decimal::from_str_exact(s.as_ref()).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse decimal.").with_source(e) - })?; - - Self::decimal(decimal) - } - - /// Try to create a decimal literal from [`Decimal`]. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// use rust_decimal::Decimal; - /// - /// let t = Datum::decimal(Decimal::new(123, 2)).unwrap(); - /// - /// assert_eq!(&format!("{t}"), "1.23"); - /// ``` - pub fn decimal(value: impl Into) -> Result { - let decimal = value.into(); - let scale = decimal.scale(); - - let r#type = Type::decimal(MAX_DECIMAL_PRECISION, scale)?; - if let Type::Primitive(p) = r#type { - Ok(Self { - r#type: p, - literal: PrimitiveLiteral::Int128(decimal.mantissa()), - }) - } else { - unreachable!("Decimal type must be primitive.") - } - } - - /// Try to create a decimal literal from [`Decimal`] with precision. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Datum; - /// use rust_decimal::Decimal; - /// - /// let t = Datum::decimal_with_precision(Decimal::new(123, 2), 30).unwrap(); - /// - /// assert_eq!(&format!("{t}"), "1.23"); - /// ``` - pub fn decimal_with_precision(value: impl Into, precision: u32) -> Result { - let decimal = value.into(); - let scale = decimal.scale(); - - let available_bytes = Type::decimal_required_bytes(precision)? as usize; - let unscaled_value = BigInt::from(decimal.mantissa()); - let actual_bytes = unscaled_value.to_signed_bytes_be(); - if actual_bytes.len() > available_bytes { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Decimal value {} is too large for precision {}", - decimal, precision - ), - )); - } - - let r#type = Type::decimal(precision, scale)?; - if let Type::Primitive(p) = r#type { - Ok(Self { - r#type: p, - literal: PrimitiveLiteral::Int128(decimal.mantissa()), - }) - } else { - unreachable!("Decimal type must be primitive.") - } - } - - fn i64_to_i32 + PartialOrd>(val: T) -> Datum { - if val > INT_MAX as i64 { - Datum::new(PrimitiveType::Int, PrimitiveLiteral::AboveMax) - } else if val < INT_MIN as i64 { - Datum::new(PrimitiveType::Int, PrimitiveLiteral::BelowMin) - } else { - Datum::int(val.into() as i32) - } - } - - fn i128_to_i32 + PartialOrd>(val: T) -> Datum { - if val > INT_MAX as i128 { - Datum::new(PrimitiveType::Int, PrimitiveLiteral::AboveMax) - } else if val < INT_MIN as i128 { - Datum::new(PrimitiveType::Int, PrimitiveLiteral::BelowMin) - } else { - Datum::int(val.into() as i32) - } - } - - fn i128_to_i64 + PartialOrd>(val: T) -> Datum { - if val > LONG_MAX as i128 { - Datum::new(PrimitiveType::Long, PrimitiveLiteral::AboveMax) - } else if val < LONG_MIN as i128 { - Datum::new(PrimitiveType::Long, PrimitiveLiteral::BelowMin) - } else { - Datum::long(val.into() as i64) - } - } - - fn string_to_i128>(s: S) -> Result { - s.as_ref().parse::().map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse string to i128.").with_source(e) - }) - } - - /// Convert the datum to `target_type`. - pub fn to(self, target_type: &Type) -> Result { - match target_type { - Type::Primitive(target_primitive_type) => { - match (&self.literal, &self.r#type, target_primitive_type) { - (PrimitiveLiteral::Int(val), _, PrimitiveType::Int) => Ok(Datum::int(*val)), - (PrimitiveLiteral::Int(val), _, PrimitiveType::Date) => Ok(Datum::date(*val)), - (PrimitiveLiteral::Int(val), _, PrimitiveType::Long) => Ok(Datum::long(*val)), - (PrimitiveLiteral::Long(val), _, PrimitiveType::Int) => { - Ok(Datum::i64_to_i32(*val)) - } - (PrimitiveLiteral::Long(val), _, PrimitiveType::Timestamp) => { - Ok(Datum::timestamp_micros(*val)) - } - (PrimitiveLiteral::Long(val), _, PrimitiveType::Timestamptz) => { - Ok(Datum::timestamptz_micros(*val)) - } - // Let's wait with nano's until this clears up: https://github.com/apache/iceberg/pull/11775 - (PrimitiveLiteral::Int128(val), _, PrimitiveType::Long) => { - Ok(Datum::i128_to_i64(*val)) - } - - (PrimitiveLiteral::String(val), _, PrimitiveType::Boolean) => { - Datum::bool_from_str(val) - } - (PrimitiveLiteral::String(val), _, PrimitiveType::Int) => { - Datum::string_to_i128(val).map(Datum::i128_to_i32) - } - (PrimitiveLiteral::String(val), _, PrimitiveType::Long) => { - Datum::string_to_i128(val).map(Datum::i128_to_i64) - } - (PrimitiveLiteral::String(val), _, PrimitiveType::Timestamp) => { - Datum::timestamp_from_str(val) - } - (PrimitiveLiteral::String(val), _, PrimitiveType::Timestamptz) => { - Datum::timestamptz_from_str(val) - } - - // TODO: implement more type conversions - (_, self_type, target_type) if self_type == target_type => Ok(self), - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Can't convert datum from {} type to {} type.", - self.r#type, target_primitive_type - ), - )), - } - } - _ => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Can't convert datum from {} type to {} type.", - self.r#type, target_type - ), - )), - } - } - - /// Get the primitive literal from datum. - pub fn literal(&self) -> &PrimitiveLiteral { - &self.literal - } - - /// Get the primitive type from datum. - pub fn data_type(&self) -> &PrimitiveType { - &self.r#type - } - - /// Returns true if the Literal represents a primitive type - /// that can be a NaN, and that it's value is NaN - pub fn is_nan(&self) -> bool { - match self.literal { - PrimitiveLiteral::Double(val) => val.is_nan(), - PrimitiveLiteral::Float(val) => val.is_nan(), - _ => false, - } - } - - /// Returns a human-readable string representation of this literal. - /// - /// For string literals, this returns the raw string value without quotes. - /// For all other literals, it falls back to [`to_string()`]. - pub(crate) fn to_human_string(&self) -> String { - match self.literal() { - PrimitiveLiteral::String(s) => s.to_string(), - _ => self.to_string(), - } - } -} - -/// Map is a collection of key-value pairs with a key type and a value type. -/// It used in Literal::Map, to make it hashable, the order of key-value pairs is stored in a separate vector -/// so that we can hash the map in a deterministic way. But it also means that the order of key-value pairs is matter -/// for the hash value. -/// -/// When converting to Arrow (e.g., for Iceberg), the map should be represented using -/// the default field name "key_value". -/// -/// Example: -/// -/// ```text -/// let key_value_field = Field::new( -/// DEFAULT_MAP_FIELD_NAME, -/// arrow_schema::DataType::Struct(vec![ -/// Arc::new(key_field.clone()), -/// Arc::new(value_field.clone()), -/// ].into()), -/// false -/// ); -/// ''' -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Map { - index: HashMap, - pair: Vec<(Literal, Option)>, -} - -impl Map { - /// Creates a new empty map. - pub fn new() -> Self { - Self { - index: HashMap::new(), - pair: Vec::new(), - } - } - - /// Return the number of key-value pairs in the map. - pub fn len(&self) -> usize { - self.pair.len() - } - - /// Returns true if the map contains no elements. - pub fn is_empty(&self) -> bool { - self.pair.is_empty() - } - - /// Inserts a key-value pair into the map. - /// If the map did not have this key present, None is returned. - /// If the map did have this key present, the value is updated, and the old value is returned. - pub fn insert(&mut self, key: Literal, value: Option) -> Option> { - if let Some(index) = self.index.get(&key) { - let old_value = std::mem::replace(&mut self.pair[*index].1, value); - Some(old_value) - } else { - self.pair.push((key.clone(), value)); - self.index.insert(key, self.pair.len() - 1); - None - } - } - - /// Returns a reference to the value corresponding to the key. - /// If the key is not present in the map, None is returned. - pub fn get(&self, key: &Literal) -> Option<&Option> { - self.index.get(key).map(|index| &self.pair[*index].1) - } - - /// The order of map is matter, so this method used to compare two maps has same key-value pairs without considering the order. - pub fn has_same_content(&self, other: &Map) -> bool { - if self.len() != other.len() { - return false; - } - - for (key, value) in &self.pair { - match other.get(key) { - Some(other_value) if value == other_value => (), - _ => return false, - } - } - - true - } -} - -impl Default for Map { - fn default() -> Self { - Self::new() - } -} - -impl Hash for Map { - fn hash(&self, state: &mut H) { - for (key, value) in &self.pair { - key.hash(state); - value.hash(state); - } - } -} - -impl FromIterator<(Literal, Option)> for Map { - fn from_iter)>>(iter: T) -> Self { - let mut map = Map::new(); - for (key, value) in iter { - map.insert(key, value); - } - map - } -} - -impl IntoIterator for Map { - type Item = (Literal, Option); - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.pair.into_iter() - } -} - -impl From<[(Literal, Option); N]> for Map { - fn from(value: [(Literal, Option); N]) -> Self { - value.iter().cloned().collect() - } -} - -/// Values present in iceberg type -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub enum Literal { - /// A primitive value - Primitive(PrimitiveLiteral), - /// A struct is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema. - /// Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type. - /// Fields may have an optional comment or doc string. Fields can have default values. - Struct(Struct), - /// A list is a collection of values with some element type. - /// The element field has an integer id that is unique in the table schema. - /// Elements can be either optional or required. Element types may be any type. - List(Vec>), - /// A map is a collection of key-value pairs with a key type and a value type. - /// Both the key field and value field each have an integer id that is unique in the table schema. - /// Map keys are required and map values can be either optional or required. Both map keys and map values may be any type, including nested types. - Map(Map), -} - -impl Literal { - /// Creates a boolean value. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// let t = Literal::bool(true); - /// - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Boolean(true)), t); - /// ``` - pub fn bool>(t: T) -> Self { - Self::Primitive(PrimitiveLiteral::Boolean(t.into())) - } - - /// Creates a boolean value from string. - /// See [Parse bool from str](https://doc.rust-lang.org/stable/std/primitive.bool.html#impl-FromStr-for-bool) for reference. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// let t = Literal::bool_from_str("false").unwrap(); - /// - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Boolean(false)), t); - /// ``` - pub fn bool_from_str>(s: S) -> Result { - let v = s.as_ref().parse::().map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse string to bool.").with_source(e) - })?; - Ok(Self::Primitive(PrimitiveLiteral::Boolean(v))) - } - - /// Creates an 32bit integer. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// let t = Literal::int(23i8); - /// - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Int(23)), t); - /// ``` - pub fn int>(t: T) -> Self { - Self::Primitive(PrimitiveLiteral::Int(t.into())) - } - - /// Creates an 64bit integer. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// let t = Literal::long(24i8); - /// - /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Long(24)), t); - /// ``` - pub fn long>(t: T) -> Self { - Self::Primitive(PrimitiveLiteral::Long(t.into())) - } - - /// Creates an 32bit floating point number. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// use ordered_float::OrderedFloat; - /// let t = Literal::float(32.1f32); - /// - /// assert_eq!( - /// Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(32.1))), - /// t - /// ); - /// ``` - pub fn float>(t: T) -> Self { - Self::Primitive(PrimitiveLiteral::Float(OrderedFloat(t.into()))) - } - - /// Creates an 32bit floating point number. - /// - /// Example: - /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// use ordered_float::OrderedFloat; - /// let t = Literal::double(32.1f64); - /// - /// assert_eq!( - /// Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(32.1))), - /// t - /// ); - /// ``` - pub fn double>(t: T) -> Self { - Self::Primitive(PrimitiveLiteral::Double(OrderedFloat(t.into()))) - } - - /// Creates date literal from number of days from unix epoch directly. - pub fn date(days: i32) -> Self { - Self::Primitive(PrimitiveLiteral::Int(days)) - } - - /// Creates a date in `%Y-%m-%d` format, assume in utc timezone. - /// - /// See [`NaiveDate::from_str`]. - /// - /// Example - /// ```rust - /// use iceberg::spec::Literal; - /// let t = Literal::date_from_str("1970-01-03").unwrap(); - /// - /// assert_eq!(Literal::date(2), t); - /// ``` - pub fn date_from_str>(s: S) -> Result { - let t = s.as_ref().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse date from string: {}", s.as_ref()), - ) - .with_source(e) - })?; - - Ok(Self::date(date_from_naive_date(t))) - } - - /// Create a date from calendar date (year, month and day). - /// - /// See [`NaiveDate::from_ymd_opt`]. - /// - /// Example: - /// - ///```rust - /// use iceberg::spec::Literal; - /// let t = Literal::date_from_ymd(1970, 1, 5).unwrap(); - /// - /// assert_eq!(Literal::date(4), t); - /// ``` - pub fn date_from_ymd(year: i32, month: u32, day: u32) -> Result { - let t = NaiveDate::from_ymd_opt(year, month, day).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't create date from year: {year}, month: {month}, day: {day}"), - ) - })?; - - Ok(Self::date(date_from_naive_date(t))) - } - - /// Creates time in microseconds directly - pub fn time(value: i64) -> Self { - Self::Primitive(PrimitiveLiteral::Long(value)) - } - - /// Creates time literal from [`chrono::NaiveTime`]. - fn time_from_naive_time(t: NaiveTime) -> Self { - let duration = t - unix_epoch().time(); - // It's safe to unwrap here since less than 24 hours will never overflow. - let micro_secs = duration.num_microseconds().unwrap(); - - Literal::time(micro_secs) - } - - /// Creates time in microseconds in `%H:%M:%S:.f` format. - /// - /// See [`NaiveTime::from_str`] for details. - /// - /// Example: - /// ```rust - /// use iceberg::spec::Literal; - /// let t = Literal::time_from_str("01:02:01.888999777").unwrap(); - /// - /// let micro_secs = { - /// 1 * 3600 * 1_000_000 + // 1 hour - /// 2 * 60 * 1_000_000 + // 2 minutes - /// 1 * 1_000_000 + // 1 second - /// 888999 // microseconds - /// }; - /// assert_eq!(Literal::time(micro_secs), t); - /// ``` - pub fn time_from_str>(s: S) -> Result { - let t = s.as_ref().parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse time from string: {}", s.as_ref()), - ) - .with_source(e) - })?; - - Ok(Self::time_from_naive_time(t)) - } - - /// Creates time literal from hour, minute, second, and microseconds. - /// - /// See [`NaiveTime::from_hms_micro_opt`]. - /// - /// Example: - /// ```rust - /// use iceberg::spec::Literal; - /// let t = Literal::time_from_hms_micro(22, 15, 33, 111).unwrap(); - /// - /// assert_eq!(Literal::time_from_str("22:15:33.000111").unwrap(), t); - /// ``` - pub fn time_from_hms_micro(hour: u32, min: u32, sec: u32, micro: u32) -> Result { - let t = NaiveTime::from_hms_micro_opt(hour, min, sec, micro) - .ok_or_else(|| Error::new( - ErrorKind::DataInvalid, - format!("Can't create time from hour: {hour}, min: {min}, second: {sec}, microsecond: {micro}"), - ))?; - Ok(Self::time_from_naive_time(t)) - } - - /// Creates a timestamp from unix epoch in microseconds. - pub fn timestamp(value: i64) -> Self { - Self::Primitive(PrimitiveLiteral::Long(value)) - } - - /// Creates a timestamp with timezone from unix epoch in microseconds. - pub fn timestamptz(value: i64) -> Self { - Self::Primitive(PrimitiveLiteral::Long(value)) - } - - /// Creates a timestamp from unix epoch in nanoseconds. - pub(crate) fn timestamp_nano(value: i64) -> Self { - Self::Primitive(PrimitiveLiteral::Long(value)) - } - - /// Creates a timestamp with timezone from unix epoch in nanoseconds. - pub(crate) fn timestamptz_nano(value: i64) -> Self { - Self::Primitive(PrimitiveLiteral::Long(value)) - } - - /// Creates a timestamp from [`DateTime`]. - pub fn timestamp_from_datetime(dt: DateTime) -> Self { - Self::timestamp(dt.with_timezone(&Utc).timestamp_micros()) - } - - /// Creates a timestamp with timezone from [`DateTime`]. - pub fn timestamptz_from_datetime(dt: DateTime) -> Self { - Self::timestamptz(dt.with_timezone(&Utc).timestamp_micros()) - } - - /// Parse a timestamp in RFC3339 format. - /// - /// See [`DateTime::from_str`]. - /// - /// Example: - /// - /// ```rust - /// use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; - /// use iceberg::spec::Literal; - /// let t = Literal::timestamp_from_str("2012-12-12 12:12:12.8899-04:00").unwrap(); - /// - /// let t2 = { - /// let date = NaiveDate::from_ymd_opt(2012, 12, 12).unwrap(); - /// let time = NaiveTime::from_hms_micro_opt(12, 12, 12, 889900).unwrap(); - /// let dt = NaiveDateTime::new(date, time); - /// Literal::timestamp_from_datetime(DateTime::::from_local( - /// dt, - /// FixedOffset::west_opt(4 * 3600).unwrap(), - /// )) - /// }; - /// - /// assert_eq!(t, t2); - /// ``` - pub fn timestamp_from_str>(s: S) -> Result { - let dt = DateTime::::from_str(s.as_ref()).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse datetime.").with_source(e) - })?; - - Ok(Self::timestamp_from_datetime(dt)) - } - - /// Similar to [`Literal::timestamp_from_str`], but return timestamp with timezone literal. - pub fn timestamptz_from_str>(s: S) -> Result { - let dt = DateTime::::from_str(s.as_ref()).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse datetime.").with_source(e) - })?; - - Ok(Self::timestamptz_from_datetime(dt)) - } - - /// Creates a string literal. - pub fn string(s: S) -> Self { - Self::Primitive(PrimitiveLiteral::String(s.to_string())) - } - - /// Creates uuid literal. - pub fn uuid(uuid: Uuid) -> Self { - Self::Primitive(PrimitiveLiteral::UInt128(uuid.as_u128())) - } - - /// Creates uuid from str. See [`Uuid::parse_str`]. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Literal; - /// use uuid::Uuid; - /// let t1 = Literal::uuid_from_str("a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8").unwrap(); - /// let t2 = Literal::uuid(Uuid::from_u128_le(0xd8d7d6d5d4d3d2d1c2c1b2b1a4a3a2a1)); - /// - /// assert_eq!(t1, t2); - /// ``` - pub fn uuid_from_str>(s: S) -> Result { - let uuid = Uuid::parse_str(s.as_ref()).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Can't parse uuid from string: {}", s.as_ref()), - ) - .with_source(e) - })?; - Ok(Self::uuid(uuid)) - } - - /// Creates a fixed literal from bytes. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// let t1 = Literal::fixed(vec![1u8, 2u8]); - /// let t2 = Literal::Primitive(PrimitiveLiteral::Binary(vec![1u8, 2u8])); - /// - /// assert_eq!(t1, t2); - /// ``` - pub fn fixed>(input: I) -> Self { - Literal::Primitive(PrimitiveLiteral::Binary(input.into_iter().collect())) - } - - /// Creates a binary literal from bytes. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::{Literal, PrimitiveLiteral}; - /// let t1 = Literal::binary(vec![1u8, 2u8]); - /// let t2 = Literal::Primitive(PrimitiveLiteral::Binary(vec![1u8, 2u8])); - /// - /// assert_eq!(t1, t2); - /// ``` - pub fn binary>(input: I) -> Self { - Literal::Primitive(PrimitiveLiteral::Binary(input.into_iter().collect())) - } - - /// Creates a decimal literal. - pub fn decimal(decimal: i128) -> Self { - Self::Primitive(PrimitiveLiteral::Int128(decimal)) - } - - /// Creates decimal literal from string. See [`Decimal::from_str_exact`]. - /// - /// Example: - /// - /// ```rust - /// use iceberg::spec::Literal; - /// use rust_decimal::Decimal; - /// let t1 = Literal::decimal(12345); - /// let t2 = Literal::decimal_from_str("123.45").unwrap(); - /// - /// assert_eq!(t1, t2); - /// ``` - pub fn decimal_from_str>(s: S) -> Result { - let decimal = Decimal::from_str_exact(s.as_ref()).map_err(|e| { - Error::new(ErrorKind::DataInvalid, "Can't parse decimal.").with_source(e) - })?; - Ok(Self::decimal(decimal.mantissa())) - } - - /// Attempts to convert the Literal to a PrimitiveLiteral - pub fn as_primitive_literal(&self) -> Option { - match self { - Literal::Primitive(primitive) => Some(primitive.clone()), - _ => None, - } - } -} - -/// The partition struct stores the tuple of partition values for each file. -/// Its type is derived from the partition fields of the partition spec used to write the manifest file. -/// In v2, the partition struct’s field ids must match the ids from the partition spec. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Struct { - /// Vector to store the field values - fields: Vec>, -} - -impl Struct { - /// Create a empty struct. - pub fn empty() -> Self { - Self { fields: Vec::new() } - } - - /// Create a iterator to read the field in order of field_value. - pub fn iter(&self) -> impl ExactSizeIterator> { - self.fields.iter().map(|field| field.as_ref()) - } - - /// returns true if the field at position `index` is null - pub fn is_null_at_index(&self, index: usize) -> bool { - self.fields[index].is_none() - } - - /// Return fields in the struct. - pub fn fields(&self) -> &[Option] { - &self.fields - } -} - -impl Index for Struct { - type Output = Option; - - fn index(&self, idx: usize) -> &Self::Output { - &self.fields[idx] - } -} - -impl IntoIterator for Struct { - type Item = Option; - - type IntoIter = std::vec::IntoIter>; - - fn into_iter(self) -> Self::IntoIter { - self.fields.into_iter() - } -} - -impl FromIterator> for Struct { - fn from_iter>>(iter: I) -> Self { - Struct { - fields: iter.into_iter().collect(), - } - } -} - -impl Literal { - /// Create iceberg value from a json value - /// - /// See [this spec](https://iceberg.apache.org/spec/#json-single-value-serialization) for reference. - pub fn try_from_json(value: JsonValue, data_type: &Type) -> Result> { - match data_type { - Type::Primitive(primitive) => match (primitive, value) { - (PrimitiveType::Boolean, JsonValue::Bool(bool)) => { - Ok(Some(Literal::Primitive(PrimitiveLiteral::Boolean(bool)))) - } - (PrimitiveType::Int, JsonValue::Number(number)) => { - Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( - number - .as_i64() - .ok_or(Error::new( - crate::ErrorKind::DataInvalid, - "Failed to convert json number to int", - ))? - .try_into()?, - )))) - } - (PrimitiveType::Long, JsonValue::Number(number)) => Ok(Some(Literal::Primitive( - PrimitiveLiteral::Long(number.as_i64().ok_or(Error::new( - crate::ErrorKind::DataInvalid, - "Failed to convert json number to long", - ))?), - ))), - (PrimitiveType::Float, JsonValue::Number(number)) => Ok(Some(Literal::Primitive( - PrimitiveLiteral::Float(OrderedFloat(number.as_f64().ok_or(Error::new( - crate::ErrorKind::DataInvalid, - "Failed to convert json number to float", - ))? as f32)), - ))), - (PrimitiveType::Double, JsonValue::Number(number)) => Ok(Some(Literal::Primitive( - PrimitiveLiteral::Double(OrderedFloat(number.as_f64().ok_or(Error::new( - crate::ErrorKind::DataInvalid, - "Failed to convert json number to double", - ))?)), - ))), - (PrimitiveType::Date, JsonValue::String(s)) => { - Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( - date::date_to_days(&NaiveDate::parse_from_str(&s, "%Y-%m-%d")?), - )))) - } - (PrimitiveType::Time, JsonValue::String(s)) => { - Ok(Some(Literal::Primitive(PrimitiveLiteral::Long( - time::time_to_microseconds(&NaiveTime::parse_from_str(&s, "%H:%M:%S%.f")?), - )))) - } - (PrimitiveType::Timestamp, JsonValue::String(s)) => Ok(Some(Literal::Primitive( - PrimitiveLiteral::Long(timestamp::datetime_to_microseconds( - &NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f")?, - )), - ))), - (PrimitiveType::Timestamptz, JsonValue::String(s)) => { - Ok(Some(Literal::Primitive(PrimitiveLiteral::Long( - timestamptz::datetimetz_to_microseconds(&Utc.from_utc_datetime( - &NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f+00:00")?, - )), - )))) - } - (PrimitiveType::String, JsonValue::String(s)) => { - Ok(Some(Literal::Primitive(PrimitiveLiteral::String(s)))) - } - (PrimitiveType::Uuid, JsonValue::String(s)) => Ok(Some(Literal::Primitive( - PrimitiveLiteral::UInt128(Uuid::parse_str(&s)?.as_u128()), - ))), - (PrimitiveType::Fixed(_), JsonValue::String(_)) => todo!(), - (PrimitiveType::Binary, JsonValue::String(_)) => todo!(), - ( - PrimitiveType::Decimal { - precision: _, - scale, - }, - JsonValue::String(s), - ) => { - let mut decimal = Decimal::from_str_exact(&s)?; - decimal.rescale(*scale); - Ok(Some(Literal::Primitive(PrimitiveLiteral::Int128( - decimal.mantissa(), - )))) - } - (_, JsonValue::Null) => Ok(None), - (i, j) => Err(Error::new( - crate::ErrorKind::DataInvalid, - format!( - "The json value {} doesn't fit to the iceberg type {}.", - j, i - ), - )), - }, - Type::Struct(schema) => { - if let JsonValue::Object(mut object) = value { - Ok(Some(Literal::Struct(Struct::from_iter( - schema.fields().iter().map(|field| { - object.remove(&field.id.to_string()).and_then(|value| { - Literal::try_from_json(value, &field.field_type) - .and_then(|value| { - value.ok_or(Error::new( - ErrorKind::DataInvalid, - "Key of map cannot be null", - )) - }) - .ok() - }) - }), - )))) - } else { - Err(Error::new( - crate::ErrorKind::DataInvalid, - "The json value for a struct type must be an object.", - )) - } - } - Type::List(list) => { - if let JsonValue::Array(array) = value { - Ok(Some(Literal::List( - array - .into_iter() - .map(|value| { - Literal::try_from_json(value, &list.element_field.field_type) - }) - .collect::>>()?, - ))) - } else { - Err(Error::new( - crate::ErrorKind::DataInvalid, - "The json value for a list type must be an array.", - )) - } - } - Type::Map(map) => { - if let JsonValue::Object(mut object) = value { - if let (Some(JsonValue::Array(keys)), Some(JsonValue::Array(values))) = - (object.remove("keys"), object.remove("values")) - { - Ok(Some(Literal::Map(Map::from_iter( - keys.into_iter() - .zip(values.into_iter()) - .map(|(key, value)| { - Ok(( - Literal::try_from_json(key, &map.key_field.field_type) - .and_then(|value| { - value.ok_or(Error::new( - ErrorKind::DataInvalid, - "Key of map cannot be null", - )) - })?, - Literal::try_from_json(value, &map.value_field.field_type)?, - )) - }) - .collect::>>()?, - )))) - } else { - Err(Error::new( - crate::ErrorKind::DataInvalid, - "The json value for a list type must be an array.", - )) - } - } else { - Err(Error::new( - crate::ErrorKind::DataInvalid, - "The json value for a list type must be an array.", - )) - } - } - } - } - - /// Converting iceberg value to json value. - /// - /// See [this spec](https://iceberg.apache.org/spec/#json-single-value-serialization) for reference. - pub fn try_into_json(self, r#type: &Type) -> Result { - match (self, r#type) { - (Literal::Primitive(prim), Type::Primitive(prim_type)) => match (prim_type, prim) { - (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(val)) => { - Ok(JsonValue::Bool(val)) - } - (PrimitiveType::Int, PrimitiveLiteral::Int(val)) => { - Ok(JsonValue::Number((val).into())) - } - (PrimitiveType::Long, PrimitiveLiteral::Long(val)) => { - Ok(JsonValue::Number((val).into())) - } - (PrimitiveType::Float, PrimitiveLiteral::Float(val)) => { - match Number::from_f64(val.0 as f64) { - Some(number) => Ok(JsonValue::Number(number)), - None => Ok(JsonValue::Null), - } - } - (PrimitiveType::Double, PrimitiveLiteral::Double(val)) => { - match Number::from_f64(val.0) { - Some(number) => Ok(JsonValue::Number(number)), - None => Ok(JsonValue::Null), - } - } - (PrimitiveType::Date, PrimitiveLiteral::Int(val)) => { - Ok(JsonValue::String(date::days_to_date(val).to_string())) - } - (PrimitiveType::Time, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( - time::microseconds_to_time(val).to_string(), - )), - (PrimitiveType::Timestamp, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( - timestamp::microseconds_to_datetime(val) - .format("%Y-%m-%dT%H:%M:%S%.f") - .to_string(), - )), - (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( - timestamptz::microseconds_to_datetimetz(val) - .format("%Y-%m-%dT%H:%M:%S%.f+00:00") - .to_string(), - )), - (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( - timestamp::nanoseconds_to_datetime(val) - .format("%Y-%m-%dT%H:%M:%S%.f") - .to_string(), - )), - (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(val)) => { - Ok(JsonValue::String( - timestamptz::nanoseconds_to_datetimetz(val) - .format("%Y-%m-%dT%H:%M:%S%.f+00:00") - .to_string(), - )) - } - (PrimitiveType::String, PrimitiveLiteral::String(val)) => { - Ok(JsonValue::String(val.clone())) - } - (_, PrimitiveLiteral::UInt128(val)) => { - Ok(JsonValue::String(Uuid::from_u128(val).to_string())) - } - (_, PrimitiveLiteral::Binary(val)) => Ok(JsonValue::String(val.iter().fold( - String::new(), - |mut acc, x| { - acc.push_str(&format!("{:x}", x)); - acc - }, - ))), - (_, PrimitiveLiteral::Int128(val)) => match r#type { - Type::Primitive(PrimitiveType::Decimal { - precision: _precision, - scale, - }) => { - let decimal = Decimal::try_from_i128_with_scale(val, *scale)?; - Ok(JsonValue::String(decimal.to_string())) - } - _ => Err(Error::new( - ErrorKind::DataInvalid, - "The iceberg type for decimal literal must be decimal.", - ))?, - }, - _ => Err(Error::new( - ErrorKind::DataInvalid, - "The iceberg value doesn't fit to the iceberg type.", - )), - }, - (Literal::Struct(s), Type::Struct(struct_type)) => { - let mut id_and_value = Vec::with_capacity(struct_type.fields().len()); - for (value, field) in s.into_iter().zip(struct_type.fields()) { - let json = match value { - Some(val) => val.try_into_json(&field.field_type)?, - None => JsonValue::Null, - }; - id_and_value.push((field.id.to_string(), json)); - } - Ok(JsonValue::Object(JsonMap::from_iter(id_and_value))) - } - (Literal::List(list), Type::List(list_type)) => Ok(JsonValue::Array( - list.into_iter() - .map(|opt| match opt { - Some(literal) => literal.try_into_json(&list_type.element_field.field_type), - None => Ok(JsonValue::Null), - }) - .collect::>>()?, - )), - (Literal::Map(map), Type::Map(map_type)) => { - let mut object = JsonMap::with_capacity(2); - let mut json_keys = Vec::with_capacity(map.len()); - let mut json_values = Vec::with_capacity(map.len()); - for (key, value) in map.into_iter() { - json_keys.push(key.try_into_json(&map_type.key_field.field_type)?); - json_values.push(match value { - Some(literal) => literal.try_into_json(&map_type.value_field.field_type)?, - None => JsonValue::Null, - }); - } - object.insert("keys".to_string(), JsonValue::Array(json_keys)); - object.insert("values".to_string(), JsonValue::Array(json_values)); - Ok(JsonValue::Object(object)) - } - (value, r#type) => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "The iceberg value {:?} doesn't fit to the iceberg type {}.", - value, r#type - ), - )), - } - } - - /// Convert Value to the any type - pub fn into_any(self) -> Box { - match self { - Literal::Primitive(prim) => match prim { - PrimitiveLiteral::Boolean(any) => Box::new(any), - PrimitiveLiteral::Int(any) => Box::new(any), - PrimitiveLiteral::Long(any) => Box::new(any), - PrimitiveLiteral::Float(any) => Box::new(any), - PrimitiveLiteral::Double(any) => Box::new(any), - PrimitiveLiteral::Binary(any) => Box::new(any), - PrimitiveLiteral::String(any) => Box::new(any), - PrimitiveLiteral::UInt128(any) => Box::new(any), - PrimitiveLiteral::Int128(any) => Box::new(any), - PrimitiveLiteral::AboveMax | PrimitiveLiteral::BelowMin => unimplemented!(), - }, - _ => unimplemented!(), - } - } -} - -mod date { - use chrono::{DateTime, NaiveDate, TimeDelta, TimeZone, Utc}; - - pub(crate) fn date_to_days(date: &NaiveDate) -> i32 { - date.signed_duration_since( - // This is always the same and shouldn't fail - NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(), - ) - .num_days() as i32 - } - - pub(crate) fn days_to_date(days: i32) -> NaiveDate { - // This shouldn't fail until the year 262000 - (chrono::DateTime::UNIX_EPOCH + TimeDelta::try_days(days as i64).unwrap()) - .naive_utc() - .date() - } - - /// Returns unix epoch. - pub(crate) fn unix_epoch() -> DateTime { - Utc.timestamp_nanos(0) - } - - /// Creates date literal from `NaiveDate`, assuming it's utc timezone. - pub(crate) fn date_from_naive_date(date: NaiveDate) -> i32 { - (date - unix_epoch().date_naive()).num_days() as i32 - } -} - -mod time { - use chrono::NaiveTime; - - pub(crate) fn time_to_microseconds(time: &NaiveTime) -> i64 { - time.signed_duration_since( - // This is always the same and shouldn't fail - NaiveTime::from_num_seconds_from_midnight_opt(0, 0).unwrap(), - ) - .num_microseconds() - .unwrap() - } - - pub(crate) fn microseconds_to_time(micros: i64) -> NaiveTime { - let (secs, rem) = (micros / 1_000_000, micros % 1_000_000); - - NaiveTime::from_num_seconds_from_midnight_opt(secs as u32, rem as u32 * 1_000).unwrap() - } -} - -mod timestamp { - use chrono::{DateTime, NaiveDateTime}; - - pub(crate) fn datetime_to_microseconds(time: &NaiveDateTime) -> i64 { - time.and_utc().timestamp_micros() - } - - pub(crate) fn microseconds_to_datetime(micros: i64) -> NaiveDateTime { - // This shouldn't fail until the year 262000 - DateTime::from_timestamp_micros(micros).unwrap().naive_utc() - } - - pub(crate) fn nanoseconds_to_datetime(nanos: i64) -> NaiveDateTime { - DateTime::from_timestamp_nanos(nanos).naive_utc() - } -} - -mod timestamptz { - use chrono::{DateTime, Utc}; - - pub(crate) fn datetimetz_to_microseconds(time: &DateTime) -> i64 { - time.timestamp_micros() - } - - pub(crate) fn microseconds_to_datetimetz(micros: i64) -> DateTime { - let (secs, rem) = (micros / 1_000_000, micros % 1_000_000); - - DateTime::from_timestamp(secs, rem as u32 * 1_000).unwrap() - } - - pub(crate) fn nanoseconds_to_datetimetz(nanos: i64) -> DateTime { - let (secs, rem) = (nanos / 1_000_000_000, nanos % 1_000_000_000); - - DateTime::from_timestamp(secs, rem as u32).unwrap() - } -} - -mod _serde { - use serde::de::Visitor; - use serde::ser::{SerializeMap, SerializeSeq, SerializeStruct}; - use serde::{Deserialize, Serialize}; - use serde_bytes::ByteBuf; - use serde_derive::{Deserialize as DeserializeDerive, Serialize as SerializeDerive}; - - use super::{Literal, Map, PrimitiveLiteral}; - use crate::spec::{MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, PrimitiveType, Type}; - use crate::{Error, ErrorKind}; - - #[derive(SerializeDerive, DeserializeDerive, Debug)] - #[serde(transparent)] - /// Raw literal representation used for serde. The serialize way is used for Avro serializer. - pub struct RawLiteral(RawLiteralEnum); - - impl RawLiteral { - /// Covert literal to raw literal. - pub fn try_from(literal: Literal, ty: &Type) -> Result { - Ok(Self(RawLiteralEnum::try_from(literal, ty)?)) - } - - /// Convert raw literal to literal. - pub fn try_into(self, ty: &Type) -> Result, Error> { - self.0.try_into(ty) - } - } - - #[derive(SerializeDerive, Clone, Debug)] - #[serde(untagged)] - enum RawLiteralEnum { - Null, - Boolean(bool), - Int(i32), - Long(i64), - Float(f32), - Double(f64), - String(String), - Bytes(ByteBuf), - List(List), - StringMap(StringMap), - Record(Record), - } - - #[derive(Clone, Debug)] - struct Record { - required: Vec<(String, RawLiteralEnum)>, - optional: Vec<(String, Option)>, - } - - impl Serialize for Record { - fn serialize(&self, serializer: S) -> Result - where S: serde::Serializer { - let len = self.required.len() + self.optional.len(); - let mut record = serializer.serialize_struct("", len)?; - for (k, v) in &self.required { - record.serialize_field(Box::leak(k.clone().into_boxed_str()), &v)?; - } - for (k, v) in &self.optional { - record.serialize_field(Box::leak(k.clone().into_boxed_str()), &v)?; - } - record.end() - } - } - - #[derive(Clone, Debug)] - struct List { - list: Vec>, - required: bool, - } - - impl Serialize for List { - fn serialize(&self, serializer: S) -> Result - where S: serde::Serializer { - let mut seq = serializer.serialize_seq(Some(self.list.len()))?; - for value in &self.list { - if self.required { - seq.serialize_element(value.as_ref().ok_or_else(|| { - serde::ser::Error::custom( - "List element is required, element cannot be null", - ) - })?)?; - } else { - seq.serialize_element(&value)?; - } - } - seq.end() - } - } - - #[derive(Clone, Debug)] - struct StringMap { - raw: Vec<(String, Option)>, - required: bool, - } - - impl Serialize for StringMap { - fn serialize(&self, serializer: S) -> Result - where S: serde::Serializer { - let mut map = serializer.serialize_map(Some(self.raw.len()))?; - for (k, v) in &self.raw { - if self.required { - map.serialize_entry( - k, - v.as_ref().ok_or_else(|| { - serde::ser::Error::custom( - "Map element is required, element cannot be null", - ) - })?, - )?; - } else { - map.serialize_entry(k, v)?; - } - } - map.end() - } - } - - impl<'de> Deserialize<'de> for RawLiteralEnum { - fn deserialize(deserializer: D) -> Result - where D: serde::Deserializer<'de> { - struct RawLiteralVisitor; - impl<'de> Visitor<'de> for RawLiteralVisitor { - type Value = RawLiteralEnum; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("expect") - } - - fn visit_bool(self, v: bool) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::Boolean(v)) - } - - fn visit_i32(self, v: i32) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::Int(v)) - } - - fn visit_i64(self, v: i64) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::Long(v)) - } - - /// Used in json - fn visit_u64(self, v: u64) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::Long(v as i64)) - } - - fn visit_f32(self, v: f32) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::Float(v)) - } - - fn visit_f64(self, v: f64) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::Double(v)) - } - - fn visit_str(self, v: &str) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::String(v.to_string())) - } - - fn visit_bytes(self, v: &[u8]) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::Bytes(ByteBuf::from(v))) - } - - fn visit_borrowed_str(self, v: &'de str) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::String(v.to_string())) - } - - fn visit_unit(self) -> Result - where E: serde::de::Error { - Ok(RawLiteralEnum::Null) - } - - fn visit_map(self, mut map: A) -> Result - where A: serde::de::MapAccess<'de> { - let mut required = Vec::new(); - while let Some(key) = map.next_key::()? { - let value = map.next_value::()?; - required.push((key, value)); - } - Ok(RawLiteralEnum::Record(Record { - required, - optional: Vec::new(), - })) - } - - fn visit_seq(self, mut seq: A) -> Result - where A: serde::de::SeqAccess<'de> { - let mut list = Vec::new(); - while let Some(value) = seq.next_element::()? { - list.push(Some(value)); - } - Ok(RawLiteralEnum::List(List { - list, - // `required` only used in serialize, just set default in deserialize. - required: false, - })) - } - } - deserializer.deserialize_any(RawLiteralVisitor) - } - } - - impl RawLiteralEnum { - pub fn try_from(literal: Literal, ty: &Type) -> Result { - let raw = match literal { - Literal::Primitive(prim) => match prim { - super::PrimitiveLiteral::Boolean(v) => RawLiteralEnum::Boolean(v), - super::PrimitiveLiteral::Int(v) => RawLiteralEnum::Int(v), - super::PrimitiveLiteral::Long(v) => RawLiteralEnum::Long(v), - super::PrimitiveLiteral::Float(v) => RawLiteralEnum::Float(v.0), - super::PrimitiveLiteral::Double(v) => RawLiteralEnum::Double(v.0), - super::PrimitiveLiteral::String(v) => RawLiteralEnum::String(v), - super::PrimitiveLiteral::UInt128(v) => { - RawLiteralEnum::Bytes(ByteBuf::from(v.to_be_bytes())) - } - super::PrimitiveLiteral::Binary(v) => RawLiteralEnum::Bytes(ByteBuf::from(v)), - super::PrimitiveLiteral::Int128(v) => { - RawLiteralEnum::Bytes(ByteBuf::from(v.to_be_bytes())) - } - super::PrimitiveLiteral::AboveMax | super::PrimitiveLiteral::BelowMin => { - return Err(Error::new( - ErrorKind::DataInvalid, - "Can't convert AboveMax or BelowMax", - )); - } - }, - Literal::Struct(r#struct) => { - let mut required = Vec::new(); - let mut optional = Vec::new(); - if let Type::Struct(struct_ty) = ty { - for (value, field) in r#struct.into_iter().zip(struct_ty.fields()) { - if field.required { - if let Some(value) = value { - required.push(( - field.name.clone(), - RawLiteralEnum::try_from(value, &field.field_type)?, - )); - } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Can't convert null to required field", - )); - } - } else if let Some(value) = value { - optional.push(( - field.name.clone(), - Some(RawLiteralEnum::try_from(value, &field.field_type)?), - )); - } else { - optional.push((field.name.clone(), None)); - } - } - } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Type {} should be a struct", ty), - )); - } - RawLiteralEnum::Record(Record { required, optional }) - } - Literal::List(list) => { - if let Type::List(list_ty) = ty { - let list = list - .into_iter() - .map(|v| { - v.map(|v| { - RawLiteralEnum::try_from(v, &list_ty.element_field.field_type) - }) - .transpose() - }) - .collect::>()?; - RawLiteralEnum::List(List { - list, - required: list_ty.element_field.required, - }) - } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Type {} should be a list", ty), - )); - } - } - Literal::Map(map) => { - if let Type::Map(map_ty) = ty { - if let Type::Primitive(PrimitiveType::String) = *map_ty.key_field.field_type - { - let mut raw = Vec::with_capacity(map.len()); - for (k, v) in map { - if let Literal::Primitive(PrimitiveLiteral::String(k)) = k { - raw.push(( - k, - v.map(|v| { - RawLiteralEnum::try_from( - v, - &map_ty.value_field.field_type, - ) - }) - .transpose()?, - )); - } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "literal type is inconsistent with type", - )); - } - } - RawLiteralEnum::StringMap(StringMap { - raw, - required: map_ty.value_field.required, - }) - } else { - let list = map.into_iter().map(|(k,v)| { - let raw_k = - RawLiteralEnum::try_from(k, &map_ty.key_field.field_type)?; - let raw_v = v - .map(|v| { - RawLiteralEnum::try_from(v, &map_ty.value_field.field_type) - }) - .transpose()?; - if map_ty.value_field.required { - Ok(Some(RawLiteralEnum::Record(Record { - required: vec![ - (MAP_KEY_FIELD_NAME.to_string(), raw_k), - (MAP_VALUE_FIELD_NAME.to_string(), raw_v.ok_or_else(||Error::new(ErrorKind::DataInvalid, "Map value is required, value cannot be null"))?), - ], - optional: vec![], - }))) - } else { - Ok(Some(RawLiteralEnum::Record(Record { - required: vec![ - (MAP_KEY_FIELD_NAME.to_string(), raw_k), - ], - optional: vec![ - (MAP_VALUE_FIELD_NAME.to_string(), raw_v) - ], - }))) - } - }).collect::>()?; - RawLiteralEnum::List(List { - list, - required: true, - }) - } - } else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Type {} should be a map", ty), - )); - } - } - }; - Ok(raw) - } - - pub fn try_into(self, ty: &Type) -> Result, Error> { - let invalid_err = |v: &str| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Unable to convert raw literal ({}) fail convert to type {} for: type mismatch", - v, ty - ), - ) - }; - let invalid_err_with_reason = |v: &str, reason: &str| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Unable to convert raw literal ({}) fail convert to type {} for: {}", - v, ty, reason - ), - ) - }; - match self { - RawLiteralEnum::Null => Ok(None), - RawLiteralEnum::Boolean(v) => Ok(Some(Literal::bool(v))), - RawLiteralEnum::Int(v) => match ty { - Type::Primitive(PrimitiveType::Int) => Ok(Some(Literal::int(v))), - Type::Primitive(PrimitiveType::Long) => Ok(Some(Literal::long(i64::from(v)))), - Type::Primitive(PrimitiveType::Date) => Ok(Some(Literal::date(v))), - _ => Err(invalid_err("int")), - }, - RawLiteralEnum::Long(v) => match ty { - Type::Primitive(PrimitiveType::Int) => Ok(Some(Literal::int( - i32::try_from(v).map_err(|_| invalid_err("long"))?, - ))), - Type::Primitive(PrimitiveType::Date) => Ok(Some(Literal::date( - i32::try_from(v).map_err(|_| invalid_err("long"))?, - ))), - Type::Primitive(PrimitiveType::Long) => Ok(Some(Literal::long(v))), - Type::Primitive(PrimitiveType::Time) => Ok(Some(Literal::time(v))), - Type::Primitive(PrimitiveType::Timestamp) => Ok(Some(Literal::timestamp(v))), - Type::Primitive(PrimitiveType::Timestamptz) => { - Ok(Some(Literal::timestamptz(v))) - } - _ => Err(invalid_err("long")), - }, - RawLiteralEnum::Float(v) => match ty { - Type::Primitive(PrimitiveType::Float) => Ok(Some(Literal::float(v))), - Type::Primitive(PrimitiveType::Double) => { - Ok(Some(Literal::double(f64::from(v)))) - } - _ => Err(invalid_err("float")), - }, - RawLiteralEnum::Double(v) => match ty { - Type::Primitive(PrimitiveType::Float) => { - let v_32 = v as f32; - if v_32.is_finite() { - let v_64 = f64::from(v_32); - if (v_64 - v).abs() > f32::EPSILON as f64 { - // there is a precision loss - return Err(invalid_err("double")); - } - } - Ok(Some(Literal::float(v_32))) - } - Type::Primitive(PrimitiveType::Double) => Ok(Some(Literal::double(v))), - _ => Err(invalid_err("double")), - }, - RawLiteralEnum::String(v) => match ty { - Type::Primitive(PrimitiveType::String) => Ok(Some(Literal::string(v))), - _ => Err(invalid_err("string")), - }, - // # TODO:https://github.com/apache/iceberg-rust/issues/86 - // rust avro don't support deserialize any bytes representation now. - RawLiteralEnum::Bytes(_) => Err(invalid_err_with_reason( - "bytes", - "todo: rust avro doesn't support deserialize any bytes representation now", - )), - RawLiteralEnum::List(v) => match ty { - Type::List(ty) => Ok(Some(Literal::List( - v.list - .into_iter() - .map(|v| { - if let Some(v) = v { - v.try_into(&ty.element_field.field_type) - } else { - Ok(None) - } - }) - .collect::>()?, - ))), - Type::Map(map_ty) => { - let key_ty = map_ty.key_field.field_type.as_ref(); - let value_ty = map_ty.value_field.field_type.as_ref(); - let mut map = Map::new(); - for k_v in v.list { - let k_v = k_v.ok_or_else(|| invalid_err_with_reason("list","In deserialize, None will be represented as Some(RawLiteral::Null), all element in list must be valid"))?; - if let RawLiteralEnum::Record(Record { - required, - optional: _, - }) = k_v - { - if required.len() != 2 { - return Err(invalid_err_with_reason( - "list", - "Record must contains two element(key and value) of array", - )); - } - let mut key = None; - let mut value = None; - required.into_iter().for_each(|(k, v)| { - if k == MAP_KEY_FIELD_NAME { - key = Some(v); - } else if k == MAP_VALUE_FIELD_NAME { - value = Some(v); - } - }); - match (key, value) { - (Some(k), Some(v)) => { - let key = k.try_into(key_ty)?.ok_or_else(|| { - invalid_err_with_reason( - "list", - "Key element in Map must be valid", - ) - })?; - let value = v.try_into(value_ty)?; - if map_ty.value_field.required && value.is_none() { - return Err(invalid_err_with_reason( - "list", - "Value element is required in this Map", - )); - } - map.insert(key, value); - } - _ => { - return Err(invalid_err_with_reason( - "list", - "The elements of record in list are not key and value", - )); - } - } - } else { - return Err(invalid_err_with_reason( - "list", - "Map should represented as record array.", - )); - } - } - Ok(Some(Literal::Map(map))) - } - Type::Primitive(PrimitiveType::Uuid) => { - if v.list.len() != 16 { - return Err(invalid_err_with_reason( - "list", - "The length of list should be 16", - )); - } - let mut bytes = [0u8; 16]; - for (i, v) in v.list.iter().enumerate() { - if let Some(RawLiteralEnum::Long(v)) = v { - bytes[i] = *v as u8; - } else { - return Err(invalid_err_with_reason( - "list", - "The element of list should be int", - )); - } - } - Ok(Some(Literal::uuid(uuid::Uuid::from_bytes(bytes)))) - } - Type::Primitive(PrimitiveType::Decimal { - precision: _, - scale: _, - }) => { - if v.list.len() != 16 { - return Err(invalid_err_with_reason( - "list", - "The length of list should be 16", - )); - } - let mut bytes = [0u8; 16]; - for (i, v) in v.list.iter().enumerate() { - if let Some(RawLiteralEnum::Long(v)) = v { - bytes[i] = *v as u8; - } else { - return Err(invalid_err_with_reason( - "list", - "The element of list should be int", - )); - } - } - Ok(Some(Literal::decimal(i128::from_be_bytes(bytes)))) - } - Type::Primitive(PrimitiveType::Binary) => { - let bytes = v - .list - .into_iter() - .map(|v| { - if let Some(RawLiteralEnum::Long(v)) = v { - Ok(v as u8) - } else { - Err(invalid_err_with_reason( - "list", - "The element of list should be int", - )) - } - }) - .collect::, Error>>()?; - Ok(Some(Literal::binary(bytes))) - } - Type::Primitive(PrimitiveType::Fixed(size)) => { - if v.list.len() != *size as usize { - return Err(invalid_err_with_reason( - "list", - "The length of list should be equal to size", - )); - } - let bytes = v - .list - .into_iter() - .map(|v| { - if let Some(RawLiteralEnum::Long(v)) = v { - Ok(v as u8) - } else { - Err(invalid_err_with_reason( - "list", - "The element of list should be int", - )) - } - }) - .collect::, Error>>()?; - Ok(Some(Literal::fixed(bytes))) - } - _ => Err(invalid_err("list")), - }, - RawLiteralEnum::Record(Record { - required, - optional: _, - }) => match ty { - Type::Struct(struct_ty) => { - let iters: Vec> = required - .into_iter() - .map(|(field_name, value)| { - let field = struct_ty - .field_by_name(field_name.as_str()) - .ok_or_else(|| { - invalid_err_with_reason( - "record", - &format!("field {} is not exist", &field_name), - ) - })?; - let value = value.try_into(&field.field_type)?; - Ok(value) - }) - .collect::>()?; - Ok(Some(Literal::Struct(super::Struct::from_iter(iters)))) - } - Type::Map(map_ty) => { - if *map_ty.key_field.field_type != Type::Primitive(PrimitiveType::String) { - return Err(invalid_err_with_reason( - "record", - "Map key must be string", - )); - } - let mut map = Map::new(); - for (k, v) in required { - let value = v.try_into(&map_ty.value_field.field_type)?; - if map_ty.value_field.required && value.is_none() { - return Err(invalid_err_with_reason( - "record", - "Value element is required in this Map", - )); - } - map.insert(Literal::string(k), value); - } - Ok(Some(Literal::Map(map))) - } - _ => Err(invalid_err("record")), - }, - RawLiteralEnum::StringMap(_) => Err(invalid_err("string map")), - } - } - } -} - -#[cfg(test)] -mod tests { - use apache_avro::to_value; - use apache_avro::types::Value; - - use super::*; - use crate::avro::schema_to_avro_schema; - use crate::spec::Schema; - use crate::spec::Type::Primitive; - use crate::spec::datatypes::{ListType, MapType, NestedField, StructType}; - - fn check_json_serde(json: &str, expected_literal: Literal, expected_type: &Type) { - let raw_json_value = serde_json::from_str::(json).unwrap(); - let desered_literal = - Literal::try_from_json(raw_json_value.clone(), expected_type).unwrap(); - assert_eq!(desered_literal, Some(expected_literal.clone())); - - let expected_json_value: JsonValue = expected_literal.try_into_json(expected_type).unwrap(); - let sered_json = serde_json::to_string(&expected_json_value).unwrap(); - let parsed_json_value = serde_json::from_str::(&sered_json).unwrap(); - - assert_eq!(parsed_json_value, raw_json_value); - } - - fn check_avro_bytes_serde( - input: Vec, - expected_datum: Datum, - expected_type: &PrimitiveType, - ) { - let raw_schema = r#""bytes""#; - let schema = apache_avro::Schema::parse_str(raw_schema).unwrap(); - - let bytes = ByteBuf::from(input); - let datum = Datum::try_from_bytes(&bytes, expected_type.clone()).unwrap(); - assert_eq!(datum, expected_datum); - - let mut writer = apache_avro::Writer::new(&schema, Vec::new()); - writer.append_ser(datum.to_bytes().unwrap()).unwrap(); - let encoded = writer.into_inner().unwrap(); - let reader = apache_avro::Reader::with_schema(&schema, &*encoded).unwrap(); - - for record in reader { - let result = apache_avro::from_value::(&record.unwrap()).unwrap(); - let desered_datum = Datum::try_from_bytes(&result, expected_type.clone()).unwrap(); - assert_eq!(desered_datum, expected_datum); - } - } - - fn check_convert_with_avro(expected_literal: Literal, expected_type: &Type) { - let fields = vec![NestedField::required(1, "col", expected_type.clone()).into()]; - let schema = Schema::builder() - .with_fields(fields.clone()) - .build() - .unwrap(); - let avro_schema = schema_to_avro_schema("test", &schema).unwrap(); - let struct_type = Type::Struct(StructType::new(fields)); - let struct_literal = - Literal::Struct(Struct::from_iter(vec![Some(expected_literal.clone())])); - - let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()); - let raw_literal = RawLiteral::try_from(struct_literal.clone(), &struct_type).unwrap(); - writer.append_ser(raw_literal).unwrap(); - let encoded = writer.into_inner().unwrap(); - - let reader = apache_avro::Reader::new(&*encoded).unwrap(); - for record in reader { - let result = apache_avro::from_value::(&record.unwrap()).unwrap(); - let desered_literal = result.try_into(&struct_type).unwrap().unwrap(); - assert_eq!(desered_literal, struct_literal); - } - } - - fn check_serialize_avro(literal: Literal, ty: &Type, expect_value: Value) { - let expect_value = Value::Record(vec![("col".to_string(), expect_value)]); - - let fields = vec![NestedField::required(1, "col", ty.clone()).into()]; - let schema = Schema::builder() - .with_fields(fields.clone()) - .build() - .unwrap(); - let avro_schema = schema_to_avro_schema("test", &schema).unwrap(); - let struct_type = Type::Struct(StructType::new(fields)); - let struct_literal = Literal::Struct(Struct::from_iter(vec![Some(literal.clone())])); - let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()); - let raw_literal = RawLiteral::try_from(struct_literal.clone(), &struct_type).unwrap(); - let value = to_value(raw_literal) - .unwrap() - .resolve(&avro_schema) - .unwrap(); - writer.append_value_ref(&value).unwrap(); - let encoded = writer.into_inner().unwrap(); - - let reader = apache_avro::Reader::new(&*encoded).unwrap(); - for record in reader { - assert_eq!(record.unwrap(), expect_value); - } - } - - #[test] - fn json_boolean() { - let record = r#"true"#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Boolean(true)), - &Type::Primitive(PrimitiveType::Boolean), - ); - } - - #[test] - fn json_int() { - let record = r#"32"#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Int(32)), - &Type::Primitive(PrimitiveType::Int), - ); - } - - #[test] - fn json_long() { - let record = r#"32"#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Long(32)), - &Type::Primitive(PrimitiveType::Long), - ); - } - - #[test] - fn json_float() { - let record = r#"1.0"#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(1.0))), - &Type::Primitive(PrimitiveType::Float), - ); - } - - #[test] - fn json_double() { - let record = r#"1.0"#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(1.0))), - &Type::Primitive(PrimitiveType::Double), - ); - } - - #[test] - fn json_date() { - let record = r#""2017-11-16""#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Int(17486)), - &Type::Primitive(PrimitiveType::Date), - ); - } - - #[test] - fn json_time() { - let record = r#""22:31:08.123456""#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Long(81068123456)), - &Type::Primitive(PrimitiveType::Time), - ); - } - - #[test] - fn json_timestamp() { - let record = r#""2017-11-16T22:31:08.123456""#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Long(1510871468123456)), - &Type::Primitive(PrimitiveType::Timestamp), - ); - } - - #[test] - fn json_timestamptz() { - let record = r#""2017-11-16T22:31:08.123456+00:00""#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Long(1510871468123456)), - &Type::Primitive(PrimitiveType::Timestamptz), - ); - } - - #[test] - fn json_string() { - let record = r#""iceberg""#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::String("iceberg".to_string())), - &Type::Primitive(PrimitiveType::String), - ); - } - - #[test] - fn json_uuid() { - let record = r#""f79c3e09-677c-4bbd-a479-3f349cb785e7""#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::UInt128( - Uuid::parse_str("f79c3e09-677c-4bbd-a479-3f349cb785e7") - .unwrap() - .as_u128(), - )), - &Type::Primitive(PrimitiveType::Uuid), - ); - } - - #[test] - fn json_decimal() { - let record = r#""14.20""#; - - check_json_serde( - record, - Literal::Primitive(PrimitiveLiteral::Int128(1420)), - &Type::decimal(28, 2).unwrap(), - ); - } - - #[test] - fn json_struct() { - let record = r#"{"1": 1, "2": "bar", "3": null}"#; - - check_json_serde( - record, - Literal::Struct(Struct::from_iter(vec![ - Some(Literal::Primitive(PrimitiveLiteral::Int(1))), - Some(Literal::Primitive(PrimitiveLiteral::String( - "bar".to_string(), - ))), - None, - ])), - &Type::Struct(StructType::new(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "address", Type::Primitive(PrimitiveType::String)).into(), - ])), - ); - } - - #[test] - fn json_list() { - let record = r#"[1, 2, 3, null]"#; - - check_json_serde( - record, - Literal::List(vec![ - Some(Literal::Primitive(PrimitiveLiteral::Int(1))), - Some(Literal::Primitive(PrimitiveLiteral::Int(2))), - Some(Literal::Primitive(PrimitiveLiteral::Int(3))), - None, - ]), - &Type::List(ListType { - element_field: NestedField::list_element( - 0, - Type::Primitive(PrimitiveType::Int), - true, - ) - .into(), - }), - ); - } - - #[test] - fn json_map() { - let record = r#"{ "keys": ["a", "b", "c"], "values": [1, 2, null] }"#; - - check_json_serde( - record, - Literal::Map(Map::from([ - ( - Literal::Primitive(PrimitiveLiteral::String("a".to_string())), - Some(Literal::Primitive(PrimitiveLiteral::Int(1))), - ), - ( - Literal::Primitive(PrimitiveLiteral::String("b".to_string())), - Some(Literal::Primitive(PrimitiveLiteral::Int(2))), - ), - ( - Literal::Primitive(PrimitiveLiteral::String("c".to_string())), - None, - ), - ])), - &Type::Map(MapType { - key_field: NestedField::map_key_element(0, Type::Primitive(PrimitiveType::String)) - .into(), - value_field: NestedField::map_value_element( - 1, - Type::Primitive(PrimitiveType::Int), - true, - ) - .into(), - }), - ); - } - - #[test] - fn avro_bytes_boolean() { - let bytes = vec![1u8]; - - check_avro_bytes_serde(bytes, Datum::bool(true), &PrimitiveType::Boolean); - } - - #[test] - fn avro_bytes_int() { - let bytes = vec![32u8, 0u8, 0u8, 0u8]; - - check_avro_bytes_serde(bytes, Datum::int(32), &PrimitiveType::Int); - } - - #[test] - fn avro_bytes_long() { - let bytes = vec![32u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8]; - - check_avro_bytes_serde(bytes, Datum::long(32), &PrimitiveType::Long); - } - - #[test] - fn avro_bytes_long_from_int() { - let bytes = vec![32u8, 0u8, 0u8, 0u8]; - - check_avro_bytes_serde(bytes, Datum::long(32), &PrimitiveType::Long); - } - - #[test] - fn avro_bytes_float() { - let bytes = vec![0u8, 0u8, 128u8, 63u8]; - - check_avro_bytes_serde(bytes, Datum::float(1.0), &PrimitiveType::Float); - } - - #[test] - fn avro_bytes_double() { - let bytes = vec![0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 240u8, 63u8]; - - check_avro_bytes_serde(bytes, Datum::double(1.0), &PrimitiveType::Double); - } - - #[test] - fn avro_bytes_double_from_float() { - let bytes = vec![0u8, 0u8, 128u8, 63u8]; - - check_avro_bytes_serde(bytes, Datum::double(1.0), &PrimitiveType::Double); - } - - #[test] - fn avro_bytes_string() { - let bytes = vec![105u8, 99u8, 101u8, 98u8, 101u8, 114u8, 103u8]; - - check_avro_bytes_serde(bytes, Datum::string("iceberg"), &PrimitiveType::String); - } - - #[test] - fn avro_bytes_decimal() { - // (input_bytes, decimal_num, expect_scale, expect_precision) - let cases = vec![ - (vec![4u8, 210u8], 1234, 2, 38), - (vec![251u8, 46u8], -1234, 2, 38), - (vec![4u8, 210u8], 1234, 3, 38), - (vec![251u8, 46u8], -1234, 3, 38), - (vec![42u8], 42, 2, 1), - (vec![214u8], -42, 2, 1), - ]; - - for (input_bytes, decimal_num, expect_scale, expect_precision) in cases { - check_avro_bytes_serde( - input_bytes, - Datum::decimal_with_precision( - Decimal::new(decimal_num, expect_scale), - expect_precision, - ) - .unwrap(), - &PrimitiveType::Decimal { - precision: expect_precision, - scale: expect_scale, - }, - ); - } - } - - #[test] - fn avro_bytes_decimal_expect_error() { - // (decimal_num, expect_scale, expect_precision) - let cases = vec![(1234, 2, 1)]; - - for (decimal_num, expect_scale, expect_precision) in cases { - let result = Datum::decimal_with_precision( - Decimal::new(decimal_num, expect_scale), - expect_precision, - ); - assert!(result.is_err(), "expect error but got {:?}", result); - assert_eq!( - result.unwrap_err().kind(), - ErrorKind::DataInvalid, - "expect error DataInvalid", - ); - } - } - - #[test] - fn avro_convert_test_int() { - check_convert_with_avro( - Literal::Primitive(PrimitiveLiteral::Int(32)), - &Type::Primitive(PrimitiveType::Int), - ); - } - - #[test] - fn avro_convert_test_long() { - check_convert_with_avro( - Literal::Primitive(PrimitiveLiteral::Long(32)), - &Type::Primitive(PrimitiveType::Long), - ); - } - - #[test] - fn avro_convert_test_float() { - check_convert_with_avro( - Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(1.0))), - &Type::Primitive(PrimitiveType::Float), - ); - } - - #[test] - fn avro_convert_test_double() { - check_convert_with_avro( - Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(1.0))), - &Type::Primitive(PrimitiveType::Double), - ); - } - - #[test] - fn avro_convert_test_string() { - check_convert_with_avro( - Literal::Primitive(PrimitiveLiteral::String("iceberg".to_string())), - &Type::Primitive(PrimitiveType::String), - ); - } - - #[test] - fn avro_convert_test_date() { - check_convert_with_avro( - Literal::Primitive(PrimitiveLiteral::Int(17486)), - &Type::Primitive(PrimitiveType::Date), - ); - } - - #[test] - fn avro_convert_test_time() { - check_convert_with_avro( - Literal::Primitive(PrimitiveLiteral::Long(81068123456)), - &Type::Primitive(PrimitiveType::Time), - ); - } - - #[test] - fn avro_convert_test_timestamp() { - check_convert_with_avro( - Literal::Primitive(PrimitiveLiteral::Long(1510871468123456)), - &Type::Primitive(PrimitiveType::Timestamp), - ); - } - - #[test] - fn avro_convert_test_timestamptz() { - check_convert_with_avro( - Literal::Primitive(PrimitiveLiteral::Long(1510871468123456)), - &Type::Primitive(PrimitiveType::Timestamptz), - ); - } - - #[test] - fn avro_convert_test_list() { - check_convert_with_avro( - Literal::List(vec![ - Some(Literal::Primitive(PrimitiveLiteral::Int(1))), - Some(Literal::Primitive(PrimitiveLiteral::Int(2))), - Some(Literal::Primitive(PrimitiveLiteral::Int(3))), - None, - ]), - &Type::List(ListType { - element_field: NestedField::list_element( - 0, - Type::Primitive(PrimitiveType::Int), - false, - ) - .into(), - }), - ); - - check_convert_with_avro( - Literal::List(vec![ - Some(Literal::Primitive(PrimitiveLiteral::Int(1))), - Some(Literal::Primitive(PrimitiveLiteral::Int(2))), - Some(Literal::Primitive(PrimitiveLiteral::Int(3))), - ]), - &Type::List(ListType { - element_field: NestedField::list_element( - 0, - Type::Primitive(PrimitiveType::Int), - true, - ) - .into(), - }), - ); - } - - fn check_convert_with_avro_map(expected_literal: Literal, expected_type: &Type) { - let fields = vec![NestedField::required(1, "col", expected_type.clone()).into()]; - let schema = Schema::builder() - .with_fields(fields.clone()) - .build() - .unwrap(); - let avro_schema = schema_to_avro_schema("test", &schema).unwrap(); - let struct_type = Type::Struct(StructType::new(fields)); - let struct_literal = - Literal::Struct(Struct::from_iter(vec![Some(expected_literal.clone())])); - - let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()); - let raw_literal = RawLiteral::try_from(struct_literal.clone(), &struct_type).unwrap(); - writer.append_ser(raw_literal).unwrap(); - let encoded = writer.into_inner().unwrap(); - - let reader = apache_avro::Reader::new(&*encoded).unwrap(); - for record in reader { - let result = apache_avro::from_value::(&record.unwrap()).unwrap(); - let desered_literal = result.try_into(&struct_type).unwrap().unwrap(); - match (&desered_literal, &struct_literal) { - (Literal::Struct(desered), Literal::Struct(expected)) => { - match (&desered.fields[0], &expected.fields[0]) { - (Some(Literal::Map(desered)), Some(Literal::Map(expected))) => { - assert!(desered.has_same_content(expected)) - } - _ => { - unreachable!() - } - } - } - _ => { - panic!("unexpected literal type"); - } - } - } - } - - #[test] - fn avro_convert_test_map() { - check_convert_with_avro_map( - Literal::Map(Map::from([ - ( - Literal::Primitive(PrimitiveLiteral::Int(1)), - Some(Literal::Primitive(PrimitiveLiteral::Long(1))), - ), - ( - Literal::Primitive(PrimitiveLiteral::Int(2)), - Some(Literal::Primitive(PrimitiveLiteral::Long(2))), - ), - (Literal::Primitive(PrimitiveLiteral::Int(3)), None), - ])), - &Type::Map(MapType { - key_field: NestedField::map_key_element(2, Type::Primitive(PrimitiveType::Int)) - .into(), - value_field: NestedField::map_value_element( - 3, - Type::Primitive(PrimitiveType::Long), - false, - ) - .into(), - }), - ); - - check_convert_with_avro_map( - Literal::Map(Map::from([ - ( - Literal::Primitive(PrimitiveLiteral::Int(1)), - Some(Literal::Primitive(PrimitiveLiteral::Long(1))), - ), - ( - Literal::Primitive(PrimitiveLiteral::Int(2)), - Some(Literal::Primitive(PrimitiveLiteral::Long(2))), - ), - ( - Literal::Primitive(PrimitiveLiteral::Int(3)), - Some(Literal::Primitive(PrimitiveLiteral::Long(3))), - ), - ])), - &Type::Map(MapType { - key_field: NestedField::map_key_element(2, Type::Primitive(PrimitiveType::Int)) - .into(), - value_field: NestedField::map_value_element( - 3, - Type::Primitive(PrimitiveType::Long), - true, - ) - .into(), - }), - ); - } - - #[test] - fn avro_convert_test_string_map() { - check_convert_with_avro_map( - Literal::Map(Map::from([ - ( - Literal::Primitive(PrimitiveLiteral::String("a".to_string())), - Some(Literal::Primitive(PrimitiveLiteral::Int(1))), - ), - ( - Literal::Primitive(PrimitiveLiteral::String("b".to_string())), - Some(Literal::Primitive(PrimitiveLiteral::Int(2))), - ), - ( - Literal::Primitive(PrimitiveLiteral::String("c".to_string())), - None, - ), - ])), - &Type::Map(MapType { - key_field: NestedField::map_key_element(2, Type::Primitive(PrimitiveType::String)) - .into(), - value_field: NestedField::map_value_element( - 3, - Type::Primitive(PrimitiveType::Int), - false, - ) - .into(), - }), - ); - - check_convert_with_avro_map( - Literal::Map(Map::from([ - ( - Literal::Primitive(PrimitiveLiteral::String("a".to_string())), - Some(Literal::Primitive(PrimitiveLiteral::Int(1))), - ), - ( - Literal::Primitive(PrimitiveLiteral::String("b".to_string())), - Some(Literal::Primitive(PrimitiveLiteral::Int(2))), - ), - ( - Literal::Primitive(PrimitiveLiteral::String("c".to_string())), - Some(Literal::Primitive(PrimitiveLiteral::Int(3))), - ), - ])), - &Type::Map(MapType { - key_field: NestedField::map_key_element(2, Type::Primitive(PrimitiveType::String)) - .into(), - value_field: NestedField::map_value_element( - 3, - Type::Primitive(PrimitiveType::Int), - true, - ) - .into(), - }), - ); - } - - #[test] - fn avro_convert_test_record() { - check_convert_with_avro( - Literal::Struct(Struct::from_iter(vec![ - Some(Literal::Primitive(PrimitiveLiteral::Int(1))), - Some(Literal::Primitive(PrimitiveLiteral::String( - "bar".to_string(), - ))), - None, - ])), - &Type::Struct(StructType::new(vec![ - NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "name", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(4, "address", Type::Primitive(PrimitiveType::String)).into(), - ])), - ); - } - - // # TODO:https://github.com/apache/iceberg-rust/issues/86 - // rust avro don't support deserialize any bytes representation now: - // - binary - // - decimal - #[test] - fn avro_convert_test_binary_ser() { - let literal = Literal::Primitive(PrimitiveLiteral::Binary(vec![1, 2, 3, 4, 5])); - let ty = Type::Primitive(PrimitiveType::Binary); - let expect_value = Value::Bytes(vec![1, 2, 3, 4, 5]); - check_serialize_avro(literal, &ty, expect_value); - } - - #[test] - fn avro_convert_test_decimal_ser() { - let literal = Literal::decimal(12345); - let ty = Type::Primitive(PrimitiveType::Decimal { - precision: 9, - scale: 8, - }); - let expect_value = Value::Decimal(apache_avro::Decimal::from(12345_i128.to_be_bytes())); - check_serialize_avro(literal, &ty, expect_value); - } - - // # TODO:https://github.com/apache/iceberg-rust/issues/86 - // rust avro can't support to convert any byte-like type to fixed in avro now. - // - uuid ser/de - // - fixed ser/de - - #[test] - fn test_parse_timestamp() { - let value = Datum::timestamp_from_str("2021-08-01T01:09:00.0899").unwrap(); - assert_eq!(&format!("{value}"), "2021-08-01 01:09:00.089900"); - - let value = Datum::timestamp_from_str("2023-01-06T00:00:00").unwrap(); - assert_eq!(&format!("{value}"), "2023-01-06 00:00:00"); - - let value = Datum::timestamp_from_str("2021-08-01T01:09:00.0899+0800"); - assert!(value.is_err(), "Parse timestamp with timezone should fail!"); - - let value = Datum::timestamp_from_str("dfa"); - assert!( - value.is_err(), - "Parse timestamp with invalid input should fail!" - ); - } - - #[test] - fn test_parse_timestamptz() { - let value = Datum::timestamptz_from_str("2021-08-01T09:09:00.0899+0800").unwrap(); - assert_eq!(&format!("{value}"), "2021-08-01 01:09:00.089900 UTC"); - - let value = Datum::timestamptz_from_str("2021-08-01T01:09:00.0899"); - assert!( - value.is_err(), - "Parse timestamptz without timezone should fail!" - ); - - let value = Datum::timestamptz_from_str("dfa"); - assert!( - value.is_err(), - "Parse timestamptz with invalid input should fail!" - ); - } - - #[test] - fn test_datum_ser_deser() { - let test_fn = |datum: Datum| { - let json = serde_json::to_value(&datum).unwrap(); - let desered_datum: Datum = serde_json::from_value(json).unwrap(); - assert_eq!(datum, desered_datum); - }; - let datum = Datum::int(1); - test_fn(datum); - let datum = Datum::long(1); - test_fn(datum); - - let datum = Datum::float(1.0); - test_fn(datum); - let datum = Datum::float(0_f32); - test_fn(datum); - let datum = Datum::float(-0_f32); - test_fn(datum); - let datum = Datum::float(f32::MAX); - test_fn(datum); - let datum = Datum::float(f32::MIN); - test_fn(datum); - - // serde_json can't serialize f32::INFINITY, f32::NEG_INFINITY, f32::NAN - let datum = Datum::float(f32::INFINITY); - let json = serde_json::to_string(&datum).unwrap(); - assert!(serde_json::from_str::(&json).is_err()); - let datum = Datum::float(f32::NEG_INFINITY); - let json = serde_json::to_string(&datum).unwrap(); - assert!(serde_json::from_str::(&json).is_err()); - let datum = Datum::float(f32::NAN); - let json = serde_json::to_string(&datum).unwrap(); - assert!(serde_json::from_str::(&json).is_err()); - - let datum = Datum::double(1.0); - test_fn(datum); - let datum = Datum::double(f64::MAX); - test_fn(datum); - let datum = Datum::double(f64::MIN); - test_fn(datum); - - // serde_json can't serialize f32::INFINITY, f32::NEG_INFINITY, f32::NAN - let datum = Datum::double(f64::INFINITY); - let json = serde_json::to_string(&datum).unwrap(); - assert!(serde_json::from_str::(&json).is_err()); - let datum = Datum::double(f64::NEG_INFINITY); - let json = serde_json::to_string(&datum).unwrap(); - assert!(serde_json::from_str::(&json).is_err()); - let datum = Datum::double(f64::NAN); - let json = serde_json::to_string(&datum).unwrap(); - assert!(serde_json::from_str::(&json).is_err()); - - let datum = Datum::string("iceberg"); - test_fn(datum); - let datum = Datum::bool(true); - test_fn(datum); - let datum = Datum::date(17486); - test_fn(datum); - let datum = Datum::time_from_hms_micro(22, 15, 33, 111).unwrap(); - test_fn(datum); - let datum = Datum::timestamp_micros(1510871468123456); - test_fn(datum); - let datum = Datum::timestamptz_micros(1510871468123456); - test_fn(datum); - let datum = Datum::uuid(Uuid::parse_str("f79c3e09-677c-4bbd-a479-3f349cb785e7").unwrap()); - test_fn(datum); - let datum = Datum::decimal(1420).unwrap(); - test_fn(datum); - let datum = Datum::binary(vec![1, 2, 3, 4, 5]); - test_fn(datum); - let datum = Datum::fixed(vec![1, 2, 3, 4, 5]); - test_fn(datum); - } - - #[test] - fn test_datum_date_convert_to_int() { - let datum_date = Datum::date(12345); - - let result = datum_date.to(&Primitive(PrimitiveType::Int)).unwrap(); - - let expected = Datum::int(12345); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_int_convert_to_date() { - let datum_int = Datum::int(12345); - - let result = datum_int.to(&Primitive(PrimitiveType::Date)).unwrap(); - - let expected = Datum::date(12345); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_long_convert_to_int() { - let datum = Datum::long(12345); - - let result = datum.to(&Primitive(PrimitiveType::Int)).unwrap(); - - let expected = Datum::int(12345); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_long_convert_to_int_above_max() { - let datum = Datum::long(INT_MAX as i64 + 1); - - let result = datum.to(&Primitive(PrimitiveType::Int)).unwrap(); - - let expected = Datum::new(PrimitiveType::Int, PrimitiveLiteral::AboveMax); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_long_convert_to_int_below_min() { - let datum = Datum::long(INT_MIN as i64 - 1); - - let result = datum.to(&Primitive(PrimitiveType::Int)).unwrap(); - - let expected = Datum::new(PrimitiveType::Int, PrimitiveLiteral::BelowMin); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_long_convert_to_timestamp() { - let datum = Datum::long(12345); - - let result = datum.to(&Primitive(PrimitiveType::Timestamp)).unwrap(); - - let expected = Datum::timestamp_micros(12345); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_long_convert_to_timestamptz() { - let datum = Datum::long(12345); - - let result = datum.to(&Primitive(PrimitiveType::Timestamptz)).unwrap(); - - let expected = Datum::timestamptz_micros(12345); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_decimal_convert_to_long() { - let datum = Datum::decimal(12345).unwrap(); - - let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap(); - - let expected = Datum::long(12345); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_decimal_convert_to_long_above_max() { - let datum = Datum::decimal(LONG_MAX as i128 + 1).unwrap(); - - let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap(); - - let expected = Datum::new(PrimitiveType::Long, PrimitiveLiteral::AboveMax); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_decimal_convert_to_long_below_min() { - let datum = Datum::decimal(LONG_MIN as i128 - 1).unwrap(); - - let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap(); - - let expected = Datum::new(PrimitiveType::Long, PrimitiveLiteral::BelowMin); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_string_convert_to_boolean() { - let datum = Datum::string("true"); - - let result = datum.to(&Primitive(PrimitiveType::Boolean)).unwrap(); - - let expected = Datum::bool(true); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_string_convert_to_int() { - let datum = Datum::string("12345"); - - let result = datum.to(&Primitive(PrimitiveType::Int)).unwrap(); - - let expected = Datum::int(12345); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_string_convert_to_long() { - let datum = Datum::string("12345"); - - let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap(); - - let expected = Datum::long(12345); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_string_convert_to_timestamp() { - let datum = Datum::string("1925-05-20T19:25:00.000"); - - let result = datum.to(&Primitive(PrimitiveType::Timestamp)).unwrap(); - - let expected = Datum::timestamp_micros(-1407990900000000); - - assert_eq!(result, expected); - } - - #[test] - fn test_datum_string_convert_to_timestamptz() { - let datum = Datum::string("1925-05-20T19:25:00.000 UTC"); - - let result = datum.to(&Primitive(PrimitiveType::Timestamptz)).unwrap(); - - let expected = Datum::timestamptz_micros(-1407990900000000); - - assert_eq!(result, expected); - } - - #[test] - fn test_iceberg_float_order() { - // Test float ordering - let float_values = vec![ - Datum::float(f32::NAN), - Datum::float(-f32::NAN), - Datum::float(f32::MAX), - Datum::float(f32::MIN), - Datum::float(f32::INFINITY), - Datum::float(-f32::INFINITY), - Datum::float(1.0), - Datum::float(-1.0), - Datum::float(0.0), - Datum::float(-0.0), - ]; - - let mut float_sorted = float_values.clone(); - float_sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); - - let float_expected = vec![ - Datum::float(-f32::NAN), - Datum::float(-f32::INFINITY), - Datum::float(f32::MIN), - Datum::float(-1.0), - Datum::float(-0.0), - Datum::float(0.0), - Datum::float(1.0), - Datum::float(f32::MAX), - Datum::float(f32::INFINITY), - Datum::float(f32::NAN), - ]; - - assert_eq!(float_sorted, float_expected); - - // Test double ordering - let double_values = vec![ - Datum::double(f64::NAN), - Datum::double(-f64::NAN), - Datum::double(f64::INFINITY), - Datum::double(-f64::INFINITY), - Datum::double(f64::MAX), - Datum::double(f64::MIN), - Datum::double(1.0), - Datum::double(-1.0), - Datum::double(0.0), - Datum::double(-0.0), - ]; - - let mut double_sorted = double_values.clone(); - double_sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); - - let double_expected = vec![ - Datum::double(-f64::NAN), - Datum::double(-f64::INFINITY), - Datum::double(f64::MIN), - Datum::double(-1.0), - Datum::double(-0.0), - Datum::double(0.0), - Datum::double(1.0), - Datum::double(f64::MAX), - Datum::double(f64::INFINITY), - Datum::double(f64::NAN), - ]; - - assert_eq!(double_sorted, double_expected); - } -} diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs new file mode 100644 index 0000000000..81c6708b9d --- /dev/null +++ b/crates/iceberg/src/spec/values/datum.rs @@ -0,0 +1,1324 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Typed literals with validation + +use std::cmp::Ordering; +use std::fmt::{Display, Formatter, LowerExp}; +use std::str::FromStr; + +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; +use ordered_float::{Float, OrderedFloat}; +use serde::de::{self, MapAccess}; +use serde::ser::SerializeStruct; +use serde::{Deserialize, Serialize}; +use serde_bytes::ByteBuf; + +use super::decimal_utils::{ + Decimal, decimal_from_i128_with_scale, decimal_from_str_exact, decimal_mantissa, + decimal_precision, decimal_scale, i128_from_be_bytes, i128_to_be_bytes_min, +}; +use super::literal::Literal; +use super::primitive::PrimitiveLiteral; +use super::serde::_serde::RawLiteral; +use super::temporal::{date, time, timestamp, timestamptz}; +use crate::ensure_data_valid; +use crate::error::{Error, ErrorKind, Result, invalid_data}; +use crate::spec::MAX_DECIMAL_PRECISION; +use crate::spec::datatypes::{PrimitiveType, Type}; + +/// Maximum value for [`PrimitiveType::Time`] type in microseconds, e.g. 23 hours 59 minutes 59 seconds 999999 microseconds. +pub(crate) const MAX_TIME_VALUE: i64 = 24 * 60 * 60 * 1_000_000i64 - 1; + +pub(crate) const INT_MAX: i32 = 2147483647; +pub(crate) const INT_MIN: i32 = -2147483648; +pub(crate) const LONG_MAX: i64 = 9223372036854775807; +pub(crate) const LONG_MIN: i64 = -9223372036854775808; + +/// Literal associated with its type. The value and type pair is checked when construction, so the type and value is +/// guaranteed to be correct when used. +/// +/// By default, we decouple the type and value of a literal, so we can use avoid the cost of storing extra type info +/// for each literal. But associate type with literal can be useful in some cases, for example, in unbound expression. +#[derive(Clone, Debug, PartialEq, Hash, Eq)] +pub struct Datum { + r#type: PrimitiveType, + literal: PrimitiveLiteral, +} + +impl Serialize for Datum { + fn serialize( + &self, + serializer: S, + ) -> std::result::Result { + let mut struct_ser = serializer + .serialize_struct("Datum", 2) + .map_err(serde::ser::Error::custom)?; + struct_ser + .serialize_field("type", &self.r#type) + .map_err(serde::ser::Error::custom)?; + struct_ser + .serialize_field( + "literal", + &RawLiteral::try_from( + Literal::Primitive(self.literal.clone()), + &Type::Primitive(self.r#type.clone()), + ) + .map_err(serde::ser::Error::custom)?, + ) + .map_err(serde::ser::Error::custom)?; + struct_ser.end() + } +} + +impl<'de> Deserialize<'de> for Datum { + fn deserialize>( + deserializer: D, + ) -> std::result::Result { + #[derive(Deserialize)] + #[serde(field_identifier, rename_all = "lowercase")] + enum Field { + Type, + Literal, + } + + struct DatumVisitor; + + impl<'de> de::Visitor<'de> for DatumVisitor { + type Value = Datum; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + formatter.write_str("struct Datum") + } + + fn visit_seq(self, mut seq: A) -> std::result::Result + where A: de::SeqAccess<'de> { + let r#type = seq + .next_element::()? + .ok_or_else(|| de::Error::invalid_length(0, &self))?; + let value = seq + .next_element::()? + .ok_or_else(|| de::Error::invalid_length(1, &self))?; + let Literal::Primitive(primitive) = value + .try_into(&Type::Primitive(r#type.clone())) + .map_err(de::Error::custom)? + .ok_or_else(|| de::Error::custom("None value"))? + else { + return Err(de::Error::custom("Invalid value")); + }; + + Ok(Datum::new(r#type, primitive)) + } + + fn visit_map(self, mut map: V) -> std::result::Result + where V: MapAccess<'de> { + let mut raw_primitive: Option = None; + let mut r#type: Option = None; + while let Some(key) = map.next_key()? { + match key { + Field::Type => { + if r#type.is_some() { + return Err(de::Error::duplicate_field("type")); + } + r#type = Some(map.next_value()?); + } + Field::Literal => { + if raw_primitive.is_some() { + return Err(de::Error::duplicate_field("literal")); + } + raw_primitive = Some(map.next_value()?); + } + } + } + let Some(r#type) = r#type else { + return Err(de::Error::missing_field("type")); + }; + let Some(raw_primitive) = raw_primitive else { + return Err(de::Error::missing_field("literal")); + }; + let Literal::Primitive(primitive) = raw_primitive + .try_into(&Type::Primitive(r#type.clone())) + .map_err(de::Error::custom)? + .ok_or_else(|| de::Error::custom("None value"))? + else { + return Err(de::Error::custom("Invalid value")); + }; + Ok(Datum::new(r#type, primitive)) + } + } + const FIELDS: &[&str] = &["type", "literal"]; + deserializer.deserialize_struct("Datum", FIELDS, DatumVisitor) + } +} + +// Compare following iceberg float ordering rules: +// -NaN < -Infinity < -value < -0 < 0 < value < Infinity < NaN +fn iceberg_float_cmp_f32(a: OrderedFloat, b: OrderedFloat) -> Option { + Some(a.total_cmp(&b)) +} + +fn iceberg_float_cmp_f64(a: OrderedFloat, b: OrderedFloat) -> Option { + Some(a.total_cmp(&b)) +} + +impl PartialOrd for Datum { + fn partial_cmp(&self, other: &Self) -> Option { + match (&self.literal, &other.literal, &self.r#type, &other.r#type) { + // generate the arm with same type and same literal + ( + PrimitiveLiteral::Boolean(val), + PrimitiveLiteral::Boolean(other_val), + PrimitiveType::Boolean, + PrimitiveType::Boolean, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Int(val), + PrimitiveLiteral::Int(other_val), + PrimitiveType::Int, + PrimitiveType::Int, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Long(val), + PrimitiveLiteral::Long(other_val), + PrimitiveType::Long, + PrimitiveType::Long, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Float(val), + PrimitiveLiteral::Float(other_val), + PrimitiveType::Float, + PrimitiveType::Float, + ) => iceberg_float_cmp_f32(*val, *other_val), + ( + PrimitiveLiteral::Double(val), + PrimitiveLiteral::Double(other_val), + PrimitiveType::Double, + PrimitiveType::Double, + ) => iceberg_float_cmp_f64(*val, *other_val), + ( + PrimitiveLiteral::Int(val), + PrimitiveLiteral::Int(other_val), + PrimitiveType::Date, + PrimitiveType::Date, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Long(val), + PrimitiveLiteral::Long(other_val), + PrimitiveType::Time, + PrimitiveType::Time, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Long(val), + PrimitiveLiteral::Long(other_val), + PrimitiveType::Timestamp, + PrimitiveType::Timestamp, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Long(val), + PrimitiveLiteral::Long(other_val), + PrimitiveType::Timestamptz, + PrimitiveType::Timestamptz, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Long(val), + PrimitiveLiteral::Long(other_val), + PrimitiveType::TimestampNs, + PrimitiveType::TimestampNs, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Long(val), + PrimitiveLiteral::Long(other_val), + PrimitiveType::TimestamptzNs, + PrimitiveType::TimestamptzNs, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::String(val), + PrimitiveLiteral::String(other_val), + PrimitiveType::String, + PrimitiveType::String, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::UInt128(val), + PrimitiveLiteral::UInt128(other_val), + PrimitiveType::Uuid, + PrimitiveType::Uuid, + ) => uuid::Uuid::from_u128(*val).partial_cmp(&uuid::Uuid::from_u128(*other_val)), + ( + PrimitiveLiteral::Binary(val), + PrimitiveLiteral::Binary(other_val), + PrimitiveType::Fixed(_), + PrimitiveType::Fixed(_), + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Binary(val), + PrimitiveLiteral::Binary(other_val), + PrimitiveType::Binary, + PrimitiveType::Binary, + ) => val.partial_cmp(other_val), + ( + PrimitiveLiteral::Binary(_), + PrimitiveLiteral::Binary(_), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_), + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_), + ) => None, + ( + PrimitiveLiteral::Int128(val), + PrimitiveLiteral::Int128(other_val), + PrimitiveType::Decimal { + precision: _, + scale, + }, + PrimitiveType::Decimal { + precision: _, + scale: other_scale, + }, + ) => { + let val = decimal_from_i128_with_scale(*val, *scale); + let other_val = decimal_from_i128_with_scale(*other_val, *other_scale); + val.partial_cmp(&other_val) + } + _ => None, + } + } +} + +impl Display for Datum { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match (&self.r#type, &self.literal) { + (_, PrimitiveLiteral::Boolean(val)) => write!(f, "{val}"), + ( + PrimitiveType::Int | PrimitiveType::Long | PrimitiveType::Unknown, + PrimitiveLiteral::Int(val), + ) => { + write!(f, "{val}") + } + ( + PrimitiveType::Int | PrimitiveType::Long | PrimitiveType::Unknown, + PrimitiveLiteral::Long(val), + ) => { + write!(f, "{val}") + } + (_, PrimitiveLiteral::Float(val)) => write!(f, "{val}"), + (_, PrimitiveLiteral::Double(val)) => write!(f, "{val}"), + (PrimitiveType::Date, PrimitiveLiteral::Int(val)) => { + write!(f, "{}", date::days_to_date(*val)) + } + (PrimitiveType::Time, PrimitiveLiteral::Long(val)) => { + write!(f, "{}", time::microseconds_to_time(*val)) + } + (PrimitiveType::Timestamp, PrimitiveLiteral::Long(val)) => { + write!(f, "{}", timestamp::microseconds_to_datetime(*val)) + } + (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(val)) => { + write!(f, "{}", timestamptz::microseconds_to_datetimetz(*val)) + } + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(val)) => { + write!(f, "{}", timestamp::nanoseconds_to_datetime(*val)) + } + (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(val)) => { + write!(f, "{}", timestamptz::nanoseconds_to_datetimetz(*val)) + } + (_, PrimitiveLiteral::String(val)) => write!(f, r#""{val}""#), + (PrimitiveType::Uuid | PrimitiveType::Unknown, PrimitiveLiteral::UInt128(val)) => { + write!(f, "{}", uuid::Uuid::from_u128(*val)) + } + (_, PrimitiveLiteral::Binary(val)) => display_bytes(val, f), + ( + PrimitiveType::Decimal { + precision: _, + scale, + }, + PrimitiveLiteral::Int128(val), + ) => { + write!(f, "{}", decimal_from_i128_with_scale(*val, *scale)) + } + // The scale lives on the dropped source type, so only the unscaled value is left. + (PrimitiveType::Unknown, PrimitiveLiteral::Int128(val)) => write!(f, "{val}"), + (_, _) => { + unreachable!() + } + } + } +} + +fn display_bytes(bytes: &[u8], f: &mut Formatter<'_>) -> std::fmt::Result { + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + s.push_str(&format!("{b:02X}")); + } + f.write_str(&s) +} + +impl From for Literal { + fn from(value: Datum) -> Self { + Literal::Primitive(value.literal) + } +} + +impl From for PrimitiveLiteral { + fn from(value: Datum) -> Self { + value.literal + } +} + +impl Datum { + /// Creates a `Datum` from a `PrimitiveType` and a `PrimitiveLiteral` + pub(crate) fn new(r#type: PrimitiveType, literal: PrimitiveLiteral) -> Self { + Datum { r#type, literal } + } + + /// Create iceberg value from bytes. + /// + /// See [this spec](https://iceberg.apache.org/spec/#binary-single-value-serialization) for reference. + pub fn try_from_bytes(bytes: &[u8], data_type: PrimitiveType) -> Result { + let literal = match data_type { + PrimitiveType::Unknown => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "Cannot create datum for unknown type from bytes", + )); + } + PrimitiveType::Boolean => { + if bytes.len() == 1 && bytes[0] == 0u8 { + PrimitiveLiteral::Boolean(false) + } else { + PrimitiveLiteral::Boolean(true) + } + } + PrimitiveType::Int => PrimitiveLiteral::Int(i32::from_le_bytes(bytes.try_into()?)), + PrimitiveType::Long => { + if bytes.len() == 4 { + // In the case of an evolved field + PrimitiveLiteral::Long(i32::from_le_bytes(bytes.try_into()?) as i64) + } else { + PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) + } + } + PrimitiveType::Float => { + PrimitiveLiteral::Float(OrderedFloat(f32::from_le_bytes(bytes.try_into()?))) + } + PrimitiveType::Double => { + if bytes.len() == 4 { + // In the case of an evolved field + PrimitiveLiteral::Double(OrderedFloat( + f32::from_le_bytes(bytes.try_into()?) as f64 + )) + } else { + PrimitiveLiteral::Double(OrderedFloat(f64::from_le_bytes(bytes.try_into()?))) + } + } + PrimitiveType::Date => PrimitiveLiteral::Int(i32::from_le_bytes(bytes.try_into()?)), + PrimitiveType::Time => PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)), + PrimitiveType::Timestamp => { + PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) + } + PrimitiveType::Timestamptz => { + PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) + } + PrimitiveType::TimestampNs => { + PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) + } + PrimitiveType::TimestamptzNs => { + PrimitiveLiteral::Long(i64::from_le_bytes(bytes.try_into()?)) + } + PrimitiveType::String => { + PrimitiveLiteral::String(std::str::from_utf8(bytes)?.to_string()) + } + PrimitiveType::Uuid => { + PrimitiveLiteral::UInt128(u128::from_be_bytes(bytes.try_into()?)) + } + PrimitiveType::Fixed(_) + | PrimitiveType::Binary + | PrimitiveType::Geometry(_) + | PrimitiveType::Geography(_) => PrimitiveLiteral::Binary(Vec::from(bytes)), + PrimitiveType::Decimal { .. } => PrimitiveLiteral::Int128( + i128_from_be_bytes(bytes) + .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?, + ), + }; + Ok(Datum::new(data_type, literal)) + } + + /// Convert the value to bytes + /// + /// See [this spec](https://iceberg.apache.org/spec/#binary-single-value-serialization) for reference. + pub fn to_bytes(&self) -> Result { + let buf = match &self.literal { + PrimitiveLiteral::Boolean(val) => { + if *val { + ByteBuf::from([1u8]) + } else { + ByteBuf::from([0u8]) + } + } + PrimitiveLiteral::Int(val) => ByteBuf::from(val.to_le_bytes()), + PrimitiveLiteral::Long(val) => ByteBuf::from(val.to_le_bytes()), + PrimitiveLiteral::Float(val) => ByteBuf::from(val.to_le_bytes()), + PrimitiveLiteral::Double(val) => ByteBuf::from(val.to_le_bytes()), + PrimitiveLiteral::String(val) => ByteBuf::from(val.as_bytes()), + PrimitiveLiteral::UInt128(val) => ByteBuf::from(val.to_be_bytes()), + PrimitiveLiteral::Binary(val) => ByteBuf::from(val.as_slice()), + PrimitiveLiteral::Int128(val) => { + let PrimitiveType::Decimal { precision, .. } = self.r#type else { + return Err(invalid_data!( + "PrimitiveLiteral Int128 must be PrimitiveType Decimal but got {}", + self.r#type + )); + }; + + // It's required by iceberg spec that we must keep the minimum + // number of bytes for the value + let Ok(required_bytes) = Type::decimal_required_bytes(precision) else { + return Err(invalid_data!( + "PrimitiveType Decimal must has valid precision but got {precision}" + )); + }; + + // The primitive literal is unscaled value. + // Convert into two's-complement byte representation in big-endian byte order. + let mut bytes = i128_to_be_bytes_min(*val); + // Truncate with required bytes to make sure. + bytes.truncate(required_bytes as usize); + + ByteBuf::from(bytes) + } + PrimitiveLiteral::AboveMax | PrimitiveLiteral::BelowMin => { + return Err(invalid_data!( + "Cannot convert AboveMax or BelowMin to bytes" + )); + } + }; + + Ok(buf) + } + + /// Creates a boolean value. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; + /// let t = Datum::bool(true); + /// + /// assert_eq!(format!("{}", t), "true".to_string()); + /// assert_eq!( + /// Literal::from(t), + /// Literal::Primitive(PrimitiveLiteral::Boolean(true)) + /// ); + /// ``` + pub fn bool>(t: T) -> Self { + Self { + r#type: PrimitiveType::Boolean, + literal: PrimitiveLiteral::Boolean(t.into()), + } + } + + /// Creates a boolean value from string. + /// See [Parse bool from str](https://doc.rust-lang.org/stable/std/primitive.bool.html#impl-FromStr-for-bool) for reference. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; + /// let t = Datum::bool_from_str("false").unwrap(); + /// + /// assert_eq!(&format!("{}", t), "false"); + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Boolean(false)), + /// t.into() + /// ); + /// ``` + pub fn bool_from_str>(s: S) -> Result { + let v = s + .as_ref() + .parse::() + .map_err(|e| invalid_data!("Can't parse string to bool.").with_source(e))?; + Ok(Self::bool(v)) + } + + /// Creates an 32bit integer. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; + /// let t = Datum::int(23i8); + /// + /// assert_eq!(&format!("{}", t), "23"); + /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Int(23)), t.into()); + /// ``` + pub fn int>(t: T) -> Self { + Self { + r#type: PrimitiveType::Int, + literal: PrimitiveLiteral::Int(t.into()), + } + } + + /// Creates an 64bit integer. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; + /// let t = Datum::long(24i8); + /// + /// assert_eq!(&format!("{t}"), "24"); + /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Long(24)), t.into()); + /// ``` + pub fn long>(t: T) -> Self { + Self { + r#type: PrimitiveType::Long, + literal: PrimitiveLiteral::Long(t.into()), + } + } + + /// Creates an 32bit floating point number. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; + /// use ordered_float::OrderedFloat; + /// let t = Datum::float(32.1f32); + /// + /// assert_eq!(&format!("{t}"), "32.1"); + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(32.1))), + /// t.into() + /// ); + /// ``` + pub fn float>(t: T) -> Self { + Self { + r#type: PrimitiveType::Float, + literal: PrimitiveLiteral::Float(OrderedFloat(t.into())), + } + } + + /// Creates an 64bit floating point number. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; + /// use ordered_float::OrderedFloat; + /// let t = Datum::double(32.1f64); + /// + /// assert_eq!(&format!("{t}"), "32.1"); + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(32.1))), + /// t.into() + /// ); + /// ``` + pub fn double>(t: T) -> Self { + Self { + r#type: PrimitiveType::Double, + literal: PrimitiveLiteral::Double(OrderedFloat(t.into())), + } + } + + /// Creates date literal from number of days from unix epoch directly. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; + /// // 2 days after 1970-01-01 + /// let t = Datum::date(2); + /// + /// assert_eq!(&format!("{t}"), "1970-01-03"); + /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Int(2)), t.into()); + /// ``` + pub fn date(days: i32) -> Self { + Self { + r#type: PrimitiveType::Date, + literal: PrimitiveLiteral::Int(days), + } + } + + /// Creates date literal in `%Y-%m-%d` format, assume in utc timezone. + /// + /// See [`NaiveDate::from_str`]. + /// + /// Example + /// ```rust + /// use iceberg::spec::{Datum, Literal}; + /// let t = Datum::date_from_str("1970-01-05").unwrap(); + /// + /// assert_eq!(&format!("{t}"), "1970-01-05"); + /// assert_eq!(Literal::date(4), t.into()); + /// ``` + pub fn date_from_str>(s: S) -> Result { + let t = s.as_ref().parse::().map_err(|e| { + invalid_data!("Can't parse date from string: {}", s.as_ref()).with_source(e) + })?; + + Ok(Self::date(date::date_from_naive_date(t))) + } + + /// Create date literal from calendar date (year, month and day). + /// + /// See [`NaiveDate::from_ymd_opt`]. + /// + /// Example: + /// + ///```rust + /// use iceberg::spec::{Datum, Literal}; + /// let t = Datum::date_from_ymd(1970, 1, 5).unwrap(); + /// + /// assert_eq!(&format!("{t}"), "1970-01-05"); + /// assert_eq!(Literal::date(4), t.into()); + /// ``` + pub fn date_from_ymd(year: i32, month: u32, day: u32) -> Result { + let t = NaiveDate::from_ymd_opt(year, month, day).ok_or_else(|| { + invalid_data!("Can't create date from year: {year}, month: {month}, day: {day}") + })?; + + Ok(Self::date(date::date_from_naive_date(t))) + } + + /// Creates time literal in microseconds directly. + /// + /// It will return error when it's negative or too large to fit in 24 hours. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::{Datum, Literal}; + /// let micro_secs = { + /// 1 * 3600 * 1_000_000 + // 1 hour + /// 2 * 60 * 1_000_000 + // 2 minutes + /// 1 * 1_000_000 + // 1 second + /// 888999 // microseconds + /// }; + /// + /// let t = Datum::time_micros(micro_secs).unwrap(); + /// + /// assert_eq!(&format!("{t}"), "01:02:01.888999"); + /// assert_eq!(Literal::time(micro_secs), t.into()); + /// + /// let negative_value = -100; + /// assert!(Datum::time_micros(negative_value).is_err()); + /// + /// let too_large_value = 36 * 60 * 60 * 1_000_000; // Too large to fit in 24 hours. + /// assert!(Datum::time_micros(too_large_value).is_err()); + /// ``` + pub fn time_micros(value: i64) -> Result { + ensure_data_valid!( + (0..=MAX_TIME_VALUE).contains(&value), + "Invalid value for Time type: {}", + value + ); + + Ok(Self { + r#type: PrimitiveType::Time, + literal: PrimitiveLiteral::Long(value), + }) + } + + /// Creates time literal from [`chrono::NaiveTime`]. + fn time_from_naive_time(t: NaiveTime) -> Self { + let duration = t - date::unix_epoch().time(); + // It's safe to unwrap here since less than 24 hours will never overflow. + let micro_secs = duration.num_microseconds().unwrap(); + + Self { + r#type: PrimitiveType::Time, + literal: PrimitiveLiteral::Long(micro_secs), + } + } + + /// Creates time literal in microseconds in `%H:%M:%S:.f` format. + /// + /// See [`NaiveTime::from_str`] for details. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Datum, Literal}; + /// let t = Datum::time_from_str("01:02:01.888999777").unwrap(); + /// + /// assert_eq!(&format!("{t}"), "01:02:01.888999"); + /// ``` + pub fn time_from_str>(s: S) -> Result { + let t = s.as_ref().parse::().map_err(|e| { + invalid_data!("Can't parse time from string: {}", s.as_ref()).with_source(e) + })?; + + Ok(Self::time_from_naive_time(t)) + } + + /// Creates time literal from hour, minute, second, and microseconds. + /// + /// See [`NaiveTime::from_hms_micro_opt`]. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Datum, Literal}; + /// let t = Datum::time_from_hms_micro(22, 15, 33, 111).unwrap(); + /// + /// assert_eq!(&format!("{t}"), "22:15:33.000111"); + /// ``` + pub fn time_from_hms_micro(hour: u32, min: u32, sec: u32, micro: u32) -> Result { + let t = NaiveTime::from_hms_micro_opt(hour, min, sec, micro) + .ok_or_else(|| invalid_data!("Can't create time from hour: {hour}, min: {min}, second: {sec}, microsecond: {micro}"))?; + Ok(Self::time_from_naive_time(t)) + } + + /// Creates a timestamp from unix epoch in microseconds. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// let t = Datum::timestamp_micros(1000); + /// + /// assert_eq!(&format!("{t}"), "1970-01-01 00:00:00.001"); + /// ``` + pub fn timestamp_micros(value: i64) -> Self { + Self { + r#type: PrimitiveType::Timestamp, + literal: PrimitiveLiteral::Long(value), + } + } + + /// Creates a timestamp from unix epoch in nanoseconds. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// let t = Datum::timestamp_nanos(1000); + /// + /// assert_eq!(&format!("{t}"), "1970-01-01 00:00:00.000001"); + /// ``` + pub fn timestamp_nanos(value: i64) -> Self { + Self { + r#type: PrimitiveType::TimestampNs, + literal: PrimitiveLiteral::Long(value), + } + } + + /// Creates a timestamp from [`DateTime`]. + /// + /// Example: + /// + /// ```rust + /// use chrono::{NaiveDate, NaiveDateTime, TimeZone, Utc}; + /// use iceberg::spec::Datum; + /// let t = Datum::timestamp_from_datetime( + /// NaiveDate::from_ymd_opt(1992, 3, 1) + /// .unwrap() + /// .and_hms_micro_opt(1, 2, 3, 88) + /// .unwrap(), + /// ); + /// + /// assert_eq!(&format!("{t}"), "1992-03-01 01:02:03.000088"); + /// ``` + pub fn timestamp_from_datetime(dt: NaiveDateTime) -> Self { + Self::timestamp_micros(dt.and_utc().timestamp_micros()) + } + + /// Parse a timestamp in `%Y-%m-%dT%H:%M:%S%.f` format. + /// + /// See [`NaiveDateTime::from_str`]. + /// + /// Example: + /// + /// ```rust + /// use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; + /// use iceberg::spec::{Datum, Literal}; + /// let t = Datum::timestamp_from_str("1992-03-01T01:02:03.000088").unwrap(); + /// + /// assert_eq!(&format!("{t}"), "1992-03-01 01:02:03.000088"); + /// ``` + pub fn timestamp_from_str>(s: S) -> Result { + let dt = s + .as_ref() + .parse::() + .map_err(|e| invalid_data!("Can't parse timestamp.").with_source(e))?; + + Ok(Self::timestamp_from_datetime(dt)) + } + + /// Creates a timestamp with timezone from unix epoch in microseconds. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// let t = Datum::timestamptz_micros(1000); + /// + /// assert_eq!(&format!("{t}"), "1970-01-01 00:00:00.001 UTC"); + /// ``` + pub fn timestamptz_micros(value: i64) -> Self { + Self { + r#type: PrimitiveType::Timestamptz, + literal: PrimitiveLiteral::Long(value), + } + } + + /// Creates a timestamp with timezone from unix epoch in nanoseconds. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// let t = Datum::timestamptz_nanos(1000); + /// + /// assert_eq!(&format!("{t}"), "1970-01-01 00:00:00.000001 UTC"); + /// ``` + pub fn timestamptz_nanos(value: i64) -> Self { + Self { + r#type: PrimitiveType::TimestamptzNs, + literal: PrimitiveLiteral::Long(value), + } + } + + /// Creates a timestamp with timezone from [`DateTime`]. + /// Example: + /// + /// ```rust + /// use chrono::{TimeZone, Utc}; + /// use iceberg::spec::Datum; + /// let t = Datum::timestamptz_from_datetime(Utc.timestamp_opt(1000, 0).unwrap()); + /// + /// assert_eq!(&format!("{t}"), "1970-01-01 00:16:40 UTC"); + /// ``` + pub fn timestamptz_from_datetime(dt: DateTime) -> Self { + Self::timestamptz_micros(dt.with_timezone(&Utc).timestamp_micros()) + } + + /// Parse timestamp with timezone in RFC3339 format. + /// + /// See [`DateTime::from_str`]. + /// + /// Example: + /// + /// ```rust + /// use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; + /// use iceberg::spec::{Datum, Literal}; + /// let t = Datum::timestamptz_from_str("1992-03-01T01:02:03.000088+08:00").unwrap(); + /// + /// assert_eq!(&format!("{t}"), "1992-02-29 17:02:03.000088 UTC"); + /// ``` + pub fn timestamptz_from_str>(s: S) -> Result { + let dt = DateTime::::from_str(s.as_ref()) + .map_err(|e| invalid_data!("Can't parse datetime.").with_source(e))?; + + Ok(Self::timestamptz_from_datetime(dt)) + } + + /// Creates a string literal. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// let t = Datum::string("ss"); + /// + /// assert_eq!(&format!("{t}"), r#""ss""#); + /// ``` + pub fn string(s: S) -> Self { + Self { + r#type: PrimitiveType::String, + literal: PrimitiveLiteral::String(s.to_string()), + } + } + + /// Creates uuid literal. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// use uuid::uuid; + /// let t = Datum::uuid(uuid!("a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8")); + /// + /// assert_eq!(&format!("{t}"), "a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8"); + /// ``` + pub fn uuid(uuid: uuid::Uuid) -> Self { + Self { + r#type: PrimitiveType::Uuid, + literal: PrimitiveLiteral::UInt128(uuid.as_u128()), + } + } + + /// Creates uuid from str. See [`uuid::Uuid::parse_str`]. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// let t = Datum::uuid_from_str("a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8").unwrap(); + /// + /// assert_eq!(&format!("{t}"), "a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8"); + /// ``` + pub fn uuid_from_str>(s: S) -> Result { + let uuid = uuid::Uuid::parse_str(s.as_ref()).map_err(|e| { + invalid_data!("Can't parse uuid from string: {}", s.as_ref()).with_source(e) + })?; + Ok(Self::uuid(uuid)) + } + + /// Creates a fixed literal from bytes. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::{Datum, Literal, PrimitiveLiteral}; + /// let t = Datum::fixed(vec![1u8, 2u8]); + /// + /// assert_eq!(&format!("{t}"), "0102"); + /// ``` + pub fn fixed>(input: I) -> Self { + let value: Vec = input.into_iter().collect(); + Self { + r#type: PrimitiveType::Fixed(value.len() as u64), + literal: PrimitiveLiteral::Binary(value), + } + } + + /// Creates a binary literal from bytes. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// let t = Datum::binary(vec![1u8, 100u8]); + /// + /// assert_eq!(&format!("{t}"), "0164"); + /// ``` + pub fn binary>(input: I) -> Self { + Self { + r#type: PrimitiveType::Binary, + literal: PrimitiveLiteral::Binary(input.into_iter().collect()), + } + } + + /// Creates decimal literal from string. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// let t = Datum::decimal_from_str("123.45").unwrap(); + /// + /// assert_eq!(&format!("{t}"), "123.45"); + /// ``` + pub fn decimal_from_str>(s: S) -> Result { + let decimal = decimal_from_str_exact(s.as_ref())?; + + Self::decimal(decimal) + } + + /// Try to create a decimal literal from [`Decimal`]. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Datum; + /// + /// let t = Datum::decimal_from_str("1.23").unwrap(); + /// + /// assert_eq!(&format!("{t}"), "1.23"); + /// ``` + pub fn decimal(value: Decimal) -> Result { + ensure_data_valid!(value.is_finite(), "Decimal value must be finite: {}", value); + let scale = decimal_scale(&value); + + let r#type = Type::decimal(MAX_DECIMAL_PRECISION, scale)?; + if let Type::Primitive(p) = r#type { + Ok(Self { + r#type: p, + literal: PrimitiveLiteral::Int128(decimal_mantissa(&value)), + }) + } else { + unreachable!("Decimal type must be primitive.") + } + } + + /// Try to create a decimal literal from [`Decimal`] with precision. + /// + /// This method allows specifying a custom precision for the decimal type, + /// which is useful when you need to control the storage requirements. + /// Use [`Datum::decimal`] if you want to use the maximum precision (38). + pub fn decimal_with_precision(value: Decimal, precision: u32) -> Result { + ensure_data_valid!(value.is_finite(), "Decimal value must be finite: {}", value); + let scale = decimal_scale(&value); + let mantissa = decimal_mantissa(&value); + + Self::decimal_from_mantissa(mantissa, precision, scale) + } + + fn i64_to_i32 + PartialOrd>(val: T) -> Datum { + if val > INT_MAX as i64 { + Datum::new(PrimitiveType::Int, PrimitiveLiteral::AboveMax) + } else if val < INT_MIN as i64 { + Datum::new(PrimitiveType::Int, PrimitiveLiteral::BelowMin) + } else { + Datum::int(val.into() as i32) + } + } + + fn i128_to_i32 + PartialOrd>(val: T) -> Datum { + if val > INT_MAX as i128 { + Datum::new(PrimitiveType::Int, PrimitiveLiteral::AboveMax) + } else if val < INT_MIN as i128 { + Datum::new(PrimitiveType::Int, PrimitiveLiteral::BelowMin) + } else { + Datum::int(val.into() as i32) + } + } + + fn i128_to_i64 + PartialOrd>(val: T) -> Datum { + if val > LONG_MAX as i128 { + Datum::new(PrimitiveType::Long, PrimitiveLiteral::AboveMax) + } else if val < LONG_MIN as i128 { + Datum::new(PrimitiveType::Long, PrimitiveLiteral::BelowMin) + } else { + Datum::long(val.into() as i64) + } + } + + fn f64_to_f32(val: f64) -> Datum { + if val > f32::MAX as f64 { + Datum::new(PrimitiveType::Float, PrimitiveLiteral::AboveMax) + } else if val < (f32::MIN as f64) { + Datum::new(PrimitiveType::Float, PrimitiveLiteral::BelowMin) + } else { + Datum::float(val as f32) + } + } + + fn string_to_i128>(s: S) -> Result { + s.as_ref() + .parse::() + .map_err(|e| invalid_data!("Can't parse string to i128.").with_source(e)) + } + + fn decimal_from_mantissa(mantissa: i128, precision: u32, scale: u32) -> Result { + let r#type = Type::decimal(precision, scale)?; + if decimal_precision(mantissa) > precision { + let value = decimal_from_i128_with_scale(mantissa, scale); + return Err(invalid_data!( + "Decimal value {value} is too large for precision {precision}" + )); + } + + if let Type::Primitive(p) = r#type { + Ok(Self { + r#type: p, + literal: PrimitiveLiteral::Int128(mantissa), + }) + } else { + unreachable!("Decimal type must be primitive.") + } + } + + /// Replaces a decimal's unscaled value while preserving its precision and scale. + /// + /// The value is not checked against precision because transform projections + /// may use boundaries outside the source type's range. + pub(crate) fn with_decimal_mantissa(&self, mantissa: i128) -> Result { + ensure_data_valid!( + matches!(self.r#type, PrimitiveType::Decimal { .. }), + "Expected a decimal datum, got: {:?}", + self.r#type + ); + Ok(Self::new( + self.r#type.clone(), + PrimitiveLiteral::Int128(mantissa), + )) + } + + /// Convert the datum to `target_type`. + pub fn to(self, target_type: &Type) -> Result { + match target_type { + Type::Primitive(target_primitive_type) => { + match (&self.literal, &self.r#type, target_primitive_type) { + (PrimitiveLiteral::Int(val), _, PrimitiveType::Int) => Ok(Datum::int(*val)), + (PrimitiveLiteral::Int(val), _, PrimitiveType::Date) => Ok(Datum::date(*val)), + (PrimitiveLiteral::Int(val), _, PrimitiveType::Long) => Ok(Datum::long(*val)), + (PrimitiveLiteral::Long(val), _, PrimitiveType::Int) => { + Ok(Datum::i64_to_i32(*val)) + } + (PrimitiveLiteral::Double(val), _, PrimitiveType::Float) => { + Ok(Datum::f64_to_f32(val.0)) + } + (PrimitiveLiteral::Float(val), _, PrimitiveType::Double) => { + Ok(Datum::double(val.0 as f64)) + } + (PrimitiveLiteral::Long(val), _, PrimitiveType::Timestamp) => { + Ok(Datum::timestamp_micros(*val)) + } + (PrimitiveLiteral::Long(val), _, PrimitiveType::Timestamptz) => { + Ok(Datum::timestamptz_micros(*val)) + } + // Let's wait with nano's until this clears up: https://github.com/apache/iceberg/pull/11775 + (PrimitiveLiteral::Int128(val), _, PrimitiveType::Long) => { + Ok(Datum::i128_to_i64(*val)) + } + ( + PrimitiveLiteral::Int128(val), + PrimitiveType::Decimal { + scale: self_scale, .. + }, + PrimitiveType::Decimal { + precision, + scale: target_scale, + }, + ) if self_scale == target_scale => { + Datum::decimal_from_mantissa(*val, *precision, *target_scale) + } + // Java's DecimalLiteral.to() is a no-op for decimal targets, even when scales + // differ. We intentionally do not mirror that here: the same mantissa with a + // different scale represents a different value, so scale conversion needs + // explicit rescaling. + ( + PrimitiveLiteral::Int128(_), + PrimitiveType::Decimal { + scale: self_scale, .. + }, + PrimitiveType::Decimal { + scale: target_scale, + .. + }, + ) => Err(invalid_data!( + "Decimal scale conversion is not supported: source scale {self_scale}, target scale {target_scale}" + )), + (PrimitiveLiteral::String(val), _, PrimitiveType::Boolean) => { + Datum::bool_from_str(val) + } + (PrimitiveLiteral::String(val), _, PrimitiveType::Int) => { + Datum::string_to_i128(val).map(Datum::i128_to_i32) + } + (PrimitiveLiteral::String(val), _, PrimitiveType::Long) => { + Datum::string_to_i128(val).map(Datum::i128_to_i64) + } + (PrimitiveLiteral::String(val), _, PrimitiveType::Timestamp) => { + Datum::timestamp_from_str(val) + } + (PrimitiveLiteral::String(val), _, PrimitiveType::Timestamptz) => { + Datum::timestamptz_from_str(val) + } + + // TODO: implement more type conversions + (_, self_type, target_type) if self_type == target_type => Ok(self), + _ => Err(invalid_data!( + "Can't convert datum from {} type to {} type.", + self.r#type, + target_primitive_type + )), + } + } + _ => Err(invalid_data!( + "Can't convert datum from {} type to {} type.", + self.r#type, + target_type + )), + } + } + + /// Get the primitive literal from datum. + pub fn literal(&self) -> &PrimitiveLiteral { + &self.literal + } + + /// Get the primitive type from datum. + pub fn data_type(&self) -> &PrimitiveType { + &self.r#type + } + + /// Returns a copy whose literal is widened to `target` for the value + /// promotions `int` -> `long` and `float` -> `double`, so a value written + /// under the narrower type compares against a predicate bound to the wider + /// one. Returns `None` when `target` does not widen this value (decimals + /// already compare across precisions), so callers can avoid a copy. + pub(crate) fn promote_to(&self, target: &PrimitiveType) -> Option { + self.literal + .promote_to(target) + .map(|literal| Datum::new(target.clone(), literal)) + } + + /// Returns true if the Literal represents a primitive type + /// that can be a NaN, and that it's value is NaN + pub fn is_nan(&self) -> bool { + match self.literal { + PrimitiveLiteral::Double(val) => val.is_nan(), + PrimitiveLiteral::Float(val) => val.is_nan(), + _ => false, + } + } + + /// Returns a human-readable string representation of this literal. + /// + /// For string literals, this returns the raw string value without quotes. + /// Float and double literals are formatted like Java's `Float.toString` / + /// `Double.toString` (JDK 19+). + /// For all other literals, it falls back to [`to_string()`](ToString::to_string). + pub fn to_human_string(&self) -> String { + match self.literal() { + PrimitiveLiteral::String(s) => s.to_string(), + PrimitiveLiteral::Float(val) => human_float(val.0), + PrimitiveLiteral::Double(val) => human_float(val.0), + _ => self.to_string(), + } + } +} + +/// Formats a float like Java's `Float.toString` / `Double.toString` (JDK 19+): +/// plain decimal when `1e-3 <= |value| < 1e7`, otherwise `.E`, +/// always with at least one digit after the point. +fn human_float(value: T) -> String { + if value.is_nan() { + return "NaN".to_string(); + } + let sign = if value.is_sign_negative() { "-" } else { "" }; + if value.is_infinite() { + return format!("{sign}Infinity"); + } + if value.is_zero() { + return format!("{sign}0.0"); + } + + let abs = value.abs(); + // `{:e}` gives the fewest digits that round-trip. Java uses at least two + // digits and, among decimals of that length that round-trip, the closest + // one, preferring an even last digit on ties (2097152.2 for 2097152.25f32, + // 4.9E-324 for f64::from_bits(1)). Exact formatting picks exactly that. + let shortest = format!("{abs:e}"); + let precision = shortest.find('e').unwrap().saturating_sub(2).max(1); + let closest = format!("{abs:.precision$e}"); + let scientific = if closest.parse::().is_ok_and(|v| v == abs) { + closest + } else { + shortest + }; + let (mantissa, exponent) = scientific.split_once('e').unwrap(); + let exponent: i32 = exponent.parse().unwrap(); + let digits = mantissa.replace('.', ""); + let digits = digits.trim_end_matches('0'); + + if (-3..7).contains(&exponent) { + if exponent < 0 { + let zeros = "0".repeat((-exponent - 1) as usize); + format!("{sign}0.{zeros}{digits}") + } else { + let int_len = exponent as usize + 1; + if digits.len() > int_len { + format!("{sign}{}.{}", &digits[..int_len], &digits[int_len..]) + } else { + format!("{sign}{digits:0 1 { &digits[1..] } else { "0" }; + format!("{sign}{}.{fraction}E{exponent}", &digits[..1]) + } +} diff --git a/crates/iceberg/src/spec/values/decimal_utils.rs b/crates/iceberg/src/spec/values/decimal_utils.rs new file mode 100644 index 0000000000..942ec69eab --- /dev/null +++ b/crates/iceberg/src/spec/values/decimal_utils.rs @@ -0,0 +1,466 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Compatibility layer for decimal operations. +//! +//! Provides rust_decimal-compatible API using fastnum's D128 internally. +//! D128 supports 38-digit precision, meeting the Iceberg spec requirement. + +use fastnum::D128; +use fastnum::decimal::Context; + +use crate::error::invalid_data; +use crate::{Result, ensure_data_valid}; + +/// Re-export D128 as the Decimal type for use throughout the crate. +pub type Decimal = D128; + +/// Create a D128 from mantissa (i128) and scale (u32). +/// +/// This is equivalent to rust_decimal's `Decimal::from_i128_with_scale`. +/// The value is computed as: mantissa * 10^(-scale) +/// +/// For example: +/// - mantissa=12345, scale=2 => 123.45 +/// - mantissa=-456, scale=3 => -0.456 +pub fn decimal_from_i128_with_scale(mantissa: i128, scale: u32) -> Decimal { + if scale == 0 { + return D128::from_i128(mantissa).expect("i128 always fits in D128"); + } + + // Convert mantissa to string and insert decimal point at the right position + let is_negative = mantissa < 0; + let abs_str = mantissa.unsigned_abs().to_string(); + let scale_usize = scale as usize; + + let decimal_str = if abs_str.len() <= scale_usize { + // Need leading zeros: e.g., mantissa=456, scale=3 => "0.456" + // Or mantissa=5, scale=3 => "0.005" + let zeros_needed = scale_usize - abs_str.len(); + format!( + "{}0.{}{}", + if is_negative { "-" } else { "" }, + "0".repeat(zeros_needed), + abs_str + ) + } else { + // Insert decimal point: e.g., mantissa=12345, scale=2 => "123.45" + let decimal_pos = abs_str.len() - scale_usize; + format!( + "{}{}.{}", + if is_negative { "-" } else { "" }, + &abs_str[..decimal_pos], + &abs_str[decimal_pos..] + ) + }; + + D128::from_str(&decimal_str, Context::default()) + .expect("constructed decimal string is always valid") +} + +/// Try to create a D128 from mantissa and scale, with validation. +/// +/// This is equivalent to rust_decimal's `Decimal::try_from_i128_with_scale`. +/// Currently always succeeds for 38-digit decimals. +pub fn try_decimal_from_i128_with_scale(mantissa: i128, scale: u32) -> Result { + // For now, always succeeds since D128 supports full 38-digit precision + Ok(decimal_from_i128_with_scale(mantissa, scale)) +} + +/// Create a D128 from i64 mantissa and scale. +/// +/// This is equivalent to rust_decimal's `Decimal::new`. +#[allow(dead_code)] +pub fn decimal_new(mantissa: i64, scale: u32) -> Decimal { + decimal_from_i128_with_scale(mantissa as i128, scale) +} + +/// Parse a decimal from string with exact representation. +/// +/// This is equivalent to rust_decimal's `Decimal::from_str_exact`. +pub fn decimal_from_str_exact(s: &str) -> Result { + let decimal = D128::from_str(s, Context::default()) + .map_err(|e| invalid_data!("Can't parse decimal: {e}"))?; + ensure_data_valid!( + decimal.is_finite(), + "Decimal value must be finite: {}", + decimal + ); + Ok(decimal) +} + +/// Get the mantissa (unscaled coefficient) as i128. +/// +/// This is equivalent to rust_decimal's `decimal.mantissa()`. +/// +/// The mantissa is signed: negative decimals return negative mantissa. +pub fn decimal_mantissa(d: &Decimal) -> i128 { + // digits() returns unsigned coefficient as UInt + // For Iceberg decimals (max 38 digits), this always fits in u128/i128 + let digits = d.digits(); + + // Convert UInt<2> to u128 - this always succeeds for Iceberg-compliant decimals + // since 38 digits requires ~127 bits and u128 has 128 bits + let unsigned: u128 = digits + .to_u128() + .expect("Iceberg decimals (max 38 digits) always fit in u128"); + + let signed = unsigned as i128; + if d.is_sign_negative() { + -signed + } else { + signed + } +} + +/// Get the decimal digit precision of a mantissa. +/// +/// A mantissa of 0 has a precision of 1. +pub fn decimal_precision(mantissa: i128) -> u32 { + mantissa + .unsigned_abs() + .checked_ilog10() + .map_or(1, |x| x + 1) +} + +/// Get the scale (number of digits after decimal point). +/// +/// This is equivalent to rust_decimal's `decimal.scale()`. +pub fn decimal_scale(d: &Decimal) -> u32 { + let frac = d.fractional_digits_count(); + if frac < 0 { 0 } else { frac as u32 } +} + +/// Rescale a decimal to the given scale, returning the rescaled value. +/// +/// This is equivalent to rust_decimal's `decimal.rescale(scale)`. +pub fn decimal_rescale(d: Decimal, scale: u32) -> Decimal { + d.rescale(scale as i16) +} + +/// Convert big-endian signed bytes to i128. +/// +/// This handles variable-length byte arrays (up to 16 bytes) with sign extension. +/// Returns None if the byte array is longer than 16 bytes. +pub fn i128_from_be_bytes(bytes: &[u8]) -> Option { + if bytes.is_empty() { + return Some(0); + } + if bytes.len() > 16 { + return None; // Too large for i128 + } + + // Check sign bit (most significant bit of first byte) + let is_negative = bytes[0] & 0x80 != 0; + + // Pad to 16 bytes with sign extension + let mut padded = if is_negative { [0xFF; 16] } else { [0; 16] }; + let start = 16 - bytes.len(); + padded[start..].copy_from_slice(bytes); + + Some(i128::from_be_bytes(padded)) +} + +/// Convert i128 to big-endian signed bytes with minimum length. +/// +/// This produces the shortest two's complement representation of the value. +/// The result is suitable for Iceberg decimal binary serialization. +pub fn i128_to_be_bytes_min(value: i128) -> Vec { + let bytes = value.to_be_bytes(); + + // Find the first significant byte + // For positive numbers, skip leading 0x00 bytes (but keep sign bit) + // For negative numbers, skip leading 0xFF bytes (but keep sign bit) + let is_negative = value < 0; + let skip_byte = if is_negative { 0xFF } else { 0x00 }; + + let mut start = 0; + while start < 15 && bytes[start] == skip_byte { + // Check if the next byte has the correct sign bit + let next_byte = bytes[start + 1]; + let next_is_negative = (next_byte & 0x80) != 0; + if next_is_negative == is_negative { + start += 1; + } else { + break; + } + } + + bytes[start..].to_vec() +} + +/// Encode an i128 as exactly `len` big-endian two's complement bytes, matching a +/// Parquet `FIXED_LEN_BYTE_ARRAY` column of that declared `type_length`. +/// +/// Returns `None` if `value` does not fit in `len` bytes, since a truncated +/// encoding would represent a different number. +pub(crate) fn decimal_to_fixed_length_bytes_exact(value: i128, len: usize) -> Option> { + if len == 0 || len > 16 { + return None; + } + + let be_bytes = value.to_be_bytes(); + let offset = 16 - len; + let sign_byte = if value < 0 { 0xFF } else { 0x00 }; + + // The bytes being dropped must be pure sign extension. + if be_bytes[..offset].iter().any(|&b| b != sign_byte) { + return None; + } + + // The retained leading byte must carry the value's sign. + if (be_bytes[offset] & 0x80 != 0) != (value < 0) { + return None; + } + + Some(be_bytes[offset..].to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_decimal_from_i128_with_scale() { + let d = decimal_from_i128_with_scale(12345, 2); + assert_eq!(d.to_string(), "123.45"); + + let d = decimal_from_i128_with_scale(-12345, 2); + assert_eq!(d.to_string(), "-123.45"); + + let d = decimal_from_i128_with_scale(0, 5); + assert_eq!(d.to_string(), "0.00000"); + } + + #[test] + fn test_decimal_new() { + let d = decimal_new(123, 2); + assert_eq!(d.to_string(), "1.23"); + + let d = decimal_new(-456, 3); + assert_eq!(d.to_string(), "-0.456"); + } + + #[test] + fn test_decimal_from_str_exact() { + let d = decimal_from_str_exact("123.45").unwrap(); + assert_eq!(d.to_string(), "123.45"); + + let d = decimal_from_str_exact("-0.001").unwrap(); + assert_eq!(d.to_string(), "-0.001"); + + let d = decimal_from_str_exact("99999999999999999999999999999999999999").unwrap(); + assert_eq!(d.to_string(), "99999999999999999999999999999999999999"); + } + + #[test] + fn test_decimal_mantissa() { + let d = decimal_from_i128_with_scale(12345, 2); + assert_eq!(decimal_mantissa(&d), 12345); + + let d = decimal_from_i128_with_scale(-12345, 2); + assert_eq!(decimal_mantissa(&d), -12345); + } + + #[test] + fn test_decimal_precision() { + assert_eq!(decimal_precision(0), 1); + assert_eq!(decimal_precision(5), 1); + assert_eq!(decimal_precision(42), 2); + assert_eq!(decimal_precision(-42), 2); + // power-of-10 boundaries, where off-by-one bugs in log10 logic live + assert_eq!(decimal_precision(9), 1); + assert_eq!(decimal_precision(10), 2); + assert_eq!(decimal_precision(99), 2); + assert_eq!(decimal_precision(100), 3); + + // max Iceberg decimal precision + assert_eq!( + decimal_precision(99999999999999999999999999999999999999), + 38 + ); + assert_eq!( + decimal_precision(-99999999999999999999999999999999999999), + 38 + ); + + // i128 extremes + assert_eq!(decimal_precision(i128::MAX), 39); + assert_eq!(decimal_precision(i128::MIN), 39); + } + + #[test] + fn test_decimal_scale() { + let d = decimal_from_i128_with_scale(12345, 2); + assert_eq!(decimal_scale(&d), 2); + + let d = decimal_from_i128_with_scale(12345, 0); + assert_eq!(decimal_scale(&d), 0); + } + + #[test] + fn test_decimal_rescale() { + let d = decimal_from_str_exact("123.45").unwrap(); + let rescaled = decimal_rescale(d, 4); + assert_eq!(decimal_scale(&rescaled), 4); + assert_eq!(decimal_mantissa(&rescaled), 1234500); + } + + #[test] + fn test_38_digit_precision() { + // Test that we can handle 38-digit decimals (Iceberg spec requirement) + let max_38_digits = "99999999999999999999999999999999999999"; + let d = decimal_from_str_exact(max_38_digits).unwrap(); + assert_eq!(d.to_string(), max_38_digits); + + let min_38_digits = "-99999999999999999999999999999999999999"; + let d = decimal_from_str_exact(min_38_digits).unwrap(); + assert_eq!(d.to_string(), min_38_digits); + } + + #[test] + fn test_i128_from_be_bytes() { + // Empty bytes + assert_eq!(i128_from_be_bytes(&[]), Some(0)); + + // Positive values + assert_eq!(i128_from_be_bytes(&[0x01]), Some(1)); + assert_eq!(i128_from_be_bytes(&[0x7F]), Some(127)); + assert_eq!(i128_from_be_bytes(&[0x00, 0xFF]), Some(255)); + assert_eq!(i128_from_be_bytes(&[0x04, 0xD2]), Some(1234)); + + // Negative values (sign extension) + assert_eq!(i128_from_be_bytes(&[0xFF]), Some(-1)); + assert_eq!(i128_from_be_bytes(&[0x80]), Some(-128)); + assert_eq!(i128_from_be_bytes(&[0xFB, 0x2E]), Some(-1234)); + + // Too large (> 16 bytes) + assert_eq!(i128_from_be_bytes(&[0; 17]), None); + } + + #[test] + fn test_i128_to_be_bytes_min() { + // Positive values + assert_eq!(i128_to_be_bytes_min(0), vec![0x00]); + assert_eq!(i128_to_be_bytes_min(1), vec![0x01]); + assert_eq!(i128_to_be_bytes_min(127), vec![0x7F]); + assert_eq!(i128_to_be_bytes_min(128), vec![0x00, 0x80]); + assert_eq!(i128_to_be_bytes_min(255), vec![0x00, 0xFF]); + assert_eq!(i128_to_be_bytes_min(1234), vec![0x04, 0xD2]); + + // Negative values + assert_eq!(i128_to_be_bytes_min(-1), vec![0xFF]); + assert_eq!(i128_to_be_bytes_min(-128), vec![0x80]); + assert_eq!(i128_to_be_bytes_min(-129), vec![0xFF, 0x7F]); + assert_eq!(i128_to_be_bytes_min(-1234), vec![0xFB, 0x2E]); + + // Round trip test + for val in [ + 0i128, + 1, + -1, + 127, + -128, + 255, + -256, + 12345, + -12345, + i128::MAX, + i128::MIN, + ] { + let bytes = i128_to_be_bytes_min(val); + assert_eq!( + i128_from_be_bytes(&bytes), + Some(val), + "Round trip failed for {val}" + ); + } + } + + #[test] + fn test_decimal_to_fixed_length_bytes_exact_positive() { + let bytes = decimal_to_fixed_length_bytes_exact(12345, 9).unwrap(); + assert_eq!(bytes.len(), 9); + // Big-endian, zero-padded on the left: 12345 = 0x3039 + assert_eq!(bytes[7], 0x30); + assert_eq!(bytes[8], 0x39); + assert!(bytes[..7].iter().all(|&b| b == 0x00)); + } + + #[test] + fn test_decimal_to_fixed_length_bytes_exact_negative() { + let bytes = decimal_to_fixed_length_bytes_exact(-1, 9).unwrap(); + assert_eq!(bytes.len(), 9); + assert!(bytes.iter().all(|&b| b == 0xFF)); + } + + #[test] + fn test_decimal_to_fixed_length_bytes_exact_round_trip() { + for value in [ + 0i128, + 1, + -1, + 12345, + -12345, + i64::MAX as i128, + i64::MIN as i128, + ] { + let bytes = decimal_to_fixed_length_bytes_exact(value, 16).unwrap(); + assert_eq!(bytes.len(), 16); + assert_eq!( + i128_from_be_bytes(&bytes), + Some(value), + "Round trip failed for value={value}" + ); + } + } + + /// The length must be honoured exactly: a value needing more bytes cannot be + /// truncated into the column's width, because the truncation would encode a + /// different number and probe the wrong bloom filter slot. + #[test] + fn test_decimal_to_fixed_length_bytes_exact_rejects_overflow() { + // 200 needs a leading zero byte to stay positive in two's complement. + assert_eq!(decimal_to_fixed_length_bytes_exact(200, 1), None); + assert_eq!( + decimal_to_fixed_length_bytes_exact(200, 2), + Some(vec![0x00, 0xC8]) + ); + + assert_eq!(decimal_to_fixed_length_bytes_exact(-200, 1), None); + assert_eq!( + decimal_to_fixed_length_bytes_exact(-200, 2), + Some(vec![0xFF, 0x38]) + ); + + // Exactly representable boundaries. + assert_eq!( + decimal_to_fixed_length_bytes_exact(127, 1), + Some(vec![0x7F]) + ); + assert_eq!(decimal_to_fixed_length_bytes_exact(128, 1), None); + assert_eq!( + decimal_to_fixed_length_bytes_exact(-128, 1), + Some(vec![0x80]) + ); + assert_eq!(decimal_to_fixed_length_bytes_exact(-129, 1), None); + + assert_eq!(decimal_to_fixed_length_bytes_exact(i128::MAX, 15), None); + assert_eq!(decimal_to_fixed_length_bytes_exact(0, 0), None); + assert_eq!(decimal_to_fixed_length_bytes_exact(0, 17), None); + } +} diff --git a/crates/iceberg/src/spec/values/literal.rs b/crates/iceberg/src/spec/values/literal.rs new file mode 100644 index 0000000000..52e4a5c731 --- /dev/null +++ b/crates/iceberg/src/spec/values/literal.rs @@ -0,0 +1,817 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Literal values in Iceberg + +use std::any::Any; +use std::str::FromStr; + +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; +use ordered_float::OrderedFloat; +use serde_json::{Map as JsonMap, Number, Value as JsonValue}; +use uuid::Uuid; + +use super::Map; +use super::decimal_utils::{ + decimal_from_str_exact, decimal_mantissa, decimal_rescale, try_decimal_from_i128_with_scale, +}; +use super::primitive::PrimitiveLiteral; +use super::struct_value::Struct; +use super::temporal::{date, time, timestamp, timestamptz}; +use crate::error::{Error, ErrorKind, Result, invalid_data}; +use crate::spec::datatypes::{PrimitiveType, Type}; + +/// Values present in iceberg type +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Literal { + /// A primitive value + Primitive(PrimitiveLiteral), + /// A struct is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema. + /// Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type. + /// Fields may have an optional comment or doc string. Fields can have default values. + Struct(Struct), + /// A list is a collection of values with some element type. + /// The element field has an integer id that is unique in the table schema. + /// Elements can be either optional or required. Element types may be any type. + List(Vec>), + /// A map is a collection of key-value pairs with a key type and a value type. + /// Both the key field and value field each have an integer id that is unique in the table schema. + /// Map keys are required and map values can be either optional or required. Both map keys and map values may be any type, including nested types. + Map(Map), +} + +impl Literal { + /// Creates a boolean value. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Literal, PrimitiveLiteral}; + /// let t = Literal::bool(true); + /// + /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Boolean(true)), t); + /// ``` + pub fn bool>(t: T) -> Self { + Self::Primitive(PrimitiveLiteral::Boolean(t.into())) + } + + /// Creates a boolean value from string. + /// See [Parse bool from str](https://doc.rust-lang.org/stable/std/primitive.bool.html#impl-FromStr-for-bool) for reference. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Literal, PrimitiveLiteral}; + /// let t = Literal::bool_from_str("false").unwrap(); + /// + /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Boolean(false)), t); + /// ``` + pub fn bool_from_str>(s: S) -> Result { + let v = s + .as_ref() + .parse::() + .map_err(|e| invalid_data!("Can't parse string to bool.").with_source(e))?; + Ok(Self::Primitive(PrimitiveLiteral::Boolean(v))) + } + + /// Creates an 32bit integer. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Literal, PrimitiveLiteral}; + /// let t = Literal::int(23i8); + /// + /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Int(23)), t); + /// ``` + pub fn int>(t: T) -> Self { + Self::Primitive(PrimitiveLiteral::Int(t.into())) + } + + /// Creates an 64bit integer. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Literal, PrimitiveLiteral}; + /// let t = Literal::long(24i8); + /// + /// assert_eq!(Literal::Primitive(PrimitiveLiteral::Long(24)), t); + /// ``` + pub fn long>(t: T) -> Self { + Self::Primitive(PrimitiveLiteral::Long(t.into())) + } + + /// Creates an 32bit floating point number. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Literal, PrimitiveLiteral}; + /// use ordered_float::OrderedFloat; + /// let t = Literal::float(32.1f32); + /// + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(32.1))), + /// t + /// ); + /// ``` + pub fn float>(t: T) -> Self { + Self::Primitive(PrimitiveLiteral::Float(OrderedFloat(t.into()))) + } + + /// Creates an 32bit floating point number. + /// + /// Example: + /// ```rust + /// use iceberg::spec::{Literal, PrimitiveLiteral}; + /// use ordered_float::OrderedFloat; + /// let t = Literal::double(32.1f64); + /// + /// assert_eq!( + /// Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(32.1))), + /// t + /// ); + /// ``` + pub fn double>(t: T) -> Self { + Self::Primitive(PrimitiveLiteral::Double(OrderedFloat(t.into()))) + } + + /// Creates date literal from number of days from unix epoch directly. + pub fn date(days: i32) -> Self { + Self::Primitive(PrimitiveLiteral::Int(days)) + } + + /// Creates a date in `%Y-%m-%d` format, assume in utc timezone. + /// + /// See [`NaiveDate::from_str`]. + /// + /// Example + /// ```rust + /// use iceberg::spec::Literal; + /// let t = Literal::date_from_str("1970-01-03").unwrap(); + /// + /// assert_eq!(Literal::date(2), t); + /// ``` + pub fn date_from_str>(s: S) -> Result { + let t = s.as_ref().parse::().map_err(|e| { + invalid_data!("Can't parse date from string: {}", s.as_ref()).with_source(e) + })?; + + Ok(Self::date(date::date_from_naive_date(t))) + } + + /// Create a date from calendar date (year, month and day). + /// + /// See [`NaiveDate::from_ymd_opt`]. + /// + /// Example: + /// + ///```rust + /// use iceberg::spec::Literal; + /// let t = Literal::date_from_ymd(1970, 1, 5).unwrap(); + /// + /// assert_eq!(Literal::date(4), t); + /// ``` + pub fn date_from_ymd(year: i32, month: u32, day: u32) -> Result { + let t = NaiveDate::from_ymd_opt(year, month, day).ok_or_else(|| { + invalid_data!("Can't create date from year: {year}, month: {month}, day: {day}") + })?; + + Ok(Self::date(date::date_from_naive_date(t))) + } + + /// Creates time in microseconds directly + pub fn time(value: i64) -> Self { + Self::Primitive(PrimitiveLiteral::Long(value)) + } + + /// Creates time literal from [`chrono::NaiveTime`]. + fn time_from_naive_time(t: NaiveTime) -> Self { + let duration = t - date::unix_epoch().time(); + // It's safe to unwrap here since less than 24 hours will never overflow. + let micro_secs = duration.num_microseconds().unwrap(); + + Literal::time(micro_secs) + } + + /// Creates time in microseconds in `%H:%M:%S:.f` format. + /// + /// See [`NaiveTime::from_str`] for details. + /// + /// Example: + /// ```rust + /// use iceberg::spec::Literal; + /// let t = Literal::time_from_str("01:02:01.888999777").unwrap(); + /// + /// let micro_secs = { + /// 1 * 3600 * 1_000_000 + // 1 hour + /// 2 * 60 * 1_000_000 + // 2 minutes + /// 1 * 1_000_000 + // 1 second + /// 888999 // microseconds + /// }; + /// assert_eq!(Literal::time(micro_secs), t); + /// ``` + pub fn time_from_str>(s: S) -> Result { + let t = s.as_ref().parse::().map_err(|e| { + invalid_data!("Can't parse time from string: {}", s.as_ref()).with_source(e) + })?; + + Ok(Self::time_from_naive_time(t)) + } + + /// Creates time literal from hour, minute, second, and microseconds. + /// + /// See [`NaiveTime::from_hms_micro_opt`]. + /// + /// Example: + /// ```rust + /// use iceberg::spec::Literal; + /// let t = Literal::time_from_hms_micro(22, 15, 33, 111).unwrap(); + /// + /// assert_eq!(Literal::time_from_str("22:15:33.000111").unwrap(), t); + /// ``` + pub fn time_from_hms_micro(hour: u32, min: u32, sec: u32, micro: u32) -> Result { + let t = NaiveTime::from_hms_micro_opt(hour, min, sec, micro) + .ok_or_else(|| invalid_data!("Can't create time from hour: {hour}, min: {min}, second: {sec}, microsecond: {micro}"))?; + Ok(Self::time_from_naive_time(t)) + } + + /// Creates a timestamp from unix epoch in microseconds. + pub fn timestamp(value: i64) -> Self { + Self::Primitive(PrimitiveLiteral::Long(value)) + } + + /// Creates a timestamp with timezone from unix epoch in microseconds. + pub fn timestamptz(value: i64) -> Self { + Self::Primitive(PrimitiveLiteral::Long(value)) + } + + /// Creates a timestamp from unix epoch in nanoseconds. + pub(crate) fn timestamp_nano(value: i64) -> Self { + Self::Primitive(PrimitiveLiteral::Long(value)) + } + + /// Creates a timestamp with timezone from unix epoch in nanoseconds. + pub(crate) fn timestamptz_nano(value: i64) -> Self { + Self::Primitive(PrimitiveLiteral::Long(value)) + } + + /// Creates a timestamp from [`DateTime`]. + pub fn timestamp_from_datetime(dt: DateTime) -> Self { + Self::timestamp(dt.with_timezone(&Utc).timestamp_micros()) + } + + /// Creates a timestamp with timezone from [`DateTime`]. + pub fn timestamptz_from_datetime(dt: DateTime) -> Self { + Self::timestamptz(dt.with_timezone(&Utc).timestamp_micros()) + } + + /// Parse a timestamp in RFC3339 format. + /// + /// See [`DateTime::from_str`]. + /// + /// Example: + /// + /// ```rust + /// use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime}; + /// use iceberg::spec::Literal; + /// let t = Literal::timestamp_from_str("2012-12-12 12:12:12.8899-04:00").unwrap(); + /// + /// let t2 = { + /// let date = NaiveDate::from_ymd_opt(2012, 12, 12).unwrap(); + /// let time = NaiveTime::from_hms_micro_opt(12, 12, 12, 889900).unwrap(); + /// let dt = NaiveDateTime::new(date, time); + /// Literal::timestamp_from_datetime(DateTime::::from_local( + /// dt, + /// FixedOffset::west_opt(4 * 3600).unwrap(), + /// )) + /// }; + /// + /// assert_eq!(t, t2); + /// ``` + pub fn timestamp_from_str>(s: S) -> Result { + let dt = DateTime::::from_str(s.as_ref()) + .map_err(|e| invalid_data!("Can't parse datetime.").with_source(e))?; + + Ok(Self::timestamp_from_datetime(dt)) + } + + /// Similar to [`Literal::timestamp_from_str`], but return timestamp with timezone literal. + pub fn timestamptz_from_str>(s: S) -> Result { + let dt = DateTime::::from_str(s.as_ref()) + .map_err(|e| invalid_data!("Can't parse datetime.").with_source(e))?; + + Ok(Self::timestamptz_from_datetime(dt)) + } + + /// Creates a string literal. + pub fn string(s: S) -> Self { + Self::Primitive(PrimitiveLiteral::String(s.to_string())) + } + + /// Creates uuid literal. + pub fn uuid(uuid: Uuid) -> Self { + Self::Primitive(PrimitiveLiteral::UInt128(uuid.as_u128())) + } + + /// Creates uuid from str. See [`Uuid::parse_str`]. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Literal; + /// use uuid::Uuid; + /// let t1 = Literal::uuid_from_str("a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8").unwrap(); + /// let t2 = Literal::uuid(Uuid::from_u128_le(0xd8d7d6d5d4d3d2d1c2c1b2b1a4a3a2a1)); + /// + /// assert_eq!(t1, t2); + /// ``` + pub fn uuid_from_str>(s: S) -> Result { + let uuid = Uuid::parse_str(s.as_ref()).map_err(|e| { + invalid_data!("Can't parse uuid from string: {}", s.as_ref()).with_source(e) + })?; + Ok(Self::uuid(uuid)) + } + + /// Creates a fixed literal from bytes. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::{Literal, PrimitiveLiteral}; + /// let t1 = Literal::fixed(vec![1u8, 2u8]); + /// let t2 = Literal::Primitive(PrimitiveLiteral::Binary(vec![1u8, 2u8])); + /// + /// assert_eq!(t1, t2); + /// ``` + pub fn fixed>(input: I) -> Self { + Literal::Primitive(PrimitiveLiteral::Binary(input.into_iter().collect())) + } + + /// Creates a binary literal from bytes. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::{Literal, PrimitiveLiteral}; + /// let t1 = Literal::binary(vec![1u8, 2u8]); + /// let t2 = Literal::Primitive(PrimitiveLiteral::Binary(vec![1u8, 2u8])); + /// + /// assert_eq!(t1, t2); + /// ``` + pub fn binary>(input: I) -> Self { + Literal::Primitive(PrimitiveLiteral::Binary(input.into_iter().collect())) + } + + /// Creates a decimal literal. + pub fn decimal(decimal: i128) -> Self { + Self::Primitive(PrimitiveLiteral::Int128(decimal)) + } + + /// Creates decimal literal from string. + /// + /// Example: + /// + /// ```rust + /// use iceberg::spec::Literal; + /// let t1 = Literal::decimal(12345); + /// let t2 = Literal::decimal_from_str("123.45").unwrap(); + /// + /// assert_eq!(t1, t2); + /// ``` + pub fn decimal_from_str>(s: S) -> Result { + let decimal = decimal_from_str_exact(s.as_ref())?; + Ok(Self::decimal(decimal_mantissa(&decimal))) + } + + /// Attempts to convert the Literal to a PrimitiveLiteral + pub fn as_primitive_literal(&self) -> Option { + match self { + Literal::Primitive(primitive) => Some(primitive.clone()), + _ => None, + } + } + + /// Create iceberg value from a json value + /// + /// See [this spec](https://iceberg.apache.org/spec/#json-single-value-serialization) for reference. + pub fn try_from_json(value: JsonValue, data_type: &Type) -> Result> { + match data_type { + Type::Primitive(primitive) => match (primitive, value) { + (PrimitiveType::Boolean, JsonValue::Bool(bool)) => { + Ok(Some(Literal::Primitive(PrimitiveLiteral::Boolean(bool)))) + } + (PrimitiveType::Int, JsonValue::Number(number)) => { + Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( + number + .as_i64() + .ok_or(invalid_data!("Failed to convert json number to int"))? + .try_into()?, + )))) + } + (PrimitiveType::Long, JsonValue::Number(number)) => { + Ok(Some(Literal::Primitive(PrimitiveLiteral::Long( + number + .as_i64() + .ok_or(invalid_data!("Failed to convert json number to long"))?, + )))) + } + (PrimitiveType::Float, JsonValue::Number(number)) => Ok(Some(Literal::Primitive( + PrimitiveLiteral::Float(OrderedFloat( + number + .as_f64() + .ok_or(invalid_data!("Failed to convert json number to float"))? + as f32, + )), + ))), + (PrimitiveType::Double, JsonValue::Number(number)) => Ok(Some(Literal::Primitive( + PrimitiveLiteral::Double(OrderedFloat( + number + .as_f64() + .ok_or(invalid_data!("Failed to convert json number to double"))?, + )), + ))), + (PrimitiveType::Date, JsonValue::String(s)) => { + Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( + date::date_to_days(&NaiveDate::parse_from_str(&s, "%Y-%m-%d")?), + )))) + } + (PrimitiveType::Date, JsonValue::Number(number)) => { + Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( + number + .as_i64() + .ok_or(invalid_data!( + "Failed to convert json number to date (days since epoch)" + ))? + .try_into()?, + )))) + } + (PrimitiveType::Time, JsonValue::String(s)) => { + Ok(Some(Literal::Primitive(PrimitiveLiteral::Long( + time::time_to_microseconds(&NaiveTime::parse_from_str(&s, "%H:%M:%S%.f")?), + )))) + } + (PrimitiveType::Timestamp, JsonValue::String(s)) => Ok(Some(Literal::Primitive( + PrimitiveLiteral::Long(timestamp::datetime_to_microseconds( + &NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f")?, + )), + ))), + (PrimitiveType::Timestamptz, JsonValue::String(s)) => { + Ok(Some(Literal::Primitive(PrimitiveLiteral::Long( + timestamptz::datetimetz_to_microseconds(&Utc.from_utc_datetime( + &NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f+00:00")?, + )), + )))) + } + (PrimitiveType::TimestampNs, JsonValue::String(s)) => { + let ndt = NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f")?; + let nanos = timestamp::datetime_to_nanoseconds(&ndt).ok_or_else(|| { + invalid_data!( + "Timestamp is outside the representable nanosecond range: {ndt}" + ) + })?; + Ok(Some(Literal::Primitive(PrimitiveLiteral::Long(nanos)))) + } + (PrimitiveType::TimestamptzNs, JsonValue::String(s)) => { + let dt = Utc.from_utc_datetime(&NaiveDateTime::parse_from_str( + &s, + "%Y-%m-%dT%H:%M:%S%.f+00:00", + )?); + let nanos = timestamptz::datetimetz_to_nanoseconds(&dt).ok_or_else(|| { + invalid_data!( + "Timestamptz is outside the representable nanosecond range: {dt}" + ) + })?; + Ok(Some(Literal::Primitive(PrimitiveLiteral::Long(nanos)))) + } + (PrimitiveType::String, JsonValue::String(s)) => { + Ok(Some(Literal::Primitive(PrimitiveLiteral::String(s)))) + } + (PrimitiveType::Uuid, JsonValue::String(s)) => Ok(Some(Literal::Primitive( + PrimitiveLiteral::UInt128(Uuid::parse_str(&s)?.as_u128()), + ))), + (PrimitiveType::Fixed(size), JsonValue::String(s)) => { + let bytes = decode_hex_bytes(&s)?; + validate_fixed_size(bytes.len(), *size)?; + Ok(Some(Literal::Primitive(PrimitiveLiteral::Binary(bytes)))) + } + (PrimitiveType::Binary, JsonValue::String(s)) => Ok(Some(Literal::Primitive( + PrimitiveLiteral::Binary(decode_hex_bytes(&s)?), + ))), + ( + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_), + JsonValue::String(_), + ) => Err(Error::new( + ErrorKind::FeatureUnsupported, + "WKT deserialization for geospatial types is not yet supported", + )), + ( + PrimitiveType::Decimal { + precision: _, + scale, + }, + JsonValue::String(s), + ) => { + let decimal = decimal_from_str_exact(&s)?; + let rescaled = decimal_rescale(decimal, *scale); + Ok(Some(Literal::Primitive(PrimitiveLiteral::Int128( + decimal_mantissa(&rescaled), + )))) + } + (PrimitiveType::Unknown, value) if !value.is_null() => Err(invalid_data!( + "Unknown type only supports null default values" + )), + (_, JsonValue::Null) => Ok(None), + (i, j) => Err(invalid_data!( + "The json value {j} doesn't fit to the iceberg type {i}." + )), + }, + Type::Struct(schema) => { + if let JsonValue::Object(mut object) = value { + let values = schema + .fields() + .iter() + .map(|field| match object.remove(&field.id.to_string()) { + Some(value) => Literal::try_from_json(value, &field.field_type), + None => Ok(None), + }) + .collect::>>()?; + Ok(Some(Literal::Struct(Struct::from_iter(values)))) + } else { + Err(invalid_data!( + "The json value for a struct type must be an object." + )) + } + } + Type::List(list) => { + if let JsonValue::Array(array) = value { + Ok(Some(Literal::List( + array + .into_iter() + .map(|value| { + Literal::try_from_json(value, &list.element_field.field_type) + }) + .collect::>>()?, + ))) + } else { + Err(invalid_data!( + "The json value for a list type must be an array." + )) + } + } + Type::Map(map) => { + if let JsonValue::Object(mut object) = value { + if let (Some(JsonValue::Array(keys)), Some(JsonValue::Array(values))) = + (object.remove("keys"), object.remove("values")) + { + if keys.len() != values.len() { + return Err(invalid_data!( + "Map keys and values must have the same length, got {} keys and {} values", + keys.len(), + values.len() + )); + } + + Ok(Some(Literal::Map(Map::from_iter( + keys.into_iter() + .zip(values) + .map(|(key, value)| { + Ok(( + Literal::try_from_json(key, &map.key_field.field_type) + .and_then(|value| { + value.ok_or(invalid_data!( + "Key of map cannot be null" + )) + })?, + Literal::try_from_json(value, &map.value_field.field_type)?, + )) + }) + .collect::>>()?, + )))) + } else { + Err(invalid_data!( + "The json value for a list type must be an array." + )) + } + } else { + Err(invalid_data!( + "The json value for a list type must be an array." + )) + } + } + Type::Variant(_) => Err(invalid_data!( + "Variant type is not supported for single-value JSON serialization" + )), + } + } + + /// Converting iceberg value to json value. + /// + /// See [this spec](https://iceberg.apache.org/spec/#json-single-value-serialization) for reference. + pub fn try_into_json(self, r#type: &Type) -> Result { + match (self, r#type) { + (Literal::Primitive(prim), Type::Primitive(prim_type)) => match (prim_type, prim) { + (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(val)) => { + Ok(JsonValue::Bool(val)) + } + (PrimitiveType::Int, PrimitiveLiteral::Int(val)) => { + Ok(JsonValue::Number((val).into())) + } + (PrimitiveType::Long, PrimitiveLiteral::Long(val)) => { + Ok(JsonValue::Number((val).into())) + } + (PrimitiveType::Float, PrimitiveLiteral::Float(val)) => { + match Number::from_f64(val.0 as f64) { + Some(number) => Ok(JsonValue::Number(number)), + None => Ok(JsonValue::Null), + } + } + (PrimitiveType::Double, PrimitiveLiteral::Double(val)) => { + match Number::from_f64(val.0) { + Some(number) => Ok(JsonValue::Number(number)), + None => Ok(JsonValue::Null), + } + } + (PrimitiveType::Date, PrimitiveLiteral::Int(val)) => { + Ok(JsonValue::String(date::days_to_date(val).to_string())) + } + (PrimitiveType::Time, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( + time::microseconds_to_time(val).to_string(), + )), + (PrimitiveType::Timestamp, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( + timestamp::microseconds_to_datetime(val) + .format("%Y-%m-%dT%H:%M:%S%.f") + .to_string(), + )), + (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( + timestamptz::microseconds_to_datetimetz(val) + .format("%Y-%m-%dT%H:%M:%S%.f+00:00") + .to_string(), + )), + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( + timestamp::nanoseconds_to_datetime(val) + .format("%Y-%m-%dT%H:%M:%S%.f") + .to_string(), + )), + (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(val)) => { + Ok(JsonValue::String( + timestamptz::nanoseconds_to_datetimetz(val) + .format("%Y-%m-%dT%H:%M:%S%.f+00:00") + .to_string(), + )) + } + (PrimitiveType::String, PrimitiveLiteral::String(val)) => { + Ok(JsonValue::String(val.clone())) + } + (_, PrimitiveLiteral::UInt128(val)) => { + Ok(JsonValue::String(Uuid::from_u128(val).to_string())) + } + (PrimitiveType::Fixed(size), PrimitiveLiteral::Binary(val)) => { + validate_fixed_size(val.len(), *size)?; + Ok(JsonValue::String(encode_hex_bytes(&val))) + } + (PrimitiveType::Binary, PrimitiveLiteral::Binary(val)) => { + Ok(JsonValue::String(encode_hex_bytes(&val))) + } + ( + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_), + PrimitiveLiteral::Binary(_), + ) => Err(Error::new( + ErrorKind::FeatureUnsupported, + "WKT serialization for geospatial types is not yet supported", + )), + (_, PrimitiveLiteral::Int128(val)) => match r#type { + Type::Primitive(PrimitiveType::Decimal { + precision: _precision, + scale, + }) => { + let decimal = try_decimal_from_i128_with_scale(val, *scale)?; + Ok(JsonValue::String(decimal.to_string())) + } + _ => Err(invalid_data!( + "The iceberg type for decimal literal must be decimal." + ))?, + }, + _ => Err(invalid_data!( + "The iceberg value doesn't fit to the iceberg type." + )), + }, + (Literal::Struct(s), Type::Struct(struct_type)) => { + let mut id_and_value = Vec::with_capacity(struct_type.fields().len()); + for (value, field) in s.into_iter().zip(struct_type.fields()) { + let json = match value { + Some(val) => val.try_into_json(&field.field_type)?, + None => JsonValue::Null, + }; + id_and_value.push((field.id.to_string(), json)); + } + Ok(JsonValue::Object(JsonMap::from_iter(id_and_value))) + } + (Literal::List(list), Type::List(list_type)) => Ok(JsonValue::Array( + list.into_iter() + .map(|opt| match opt { + Some(literal) => literal.try_into_json(&list_type.element_field.field_type), + None => Ok(JsonValue::Null), + }) + .collect::>>()?, + )), + (Literal::Map(map), Type::Map(map_type)) => { + let mut object = JsonMap::with_capacity(2); + let mut json_keys = Vec::with_capacity(map.len()); + let mut json_values = Vec::with_capacity(map.len()); + for (key, value) in map.into_iter() { + json_keys.push(key.try_into_json(&map_type.key_field.field_type)?); + json_values.push(match value { + Some(literal) => literal.try_into_json(&map_type.value_field.field_type)?, + None => JsonValue::Null, + }); + } + object.insert("keys".to_string(), JsonValue::Array(json_keys)); + object.insert("values".to_string(), JsonValue::Array(json_values)); + Ok(JsonValue::Object(object)) + } + (value, r#type) => Err(invalid_data!( + "The iceberg value {value:?} doesn't fit to the iceberg type {type}." + )), + } + } + + /// Convert Value to the any type + pub fn into_any(self) -> Box { + match self { + Literal::Primitive(prim) => match prim { + PrimitiveLiteral::Boolean(any) => Box::new(any), + PrimitiveLiteral::Int(any) => Box::new(any), + PrimitiveLiteral::Long(any) => Box::new(any), + PrimitiveLiteral::Float(any) => Box::new(any), + PrimitiveLiteral::Double(any) => Box::new(any), + PrimitiveLiteral::Binary(any) => Box::new(any), + PrimitiveLiteral::String(any) => Box::new(any), + PrimitiveLiteral::UInt128(any) => Box::new(any), + PrimitiveLiteral::Int128(any) => Box::new(any), + PrimitiveLiteral::AboveMax | PrimitiveLiteral::BelowMin => unimplemented!(), + }, + _ => unimplemented!(), + } + } +} + +fn decode_hex_bytes(value: &str) -> Result> { + if !value.len().is_multiple_of(2) { + return Err(invalid_data!( + "Hex string must have an even number of characters: {value:?}" + )); + } + + value + .as_bytes() + .chunks_exact(2) + .map(|chunk| { + let high = decode_hex_digit(chunk[0], value)?; + let low = decode_hex_digit(chunk[1], value)?; + Ok((high << 4) | low) + }) + .collect() +} + +fn decode_hex_digit(digit: u8, value: &str) -> Result { + match digit { + b'0'..=b'9' => Ok(digit - b'0'), + b'a'..=b'f' => Ok(digit - b'a' + 10), + b'A'..=b'F' => Ok(digit - b'A' + 10), + _ => Err(invalid_data!( + "Hex string contains invalid character: {value:?}" + )), + } +} + +fn encode_hex_bytes(bytes: &[u8]) -> String { + const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef"; + + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(HEX_DIGITS[(byte >> 4) as usize] as char); + output.push(HEX_DIGITS[(byte & 0x0f) as usize] as char); + } + output +} + +fn validate_fixed_size(actual: usize, expected: u64) -> Result<()> { + if actual as u64 == expected { + Ok(()) + } else { + Err(invalid_data!( + "Fixed type must be exactly {expected} bytes, got {actual}" + )) + } +} diff --git a/crates/iceberg/src/spec/values/map.rs b/crates/iceberg/src/spec/values/map.rs new file mode 100644 index 0000000000..e0f75205f0 --- /dev/null +++ b/crates/iceberg/src/spec/values/map.rs @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Map collection type for Iceberg + +use std::collections::HashMap; +use std::hash::Hash; + +use super::Literal; + +/// Map is a collection of key-value pairs with a key type and a value type. +/// It used in Literal::Map, to make it hashable, the order of key-value pairs is stored in a separate vector +/// so that we can hash the map in a deterministic way. But it also means that the order of key-value pairs is matter +/// for the hash value. +/// +/// When converting to Arrow (e.g., for Iceberg), the map should be represented using +/// the default field name "key_value". +/// +/// Example: +/// +/// ```text +/// let key_value_field = Field::new( +/// DEFAULT_MAP_FIELD_NAME, +/// arrow_schema::DataType::Struct(vec![ +/// Arc::new(key_field.clone()), +/// Arc::new(value_field.clone()), +/// ].into()), +/// false +/// ); +/// ''' +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Map { + index: HashMap, + pair: Vec<(Literal, Option)>, +} + +impl Map { + /// Creates a new empty map. + pub fn new() -> Self { + Self { + index: HashMap::new(), + pair: Vec::new(), + } + } + + /// Return the number of key-value pairs in the map. + pub fn len(&self) -> usize { + self.pair.len() + } + + /// Returns true if the map contains no elements. + pub fn is_empty(&self) -> bool { + self.pair.is_empty() + } + + /// Inserts a key-value pair into the map. + /// If the map did not have this key present, None is returned. + /// If the map did have this key present, the value is updated, and the old value is returned. + pub fn insert(&mut self, key: Literal, value: Option) -> Option> { + if let Some(index) = self.index.get(&key) { + let old_value = std::mem::replace(&mut self.pair[*index].1, value); + Some(old_value) + } else { + self.pair.push((key.clone(), value)); + self.index.insert(key, self.pair.len() - 1); + None + } + } + + /// Returns a reference to the value corresponding to the key. + /// If the key is not present in the map, None is returned. + pub fn get(&self, key: &Literal) -> Option<&Option> { + self.index.get(key).map(|index| &self.pair[*index].1) + } + + /// The order of map is matter, so this method used to compare two maps has same key-value pairs without considering the order. + pub fn has_same_content(&self, other: &Map) -> bool { + if self.len() != other.len() { + return false; + } + + for (key, value) in &self.pair { + match other.get(key) { + Some(other_value) if value == other_value => (), + _ => return false, + } + } + + true + } +} + +impl Default for Map { + fn default() -> Self { + Self::new() + } +} + +impl Hash for Map { + fn hash(&self, state: &mut H) { + for (key, value) in &self.pair { + key.hash(state); + value.hash(state); + } + } +} + +impl FromIterator<(Literal, Option)> for Map { + fn from_iter)>>(iter: T) -> Self { + let mut map = Map::new(); + for (key, value) in iter { + map.insert(key, value); + } + map + } +} + +impl IntoIterator for Map { + type Item = (Literal, Option); + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.pair.into_iter() + } +} + +impl From<[(Literal, Option); N]> for Map { + fn from(value: [(Literal, Option); N]) -> Self { + value.iter().cloned().collect() + } +} diff --git a/crates/iceberg/src/spec/values/mod.rs b/crates/iceberg/src/spec/values/mod.rs new file mode 100644 index 0000000000..d5dcb73aba --- /dev/null +++ b/crates/iceberg/src/spec/values/mod.rs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! This module contains Iceberg value types + +pub(crate) mod datum; +pub(crate) mod decimal_utils; +mod literal; +mod map; +mod primitive; +pub(crate) mod serde; +mod struct_value; +pub(crate) mod temporal; + +#[cfg(test)] +mod tests; + +// Re-export all public types +pub use datum::Datum; +pub use decimal_utils::Decimal; +pub use literal::Literal; +pub use map::Map; +pub use primitive::PrimitiveLiteral; +pub(crate) use serde::_serde::RawLiteral; +pub use struct_value::Struct; diff --git a/crates/iceberg/src/spec/values/primitive.rs b/crates/iceberg/src/spec/values/primitive.rs new file mode 100644 index 0000000000..2204e22427 --- /dev/null +++ b/crates/iceberg/src/spec/values/primitive.rs @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Primitive literal types + +use std::cmp::Ordering; + +use ordered_float::{FloatCore, OrderedFloat}; + +use crate::spec::PrimitiveType; + +/// Values present in iceberg type +/// +/// `Float` and `Double` compare as Java's `Float.compare` and `Double.compare` do: `-0.0` is +/// less than `0.0`, and every NaN is the same value, greater than all others. iceberg-java +/// compares partition values this way, so `-0.0` and `0.0` are different partitions. The +/// [spec](https://iceberg.apache.org/spec/#scan-planning) states the same rule: floating point +/// partition values are equal if their IEEE 754 bit layouts are equal, with NaNs normalized. +// The derived `Hash` gives `-0.0` and `0.0` the same hash. That is coarser than `eq`, which is +// allowed: equal values still hash alike. +#[allow(clippy::derived_hash_with_manual_eq)] +#[derive(Clone, Debug, Hash, Eq)] +pub enum PrimitiveLiteral { + /// 0x00 for false, non-zero byte for true + Boolean(bool), + /// Stored as 4-byte little-endian + Int(i32), + /// Stored as 8-byte little-endian + Long(i64), + /// Stored as 4-byte little-endian + Float(OrderedFloat), + /// Stored as 8-byte little-endian + Double(OrderedFloat), + /// UTF-8 bytes (without length) + String(String), + /// Binary value (without length) + Binary(Vec), + /// Stored as 16-byte big-endian + Int128(i128), + /// Stored as 16-byte big-endian + UInt128(u128), + /// When a number is larger than it can hold + AboveMax, + /// When a number is smaller than it can hold + BelowMin, +} + +impl PartialEq for PrimitiveLiteral { + fn eq(&self, other: &Self) -> bool { + self.partial_cmp(other) == Some(Ordering::Equal) + } +} + +impl PartialOrd for PrimitiveLiteral { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Boolean(a), Self::Boolean(b)) => a.partial_cmp(b), + (Self::Int(a), Self::Int(b)) => a.partial_cmp(b), + (Self::Long(a), Self::Long(b)) => a.partial_cmp(b), + (Self::Float(a), Self::Float(b)) => Some(float_cmp(a, b)), + (Self::Double(a), Self::Double(b)) => Some(float_cmp(a, b)), + (Self::String(a), Self::String(b)) => a.partial_cmp(b), + (Self::Binary(a), Self::Binary(b)) => a.partial_cmp(b), + (Self::Int128(a), Self::Int128(b)) => a.partial_cmp(b), + (Self::UInt128(a), Self::UInt128(b)) => a.partial_cmp(b), + (Self::AboveMax, Self::AboveMax) | (Self::BelowMin, Self::BelowMin) => { + Some(Ordering::Equal) + } + // Different variants order by declaration, as the derived impl did. A variant that + // lacks an arm above lands here against itself; fail closed instead of calling the + // two values equal. + _ => { + debug_assert_ne!(std::mem::discriminant(self), std::mem::discriminant(other)); + match self.variant_index().cmp(&other.variant_index()) { + Ordering::Equal => None, + ordering => Some(ordering), + } + } + } + } +} + +/// Compares floats as Java's `Float.compare` and `Double.compare` do. `OrderedFloat` already +/// treats every NaN as one value above all others; it only lacks `-0.0` before `0.0`. +fn float_cmp(a: &OrderedFloat, b: &OrderedFloat) -> Ordering { + a.cmp(b).then_with(|| { + if a.is_nan() { + Ordering::Equal + } else { + b.is_sign_negative().cmp(&a.is_sign_negative()) + } + }) +} + +impl PrimitiveLiteral { + /// Widens this literal to its promotion ceiling (`int` -> `long`, `float` -> + /// `double`), returning `None` for any other literal. Used when the target type + /// is unknown (the partition source column was dropped) so a value and its + /// deletes still normalize to the same type. + pub(crate) fn widen(&self) -> Option { + match self { + PrimitiveLiteral::Int(value) => Some(PrimitiveLiteral::Long(i64::from(*value))), + PrimitiveLiteral::Float(value) => { + Some(PrimitiveLiteral::Double(OrderedFloat(f64::from(value.0)))) + } + _ => None, + } + } + + /// Widens this literal to `target` for the value promotions `int` -> `long` + /// and `float` -> `double`, returning `None` when `target` does not promote it. + pub(crate) fn promote_to(&self, target: &PrimitiveType) -> Option { + match (self, target) { + (PrimitiveLiteral::Int(_), PrimitiveType::Long) + | (PrimitiveLiteral::Float(_), PrimitiveType::Double) => self.widen(), + _ => None, + } + } + + /// Must follow the declaration order of the variants: `partial_cmp` uses it to order + /// different variants the way the derived `PartialOrd` did. + fn variant_index(&self) -> u8 { + match self { + Self::Boolean(_) => 0, + Self::Int(_) => 1, + Self::Long(_) => 2, + Self::Float(_) => 3, + Self::Double(_) => 4, + Self::String(_) => 5, + Self::Binary(_) => 6, + Self::Int128(_) => 7, + Self::UInt128(_) => 8, + Self::AboveMax => 9, + Self::BelowMin => 10, + } + } + + /// Returns true if the Literal represents a primitive type + /// that can be a NaN, and that it's value is NaN + pub fn is_nan(&self) -> bool { + match self { + PrimitiveLiteral::Double(val) => val.is_nan(), + PrimitiveLiteral::Float(val) => val.is_nan(), + _ => false, + } + } +} diff --git a/crates/iceberg/src/spec/values/serde.rs b/crates/iceberg/src/spec/values/serde.rs new file mode 100644 index 0000000000..028c2d796e --- /dev/null +++ b/crates/iceberg/src/spec/values/serde.rs @@ -0,0 +1,748 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//\! Serialization and deserialization support for Iceberg values + +pub(crate) mod _serde { + use std::collections::HashMap; + + use serde::de::Visitor; + use serde::ser::{SerializeMap, SerializeSeq, SerializeStruct}; + use serde::{Deserialize, Serialize}; + use serde_bytes::ByteBuf; + use serde_derive::{Deserialize as DeserializeDerive, Serialize as SerializeDerive}; + + use crate::Error; + use crate::error::invalid_data; + use crate::spec::values::{Literal, Map, PrimitiveLiteral, Struct}; + use crate::spec::{MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, PrimitiveType, StructType, Type}; + + #[derive(SerializeDerive, DeserializeDerive, Debug)] + #[serde(transparent)] + /// Raw literal representation used for serde. The serialize way is used for Avro serializer. + pub struct RawLiteral(RawLiteralEnum); + + impl RawLiteral { + /// Covert literal to raw literal. + pub fn try_from(literal: Literal, ty: &Type) -> Result { + Ok(Self(RawLiteralEnum::try_from(literal, ty)?)) + } + + /// Convert raw literal to literal. + pub fn try_into(self, ty: &Type) -> Result, Error> { + self.0.try_into(ty) + } + + /// Matches the fields of a record to `struct_type` by name, the way Avro + /// schema resolution matches record fields. The result has + /// `struct_type`'s fields in its order, with null for an optional field the + /// record lacks. Record fields that `struct_type` lacks are dropped. Values + /// other than records are returned unchanged. + /// + /// Fields aren't matched by field ID. A writer that stores a field under + /// another name, such as Java replacing characters that Avro names don't + /// allow, reads as a missing field. + pub fn project_by_name(self, struct_type: &StructType) -> Result { + let (mut required, optional) = match self.0 { + RawLiteralEnum::Record(Record { required, optional }) => (required, optional), + other => return Ok(Self(other)), + }; + required.extend( + optional + .into_iter() + .map(|(name, value)| (name, value.unwrap_or(RawLiteralEnum::Null))), + ); + let fields = struct_type.fields(); + let required = if required.len() == fields.len() + && required + .iter() + .zip(fields) + .all(|((name, _), field)| *name == field.name) + { + required + } else { + let mut values: HashMap = required.into_iter().collect(); + fields + .iter() + .map(|field| match values.remove(&field.name) { + Some(value) => Ok((field.name.clone(), value)), + None if field.required => Err(invalid_data!( + "Record has no value for required field {}", + field.name + )), + None => Ok((field.name.clone(), RawLiteralEnum::Null)), + }) + .collect::>()? + }; + Ok(Self(RawLiteralEnum::Record(Record { + required, + optional: Vec::new(), + }))) + } + } + + #[derive(SerializeDerive, Clone, Debug)] + #[serde(untagged)] + enum RawLiteralEnum { + Null, + Boolean(bool), + Int(i32), + Long(i64), + Float(f32), + Double(f64), + String(String), + Bytes(ByteBuf), + List(List), + StringMap(StringMap), + Record(Record), + } + + #[derive(Clone, Debug)] + struct Record { + required: Vec<(String, RawLiteralEnum)>, + optional: Vec<(String, Option)>, + } + + impl Serialize for Record { + fn serialize(&self, serializer: S) -> Result + where S: serde::Serializer { + let len = self.required.len() + self.optional.len(); + let mut record = serializer.serialize_struct("", len)?; + for (k, v) in &self.required { + record.serialize_field(Box::leak(k.clone().into_boxed_str()), &v)?; + } + for (k, v) in &self.optional { + record.serialize_field(Box::leak(k.clone().into_boxed_str()), &v)?; + } + record.end() + } + } + + #[derive(Clone, Debug)] + struct List { + list: Vec>, + required: bool, + } + + impl Serialize for List { + fn serialize(&self, serializer: S) -> Result + where S: serde::Serializer { + let mut seq = serializer.serialize_seq(Some(self.list.len()))?; + for value in &self.list { + if self.required { + seq.serialize_element(value.as_ref().ok_or_else(|| { + serde::ser::Error::custom( + "List element is required, element cannot be null", + ) + })?)?; + } else { + seq.serialize_element(&value)?; + } + } + seq.end() + } + } + + #[derive(Clone, Debug)] + struct StringMap { + raw: Vec<(String, Option)>, + required: bool, + } + + impl Serialize for StringMap { + fn serialize(&self, serializer: S) -> Result + where S: serde::Serializer { + let mut map = serializer.serialize_map(Some(self.raw.len()))?; + for (k, v) in &self.raw { + if self.required { + map.serialize_entry( + k, + v.as_ref().ok_or_else(|| { + serde::ser::Error::custom( + "Map element is required, element cannot be null", + ) + })?, + )?; + } else { + map.serialize_entry(k, v)?; + } + } + map.end() + } + } + + impl<'de> Deserialize<'de> for RawLiteralEnum { + fn deserialize(deserializer: D) -> Result + where D: serde::Deserializer<'de> { + struct RawLiteralVisitor; + impl<'de> Visitor<'de> for RawLiteralVisitor { + type Value = RawLiteralEnum; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("expect") + } + + fn visit_bool(self, v: bool) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::Boolean(v)) + } + + fn visit_i32(self, v: i32) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::Int(v)) + } + + fn visit_i64(self, v: i64) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::Long(v)) + } + + /// Used in json + fn visit_u64(self, v: u64) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::Long(v as i64)) + } + + fn visit_f32(self, v: f32) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::Float(v)) + } + + fn visit_f64(self, v: f64) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::Double(v)) + } + + fn visit_str(self, v: &str) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::String(v.to_string())) + } + + fn visit_bytes(self, v: &[u8]) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::Bytes(ByteBuf::from(v))) + } + + fn visit_borrowed_str(self, v: &'de str) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::String(v.to_string())) + } + + fn visit_unit(self) -> Result + where E: serde::de::Error { + Ok(RawLiteralEnum::Null) + } + + fn visit_map(self, mut map: A) -> Result + where A: serde::de::MapAccess<'de> { + let mut required = Vec::new(); + while let Some(key) = map.next_key::()? { + let value = map.next_value::()?; + required.push((key, value)); + } + Ok(RawLiteralEnum::Record(Record { + required, + optional: Vec::new(), + })) + } + + fn visit_seq(self, mut seq: A) -> Result + where A: serde::de::SeqAccess<'de> { + let mut list = Vec::new(); + while let Some(value) = seq.next_element::()? { + list.push(Some(value)); + } + Ok(RawLiteralEnum::List(List { + list, + // `required` only used in serialize, just set default in deserialize. + required: false, + })) + } + } + deserializer.deserialize_any(RawLiteralVisitor) + } + } + + impl RawLiteralEnum { + pub fn try_from(literal: Literal, ty: &Type) -> Result { + let raw = match literal { + Literal::Primitive(prim) => match prim { + PrimitiveLiteral::Boolean(v) => RawLiteralEnum::Boolean(v), + PrimitiveLiteral::Int(v) => RawLiteralEnum::Int(v), + PrimitiveLiteral::Long(v) => RawLiteralEnum::Long(v), + PrimitiveLiteral::Float(v) => RawLiteralEnum::Float(v.0), + PrimitiveLiteral::Double(v) => RawLiteralEnum::Double(v.0), + PrimitiveLiteral::String(v) => RawLiteralEnum::String(v), + PrimitiveLiteral::UInt128(v) => { + RawLiteralEnum::Bytes(ByteBuf::from(v.to_be_bytes())) + } + PrimitiveLiteral::Binary(v) => RawLiteralEnum::Bytes(ByteBuf::from(v)), + PrimitiveLiteral::Int128(v) => { + RawLiteralEnum::Bytes(ByteBuf::from(v.to_be_bytes())) + } + PrimitiveLiteral::AboveMax | PrimitiveLiteral::BelowMin => { + return Err(invalid_data!("Can't convert AboveMax or BelowMax")); + } + }, + Literal::Struct(r#struct) => { + let mut required = Vec::new(); + let mut optional = Vec::new(); + if let Type::Struct(struct_ty) = ty { + for (value, field) in r#struct.into_iter().zip(struct_ty.fields()) { + if field.required { + if let Some(value) = value { + required.push(( + field.name.clone(), + RawLiteralEnum::try_from(value, &field.field_type)?, + )); + } else { + return Err(invalid_data!( + "Can't convert null to required field" + )); + } + } else if let Some(value) = value { + optional.push(( + field.name.clone(), + Some(RawLiteralEnum::try_from(value, &field.field_type)?), + )); + } else { + optional.push((field.name.clone(), None)); + } + } + } else { + return Err(invalid_data!("Type {ty} should be a struct")); + } + RawLiteralEnum::Record(Record { required, optional }) + } + Literal::List(list) => { + if let Type::List(list_ty) = ty { + let list = list + .into_iter() + .map(|v| { + v.map(|v| { + RawLiteralEnum::try_from(v, &list_ty.element_field.field_type) + }) + .transpose() + }) + .collect::>()?; + RawLiteralEnum::List(List { + list, + required: list_ty.element_field.required, + }) + } else { + return Err(invalid_data!("Type {ty} should be a list")); + } + } + Literal::Map(map) => { + if let Type::Map(map_ty) = ty { + if let Type::Primitive(PrimitiveType::String) = *map_ty.key_field.field_type + { + let mut raw = Vec::with_capacity(map.len()); + for (k, v) in map { + if let Literal::Primitive(PrimitiveLiteral::String(k)) = k { + raw.push(( + k, + v.map(|v| { + RawLiteralEnum::try_from( + v, + &map_ty.value_field.field_type, + ) + }) + .transpose()?, + )); + } else { + return Err(invalid_data!( + "literal type is inconsistent with type" + )); + } + } + RawLiteralEnum::StringMap(StringMap { + raw, + required: map_ty.value_field.required, + }) + } else { + let list = map.into_iter().map(|(k,v)| { + let raw_k = + RawLiteralEnum::try_from(k, &map_ty.key_field.field_type)?; + let raw_v = v + .map(|v| { + RawLiteralEnum::try_from(v, &map_ty.value_field.field_type) + }) + .transpose()?; + if map_ty.value_field.required { + Ok(Some(RawLiteralEnum::Record(Record { + required: vec![ + (MAP_KEY_FIELD_NAME.to_string(), raw_k), + (MAP_VALUE_FIELD_NAME.to_string(), raw_v.ok_or_else(||invalid_data!("Map value is required, value cannot be null"))?), + ], + optional: vec![], + }))) + } else { + Ok(Some(RawLiteralEnum::Record(Record { + required: vec![ + (MAP_KEY_FIELD_NAME.to_string(), raw_k), + ], + optional: vec![ + (MAP_VALUE_FIELD_NAME.to_string(), raw_v) + ], + }))) + } + }).collect::>()?; + RawLiteralEnum::List(List { + list, + required: true, + }) + } + } else { + return Err(invalid_data!("Type {ty} should be a map")); + } + } + }; + Ok(raw) + } + + pub fn try_into(self, ty: &Type) -> Result, Error> { + let invalid_err = |v: &str| { + invalid_data!( + "Unable to convert raw literal ({v}) fail convert to type {ty} for: type mismatch" + ) + }; + let invalid_err_with_reason = |v: &str, reason: &str| { + invalid_data!( + "Unable to convert raw literal ({v}) fail convert to type {ty} for: {reason}" + ) + }; + match self { + RawLiteralEnum::Null => Ok(None), + RawLiteralEnum::Boolean(v) => match ty { + Type::Primitive(PrimitiveType::Boolean) => Ok(Some(Literal::bool(v))), + _ => Err(invalid_err("boolean")), + }, + RawLiteralEnum::Int(v) => match ty { + Type::Primitive(PrimitiveType::Int) => Ok(Some(Literal::int(v))), + Type::Primitive(PrimitiveType::Long) => Ok(Some(Literal::long(i64::from(v)))), + Type::Primitive(PrimitiveType::Date) => Ok(Some(Literal::date(v))), + _ => Err(invalid_err("int")), + }, + RawLiteralEnum::Long(v) => match ty { + Type::Primitive(PrimitiveType::Int) => Ok(Some(Literal::int( + i32::try_from(v).map_err(|_| invalid_err("long"))?, + ))), + Type::Primitive(PrimitiveType::Date) => Ok(Some(Literal::date( + i32::try_from(v).map_err(|_| invalid_err("long"))?, + ))), + Type::Primitive(PrimitiveType::Long) => Ok(Some(Literal::long(v))), + Type::Primitive(PrimitiveType::Time) => Ok(Some(Literal::time(v))), + Type::Primitive(PrimitiveType::Timestamp) => Ok(Some(Literal::timestamp(v))), + Type::Primitive(PrimitiveType::Timestamptz) => { + Ok(Some(Literal::timestamptz(v))) + } + Type::Primitive(PrimitiveType::TimestampNs) => { + Ok(Some(Literal::timestamp_nano(v))) + } + Type::Primitive(PrimitiveType::TimestamptzNs) => { + Ok(Some(Literal::timestamptz_nano(v))) + } + _ => Err(invalid_err("long")), + }, + RawLiteralEnum::Float(v) => match ty { + Type::Primitive(PrimitiveType::Float) => Ok(Some(Literal::float(v))), + Type::Primitive(PrimitiveType::Double) => { + Ok(Some(Literal::double(f64::from(v)))) + } + _ => Err(invalid_err("float")), + }, + RawLiteralEnum::Double(v) => match ty { + Type::Primitive(PrimitiveType::Float) => Ok(Some(Literal::float(v as f32))), + Type::Primitive(PrimitiveType::Double) => Ok(Some(Literal::double(v))), + _ => Err(invalid_err("double")), + }, + RawLiteralEnum::String(v) => match ty { + Type::Primitive(PrimitiveType::String) => Ok(Some(Literal::string(v))), + _ => Err(invalid_err("string")), + }, + RawLiteralEnum::Bytes(v) => match ty { + Type::Primitive(PrimitiveType::Binary) => Ok(Some(Literal::binary(v.to_vec()))), + Type::Primitive(PrimitiveType::Fixed(expected_len)) => { + if v.len() == *expected_len as usize { + Ok(Some(Literal::fixed(v.to_vec()))) + } else { + Err(invalid_err_with_reason( + "bytes", + &format!( + "Fixed type must be exactly {} bytes, got {}", + expected_len, + v.len() + ), + )) + } + } + Type::Primitive(PrimitiveType::Uuid) => { + if v.len() == 16 { + let bytes: [u8; 16] = v.as_slice().try_into().map_err(|_| { + invalid_err_with_reason("bytes", "UUID must be exactly 16 bytes") + })?; + Ok(Some(Literal::uuid(uuid::Uuid::from_bytes(bytes)))) + } else { + Err(invalid_err_with_reason( + "bytes", + "UUID must be exactly 16 bytes", + )) + } + } + Type::Primitive(PrimitiveType::Decimal { precision, .. }) => { + let required_bytes = Type::decimal_required_bytes(*precision)? as usize; + + if v.len() == required_bytes { + // Pad the bytes to 16 bytes (i128 size) with sign extension + let mut padded_bytes = [0u8; 16]; + let start_idx = 16 - v.len(); + + // Copy the input bytes to the end of the array + padded_bytes[start_idx..].copy_from_slice(&v); + + // Sign extend if the number is negative (MSB is 1) + if !v.is_empty() && (v[0] & 0x80) != 0 { + // Fill the padding with 0xFF for negative numbers + for byte in &mut padded_bytes[..start_idx] { + *byte = 0xFF; + } + } + + Ok(Some(Literal::Primitive(PrimitiveLiteral::Int128( + i128::from_be_bytes(padded_bytes), + )))) + } else { + Err(invalid_err_with_reason( + "bytes", + &format!( + "Decimal with precision {} must be exactly {} bytes, got {}", + precision, + required_bytes, + v.len() + ), + )) + } + } + _ => Err(invalid_err("bytes")), + }, + RawLiteralEnum::List(v) => match ty { + Type::List(ty) => Ok(Some(Literal::List( + v.list + .into_iter() + .map(|v| { + if let Some(v) = v { + v.try_into(&ty.element_field.field_type) + } else { + Ok(None) + } + }) + .collect::>()?, + ))), + Type::Map(map_ty) => { + let key_ty = map_ty.key_field.field_type.as_ref(); + let value_ty = map_ty.value_field.field_type.as_ref(); + let mut map = Map::new(); + for k_v in v.list { + let k_v = k_v.ok_or_else(|| invalid_err_with_reason("list","In deserialize, None will be represented as Some(RawLiteral::Null), all element in list must be valid"))?; + if let RawLiteralEnum::Record(Record { + required, + optional: _, + }) = k_v + { + if required.len() != 2 { + return Err(invalid_err_with_reason( + "list", + "Record must contains two element(key and value) of array", + )); + } + let mut key = None; + let mut value = None; + required.into_iter().for_each(|(k, v)| { + if k == MAP_KEY_FIELD_NAME { + key = Some(v); + } else if k == MAP_VALUE_FIELD_NAME { + value = Some(v); + } + }); + match (key, value) { + (Some(k), Some(v)) => { + let key = k.try_into(key_ty)?.ok_or_else(|| { + invalid_err_with_reason( + "list", + "Key element in Map must be valid", + ) + })?; + let value = v.try_into(value_ty)?; + if map_ty.value_field.required && value.is_none() { + return Err(invalid_err_with_reason( + "list", + "Value element is required in this Map", + )); + } + map.insert(key, value); + } + _ => { + return Err(invalid_err_with_reason( + "list", + "The elements of record in list are not key and value", + )); + } + } + } else { + return Err(invalid_err_with_reason( + "list", + "Map should represented as record array.", + )); + } + } + Ok(Some(Literal::Map(map))) + } + Type::Primitive(PrimitiveType::Uuid) => { + if v.list.len() != 16 { + return Err(invalid_err_with_reason( + "list", + "The length of list should be 16", + )); + } + let mut bytes = [0u8; 16]; + for (i, v) in v.list.iter().enumerate() { + if let Some(RawLiteralEnum::Long(v)) = v { + bytes[i] = *v as u8; + } else { + return Err(invalid_err_with_reason( + "list", + "The element of list should be int", + )); + } + } + Ok(Some(Literal::uuid(uuid::Uuid::from_bytes(bytes)))) + } + Type::Primitive(PrimitiveType::Decimal { + precision: _, + scale: _, + }) => { + if v.list.len() != 16 { + return Err(invalid_err_with_reason( + "list", + "The length of list should be 16", + )); + } + let mut bytes = [0u8; 16]; + for (i, v) in v.list.iter().enumerate() { + if let Some(RawLiteralEnum::Long(v)) = v { + bytes[i] = *v as u8; + } else { + return Err(invalid_err_with_reason( + "list", + "The element of list should be int", + )); + } + } + Ok(Some(Literal::decimal(i128::from_be_bytes(bytes)))) + } + Type::Primitive(PrimitiveType::Binary) => { + let bytes = v + .list + .into_iter() + .map(|v| { + if let Some(RawLiteralEnum::Long(v)) = v { + Ok(v as u8) + } else { + Err(invalid_err_with_reason( + "list", + "The element of list should be int", + )) + } + }) + .collect::, Error>>()?; + Ok(Some(Literal::binary(bytes))) + } + Type::Primitive(PrimitiveType::Fixed(size)) => { + if v.list.len() != *size as usize { + return Err(invalid_err_with_reason( + "list", + "The length of list should be equal to size", + )); + } + let bytes = v + .list + .into_iter() + .map(|v| { + if let Some(RawLiteralEnum::Long(v)) = v { + Ok(v as u8) + } else { + Err(invalid_err_with_reason( + "list", + "The element of list should be int", + )) + } + }) + .collect::, Error>>()?; + Ok(Some(Literal::fixed(bytes))) + } + _ => Err(invalid_err("list")), + }, + RawLiteralEnum::Record(Record { + required, + optional: _, + }) => match ty { + Type::Struct(struct_ty) => { + let iters: Vec> = required + .into_iter() + .map(|(field_name, value)| { + let field = struct_ty + .field_by_name(field_name.as_str()) + .ok_or_else(|| { + invalid_err_with_reason( + "record", + &format!("field {} is not exist", field_name), + ) + })?; + let value = value.try_into(&field.field_type)?; + Ok(value) + }) + .collect::>()?; + Ok(Some(Literal::Struct(Struct::from_iter(iters)))) + } + Type::Map(map_ty) => { + if *map_ty.key_field.field_type != Type::Primitive(PrimitiveType::String) { + return Err(invalid_err_with_reason( + "record", + "Map key must be string", + )); + } + let mut map = Map::new(); + for (k, v) in required { + let value = v.try_into(&map_ty.value_field.field_type)?; + if map_ty.value_field.required && value.is_none() { + return Err(invalid_err_with_reason( + "record", + "Value element is required in this Map", + )); + } + map.insert(Literal::string(k), value); + } + Ok(Some(Literal::Map(map))) + } + _ => Err(invalid_err("record")), + }, + RawLiteralEnum::StringMap(_) => Err(invalid_err("string map")), + } + } + } +} diff --git a/crates/iceberg/src/spec/values/struct_value.rs b/crates/iceberg/src/spec/values/struct_value.rs new file mode 100644 index 0000000000..2b44961edf --- /dev/null +++ b/crates/iceberg/src/spec/values/struct_value.rs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Struct type for Iceberg values + +use std::ops::Index; + +use super::Literal; + +/// The partition struct stores the tuple of partition values for each file. +/// Its type is derived from the partition fields of the partition spec used to write the manifest file. +/// In v2, the partition struct's field ids must match the ids from the partition spec. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Struct { + /// Vector to store the field values + fields: Vec>, +} + +impl Struct { + /// Create a empty struct. + pub fn empty() -> Self { + Self { fields: Vec::new() } + } + + /// Create a iterator to read the field in order of field_value. + pub fn iter(&self) -> impl ExactSizeIterator> { + self.fields.iter().map(|field| field.as_ref()) + } + + /// returns true if the field at position `index` is null + pub fn is_null_at_index(&self, index: usize) -> bool { + self.fields[index].is_none() + } + + /// Return fields in the struct. + pub fn fields(&self) -> &[Option] { + &self.fields + } +} + +impl Index for Struct { + type Output = Option; + + fn index(&self, idx: usize) -> &Self::Output { + &self.fields[idx] + } +} + +impl IntoIterator for Struct { + type Item = Option; + + type IntoIter = std::vec::IntoIter>; + + fn into_iter(self) -> Self::IntoIter { + self.fields.into_iter() + } +} + +impl FromIterator> for Struct { + fn from_iter>>(iter: I) -> Self { + Struct { + fields: iter.into_iter().collect(), + } + } +} diff --git a/crates/iceberg/src/spec/values/temporal.rs b/crates/iceberg/src/spec/values/temporal.rs new file mode 100644 index 0000000000..76b2c897e1 --- /dev/null +++ b/crates/iceberg/src/spec/values/temporal.rs @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Temporal value conversions for dates, times, and timestamps + +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeDelta, TimeZone, Utc}; + +pub(crate) mod date { + use super::*; + + pub(crate) fn date_to_days(date: &NaiveDate) -> i32 { + date.signed_duration_since( + // This is always the same and shouldn't fail + NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(), + ) + .num_days() as i32 + } + + pub(crate) fn days_to_date(days: i32) -> NaiveDate { + // This shouldn't fail until the year 262000 + (DateTime::UNIX_EPOCH + TimeDelta::try_days(days as i64).unwrap()) + .naive_utc() + .date() + } + + /// Returns unix epoch. + pub(crate) fn unix_epoch() -> DateTime { + Utc.timestamp_nanos(0) + } + + /// Creates date literal from `NaiveDate`, assuming it's utc timezone. + pub(crate) fn date_from_naive_date(date: NaiveDate) -> i32 { + (date - unix_epoch().date_naive()).num_days() as i32 + } +} + +pub(crate) mod time { + use super::*; + + pub(crate) fn time_to_microseconds(time: &NaiveTime) -> i64 { + time.signed_duration_since( + // This is always the same and shouldn't fail + NaiveTime::from_num_seconds_from_midnight_opt(0, 0).unwrap(), + ) + .num_microseconds() + .unwrap() + } + + pub(crate) fn microseconds_to_time(micros: i64) -> NaiveTime { + let (secs, rem) = (micros / 1_000_000, micros % 1_000_000); + + NaiveTime::from_num_seconds_from_midnight_opt(secs as u32, rem as u32 * 1_000).unwrap() + } +} + +pub(crate) mod timestamp { + use super::*; + + pub(crate) fn datetime_to_microseconds(time: &NaiveDateTime) -> i64 { + time.and_utc().timestamp_micros() + } + + pub(crate) fn microseconds_to_datetime(micros: i64) -> NaiveDateTime { + // This shouldn't fail until the year 262000 + DateTime::from_timestamp_micros(micros).unwrap().naive_utc() + } + + pub(crate) fn nanoseconds_to_datetime(nanos: i64) -> NaiveDateTime { + DateTime::from_timestamp_nanos(nanos).naive_utc() + } + + /// Nanoseconds since the Unix epoch, or `None` if outside the representable `i64` range + /// (roughly the years 1678–2262). + pub(crate) fn datetime_to_nanoseconds(time: &NaiveDateTime) -> Option { + time.and_utc().timestamp_nanos_opt() + } +} + +pub(crate) mod timestamptz { + use super::*; + + pub(crate) fn datetimetz_to_microseconds(time: &DateTime) -> i64 { + time.timestamp_micros() + } + + pub(crate) fn microseconds_to_datetimetz(micros: i64) -> DateTime { + // Returns None only outside chrono's range (~year -262144 to +262142); + // unreachable for any realistic Iceberg timestamp. + DateTime::from_timestamp_micros(micros).expect( + "timestamp micros outside chrono's representable range (~year -262144..+262142)", + ) + } + + pub(crate) fn nanoseconds_to_datetimetz(nanos: i64) -> DateTime { + DateTime::from_timestamp_nanos(nanos) + } + + /// Nanoseconds since the Unix epoch, or `None` if outside the representable `i64` range + /// (roughly the years 1678–2262). + pub(crate) fn datetimetz_to_nanoseconds(time: &DateTime) -> Option { + time.timestamp_nanos_opt() + } +} diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs new file mode 100644 index 0000000000..67fc82a560 --- /dev/null +++ b/crates/iceberg/src/spec/values/tests.rs @@ -0,0 +1,2239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Tests for Iceberg value types + +use std::collections::HashSet; + +use apache_avro::to_value; +use apache_avro::types::Value; +use ordered_float::OrderedFloat; +use serde_bytes::ByteBuf; +use serde_json::Value as JsonValue; +use uuid::Uuid; + +use super::decimal_utils::{decimal_from_i128_with_scale, decimal_new}; +use crate::ErrorKind; +use crate::avro::schema_to_avro_schema; +use crate::spec::Schema; +use crate::spec::Type::Primitive; +use crate::spec::datatypes::{ + ListType, MapType, NestedField, PrimitiveType, StructType, Type, VariantType, +}; +use crate::spec::values::datum::{INT_MAX, INT_MIN, LONG_MAX, LONG_MIN}; +use crate::spec::values::{Datum, Decimal, Literal, Map, PrimitiveLiteral, RawLiteral, Struct}; + +fn check_json_serde(json: &str, expected_literal: Literal, expected_type: &Type) { + let raw_json_value = serde_json::from_str::(json).unwrap(); + let desered_literal = Literal::try_from_json(raw_json_value.clone(), expected_type).unwrap(); + assert_eq!(desered_literal, Some(expected_literal.clone())); + + let expected_json_value: JsonValue = expected_literal.try_into_json(expected_type).unwrap(); + let sered_json = serde_json::to_string(&expected_json_value).unwrap(); + let parsed_json_value = serde_json::from_str::(&sered_json).unwrap(); + + assert_eq!(parsed_json_value, raw_json_value); +} + +fn check_raw_literal_json_serde(expected_literal: Literal, expected_type: &Type) { + let raw_literal = RawLiteral::try_from(expected_literal.clone(), expected_type).unwrap(); + let serialized = serde_json::to_string(&raw_literal).unwrap(); + let deserialized: RawLiteral = serde_json::from_str(&serialized).unwrap(); + + assert_eq!( + deserialized.try_into(expected_type).unwrap(), + Some(expected_literal) + ); +} + +fn check_avro_bytes_serde(input: Vec, expected_datum: Datum, expected_type: &PrimitiveType) { + let raw_schema = r#""bytes""#; + let schema = apache_avro::Schema::parse_str(raw_schema).unwrap(); + + let bytes = ByteBuf::from(input); + let datum = Datum::try_from_bytes(&bytes, expected_type.clone()).unwrap(); + assert_eq!(datum, expected_datum); + + let mut writer = apache_avro::Writer::new(&schema, Vec::new()).unwrap(); + writer.append_ser(datum.to_bytes().unwrap()).unwrap(); + let encoded = writer.into_inner().unwrap(); + let reader = apache_avro::Reader::builder(&*encoded) + .reader_schema(&schema) + .build() + .unwrap(); + + for record in reader { + let result = apache_avro::from_value::(&record.unwrap()).unwrap(); + let desered_datum = Datum::try_from_bytes(&result, expected_type.clone()).unwrap(); + assert_eq!(desered_datum, expected_datum); + } +} + +fn check_convert_with_avro(expected_literal: Literal, expected_type: &Type) { + let fields = vec![NestedField::required(1, "col", expected_type.clone()).into()]; + let schema = Schema::builder() + .with_fields(fields.clone()) + .build() + .unwrap(); + let avro_schema = schema_to_avro_schema("test", &schema).unwrap(); + let struct_type = Type::Struct(StructType::new(fields)); + let struct_literal = Literal::Struct(Struct::from_iter(vec![Some(expected_literal.clone())])); + + let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()).unwrap(); + let raw_literal = RawLiteral::try_from(struct_literal.clone(), &struct_type).unwrap(); + writer.append_ser(raw_literal).unwrap(); + let encoded = writer.into_inner().unwrap(); + + let reader = apache_avro::Reader::new(&*encoded).unwrap(); + for record in reader { + let result = apache_avro::from_value::(&record.unwrap()).unwrap(); + let desered_literal = result.try_into(&struct_type).unwrap().unwrap(); + assert_eq!(desered_literal, struct_literal); + } +} + +fn check_serialize_avro(literal: Literal, ty: &Type, expect_value: Value) { + let expect_value = Value::Record(vec![("col".to_string(), expect_value)]); + + let fields = vec![NestedField::required(1, "col", ty.clone()).into()]; + let schema = Schema::builder() + .with_fields(fields.clone()) + .build() + .unwrap(); + let avro_schema = schema_to_avro_schema("test", &schema).unwrap(); + let struct_type = Type::Struct(StructType::new(fields)); + let struct_literal = Literal::Struct(Struct::from_iter(vec![Some(literal.clone())])); + let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()).unwrap(); + let raw_literal = RawLiteral::try_from(struct_literal.clone(), &struct_type).unwrap(); + let value = to_value(raw_literal) + .unwrap() + .resolve(&avro_schema) + .unwrap(); + writer.append_value_ref(&value).unwrap(); + let encoded = writer.into_inner().unwrap(); + + let reader = apache_avro::Reader::new(&*encoded).unwrap(); + for record in reader { + assert_eq!(record.unwrap(), expect_value); + } +} + +#[test] +fn json_boolean() { + let record = r#"true"#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Boolean(true)), + &Primitive(PrimitiveType::Boolean), + ); +} + +#[test] +fn json_int() { + let record = r#"32"#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Int(32)), + &Primitive(PrimitiveType::Int), + ); +} + +#[test] +fn json_long() { + let record = r#"32"#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Long(32)), + &Primitive(PrimitiveType::Long), + ); +} + +#[test] +fn json_float() { + let record = r#"1.0"#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(1.0))), + &Primitive(PrimitiveType::Float), + ); +} + +#[test] +fn json_double() { + let record = r#"1.0"#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(1.0))), + &Primitive(PrimitiveType::Double), + ); +} + +const FLOAT_VALUES: [f32; 6] = [ + 100.1, + 123.456, + -123.456, + f32::MAX, + f32::MIN, + f32::from_bits(1), +]; + +#[test] +fn raw_literal_json_number_to_float() { + for value in FLOAT_VALUES { + let raw: RawLiteral = serde_json::from_str(&format!("{value:e}")).unwrap(); + + assert_eq!( + raw.try_into(&Primitive(PrimitiveType::Float)).unwrap(), + Some(Literal::float(value)) + ); + } +} + +#[test] +fn raw_literal_avro_double_to_float() { + for value in FLOAT_VALUES { + for double in [f64::from(value), format!("{value:e}").parse().unwrap()] { + let raw: RawLiteral = apache_avro::from_value(&Value::Double(double)).unwrap(); + + assert_eq!( + raw.try_into(&Primitive(PrimitiveType::Float)).unwrap(), + Some(Literal::float(value)) + ); + } + } +} + +#[test] +fn raw_literal_double_without_exact_float_narrows_to_nearest() { + for (double, expected) in [(16777217.0_f64, 16777216.0_f32), (1.0000000001, 1.0)] { + let raw: RawLiteral = serde_json::from_str(&format!("{double:?}")).unwrap(); + + assert_eq!( + raw.try_into(&Primitive(PrimitiveType::Float)).unwrap(), + Some(Literal::float(expected)) + ); + } +} + +#[test] +fn raw_literal_avro_non_finite_and_negative_zero_double_to_float() { + for (double, expected) in [ + (f64::NAN, f32::NAN), + (f64::INFINITY, f32::INFINITY), + (f64::NEG_INFINITY, f32::NEG_INFINITY), + (-0.0, -0.0_f32), + ] { + let raw: RawLiteral = apache_avro::from_value(&Value::Double(double)).unwrap(); + let value = match raw.try_into(&Primitive(PrimitiveType::Float)).unwrap() { + Some(Literal::Primitive(PrimitiveLiteral::Float(value))) => value.0, + _ => panic!("unexpected literal type"), + }; + + if expected.is_nan() { + assert!(value.is_nan()); + } else { + assert_eq!(value.to_bits(), expected.to_bits()); + } + } +} + +#[test] +fn json_date() { + let record = r#""2017-11-16""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Int(17486)), + &Primitive(PrimitiveType::Date), + ); +} + +#[test] +fn json_time() { + let record = r#""22:31:08.123456""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Long(81068123456)), + &Primitive(PrimitiveType::Time), + ); +} + +#[test] +fn json_timestamp() { + let record = r#""2017-11-16T22:31:08.123456""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Long(1510871468123456)), + &Primitive(PrimitiveType::Timestamp), + ); +} + +#[test] +fn json_timestamptz() { + let record = r#""2017-11-16T22:31:08.123456+00:00""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Long(1510871468123456)), + &Primitive(PrimitiveType::Timestamptz), + ); +} + +#[test] +fn json_timestamp_ns() { + let record = r#""2017-11-16T22:31:08.123456789""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Long(1510871468123456789)), + &Primitive(PrimitiveType::TimestampNs), + ); +} + +#[test] +fn json_timestamptz_ns() { + let record = r#""2017-11-16T22:31:08.123456789+00:00""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Long(1510871468123456789)), + &Primitive(PrimitiveType::TimestamptzNs), + ); +} + +#[test] +fn raw_literal_json_serde_nanosecond_timestamps() { + check_raw_literal_json_serde( + Literal::timestamp_nano(1510871468123456789), + &Primitive(PrimitiveType::TimestampNs), + ); + check_raw_literal_json_serde( + Literal::timestamptz_nano(1510871468123456789), + &Primitive(PrimitiveType::TimestamptzNs), + ); +} + +#[test] +fn json_timestamptz_ns_rejects_non_utc_offset() { + // Per the spec, timestamptz_ns single-value serialization must use offset "+00:00"; Java's + // SingleValueParser enforces the same (DateTimeUtil.isUTCTimestamptz). A non-UTC offset is not a + // valid encoding and must be rejected, not silently re-based to UTC. + let record = serde_json::Value::String("2017-11-16T22:31:08.123456789+05:00".to_string()); + let result = Literal::try_from_json(record, &Primitive(PrimitiveType::TimestamptzNs)); + assert!( + result.is_err(), + "non-UTC offset must be rejected for timestamptz_ns, got {result:?}" + ); +} + +#[test] +fn json_timestamptz_rejects_non_utc_offset() { + // Micros-precision counterpart, mirroring Java's TestSingleValueParser.testInvalidTimestamptz: + // the offset must be "+00:00", so a non-UTC offset is rejected. + let record = serde_json::Value::String("2017-11-16T22:31:08.123456+05:00".to_string()); + let result = Literal::try_from_json(record, &Primitive(PrimitiveType::Timestamptz)); + assert!( + result.is_err(), + "non-UTC offset must be rejected for timestamptz, got {result:?}" + ); +} + +#[test] +fn json_string() { + let record = r#""iceberg""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::String("iceberg".to_string())), + &Primitive(PrimitiveType::String), + ); +} + +#[test] +fn json_uuid() { + let record = r#""f79c3e09-677c-4bbd-a479-3f349cb785e7""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::UInt128( + Uuid::parse_str("f79c3e09-677c-4bbd-a479-3f349cb785e7") + .unwrap() + .as_u128(), + )), + &Primitive(PrimitiveType::Uuid), + ); +} + +#[test] +fn json_decimal() { + let record = r#""14.20""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Int128(1420)), + &Type::decimal(28, 2).unwrap(), + ); +} + +#[test] +fn json_binary() { + let record = r#""00010fff""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Binary(vec![0, 1, 15, 255])), + &Primitive(PrimitiveType::Binary), + ); +} + +#[test] +fn json_fixed() { + let record = r#""00010fff""#; + + check_json_serde( + record, + Literal::Primitive(PrimitiveLiteral::Binary(vec![0, 1, 15, 255])), + &Primitive(PrimitiveType::Fixed(4)), + ); +} + +#[test] +fn json_unknown_only_accepts_null() { + let unknown = Primitive(PrimitiveType::Unknown); + + assert_eq!( + Literal::try_from_json(JsonValue::Null, &unknown).unwrap(), + None + ); + let error = Literal::try_from_json(serde_json::json!(1), &unknown).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!( + error + .message() + .contains("Unknown type only supports null default values") + ); +} + +#[test] +fn test_should_parse_json_binary_if_hex_uses_uppercase_digits() { + let result = Literal::try_from_json( + serde_json::json!("00010FFF"), + &Primitive(PrimitiveType::Binary), + ) + .unwrap(); + + assert_eq!( + result, + Some(Literal::Primitive(PrimitiveLiteral::Binary(vec![ + 0, 1, 15, 255 + ]))) + ); +} + +#[test] +fn test_should_reject_json_binary_if_hex_is_invalid() { + assert!( + Literal::try_from_json(serde_json::json!("f"), &Primitive(PrimitiveType::Binary),).is_err() + ); + assert!( + Literal::try_from_json(serde_json::json!("fg"), &Primitive(PrimitiveType::Binary),) + .is_err() + ); +} + +#[test] +fn test_should_reject_json_fixed_if_length_does_not_match() { + assert!( + Literal::try_from_json(serde_json::json!("ff"), &Primitive(PrimitiveType::Fixed(2)),) + .is_err() + ); + assert!( + Literal::Primitive(PrimitiveLiteral::Binary(vec![255])) + .try_into_json(&Primitive(PrimitiveType::Fixed(2))) + .is_err() + ); +} + +#[test] +fn json_struct() { + let record = r#"{"1": 1, "2": "bar", "3": null}"#; + + check_json_serde( + record, + Literal::Struct(Struct::from_iter(vec![ + Some(Literal::Primitive(PrimitiveLiteral::Int(1))), + Some(Literal::Primitive(PrimitiveLiteral::String( + "bar".to_string(), + ))), + None, + ])), + &Type::Struct(StructType::new(vec![ + NestedField::required(1, "id", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "address", Primitive(PrimitiveType::String)).into(), + ])), + ); +} + +#[test] +fn json_list() { + let record = r#"[1, 2, 3, null]"#; + + check_json_serde( + record, + Literal::List(vec![ + Some(Literal::Primitive(PrimitiveLiteral::Int(1))), + Some(Literal::Primitive(PrimitiveLiteral::Int(2))), + Some(Literal::Primitive(PrimitiveLiteral::Int(3))), + None, + ]), + &Type::List(ListType { + element_field: NestedField::list_element(0, Primitive(PrimitiveType::Int), true).into(), + }), + ); +} + +#[test] +fn json_map() { + let record = r#"{ "keys": ["a", "b", "c"], "values": [1, 2, null] }"#; + + check_json_serde( + record, + Literal::Map(Map::from([ + ( + Literal::Primitive(PrimitiveLiteral::String("a".to_string())), + Some(Literal::Primitive(PrimitiveLiteral::Int(1))), + ), + ( + Literal::Primitive(PrimitiveLiteral::String("b".to_string())), + Some(Literal::Primitive(PrimitiveLiteral::Int(2))), + ), + ( + Literal::Primitive(PrimitiveLiteral::String("c".to_string())), + None, + ), + ])), + &Type::Map(MapType { + key_field: NestedField::map_key_element(0, Primitive(PrimitiveType::String)).into(), + value_field: NestedField::map_value_element(1, Primitive(PrimitiveType::Int), true) + .into(), + }), + ); +} + +#[test] +fn json_map_rejects_mismatched_key_value_lengths() { + let map_type = Type::Map(MapType { + key_field: NestedField::map_key_element(0, Primitive(PrimitiveType::String)).into(), + value_field: NestedField::map_value_element(1, Primitive(PrimitiveType::Int), true).into(), + }); + + for record in [ + r#"{"keys":["a","b"],"values":[1]}"#, + r#"{"keys":["a"],"values":[1,2]}"#, + ] { + let value = serde_json::from_str::(record).unwrap(); + let error = Literal::try_from_json(value, &map_type).unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.to_string().contains("must have the same length")); + } +} + +#[test] +fn avro_bytes_boolean() { + let bytes = vec![1u8]; + + check_avro_bytes_serde(bytes, Datum::bool(true), &PrimitiveType::Boolean); +} + +#[test] +fn avro_bytes_int() { + let bytes = vec![32u8, 0u8, 0u8, 0u8]; + + check_avro_bytes_serde(bytes, Datum::int(32), &PrimitiveType::Int); +} + +#[test] +fn avro_bytes_long() { + let bytes = vec![32u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8]; + + check_avro_bytes_serde(bytes, Datum::long(32), &PrimitiveType::Long); +} + +#[test] +fn avro_bytes_long_from_int() { + let bytes = vec![32u8, 0u8, 0u8, 0u8]; + + check_avro_bytes_serde(bytes, Datum::long(32), &PrimitiveType::Long); +} + +#[test] +fn avro_bytes_float() { + let bytes = vec![0u8, 0u8, 128u8, 63u8]; + + check_avro_bytes_serde(bytes, Datum::float(1.0_f32), &PrimitiveType::Float); +} + +#[test] +fn avro_bytes_double() { + let bytes = vec![0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 240u8, 63u8]; + + check_avro_bytes_serde(bytes, Datum::double(1.0), &PrimitiveType::Double); +} + +#[test] +fn avro_bytes_double_from_float() { + let bytes = vec![0u8, 0u8, 128u8, 63u8]; + + check_avro_bytes_serde(bytes, Datum::double(1.0), &PrimitiveType::Double); +} + +#[test] +fn avro_bytes_string() { + let bytes = vec![105u8, 99u8, 101u8, 98u8, 101u8, 114u8, 103u8]; + + check_avro_bytes_serde(bytes, Datum::string("iceberg"), &PrimitiveType::String); +} + +#[test] +fn avro_bytes_geometry() { + let bytes = vec![1u8, 2u8, 3u8, 4u8]; + let ty = PrimitiveType::Geometry(Default::default()); + check_avro_bytes_serde( + bytes.clone(), + Datum::new(ty.clone(), PrimitiveLiteral::Binary(bytes)), + &ty, + ); +} + +#[test] +fn geospatial_wkt_json_is_not_yet_supported() { + for primitive_type in [ + PrimitiveType::Geometry(Default::default()), + PrimitiveType::Geography(Default::default()), + ] { + let err = Literal::try_from_json( + serde_json::json!("POINT (1 2)"), + &Primitive(primitive_type.clone()), + ) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + assert_eq!( + err.message(), + "WKT deserialization for geospatial types is not yet supported" + ); + + let err = Literal::Primitive(PrimitiveLiteral::Binary(vec![1, 2, 3])) + .try_into_json(&Primitive(primitive_type)) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + assert_eq!( + err.message(), + "WKT serialization for geospatial types is not yet supported" + ); + } +} + +#[test] +fn geospatial_datums_are_not_orderable() { + let geometry_type = PrimitiveType::Geometry(Default::default()); + let geography_type = PrimitiveType::Geography(Default::default()); + let geometry = Datum::new( + geometry_type.clone(), + PrimitiveLiteral::Binary(vec![1, 2, 3]), + ); + let other_geometry = Datum::new(geometry_type, PrimitiveLiteral::Binary(vec![4, 5, 6])); + let geography = Datum::new( + geography_type.clone(), + PrimitiveLiteral::Binary(vec![1, 2, 3]), + ); + let other_geography = Datum::new(geography_type, PrimitiveLiteral::Binary(vec![4, 5, 6])); + + assert_eq!(geometry.partial_cmp(&other_geometry), None); + assert_eq!(geography.partial_cmp(&other_geography), None); +} + +#[test] +fn avro_bytes_decimal() { + // (input_bytes, decimal_num, expect_scale, expect_precision) + let cases = vec![ + (vec![4u8, 210u8], 1234, 2, 38), + (vec![251u8, 46u8], -1234, 2, 38), + (vec![4u8, 210u8], 1234, 3, 38), + (vec![251u8, 46u8], -1234, 3, 38), + (vec![42u8], 42, 2, 2), + (vec![214u8], -42, 2, 2), + ]; + + for (input_bytes, decimal_num, expect_scale, expect_precision) in cases { + check_avro_bytes_serde( + input_bytes, + Datum::decimal_with_precision(decimal_new(decimal_num, expect_scale), expect_precision) + .unwrap(), + &PrimitiveType::Decimal { + precision: expect_precision, + scale: expect_scale, + }, + ); + } +} + +#[test] +fn avro_bytes_decimal_expect_error() { + // (decimal_num, expect_scale, expect_precision) + let cases = vec![(1234, 2, 1)]; + + for (decimal_num, expect_scale, expect_precision) in cases { + let result = + Datum::decimal_with_precision(decimal_new(decimal_num, expect_scale), expect_precision); + assert!(result.is_err(), "expect error but got {result:?}"); + assert_eq!( + result.unwrap_err().kind(), + ErrorKind::DataInvalid, + "expect error DataInvalid", + ); + } +} + +fn check_raw_literal_bytes_serde_via_avro( + input_bytes: Vec, + expected_literal: Literal, + expected_type: &Type, +) { + use apache_avro::types::Value; + + // Create an Avro bytes value and deserialize it through the RawLiteral path + let avro_value = Value::Bytes(input_bytes); + let raw_literal: RawLiteral = apache_avro::from_value(&avro_value).unwrap(); + let result = raw_literal.try_into(expected_type).unwrap(); + assert_eq!(result, Some(expected_literal)); +} + +fn check_raw_literal_bytes_error_via_avro(input_bytes: Vec, expected_type: &Type) { + use apache_avro::types::Value; + + let avro_value = Value::Bytes(input_bytes); + let raw_literal: RawLiteral = apache_avro::from_value(&avro_value).unwrap(); + let result = raw_literal.try_into(expected_type); + assert!(result.is_err(), "Expected error but got: {result:?}"); +} + +#[test] +fn test_raw_literal_bytes_binary() { + let bytes = vec![1u8, 2u8, 3u8, 4u8, 5u8]; + check_raw_literal_bytes_serde_via_avro( + bytes.clone(), + Literal::binary(bytes), + &Primitive(PrimitiveType::Binary), + ); +} + +#[test] +fn test_raw_literal_bytes_binary_empty() { + let bytes = vec![]; + check_raw_literal_bytes_serde_via_avro( + bytes.clone(), + Literal::binary(bytes), + &Primitive(PrimitiveType::Binary), + ); +} + +#[test] +fn test_raw_literal_bytes_fixed_correct_length() { + let bytes = vec![1u8, 2u8, 3u8, 4u8]; + check_raw_literal_bytes_serde_via_avro( + bytes.clone(), + Literal::fixed(bytes), + &Primitive(PrimitiveType::Fixed(4)), + ); +} + +#[test] +fn test_raw_literal_bytes_fixed_wrong_length() { + let bytes = vec![1u8, 2u8, 3u8]; // 3 bytes, but expecting 4 + check_raw_literal_bytes_error_via_avro(bytes, &Primitive(PrimitiveType::Fixed(4))); +} + +#[test] +fn test_raw_literal_bytes_fixed_empty_correct_length() { + let bytes = vec![]; + check_raw_literal_bytes_serde_via_avro( + bytes.clone(), + Literal::fixed(bytes), + &Primitive(PrimitiveType::Fixed(0)), + ); +} + +#[test] +fn test_raw_literal_bytes_uuid_correct_length() { + let uuid_bytes = vec![ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, + 0xef, + ]; + let expected_uuid = u128::from_be_bytes([ + 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, + 0xef, + ]); + check_raw_literal_bytes_serde_via_avro( + uuid_bytes, + Literal::Primitive(PrimitiveLiteral::UInt128(expected_uuid)), + &Primitive(PrimitiveType::Uuid), + ); +} + +#[test] +fn test_raw_literal_bytes_uuid_wrong_length() { + let bytes = vec![1u8, 2u8, 3u8]; // 3 bytes, but UUID needs 16 + check_raw_literal_bytes_error_via_avro(bytes, &Primitive(PrimitiveType::Uuid)); +} + +#[test] +fn test_raw_literal_bytes_decimal_precision_4_scale_2() { + // Precision 4 requires 2 bytes + let decimal_bytes = vec![0x04, 0xd2]; // 1234 in 2 bytes + let expected_decimal = 1234i128; + check_raw_literal_bytes_serde_via_avro( + decimal_bytes, + Literal::Primitive(PrimitiveLiteral::Int128(expected_decimal)), + &Primitive(PrimitiveType::Decimal { + precision: 4, + scale: 2, + }), + ); +} + +#[test] +fn test_raw_literal_bytes_decimal_precision_4_negative() { + // Precision 4 requires 2 bytes, negative number + let decimal_bytes = vec![0xfb, 0x2e]; // -1234 in 2 bytes + let expected_decimal = -1234i128; + check_raw_literal_bytes_serde_via_avro( + decimal_bytes, + Literal::Primitive(PrimitiveLiteral::Int128(expected_decimal)), + &Primitive(PrimitiveType::Decimal { + precision: 4, + scale: 2, + }), + ); +} + +#[test] +fn test_raw_literal_bytes_decimal_precision_9_scale_2() { + // Precision 9 requires 4 bytes + let decimal_bytes = vec![0x00, 0x12, 0xd6, 0x87]; // 1234567 in 4 bytes + let expected_decimal = 1234567i128; + check_raw_literal_bytes_serde_via_avro( + decimal_bytes, + Literal::Primitive(PrimitiveLiteral::Int128(expected_decimal)), + &Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 2, + }), + ); +} + +#[test] +fn test_raw_literal_bytes_decimal_precision_18_scale_2() { + // Precision 18 requires 8 bytes + let decimal_bytes = vec![0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xd2]; // 1234 in 8 bytes + let expected_decimal = 1234i128; + check_raw_literal_bytes_serde_via_avro( + decimal_bytes, + Literal::Primitive(PrimitiveLiteral::Int128(expected_decimal)), + &Primitive(PrimitiveType::Decimal { + precision: 18, + scale: 2, + }), + ); +} + +#[test] +fn test_raw_literal_bytes_decimal_precision_38_scale_2() { + // Precision 38 requires 16 bytes (maximum precision) + let decimal_bytes = vec![ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0xd2, // 1234 in 16 bytes + ]; + let expected_decimal = 1234i128; + check_raw_literal_bytes_serde_via_avro( + decimal_bytes, + Literal::Primitive(PrimitiveLiteral::Int128(expected_decimal)), + &Primitive(PrimitiveType::Decimal { + precision: 38, + scale: 2, + }), + ); +} + +#[test] +fn test_raw_literal_bytes_decimal_precision_1_scale_0() { + // Precision 1 requires 1 byte + let decimal_bytes = vec![0x07]; // 7 in 1 byte + let expected_decimal = 7i128; + check_raw_literal_bytes_serde_via_avro( + decimal_bytes, + Literal::Primitive(PrimitiveLiteral::Int128(expected_decimal)), + &Primitive(PrimitiveType::Decimal { + precision: 1, + scale: 0, + }), + ); +} + +#[test] +fn test_raw_literal_bytes_decimal_precision_1_negative() { + // Precision 1 requires 1 byte, negative number + let decimal_bytes = vec![0xf9]; // -7 in 1 byte (two's complement) + let expected_decimal = -7i128; + check_raw_literal_bytes_serde_via_avro( + decimal_bytes, + Literal::Primitive(PrimitiveLiteral::Int128(expected_decimal)), + &Primitive(PrimitiveType::Decimal { + precision: 1, + scale: 0, + }), + ); +} + +#[test] +fn test_raw_literal_bytes_decimal_wrong_length() { + // 3 bytes provided, but precision 4 requires 2 bytes + let bytes = vec![1u8, 2u8, 3u8]; + check_raw_literal_bytes_error_via_avro( + bytes, + &Primitive(PrimitiveType::Decimal { + precision: 4, + scale: 2, + }), + ); +} + +#[test] +fn test_raw_literal_bytes_decimal_wrong_length_too_few() { + // 1 byte provided, but precision 9 requires 4 bytes + let bytes = vec![0x42]; + check_raw_literal_bytes_error_via_avro( + bytes, + &Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 2, + }), + ); +} + +#[test] +fn test_raw_literal_bytes_unsupported_type() { + let bytes = vec![1u8, 2u8, 3u8, 4u8]; + check_raw_literal_bytes_error_via_avro(bytes, &Primitive(PrimitiveType::Int)); +} + +#[test] +fn test_raw_literal_boolean_round_trip() { + for value in [false, true] { + let ty = Primitive(PrimitiveType::Boolean); + check_raw_literal_json_serde(Literal::bool(value), &ty); + check_convert_with_avro(Literal::bool(value), &ty); + + for json in [ + serde_json::json!({"type": "boolean", "literal": value}), + serde_json::json!(["boolean", value]), + ] { + assert_eq!( + serde_json::from_value::(json).unwrap(), + Datum::bool(value) + ); + } + } +} + +#[test] +fn test_datum_json_boolean_rejects_type_mismatch() { + for ty in ["int", "long", "string", "unknown"] { + for value in [false, true] { + for json in [ + serde_json::json!({"type": ty, "literal": value}), + serde_json::json!([ty, value]), + ] { + let error = serde_json::from_value::(json).unwrap_err(); + assert!(error.to_string().contains("raw literal (boolean)")); + assert!(error.to_string().contains("type mismatch")); + } + } + } +} + +#[test] +fn test_raw_literal_null_for_boolean_type() { + let ty = Primitive(PrimitiveType::Boolean); + let avro_literal: RawLiteral = apache_avro::from_value(&Value::Null).unwrap(); + let json_literal: RawLiteral = serde_json::from_str("null").unwrap(); + + for literal in [avro_literal, json_literal] { + assert_eq!(literal.try_into(&ty).unwrap(), None); + } +} + +#[test] +fn test_raw_literal_boolean_rejects_type_mismatch() { + let types = [ + Primitive(PrimitiveType::Int), + Primitive(PrimitiveType::Long), + Primitive(PrimitiveType::Float), + Primitive(PrimitiveType::Double), + Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 2, + }), + Primitive(PrimitiveType::Date), + Primitive(PrimitiveType::Time), + Primitive(PrimitiveType::Timestamp), + Primitive(PrimitiveType::Timestamptz), + Primitive(PrimitiveType::TimestampNs), + Primitive(PrimitiveType::TimestamptzNs), + Primitive(PrimitiveType::String), + Primitive(PrimitiveType::Uuid), + Primitive(PrimitiveType::Fixed(1)), + Primitive(PrimitiveType::Binary), + Primitive(PrimitiveType::Unknown), + Type::Struct(StructType::new(vec![])), + Type::List(ListType::new( + NestedField::list_element(1, Primitive(PrimitiveType::Boolean), false).into(), + )), + Type::Map(MapType::new( + NestedField::map_key_element(1, Primitive(PrimitiveType::String)).into(), + NestedField::map_value_element(2, Primitive(PrimitiveType::Boolean), false).into(), + )), + Type::Variant(VariantType), + ]; + + for value in [false, true] { + for ty in &types { + let avro_literal: RawLiteral = apache_avro::from_value(&Value::Boolean(value)).unwrap(); + let json_literal: RawLiteral = + serde_json::from_value(serde_json::json!(value)).unwrap(); + + for literal in [avro_literal, json_literal] { + let error = literal.try_into(ty).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.to_string().contains("raw literal (boolean)")); + assert!(error.to_string().contains("type mismatch")); + } + } + } +} + +#[test] +fn test_raw_literal_boolean_rejects_nested_type_mismatch() { + let cases = [ + ( + serde_json::json!({"col": true}), + Value::Record(vec![("col".to_string(), Value::Boolean(true))]), + Type::Struct(StructType::new(vec![ + NestedField::required(1, "col", Primitive(PrimitiveType::Int)).into(), + ])), + ), + ( + serde_json::json!([false]), + Value::Array(vec![Value::Boolean(false)]), + Type::List(ListType::new( + NestedField::list_element(1, Primitive(PrimitiveType::Int), false).into(), + )), + ), + ( + serde_json::json!({"key": true}), + Value::Map([("key".to_string(), Value::Boolean(true))].into()), + Type::Map(MapType::new( + NestedField::map_key_element(1, Primitive(PrimitiveType::String)).into(), + NestedField::map_value_element(2, Primitive(PrimitiveType::Int), false).into(), + )), + ), + ( + serde_json::json!([{"key": false, "value": 1}]), + Value::Array(vec![Value::Record(vec![ + ("key".to_string(), Value::Boolean(false)), + ("value".to_string(), Value::Int(1)), + ])]), + Type::Map(MapType::new( + NestedField::map_key_element(1, Primitive(PrimitiveType::Int)).into(), + NestedField::map_value_element(2, Primitive(PrimitiveType::Int), false).into(), + )), + ), + ]; + + for (json_value, avro_value, ty) in cases { + let avro_literal: RawLiteral = apache_avro::from_value(&avro_value).unwrap(); + let json_literal: RawLiteral = serde_json::from_value(json_value).unwrap(); + + for literal in [avro_literal, json_literal] { + let error = literal.try_into(&ty).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.to_string().contains("raw literal (boolean)")); + assert!(error.to_string().contains("type mismatch")); + } + } +} + +#[test] +fn avro_convert_test_int() { + check_convert_with_avro( + Literal::Primitive(PrimitiveLiteral::Int(32)), + &Primitive(PrimitiveType::Int), + ); +} + +#[test] +fn avro_convert_test_long() { + check_convert_with_avro( + Literal::Primitive(PrimitiveLiteral::Long(32)), + &Primitive(PrimitiveType::Long), + ); +} + +#[test] +fn avro_convert_test_float() { + check_convert_with_avro( + Literal::Primitive(PrimitiveLiteral::Float(OrderedFloat(1.0))), + &Primitive(PrimitiveType::Float), + ); +} + +#[test] +fn avro_convert_test_double() { + check_convert_with_avro( + Literal::Primitive(PrimitiveLiteral::Double(OrderedFloat(1.0))), + &Primitive(PrimitiveType::Double), + ); +} + +#[test] +fn avro_convert_test_string() { + check_convert_with_avro( + Literal::Primitive(PrimitiveLiteral::String("iceberg".to_string())), + &Primitive(PrimitiveType::String), + ); +} + +#[test] +fn avro_convert_test_date() { + check_convert_with_avro( + Literal::Primitive(PrimitiveLiteral::Int(17486)), + &Primitive(PrimitiveType::Date), + ); +} + +#[test] +fn avro_convert_test_time() { + check_convert_with_avro( + Literal::Primitive(PrimitiveLiteral::Long(81068123456)), + &Primitive(PrimitiveType::Time), + ); +} + +#[test] +fn avro_convert_test_timestamp() { + check_convert_with_avro( + Literal::Primitive(PrimitiveLiteral::Long(1510871468123456)), + &Primitive(PrimitiveType::Timestamp), + ); +} + +#[test] +fn avro_convert_test_timestamptz() { + check_convert_with_avro( + Literal::Primitive(PrimitiveLiteral::Long(1510871468123456)), + &Primitive(PrimitiveType::Timestamptz), + ); +} + +#[test] +fn avro_convert_test_list() { + check_convert_with_avro( + Literal::List(vec![ + Some(Literal::Primitive(PrimitiveLiteral::Int(1))), + Some(Literal::Primitive(PrimitiveLiteral::Int(2))), + Some(Literal::Primitive(PrimitiveLiteral::Int(3))), + None, + ]), + &Type::List(ListType { + element_field: NestedField::list_element(0, Primitive(PrimitiveType::Int), false) + .into(), + }), + ); + + check_convert_with_avro( + Literal::List(vec![ + Some(Literal::Primitive(PrimitiveLiteral::Int(1))), + Some(Literal::Primitive(PrimitiveLiteral::Int(2))), + Some(Literal::Primitive(PrimitiveLiteral::Int(3))), + ]), + &Type::List(ListType { + element_field: NestedField::list_element(0, Primitive(PrimitiveType::Int), true).into(), + }), + ); +} + +fn check_convert_with_avro_map(expected_literal: Literal, expected_type: &Type) { + let fields = vec![NestedField::required(1, "col", expected_type.clone()).into()]; + let schema = Schema::builder() + .with_fields(fields.clone()) + .build() + .unwrap(); + let avro_schema = schema_to_avro_schema("test", &schema).unwrap(); + let struct_type = Type::Struct(StructType::new(fields)); + let struct_literal = Literal::Struct(Struct::from_iter(vec![Some(expected_literal.clone())])); + + let mut writer = apache_avro::Writer::new(&avro_schema, Vec::new()).unwrap(); + let raw_literal = RawLiteral::try_from(struct_literal.clone(), &struct_type).unwrap(); + writer.append_ser(raw_literal).unwrap(); + let encoded = writer.into_inner().unwrap(); + + let reader = apache_avro::Reader::new(&*encoded).unwrap(); + for record in reader { + let result = apache_avro::from_value::(&record.unwrap()).unwrap(); + let desered_literal = result.try_into(&struct_type).unwrap().unwrap(); + match (&desered_literal, &struct_literal) { + (Literal::Struct(desered), Literal::Struct(expected)) => { + match (&desered.fields()[0], &expected.fields()[0]) { + (Some(Literal::Map(desered)), Some(Literal::Map(expected))) => { + assert!(desered.has_same_content(expected)) + } + _ => { + unreachable!() + } + } + } + _ => { + panic!("unexpected literal type"); + } + } + } +} + +#[test] +fn avro_convert_test_map() { + check_convert_with_avro_map( + Literal::Map(Map::from([ + ( + Literal::Primitive(PrimitiveLiteral::Int(1)), + Some(Literal::Primitive(PrimitiveLiteral::Long(1))), + ), + ( + Literal::Primitive(PrimitiveLiteral::Int(2)), + Some(Literal::Primitive(PrimitiveLiteral::Long(2))), + ), + (Literal::Primitive(PrimitiveLiteral::Int(3)), None), + ])), + &Type::Map(MapType { + key_field: NestedField::map_key_element(2, Primitive(PrimitiveType::Int)).into(), + value_field: NestedField::map_value_element(3, Primitive(PrimitiveType::Long), false) + .into(), + }), + ); + + check_convert_with_avro_map( + Literal::Map(Map::from([ + ( + Literal::Primitive(PrimitiveLiteral::Int(1)), + Some(Literal::Primitive(PrimitiveLiteral::Long(1))), + ), + ( + Literal::Primitive(PrimitiveLiteral::Int(2)), + Some(Literal::Primitive(PrimitiveLiteral::Long(2))), + ), + ( + Literal::Primitive(PrimitiveLiteral::Int(3)), + Some(Literal::Primitive(PrimitiveLiteral::Long(3))), + ), + ])), + &Type::Map(MapType { + key_field: NestedField::map_key_element(2, Primitive(PrimitiveType::Int)).into(), + value_field: NestedField::map_value_element(3, Primitive(PrimitiveType::Long), true) + .into(), + }), + ); +} + +#[test] +fn avro_convert_test_string_map() { + check_convert_with_avro_map( + Literal::Map(Map::from([ + ( + Literal::Primitive(PrimitiveLiteral::String("a".to_string())), + Some(Literal::Primitive(PrimitiveLiteral::Int(1))), + ), + ( + Literal::Primitive(PrimitiveLiteral::String("b".to_string())), + Some(Literal::Primitive(PrimitiveLiteral::Int(2))), + ), + ( + Literal::Primitive(PrimitiveLiteral::String("c".to_string())), + None, + ), + ])), + &Type::Map(MapType { + key_field: NestedField::map_key_element(2, Primitive(PrimitiveType::String)).into(), + value_field: NestedField::map_value_element(3, Primitive(PrimitiveType::Int), false) + .into(), + }), + ); + + check_convert_with_avro_map( + Literal::Map(Map::from([ + ( + Literal::Primitive(PrimitiveLiteral::String("a".to_string())), + Some(Literal::Primitive(PrimitiveLiteral::Int(1))), + ), + ( + Literal::Primitive(PrimitiveLiteral::String("b".to_string())), + Some(Literal::Primitive(PrimitiveLiteral::Int(2))), + ), + ( + Literal::Primitive(PrimitiveLiteral::String("c".to_string())), + Some(Literal::Primitive(PrimitiveLiteral::Int(3))), + ), + ])), + &Type::Map(MapType { + key_field: NestedField::map_key_element(2, Primitive(PrimitiveType::String)).into(), + value_field: NestedField::map_value_element(3, Primitive(PrimitiveType::Int), true) + .into(), + }), + ); +} + +#[test] +fn avro_convert_test_record() { + check_convert_with_avro( + Literal::Struct(Struct::from_iter(vec![ + Some(Literal::Primitive(PrimitiveLiteral::Int(1))), + Some(Literal::Primitive(PrimitiveLiteral::String( + "bar".to_string(), + ))), + None, + ])), + &Type::Struct(StructType::new(vec![ + NestedField::required(2, "id", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "name", Primitive(PrimitiveType::String)).into(), + NestedField::optional(4, "address", Primitive(PrimitiveType::String)).into(), + ])), + ); +} + +// # TODO:https://github.com/apache/iceberg-rust/issues/86 +// rust avro don't support deserialize any bytes representation now: +// - binary +// - decimal +#[test] +fn avro_convert_test_binary_ser() { + let literal = Literal::Primitive(PrimitiveLiteral::Binary(vec![1, 2, 3, 4, 5])); + let ty = Primitive(PrimitiveType::Binary); + let expect_value = Value::Bytes(vec![1, 2, 3, 4, 5]); + check_serialize_avro(literal, &ty, expect_value); +} + +#[test] +fn avro_convert_test_decimal_ser() { + let literal = Literal::decimal(12345); + let ty = Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 8, + }); + let expect_value = Value::Decimal(apache_avro::Decimal::from(12345_i128.to_be_bytes())); + check_serialize_avro(literal, &ty, expect_value); +} + +// # TODO:https://github.com/apache/iceberg-rust/issues/86 +// rust avro can't support to convert any byte-like type to fixed in avro now. +// - uuid ser/de +// - fixed ser/de + +#[test] +fn test_parse_timestamp() { + let value = Datum::timestamp_from_str("2021-08-01T01:09:00.0899").unwrap(); + assert_eq!(&format!("{value}"), "2021-08-01 01:09:00.089900"); + + let value = Datum::timestamp_from_str("2023-01-06T00:00:00").unwrap(); + assert_eq!(&format!("{value}"), "2023-01-06 00:00:00"); + + let value = Datum::timestamp_from_str("2021-08-01T01:09:00.0899+0800"); + assert!(value.is_err(), "Parse timestamp with timezone should fail!"); + + let value = Datum::timestamp_from_str("dfa"); + assert!( + value.is_err(), + "Parse timestamp with invalid input should fail!" + ); +} + +#[test] +fn test_parse_timestamptz() { + let value = Datum::timestamptz_from_str("2021-08-01T09:09:00.0899+0800").unwrap(); + assert_eq!(&format!("{value}"), "2021-08-01 01:09:00.089900 UTC"); + + let value = Datum::timestamptz_from_str("2021-08-01T01:09:00.0899"); + assert!( + value.is_err(), + "Parse timestamptz without timezone should fail!" + ); + + let value = Datum::timestamptz_from_str("dfa"); + assert!( + value.is_err(), + "Parse timestamptz with invalid input should fail!" + ); +} + +#[test] +fn test_pre_epoch_timestamptz_formatting() { + assert_eq!( + Datum::timestamptz_micros(-1).to_string(), + "1969-12-31 23:59:59.999999 UTC" + ); + assert_eq!( + Datum::timestamptz_nanos(-1).to_string(), + "1969-12-31 23:59:59.999999999 UTC" + ); + assert_eq!( + Datum::timestamptz_micros(-1_500_000).to_string(), + "1969-12-31 23:59:58.500 UTC" + ); + + check_json_serde( + r#""1969-12-31T23:59:59.999999+00:00""#, + Literal::long(-1), + &Primitive(PrimitiveType::Timestamptz), + ); + check_json_serde( + r#""1969-12-31T23:59:58.500+00:00""#, + Literal::long(-1_500_000), + &Primitive(PrimitiveType::Timestamptz), + ); + check_json_serde( + r#""1969-12-31T23:59:59.999999999+00:00""#, + Literal::long(-1), + &Primitive(PrimitiveType::TimestamptzNs), + ); +} + +#[test] +fn test_datum_ser_deser() { + let test_fn = |datum: Datum| { + let json = serde_json::to_value(&datum).unwrap(); + let desered_datum: Datum = serde_json::from_value(json).unwrap(); + assert_eq!(datum, desered_datum); + }; + let datum = Datum::int(1); + test_fn(datum); + let datum = Datum::long(1); + test_fn(datum); + + let datum = Datum::float(1.0_f32); + test_fn(datum); + let datum = Datum::float(0_f32); + test_fn(datum); + let datum = Datum::float(-0_f32); + test_fn(datum); + let datum = Datum::float(f32::MAX); + test_fn(datum); + let datum = Datum::float(f32::MIN); + test_fn(datum); + + // serde_json can't serialize f32::INFINITY, f32::NEG_INFINITY, f32::NAN + let datum = Datum::float(f32::INFINITY); + let json = serde_json::to_string(&datum).unwrap(); + assert!(serde_json::from_str::(&json).is_err()); + let datum = Datum::float(f32::NEG_INFINITY); + let json = serde_json::to_string(&datum).unwrap(); + assert!(serde_json::from_str::(&json).is_err()); + let datum = Datum::float(f32::NAN); + let json = serde_json::to_string(&datum).unwrap(); + assert!(serde_json::from_str::(&json).is_err()); + + let datum = Datum::double(1.0); + test_fn(datum); + let datum = Datum::double(f64::MAX); + test_fn(datum); + let datum = Datum::double(f64::MIN); + test_fn(datum); + + // serde_json can't serialize f32::INFINITY, f32::NEG_INFINITY, f32::NAN + let datum = Datum::double(f64::INFINITY); + let json = serde_json::to_string(&datum).unwrap(); + assert!(serde_json::from_str::(&json).is_err()); + let datum = Datum::double(f64::NEG_INFINITY); + let json = serde_json::to_string(&datum).unwrap(); + assert!(serde_json::from_str::(&json).is_err()); + let datum = Datum::double(f64::NAN); + let json = serde_json::to_string(&datum).unwrap(); + assert!(serde_json::from_str::(&json).is_err()); + + let datum = Datum::string("iceberg"); + test_fn(datum); + let datum = Datum::bool(true); + test_fn(datum); + let datum = Datum::date(17486); + test_fn(datum); + let datum = Datum::time_from_hms_micro(22, 15, 33, 111).unwrap(); + test_fn(datum); + let datum = Datum::timestamp_micros(1510871468123456); + test_fn(datum); + let datum = Datum::timestamptz_micros(1510871468123456); + test_fn(datum); + let datum = Datum::uuid(Uuid::parse_str("f79c3e09-677c-4bbd-a479-3f349cb785e7").unwrap()); + test_fn(datum); + let datum = Datum::decimal(decimal_new(1420, 0)).unwrap(); + test_fn(datum); + let datum = Datum::binary(vec![1, 2, 3, 4, 5]); + test_fn(datum); + let datum = Datum::fixed(vec![1, 2, 3, 4, 5]); + test_fn(datum); +} + +#[test] +fn test_datum_date_convert_to_int() { + let datum_date = Datum::date(12345); + + let result = datum_date.to(&Primitive(PrimitiveType::Int)).unwrap(); + + let expected = Datum::int(12345); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_int_convert_to_date() { + let datum_int = Datum::int(12345); + + let result = datum_int.to(&Primitive(PrimitiveType::Date)).unwrap(); + + let expected = Datum::date(12345); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_long_convert_to_int() { + let datum = Datum::long(12345); + + let result = datum.to(&Primitive(PrimitiveType::Int)).unwrap(); + + let expected = Datum::int(12345); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_long_convert_to_int_above_max() { + let datum = Datum::long(INT_MAX as i64 + 1); + + let result = datum.to(&Primitive(PrimitiveType::Int)).unwrap(); + + let expected = Datum::new(PrimitiveType::Int, PrimitiveLiteral::AboveMax); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_long_convert_to_int_below_min() { + let datum = Datum::long(INT_MIN as i64 - 1); + + let result = datum.to(&Primitive(PrimitiveType::Int)).unwrap(); + + let expected = Datum::new(PrimitiveType::Int, PrimitiveLiteral::BelowMin); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_double_convert_to_float() { + let datum = Datum::double(2.5); + + let result = datum.to(&Primitive(PrimitiveType::Float)).unwrap(); + + let expected = Datum::float(2.5_f32); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_double_convert_to_float_above_max() { + let datum = Datum::double(1e39); + + let result = datum.to(&Primitive(PrimitiveType::Float)).unwrap(); + + let expected = Datum::new(PrimitiveType::Float, PrimitiveLiteral::AboveMax); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_double_convert_to_float_below_min() { + let datum = Datum::double(-1e39); + + let result = datum.to(&Primitive(PrimitiveType::Float)).unwrap(); + + let expected = Datum::new(PrimitiveType::Float, PrimitiveLiteral::BelowMin); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_float_convert_to_double() { + let datum = Datum::float(2.5_f32); + + let result = datum.to(&Primitive(PrimitiveType::Double)).unwrap(); + + let expected = Datum::double(2.5); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_long_convert_to_timestamp() { + let datum = Datum::long(12345); + + let result = datum.to(&Primitive(PrimitiveType::Timestamp)).unwrap(); + + let expected = Datum::timestamp_micros(12345); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_long_convert_to_timestamptz() { + let datum = Datum::long(12345); + + let result = datum.to(&Primitive(PrimitiveType::Timestamptz)).unwrap(); + + let expected = Datum::timestamptz_micros(12345); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_decimal_convert_to_long() { + let datum = Datum::decimal(decimal_new(12345, 0)).unwrap(); + + let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap(); + + let expected = Datum::long(12345); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_decimal_convert_to_long_above_max() { + let datum = Datum::decimal(decimal_from_i128_with_scale(LONG_MAX as i128 + 1, 0)).unwrap(); + + let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap(); + + let expected = Datum::new(PrimitiveType::Long, PrimitiveLiteral::AboveMax); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_decimal_convert_to_long_below_min() { + let datum = Datum::decimal(decimal_from_i128_with_scale(LONG_MIN as i128 - 1, 0)).unwrap(); + + let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap(); + + let expected = Datum::new(PrimitiveType::Long, PrimitiveLiteral::BelowMin); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_string_convert_to_boolean() { + let datum = Datum::string("true"); + + let result = datum.to(&Primitive(PrimitiveType::Boolean)).unwrap(); + + let expected = Datum::bool(true); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_string_convert_to_int() { + let datum = Datum::string("12345"); + + let result = datum.to(&Primitive(PrimitiveType::Int)).unwrap(); + + let expected = Datum::int(12345); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_string_convert_to_long() { + let datum = Datum::string("12345"); + + let result = datum.to(&Primitive(PrimitiveType::Long)).unwrap(); + + let expected = Datum::long(12345); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_string_convert_to_timestamp() { + let datum = Datum::string("1925-05-20T19:25:00.000"); + + let result = datum.to(&Primitive(PrimitiveType::Timestamp)).unwrap(); + + let expected = Datum::timestamp_micros(-1407990900000000); + + assert_eq!(result, expected); +} + +#[test] +fn test_datum_string_convert_to_timestamptz() { + let datum = Datum::string("1925-05-20T19:25:00.000 UTC"); + + let result = datum.to(&Primitive(PrimitiveType::Timestamptz)).unwrap(); + + let expected = Datum::timestamptz_micros(-1407990900000000); + + assert_eq!(result, expected); +} + +#[test] +fn test_iceberg_float_order() { + // Test float ordering + let float_values = vec![ + Datum::float(f32::NAN), + Datum::float(-f32::NAN), + Datum::float(f32::MAX), + Datum::float(f32::MIN), + Datum::float(f32::INFINITY), + Datum::float(-f32::INFINITY), + Datum::float(1.0_f32), + Datum::float(-1.0_f32), + Datum::float(0.0_f32), + Datum::float(-0.0_f32), + ]; + + let mut float_sorted = float_values.clone(); + float_sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + + let float_expected = vec![ + Datum::float(-f32::NAN), + Datum::float(-f32::INFINITY), + Datum::float(f32::MIN), + Datum::float(-1.0_f32), + Datum::float(-0.0_f32), + Datum::float(0.0_f32), + Datum::float(1.0_f32), + Datum::float(f32::MAX), + Datum::float(f32::INFINITY), + Datum::float(f32::NAN), + ]; + + assert_eq!(float_sorted, float_expected); + + // Test double ordering + let double_values = vec![ + Datum::double(f64::NAN), + Datum::double(-f64::NAN), + Datum::double(f64::INFINITY), + Datum::double(-f64::INFINITY), + Datum::double(f64::MAX), + Datum::double(f64::MIN), + Datum::double(1.0), + Datum::double(-1.0), + Datum::double(0.0), + Datum::double(-0.0), + ]; + + let mut double_sorted = double_values.clone(); + double_sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + + let double_expected = vec![ + Datum::double(-f64::NAN), + Datum::double(-f64::INFINITY), + Datum::double(f64::MIN), + Datum::double(-1.0), + Datum::double(-0.0), + Datum::double(0.0), + Datum::double(1.0), + Datum::double(f64::MAX), + Datum::double(f64::INFINITY), + Datum::double(f64::NAN), + ]; + + assert_eq!(double_sorted, double_expected); +} + +#[test] +fn test_negative_zero_less_than_positive_zero() { + { + let neg_zero = Datum::float(-0.0_f32); + let pos_zero = Datum::float(0.0_f32); + + assert_eq!( + neg_zero.partial_cmp(&pos_zero), + Some(std::cmp::Ordering::Less), + "IEEE 754 totalOrder requires -0.0 < +0.0 on F32" + ); + } + + { + let neg_zero = Datum::double(-0.0); + let pos_zero = Datum::double(0.0); + + assert_eq!( + neg_zero.partial_cmp(&pos_zero), + Some(std::cmp::Ordering::Less), + "IEEE 754 totalOrder requires -0.0 < +0.0 on F64" + ); + } +} + +#[test] +fn test_signed_zero_literals_are_distinct() { + for (neg_zero, pos_zero) in [ + ( + PrimitiveLiteral::Float(OrderedFloat(-0.0)), + PrimitiveLiteral::Float(OrderedFloat(0.0)), + ), + ( + PrimitiveLiteral::Double(OrderedFloat(-0.0)), + PrimitiveLiteral::Double(OrderedFloat(0.0)), + ), + ] { + assert_ne!(neg_zero, pos_zero); + assert_eq!( + neg_zero.partial_cmp(&pos_zero), + Some(std::cmp::Ordering::Less) + ); + assert_eq!(HashSet::from([neg_zero, pos_zero]).len(), 2); + } + + assert_ne!(Datum::float(-0.0_f32), Datum::float(0.0_f32)); + assert_ne!(Datum::double(-0.0), Datum::double(0.0)); + + // A partition tuple is a `Struct`, which the partitioned writers use as a map key. + let partitions = HashSet::from([ + Struct::from_iter([Some(Literal::double(-0.0))]), + Struct::from_iter([Some(Literal::double(0.0))]), + ]); + assert_eq!(partitions.len(), 2); +} + +#[test] +fn test_nan_literals_are_equal() { + for (nan, other_nan) in [ + ( + PrimitiveLiteral::Float(OrderedFloat(f32::NAN)), + PrimitiveLiteral::Float(OrderedFloat(-f32::NAN)), + ), + ( + PrimitiveLiteral::Double(OrderedFloat(f64::NAN)), + PrimitiveLiteral::Double(OrderedFloat(-f64::NAN)), + ), + ] { + assert_eq!(nan, other_nan); + assert_eq!(nan.partial_cmp(&other_nan), Some(std::cmp::Ordering::Equal)); + assert_eq!(HashSet::from([nan, other_nan]).len(), 1); + } +} + +#[test] +fn test_primitive_literal_eq_across_variants() { + assert_eq!(PrimitiveLiteral::AboveMax, PrimitiveLiteral::AboveMax); + assert_eq!(PrimitiveLiteral::BelowMin, PrimitiveLiteral::BelowMin); + assert_ne!(PrimitiveLiteral::AboveMax, PrimitiveLiteral::BelowMin); + assert_ne!(PrimitiveLiteral::Int(1), PrimitiveLiteral::Long(1)); +} + +/// Test Date deserialization from JSON as number (days since epoch). +/// +/// This reproduces the scenario from Iceberg Java's TestAddFilesProcedure where: +/// - Date partition columns have initial_default values in manifests +/// - These values are serialized as days since epoch (e.g., 18628 for 2021-01-01) +/// - The JSON schema includes: {"type":"date","initial-default":18628} +/// +/// Prior to this fix, Date values in JSON were only parsed from String format ("2021-01-01"), +/// causing initial_default values to be lost during schema deserialization. +/// +/// This test ensures both formats are supported: +/// - String format: "2021-01-01" (used in table metadata) +/// - Number format: 18628 (used in initial-default values from add_files) +/// +/// See: Iceberg Java TestAddFilesProcedure.addDataPartitionedByDateToPartitioned() +#[test] +fn test_date_from_json_as_number() { + use serde_json::json; + + // Test Date as number (days since epoch) - used in initial-default from add_files + let date_number = json!(18628); // 2021-01-01 is 18628 days since 1970-01-01 + let result = Literal::try_from_json(date_number, &Primitive(PrimitiveType::Date)).unwrap(); + assert_eq!( + result, + Some(Literal::Primitive(PrimitiveLiteral::Int(18628))) + ); + + // Test Date as string - traditional format + let date_string = json!("2021-01-01"); + let result = Literal::try_from_json(date_string, &Primitive(PrimitiveType::Date)).unwrap(); + assert_eq!( + result, + Some(Literal::Primitive(PrimitiveLiteral::Int(18628))) + ); + + // Both formats should produce the same Literal value +} + +#[test] +fn test_decimal_from_str_rejects_non_finite_values() { + for input in [ + "NaN", + "nan", + "NAN", + "Inf", + "inf", + "+Inf", + "-Inf", + "Infinity", + "+Infinity", + "-Infinity", + "infinity", + "INFINITY", + ] { + assert_eq!( + Datum::decimal_from_str(input).unwrap_err().kind(), + ErrorKind::DataInvalid, + "{input}" + ); + assert_eq!( + Literal::decimal_from_str(input).unwrap_err().kind(), + ErrorKind::DataInvalid, + "{input}" + ); + } +} + +#[test] +fn test_decimal_constructors_reject_non_finite_values() { + for value in [Decimal::NAN, Decimal::INFINITY, Decimal::NEG_INFINITY] { + assert_eq!( + Datum::decimal(value).unwrap_err().kind(), + ErrorKind::DataInvalid + ); + assert_eq!( + Datum::decimal_with_precision(value, 9).unwrap_err().kind(), + ErrorKind::DataInvalid + ); + } +} + +#[test] +fn test_decimal_from_json_rejects_non_finite_values() { + let decimal_type = Type::decimal(9, 2).unwrap(); + for input in ["NaN", "Infinity", "-Infinity"] { + assert_eq!( + Literal::try_from_json(JsonValue::String(input.to_string()), &decimal_type) + .unwrap_err() + .kind(), + ErrorKind::DataInvalid, + "{input}" + ); + } +} + +#[test] +fn test_decimal_constructors_accept_finite_values() { + for (input, mantissa, scale) in [ + ("0", 0, 0), + ("-0", 0, 0), + ("0.00", 0, 2), + ("-0.00", 0, 2), + ("123.45", 12345, 2), + ("-123.45", -12345, 2), + ] { + let datum = Datum::decimal_from_str(input).unwrap(); + assert_eq!(datum.literal(), &PrimitiveLiteral::Int128(mantissa)); + assert_eq!(datum.data_type(), &PrimitiveType::Decimal { + precision: 38, + scale, + }); + assert_eq!( + Literal::decimal_from_str(input).unwrap(), + Literal::decimal(mantissa) + ); + let value = Decimal::from_str(input, fastnum::decimal::Context::default()).unwrap(); + assert_eq!(Datum::decimal(value).unwrap(), datum); + let datum = Datum::decimal_with_precision(value, 9).unwrap(); + assert_eq!(datum.literal(), &PrimitiveLiteral::Int128(mantissa)); + assert_eq!(datum.data_type(), &PrimitiveType::Decimal { + precision: 9, + scale, + }); + } +} + +#[test] +fn test_datum_to_decimal_narrows_precision_when_scale_matches() { + let target_type = Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 2, + }); + let datum = Datum::decimal_from_str("123.45").unwrap(); + + let converted = datum.to(&target_type).unwrap(); + + assert_eq!(converted.data_type(), &PrimitiveType::Decimal { + precision: 9, + scale: 2, + }); + assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(12345)); +} + +#[test] +fn test_datum_to_decimal_widens_precision_when_scale_matches() { + let target_type = Primitive(PrimitiveType::Decimal { + precision: 38, + scale: 2, + }); + let datum = Datum::decimal_with_precision(decimal_from_i128_with_scale(12345, 2), 9).unwrap(); + + let converted = datum.to(&target_type).unwrap(); + + assert_eq!(converted.data_type(), &PrimitiveType::Decimal { + precision: 38, + scale: 2, + }); + assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(12345)); +} + +#[test] +fn test_datum_to_decimal_accepts_zero_mantissa() { + let target_type = Primitive(PrimitiveType::Decimal { + precision: 1, + scale: 0, + }); + let datum = Datum::decimal_with_precision(decimal_from_i128_with_scale(0, 0), 9).unwrap(); + + let converted = datum.to(&target_type).unwrap(); + + assert_eq!(converted.data_type(), &PrimitiveType::Decimal { + precision: 1, + scale: 0, + }); + assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(0)); +} + +#[test] +fn test_datum_to_decimal_accepts_negative_mantissa() { + let target_type = Primitive(PrimitiveType::Decimal { + precision: 2, + scale: 1, + }); + let datum = Datum::decimal_from_str("-1.5").unwrap(); + + let converted = datum.to(&target_type).unwrap(); + + assert_eq!(converted.data_type(), &PrimitiveType::Decimal { + precision: 2, + scale: 1, + }); + assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(-15)); +} + +#[test] +fn test_datum_to_decimal_rejects_precision_too_narrow() { + let target_type = Primitive(PrimitiveType::Decimal { + precision: 1, + scale: 1, + }); + let datum = Datum::decimal_from_str("1.5").unwrap(); + + let result = datum.to(&target_type); + + assert!(result.is_err(), "expect error but got {result:?}"); + assert_eq!(result.unwrap_err().kind(), ErrorKind::DataInvalid); +} + +#[test] +fn test_datum_to_decimal_rejects_value_that_fits_storage_bytes_but_not_precision() { + let target_type = Primitive(PrimitiveType::Decimal { + precision: 1, + scale: 1, + }); + let datum = Datum::decimal_from_str("4.2").unwrap(); + + let result = datum.to(&target_type); + + assert!(result.is_err(), "expect error but got {result:?}"); + assert_eq!(result.unwrap_err().kind(), ErrorKind::DataInvalid); +} + +#[test] +fn test_datum_to_decimal_accepts_single_digit_mantissa_for_precision_one() { + let target_type = Primitive(PrimitiveType::Decimal { + precision: 1, + scale: 1, + }); + let datum = Datum::decimal_from_str("0.5").unwrap(); + + let converted = datum.to(&target_type).unwrap(); + + assert_eq!(converted.data_type(), &PrimitiveType::Decimal { + precision: 1, + scale: 1, + }); + assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(5)); +} + +#[test] +fn test_datum_decimal_with_precision_rejects_value_that_exceeds_digit_precision() { + let result = Datum::decimal_with_precision(decimal_from_i128_with_scale(42, 2), 1); + + assert!(result.is_err(), "expect error but got {result:?}"); + assert_eq!(result.unwrap_err().kind(), ErrorKind::DataInvalid); +} + +#[test] +fn test_datum_decimal_with_precision_accepts_value_that_fits_digit_precision() { + let datum = Datum::decimal_with_precision(decimal_from_i128_with_scale(5, 1), 1).unwrap(); + + assert_eq!(datum.data_type(), &PrimitiveType::Decimal { + precision: 1, + scale: 1, + }); + assert_eq!(datum.literal(), &PrimitiveLiteral::Int128(5)); +} + +#[test] +fn test_datum_to_decimal_rejects_scale_change() { + let target_type = Primitive(PrimitiveType::Decimal { + precision: 9, + scale: 3, + }); + let datum = Datum::decimal_from_str("123.45").unwrap(); + + let result = datum.to(&target_type); + assert!(result.is_err(), "expect error but got {result:?}"); + + let err = result.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("Decimal scale conversion is not supported") + ); +} + +#[test] +fn test_raw_literal_project_by_name_reorders_and_fills_fields() { + let written = StructType::new(vec![ + NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "b", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(3, "extra", Primitive(PrimitiveType::Int)).into(), + ]); + let raw = RawLiteral::try_from( + Literal::Struct(Struct::from_iter([ + Some(Literal::int(1)), + Some(Literal::int(2)), + Some(Literal::int(3)), + ])), + &Type::Struct(written), + ) + .unwrap(); + let expected = StructType::new(vec![ + NestedField::optional(4, "c", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "b", Primitive(PrimitiveType::Int)).into(), + NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(), + ]); + + let projected = raw + .project_by_name(&expected) + .unwrap() + .try_into(&Type::Struct(expected)) + .unwrap(); + + assert_eq!( + projected, + Some(Literal::Struct(Struct::from_iter([ + None, + Some(Literal::int(2)), + Some(Literal::int(1)), + ]))) + ); +} + +#[test] +fn test_raw_literal_project_by_name_rejects_missing_required_field() { + let written = StructType::new(vec![ + NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(), + ]); + let raw = RawLiteral::try_from( + Literal::Struct(Struct::from_iter([Some(Literal::int(1))])), + &Type::Struct(written), + ) + .unwrap(); + let expected = StructType::new(vec![ + NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "b", Primitive(PrimitiveType::Int)).into(), + ]); + + let err = raw.project_by_name(&expected).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); +} + +/// Expected values are the output of Java's `Double.toString` / `Float.toString` (JDK 21). +#[test] +fn test_datum_to_human_string_float_matches_java() { + let double = |value: f64| Datum::double(value).to_human_string(); + assert_eq!(double(1.0), "1.0"); + assert_eq!(double(100.0), "100.0"); + assert_eq!(double(0.0), "0.0"); + assert_eq!(double(-0.0), "-0.0"); + assert_eq!(double(1.5), "1.5"); + assert_eq!(double(0.001), "0.001"); + assert_eq!(double(9999999.0), "9999999.0"); + assert_eq!(double(1.0e7), "1.0E7"); + assert_eq!(double(1.0e-5), "1.0E-5"); + assert_eq!(double(1.23456789e20), "1.23456789E20"); + assert_eq!(double(1.0e23), "1.0E23"); + assert_eq!(double(f64::INFINITY), "Infinity"); + assert_eq!(double(f64::NEG_INFINITY), "-Infinity"); + assert_eq!(double(f64::NAN), "NaN"); + assert_eq!(double(f64::MAX), "1.7976931348623157E308"); + // Java uses at least two significant digits. + assert_eq!(double(f64::from_bits(1)), "4.9E-324"); + // Halfway between two 17-digit decimals: Java picks the even one. + assert_eq!(double(2f64.powi(-25)), "2.9802322387695312E-8"); + + let float = |value: f32| Datum::float(value).to_human_string(); + assert_eq!(float(1.0), "1.0"); + assert_eq!(float(-0.0), "-0.0"); + assert_eq!(float(0.1), "0.1"); + assert_eq!(float(1.0e7), "1.0E7"); + assert_eq!(float(f32::NEG_INFINITY), "-Infinity"); + assert_eq!(float(f32::from_bits(1)), "1.4E-45"); + // 2097152.25, halfway between 2097152.2 and 2097152.3. + assert_eq!(float(f32::from_bits(0x4a00_0001)), "2097152.2"); +} diff --git a/crates/iceberg/src/spec/view_metadata.rs b/crates/iceberg/src/spec/view_metadata.rs index dafca4190e..4a816140c3 100644 --- a/crates/iceberg/src/spec/view_metadata.rs +++ b/crates/iceberg/src/spec/view_metadata.rs @@ -32,8 +32,7 @@ use uuid::Uuid; pub use super::view_metadata_builder::ViewMetadataBuilder; use super::view_version::{ViewVersionId, ViewVersionRef}; use super::{SchemaId, SchemaRef}; -use crate::error::{Result, timestamp_ms_to_utc}; -use crate::{Error, ErrorKind}; +use crate::error::{Result, invalid_data, timestamp_ms_to_utc}; /// Reference to [`ViewMetadata`]. pub type ViewMetadataRef = Arc; @@ -170,12 +169,9 @@ impl ViewMetadata { fn validate_current_version_id(&self) -> Result<()> { if !self.versions.contains_key(&self.current_version_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "No version exists with the current version id {}.", - self.current_version_id - ), + return Err(invalid_data!( + "No version exists with the current version id {}.", + self.current_version_id )); } Ok(()) @@ -184,9 +180,8 @@ impl ViewMetadata { fn validate_current_schema_id(&self) -> Result<()> { let schema_id = self.current_version().schema_id(); if !self.schemas.contains_key(&schema_id) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("No schema exists with the schema id {}.", schema_id), + return Err(invalid_data!( + "No schema exists with the schema id {schema_id}." )); } Ok(()) @@ -225,7 +220,7 @@ impl ViewVersionLog { self.timestamp_ms } - /// Returns the last updated timestamp as a DateTime with millisecond precision. + /// Returns the last updated timestamp as a `DateTime` with millisecond precision. pub fn timestamp(&self) -> Result> { timestamp_ms_to_utc(self.timestamp_ms) } @@ -271,7 +266,7 @@ pub(super) mod _serde { pub(super) versions: Vec, pub(super) version_log: Vec, pub(super) schemas: Vec, - pub(super) properties: Option>, + pub(super) properties: Option>, } impl Serialize for ViewMetadata { @@ -305,7 +300,7 @@ pub(super) mod _serde { impl TryFrom for ViewMetadata { type Error = Error; - fn try_from(value: ViewMetadataV1) -> Result { + fn try_from(value: ViewMetadataV1) -> Result { let schemas = HashMap::from_iter( value .schemas @@ -426,7 +421,7 @@ pub(crate) mod tests { } pub(crate) fn get_test_view_metadata(file_name: &str) -> ViewMetadata { - let path = format!("testdata/view_metadata/{}", file_name); + let path = format!("testdata/view_metadata/{file_name}"); let metadata: String = fs::read_to_string(path).unwrap(); serde_json::from_str(&metadata).unwrap() diff --git a/crates/iceberg/src/spec/view_metadata_builder.rs b/crates/iceberg/src/spec/view_metadata_builder.rs index dc5c104029..a59822e70c 100644 --- a/crates/iceberg/src/spec/view_metadata_builder.rs +++ b/crates/iceberg/src/spec/view_metadata_builder.rs @@ -31,7 +31,7 @@ use super::{ }; use crate::ViewCreation; use crate::catalog::ViewUpdate; -use crate::error::{Error, ErrorKind, Result}; +use crate::error::{Result, invalid_data}; use crate::io::is_truthy; /// Manipulating view metadata. @@ -142,12 +142,10 @@ impl ViewMetadataBuilder { /// - Cannot downgrade to older format versions. pub fn upgrade_format_version(self, format_version: ViewFormatVersion) -> Result { if format_version < self.metadata.format_version { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot downgrade ViewFormatVersion from {} to {}", - self.metadata.format_version, format_version - ), + return Err(invalid_data!( + "Cannot downgrade ViewFormatVersion from {} to {}", + self.metadata.format_version, + format_version )); } @@ -183,9 +181,8 @@ impl ViewMetadataBuilder { pub fn set_current_version_id(mut self, mut version_id: i32) -> Result { if version_id == Self::LAST_ADDED { let Some(last_added_id) = self.last_added_version_id else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot set current version id to last added version: no version has been added.", + return Err(invalid_data!( + "Cannot set current version id to last added version: no version has been added." )); }; version_id = last_added_id; @@ -198,13 +195,7 @@ impl ViewMetadataBuilder { } let version = self.metadata.versions.get(&version_id).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot set current version to unknown version with id: {}", - version_id - ), - ) + invalid_data!("Cannot set current version to unknown version with id: {version_id}") })?; self.metadata.current_version_id = version_id; @@ -279,10 +270,7 @@ impl ViewMetadataBuilder { let view_version = if view_version.schema_id() == Self::LAST_ADDED { let last_added_schema_id = self.last_added_schema_id.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Cannot set last added schema: no schema has been added", - ) + invalid_data!("Cannot set last added schema: no schema has been added") })?; view_version.with_schema_id(last_added_schema_id) } else { @@ -294,12 +282,9 @@ impl ViewMetadataBuilder { .schemas .contains_key(&view_version.schema_id()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add version with unknown schema: {}", - view_version.schema_id() - ), + return Err(invalid_data!( + "Cannot add version with unknown schema: {}", + view_version.schema_id() )); } @@ -311,13 +296,10 @@ impl ViewMetadataBuilder { // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew if view_version.timestamp_ms() - last.timestamp_ms() < -ONE_MINUTE_MS { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid snapshot timestamp {}: before last snapshot timestamp {}", - view_version.timestamp_ms(), - last.timestamp_ms() - ), + return Err(invalid_data!( + "Invalid snapshot timestamp {}: before last snapshot timestamp {}", + view_version.timestamp_ms(), + last.timestamp_ms() )); } } @@ -430,12 +412,8 @@ impl ViewMetadataBuilder { .and_then(|v| v.parse::().ok()) .unwrap_or(1); if num_versions_to_keep < 0 { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "{} must be positive but was {}", - VIEW_PROPERTY_VERSION_HISTORY_SIZE, num_versions_to_keep - ), + return Err(invalid_data!( + "{VIEW_PROPERTY_VERSION_HISTORY_SIZE} must be positive but was {num_versions_to_keep}" )); } @@ -482,10 +460,10 @@ impl ViewMetadataBuilder { // as it might panic if the metadata is invalid. self.metadata.validate()?; - if let Some(previous) = self.previous_view_version.take() { - if !allow_replace_drop_dialects(&self.metadata.properties) { - require_no_dialect_dropped(&previous, self.metadata.current_version())?; - } + if let Some(previous) = self.previous_view_version.take() + && !allow_replace_drop_dialects(&self.metadata.properties) + { + require_no_dialect_dropped(&previous, self.metadata.current_version())?; } let _expired_versions = self.expire_versions(); @@ -591,14 +569,11 @@ fn require_no_dialect_dropped(previous: &ViewVersion, current: &ViewVersion) -> let updated_dialects = lowercase_sql_dialects_for(current); if !updated_dialects.is_superset(&base_dialects) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot replace view due to loss of view dialects: \nPrevious dialects: {:?}\nNew dialects: {:?}\nSet {} to true to allow dropping dialects.", - Vec::from_iter(base_dialects), - Vec::from_iter(updated_dialects), - VIEW_PROPERTY_REPLACE_DROP_DIALECT_ALLOWED - ), + return Err(invalid_data!( + "Cannot replace view due to loss of view dialects: \nPrevious dialects: {:?}\nNew dialects: {:?}\nSet {} to true to allow dropping dialects.", + Vec::from_iter(base_dialects), + Vec::from_iter(updated_dialects), + VIEW_PROPERTY_REPLACE_DROP_DIALECT_ALLOWED )); } @@ -621,12 +596,9 @@ pub(super) fn require_unique_dialects(view_version: &ViewVersion) -> Result<()> match repr { ViewRepresentation::Sql(sql_repr) => { if !seen_dialects.insert(sql_repr.dialect.to_lowercase()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid view version: Cannot add multiple queries for dialect {}", - sql_repr.dialect - ), + return Err(invalid_data!( + "Invalid view version: Cannot add multiple queries for dialect {}", + sql_repr.dialect )); } } diff --git a/crates/iceberg/src/spec/view_version.rs b/crates/iceberg/src/spec/view_version.rs index a02d58c4ef..af95f74e66 100644 --- a/crates/iceberg/src/spec/view_version.rs +++ b/crates/iceberg/src/spec/view_version.rs @@ -29,9 +29,8 @@ use typed_builder::TypedBuilder; use super::INITIAL_VIEW_VERSION_ID; use super::view_metadata::ViewVersionLog; use crate::catalog::NamespaceIdent; -use crate::error::{Result, timestamp_ms_to_utc}; +use crate::error::{Result, invalid_data, timestamp_ms_to_utc}; use crate::spec::{SchemaId, SchemaRef, ViewMetadata}; -use crate::{Error, ErrorKind}; /// Reference to [`ViewVersion`]. pub type ViewVersionRef = Arc; @@ -116,12 +115,7 @@ impl ViewVersion { pub fn schema(&self, view_metadata: &ViewMetadata) -> Result { view_metadata .schema_by_id(self.schema_id()) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Schema with id {} not found", self.schema_id()), - ) - }) + .ok_or_else(|| invalid_data!("Schema with id {} not found", self.schema_id())) .cloned() } diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index d4e696ce84..f33c246eb7 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -20,12 +20,18 @@ use std::sync::Arc; use crate::arrow::ArrowReaderBuilder; +use crate::encryption::EncryptionManager; +use crate::encryption::kms::KeyManagementClient; +use crate::error::invalid_data; use crate::inspect::MetadataTable; use crate::io::FileIO; use crate::io::object_cache::ObjectCache; +use crate::runtime::Runtime; use crate::scan::TableScanBuilder; -use crate::spec::{TableMetadata, TableMetadataRef}; -use crate::{Error, ErrorKind, Result, TableIdent}; +use crate::spec::{ + ManifestListReader, ManifestReader, SchemaRef, SnapshotRef, TableMetadata, TableMetadataRef, +}; +use crate::{Result, TableIdent}; /// Builder to create table scan. pub struct TableBuilder { @@ -33,9 +39,11 @@ pub struct TableBuilder { metadata_location: Option, metadata: Option, identifier: Option, + kms_client: Option>, readonly: bool, disable_cache: bool, cache_size_bytes: Option, + runtime: Option, } impl TableBuilder { @@ -45,9 +53,11 @@ impl TableBuilder { metadata_location: None, metadata: None, identifier: None, + kms_client: None, readonly: false, disable_cache: false, cache_size_bytes: None, + runtime: None, } } @@ -95,6 +105,22 @@ impl TableBuilder { self } + /// Set the Runtime for this table to use when spawning tasks. + pub fn runtime(mut self, runtime: Runtime) -> Self { + self.runtime = Some(runtime); + self + } + + /// optional - sets the KMS client used to unwrap keys for table encryption. + /// + /// If the table metadata has the `encryption.key-id` property set, a + /// [`KeyManagementClient`] must be provided here so the table can build + /// an [`EncryptionManager`]; otherwise [`Self::build`] will return an error. + pub fn kms_client(mut self, kms_client: Arc) -> Self { + self.kms_client = Some(kms_client); + self + } + /// build the Table pub fn build(self) -> Result
{ let Self { @@ -102,41 +128,56 @@ impl TableBuilder { metadata_location, metadata, identifier, + kms_client, readonly, disable_cache, cache_size_bytes, + runtime, } = self; let Some(file_io) = file_io else { - return Err(Error::new( - ErrorKind::DataInvalid, - "FileIO must be provided with TableBuilder.file_io()", + return Err(invalid_data!( + "FileIO must be provided with TableBuilder.file_io()" )); }; let Some(metadata) = metadata else { - return Err(Error::new( - ErrorKind::DataInvalid, - "TableMetadataRef must be provided with TableBuilder.metadata()", + return Err(invalid_data!( + "TableMetadataRef must be provided with TableBuilder.metadata()" )); }; let Some(identifier) = identifier else { - return Err(Error::new( - ErrorKind::DataInvalid, - "TableIdent must be provided with TableBuilder.identifier()", + return Err(invalid_data!( + "TableIdent must be provided with TableBuilder.identifier()" )); }; + let Some(runtime) = runtime else { + return Err(invalid_data!( + "Runtime must be provided with TableBuilder.runtime()" + )); + }; + + let encryption_manager = + EncryptionManager::from_table_metadata(kms_client.as_ref(), &metadata)?; + let object_cache = if disable_cache { - Arc::new(ObjectCache::with_disabled_cache(file_io.clone())) + Arc::new(ObjectCache::with_disabled_cache( + file_io.clone(), + encryption_manager.clone(), + )) } else if let Some(cache_size_bytes) = cache_size_bytes { Arc::new(ObjectCache::new_with_capacity( file_io.clone(), cache_size_bytes, + encryption_manager.clone(), )) } else { - Arc::new(ObjectCache::new(file_io.clone())) + Arc::new(ObjectCache::new( + file_io.clone(), + encryption_manager.clone(), + )) }; Ok(Table { @@ -146,6 +187,8 @@ impl TableBuilder { identifier, readonly, object_cache, + runtime, + encryption_manager, }) } } @@ -159,6 +202,8 @@ pub struct Table { identifier: TableIdent, readonly: bool, object_cache: Arc, + runtime: Runtime, + encryption_manager: Option>, } impl Table { @@ -174,6 +219,15 @@ impl Table { self } + /// Returns this table bound to `runtime`, sharing its manifest cache. + /// + /// Tasks such as scan planning and reads are spawned on `runtime`. A table rebuilt + /// through [`Table::builder`] starts with an empty cache. + pub fn with_runtime(mut self, runtime: Runtime) -> Self { + self.runtime = runtime; + self + } + /// Returns a TableBuilder to build a table pub fn builder() -> TableBuilder { TableBuilder::new() @@ -200,12 +254,9 @@ impl Table { /// Returns current metadata location in a result. pub fn metadata_location_result(&self) -> Result<&str> { - self.metadata_location.as_deref().ok_or(Error::new( - ErrorKind::DataInvalid, - format!( - "Metadata location does not exist for table: {}", - self.identifier - ), + self.metadata_location.as_deref().ok_or(invalid_data!( + "Metadata location does not exist for table: {}", + self.identifier )) } @@ -219,6 +270,16 @@ impl Table { self.object_cache.clone() } + /// Returns the [`EncryptionManager`] for this table, if encryption is + /// configured. + /// + /// A manager is present iff the table metadata has the + /// `encryption.key-id` property set and a [`KeyManagementClient`] was + /// supplied to the [`TableBuilder`]. + pub fn encryption_manager(&self) -> Option<&Arc> { + self.encryption_manager.as_ref() + } + /// Creates a table scan. pub fn scan(&self) -> TableScanBuilder<'_> { TableScanBuilder::new(self) @@ -230,14 +291,39 @@ impl Table { MetadataTable::new(self) } + /// Returns the [`Runtime`] for this table. + pub(crate) fn runtime(&self) -> &Runtime { + &self.runtime + } + /// Returns the flag indicating whether the `Table` is readonly or not pub fn readonly(&self) -> bool { self.readonly } + /// Returns the current schema as a shared reference. + pub fn current_schema_ref(&self) -> SchemaRef { + self.metadata.current_schema().clone() + } + + /// Creates a [`ManifestListReader`] for the given snapshot. + pub fn manifest_list_reader(&self, snapshot: &SnapshotRef) -> ManifestListReader { + ManifestListReader::new( + snapshot.clone(), + self.file_io.clone(), + self.metadata.clone(), + self.encryption_manager.clone(), + ) + } + + /// Creates a [`ManifestReader`] for loading manifests referenced by this table. + pub fn manifest_reader(&self) -> ManifestReader { + ManifestReader::new(self.file_io.clone()) + } + /// Create a reader for the table. pub fn reader_builder(&self) -> ArrowReaderBuilder { - ArrowReaderBuilder::new(self.file_io.clone()) + ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone()) } } @@ -251,10 +337,7 @@ impl Table { /// # use iceberg::TableIdent; /// # async fn example() { /// let metadata_file_location = "s3://bucket_name/path/to/metadata.json"; -/// let file_io = FileIO::from_path(&metadata_file_location) -/// .unwrap() -/// .build() -/// .unwrap(); +/// let file_io = FileIO::new_with_fs(); /// let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap(); /// let static_table = /// StaticTable::from_metadata_file(&metadata_file_location, static_identifier, file_io) @@ -281,6 +364,7 @@ impl StaticTable { .metadata(metadata) .identifier(table_ident) .file_io(file_io.clone()) + .runtime(Runtime::try_current()?) .readonly(true) .build(); @@ -292,15 +376,14 @@ impl StaticTable { table_ident: TableIdent, file_io: FileIO, ) -> Result { - let metadata_file = file_io.new_input(metadata_location)?; - let metadata_file_content = metadata_file.read().await?; - let metadata = serde_json::from_slice::(&metadata_file_content)?; + let metadata = TableMetadata::read_from(&file_io, metadata_location).await?; let table = Table::builder() .metadata(metadata) .metadata_location(metadata_location) .identifier(table_ident) .file_io(file_io.clone()) + .runtime(Runtime::try_current()?) .readonly(true) .build(); @@ -326,13 +409,29 @@ impl StaticTable { /// Create a reader for the table. pub fn reader_builder(&self) -> ArrowReaderBuilder { - ArrowReaderBuilder::new(self.0.file_io.clone()) + self.0.reader_builder() } } #[cfg(test)] mod tests { + use std::fs; + use super::*; + use crate::ErrorKind; + use crate::encryption::SensitiveBytes; + use crate::encryption::kms::MemoryKeyManagementClient; + use crate::spec::TableProperties; + + fn load_test_metadata(filename: &str) -> TableMetadata { + let path = format!( + "{}/testdata/table_metadata/{}", + env!("CARGO_MANIFEST_DIR"), + filename + ); + let json = fs::read_to_string(path).unwrap(); + serde_json::from_str(&json).unwrap() + } #[tokio::test] async fn test_static_table_from_file() { @@ -342,10 +441,7 @@ mod tests { env!("CARGO_MANIFEST_DIR"), metadata_file_name ); - let file_io = FileIO::from_path(&metadata_file_path) - .unwrap() - .build() - .unwrap(); + let file_io = FileIO::new_with_fs(); let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap(); let static_table = StaticTable::from_metadata_file(&metadata_file_path, static_identifier, file_io) @@ -370,10 +466,7 @@ mod tests { env!("CARGO_MANIFEST_DIR"), metadata_file_name ); - let file_io = FileIO::from_path(&metadata_file_path) - .unwrap() - .build() - .unwrap(); + let file_io = FileIO::new_with_fs(); let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap(); let static_table = StaticTable::from_metadata_file(&metadata_file_path, static_identifier, file_io) @@ -396,10 +489,7 @@ mod tests { env!("CARGO_MANIFEST_DIR"), metadata_file_name ); - let file_io = FileIO::from_path(&metadata_file_path) - .unwrap() - .build() - .unwrap(); + let file_io = FileIO::new_with_fs(); let metadata_file = file_io.new_input(metadata_file_path).unwrap(); let metadata_file_content = metadata_file.read().await.unwrap(); let table_metadata = @@ -409,9 +499,138 @@ mod tests { .metadata(table_metadata) .identifier(static_identifier) .file_io(file_io.clone()) + .runtime(Runtime::try_current().unwrap()) .build() .unwrap(); assert!(!table.readonly()); assert_eq!(table.identifier.name(), "table"); } + + #[tokio::test] + async fn test_table_with_runtime_moves_work_and_keeps_object_cache() { + let old = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let table = Table::builder() + .metadata(load_test_metadata("TableMetadataV2Valid.json")) + .identifier(TableIdent::from_strs(["ns", "table"]).unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(Runtime::new(&old)) + .build() + .unwrap(); + // With the old runtime shut down, the spawn below succeeds only if + // `rebound` uses the new one. + old.shutdown_background(); + + let rebound = table.clone().with_runtime(Runtime::try_current().unwrap()); + + assert!(Arc::ptr_eq(&table.object_cache(), &rebound.object_cache())); + assert_eq!( + rebound.runtime().io().spawn(async { 42 }).await.unwrap(), + 42 + ); + } + + fn make_kms() -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key("master-1").unwrap(); + Arc::new(kms) + } + + #[tokio::test] + async fn table_decrypts_manifest_list_via_object_cache() { + // The fixture contains a snapshot with key-id, encryption-keys (KEK + wrapped DEK), + // all generated with the master key bytes below. + let mut metadata: TableMetadata = load_test_metadata("TableMetadataV3ValidEncryption.json"); + + // Point the snapshot's manifest-list at the testdata file on disk. + let manifest_list_path = format!( + "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro", + env!("CARGO_MANIFEST_DIR"), + ); + let snapshot = metadata.snapshots.get_mut(&1).unwrap(); + let mut patched = snapshot.as_ref().clone(); + patched.manifest_list = manifest_list_path; + *snapshot = Arc::new(patched); + + // Seed the KMS with the same master key bytes used to generate the fixture. + let kms: Arc = { + let k = MemoryKeyManagementClient::new(); + k.add_master_key_bytes( + "master-1", + SensitiveBytes::new([ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, + 0x0d, 0x0e, 0x0f, + ]), + ) + .unwrap(); + Arc::new(k) + }; + + let table = Table::builder() + .file_io(FileIO::new_with_fs()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .kms_client(kms) + .runtime(Runtime::try_current().unwrap()) + .build() + .unwrap(); + + let snapshot_ref = table.metadata().current_snapshot().unwrap(); + let manifest_list = table + .object_cache() + .get_manifest_list(snapshot_ref, &table.metadata_ref()) + .await + .unwrap(); + assert_eq!(manifest_list.entries().len(), 0); + } + + #[tokio::test] + async fn table_builder_errors_when_encryption_key_id_set_but_no_kms() { + let metadata: TableMetadata = load_test_metadata("TableMetadataV3ValidEncryption.json"); + + let err = Table::builder() + .file_io(FileIO::new_with_memory()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .runtime(Runtime::try_current().unwrap()) + .build() + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + } + + #[tokio::test] + async fn table_builder_skips_encryption_on_pre_v3_table() { + // Encryption is a v3 spec feature; pre-v3 tables silently skip + // encryption even if encryption.key-id is set. + let mut metadata: TableMetadata = load_test_metadata("TableMetadataV2ValidMinimal.json"); + metadata.properties.insert( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "master-1".to_string(), + ); + + let table = Table::builder() + .file_io(FileIO::new_with_memory()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap()) + .kms_client(make_kms()) + .runtime(Runtime::try_current().unwrap()) + .build() + .unwrap(); + assert!(table.encryption_manager().is_none()); + } + + #[tokio::test] + async fn table_builder_skips_encryption_when_property_absent() { + let metadata: TableMetadata = load_test_metadata("TableMetadataV3ValidMinimal.json"); + let table = Table::builder() + .file_io(FileIO::new_with_memory()) + .metadata(metadata) + .identifier(TableIdent::from_strs(["ns", "plain"]).unwrap()) + .kms_client(make_kms()) + .runtime(Runtime::try_current().unwrap()) + .build() + .unwrap(); + assert!(table.encryption_manager().is_none()); + } } diff --git a/crates/integration_tests/tests/shared.rs b/crates/iceberg/src/test_utils/delete_vector.rs similarity index 58% rename from crates/integration_tests/tests/shared.rs rename to crates/iceberg/src/test_utils/delete_vector.rs index 6bdddaa6cf..b06bd74983 100644 --- a/crates/integration_tests/tests/shared.rs +++ b/crates/iceberg/src/test_utils/delete_vector.rs @@ -15,22 +15,18 @@ // specific language governing permissions and limitations // under the License. -use std::sync::{Arc, OnceLock}; +//! Fixtures for tests that need encoded deletion vectors. -use ctor::dtor; -use iceberg_integration_tests::{TestFixture, set_test_fixture}; +use roaring::RoaringTreemap; -pub mod shared_tests; - -static DOCKER_CONTAINERS: OnceLock> = OnceLock::new(); - -pub fn get_shared_containers() -> &'static Arc { - DOCKER_CONTAINERS.get_or_init(|| Arc::new(set_test_fixture("shared_tests"))) -} - -#[dtor] -fn shutdown() { - if let Some(fixture) = DOCKER_CONTAINERS.get() { - fixture._docker_compose.down() +/// Encodes a `deletion-vector-v1` Puffin blob for the given positions, matching the framing in +/// [`DeleteVector::deserialize`](crate::delete_vector::DeleteVector::deserialize). +pub(crate) fn encode_dv_blob(positions: impl IntoIterator) -> Vec { + let mut bitmap = RoaringTreemap::new(); + for pos in positions { + bitmap.insert(pos); } + let mut vector = Vec::new(); + bitmap.serialize_into(&mut vector).unwrap(); + crate::delete_vector::frame_dv_blob(&vector) } diff --git a/crates/iceberg/src/test_utils/encryption.rs b/crates/iceberg/src/test_utils/encryption.rs new file mode 100644 index 0000000000..2accd5255c --- /dev/null +++ b/crates/iceberg/src/test_utils/encryption.rs @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Fixtures for tests that need an [`EncryptionManager`] or an encrypted table. + +use std::sync::Arc; + +use super::test_runtime; +use crate::TableIdent; +use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient}; +use crate::encryption::{EncryptionManager, SensitiveBytes}; +use crate::io::FileIO; +use crate::spec::TableMetadata; +use crate::table::Table; + +/// An [`EncryptionManager`] backed by an in-memory KMS holding `table_key_id`. +pub(crate) fn make_encryption_manager(table_key_id: &str) -> Arc { + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key(table_key_id).unwrap(); + Arc::new( + EncryptionManager::builder() + .kms_client(Arc::new(kms) as Arc) + .table_key_id(table_key_id) + .build(), + ) +} + +/// Build a table backed by the V3 encryption fixture and an in-memory KMS, +/// so it has an [`EncryptionManager`](crate::encryption::EncryptionManager). +/// +/// The fixture's snapshot references an encrypted manifest list; its bytes +/// (the `manifest-list-v3-encrypted.avro` testdata, an encrypted empty list) +/// are seeded into the in-memory `FileIO` at that path so callers can read +/// the current snapshot's manifest list. +pub(crate) async fn make_encrypted_table() -> Table { + let metadata_json = std::fs::read_to_string(format!( + "{}/testdata/table_metadata/TableMetadataV3ValidEncryption.json", + env!("CARGO_MANIFEST_DIR"), + )) + .unwrap(); + let metadata: TableMetadata = serde_json::from_str(&metadata_json).unwrap(); + + let kms: Arc = { + let k = MemoryKeyManagementClient::new(); + k.add_master_key_bytes( + "master-1", + SensitiveBytes::new([ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, + 0x0e, 0x0f, + ]), + ) + .unwrap(); + Arc::new(k) + }; + + let file_io = FileIO::new_with_memory(); + + // Seed the encrypted (empty) manifest list at the path the snapshot references. + let manifest_list_bytes = std::fs::read(format!( + "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro", + env!("CARGO_MANIFEST_DIR"), + )) + .unwrap(); + file_io + .new_output(metadata.current_snapshot().unwrap().manifest_list()) + .unwrap() + .write(manifest_list_bytes.into()) + .await + .unwrap(); + + Table::builder() + .metadata(metadata) + .metadata_location("memory:///table/metadata/v1.json") + .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) + .file_io(file_io) + .kms_client(kms) + .runtime(test_runtime()) + .build() + .unwrap() +} diff --git a/crates/iceberg/src/test_utils/mod.rs b/crates/iceberg/src/test_utils/mod.rs new file mode 100644 index 0000000000..94310dd379 --- /dev/null +++ b/crates/iceberg/src/test_utils/mod.rs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Test utilities for this crate's own tests. +//! +//! Compiled under `cfg(test)` only, so nothing here is public API. Fixtures +//! that other crates in the workspace need from their own tests live in the +//! `iceberg_test_utils` crate instead. + +mod delete_vector; +mod encryption; +mod record_batch; +mod runtime; +pub(crate) mod scan; +mod transaction; + +pub(crate) use delete_vector::encode_dv_blob; +pub(crate) use encryption::{make_encrypted_table, make_encryption_manager}; +pub(crate) use record_batch::check_record_batches; +pub(crate) use runtime::test_runtime; +pub(crate) use transaction::commit_with_fresh_state; diff --git a/crates/iceberg/src/test_utils.rs b/crates/iceberg/src/test_utils/record_batch.rs similarity index 94% rename from crates/iceberg/src/test_utils.rs rename to crates/iceberg/src/test_utils/record_batch.rs index 527d37bb62..7e37f260df 100644 --- a/crates/iceberg/src/test_utils.rs +++ b/crates/iceberg/src/test_utils/record_batch.rs @@ -15,9 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Test utilities. -//! This module is pub just for internal testing. -//! It is subject to change and is not intended to be used by external users. +//! Snapshot assertions over [`RecordBatch`] results. use arrow_array::RecordBatch; use expect_test::Expect; @@ -31,7 +29,7 @@ use itertools::Itertools; /// Check the doc of [`expect_test`] for more details. /// - `ignore_check_columns`: Some columns are not stable, so we can skip them. /// - `sort_column`: The order of the data might be non-deterministic, so we can sort it by a column. -pub fn check_record_batches( +pub(crate) fn check_record_batches( record_batches: Vec, expected_schema: Expect, expected_data: Expect, diff --git a/crates/iceberg/src/test_utils/runtime.rs b/crates/iceberg/src/test_utils/runtime.rs new file mode 100644 index 0000000000..2070e6cb26 --- /dev/null +++ b/crates/iceberg/src/test_utils/runtime.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A shared tokio runtime for tests that build a [`Table`](crate::table::Table). +//! +//! Twin of `iceberg_test_utils::test_runtime`, which serves other crates in the +//! workspace. The two cannot be merged: a dev-dependency links a separate +//! instance of this crate, so a `Runtime` built there is a distinct type from +//! `crate::Runtime` here. + +use std::sync::OnceLock; + +use crate::runtime::Runtime; + +/// Returns a process-wide [`Runtime`] suitable for tests that need to construct +/// a [`Table`](crate::table::Table) outside a tokio context. +/// +/// The returned [`Runtime`] wraps a single shared multi-thread tokio runtime +/// that is lazily built on first call and lives until process exit. Cloning is +/// cheap, so test code can call this every time it needs a runtime to feed +/// into [`TableBuilder::runtime`](crate::table::TableBuilder::runtime). +pub(crate) fn test_runtime() -> Runtime { + static TOKIO_RT: OnceLock = OnceLock::new(); + let tokio_rt = TOKIO_RT.get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build test tokio runtime") + }); + Runtime::new(tokio_rt) +} diff --git a/crates/iceberg/src/test_utils/scan.rs b/crates/iceberg/src/test_utils/scan.rs new file mode 100644 index 0000000000..d8e7753d5b --- /dev/null +++ b/crates/iceberg/src/test_utils/scan.rs @@ -0,0 +1,1632 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared test fixtures for the table scan API. + +use std::collections::HashMap; +use std::fs; +use std::fs::File; +use std::sync::Arc; + +use arrow_array::cast::AsArray; +use arrow_array::{ + Array, ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray, +}; +use minijinja::value::Value; +use minijinja::{AutoEscape, Environment, context}; +use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY}; +use parquet::basic::Compression; +use parquet::file::properties::WriterProperties; +use tempfile::TempDir; +use uuid::Uuid; + +use crate::TableIdent; +use crate::io::{FileIO, OutputFile}; +use crate::metadata_columns::{ + RESERVED_COL_NAME_DELETE_FILE_PATH, RESERVED_COL_NAME_DELETE_FILE_POS, + RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_FIELD_ID_DELETE_FILE_PATH, + RESERVED_FIELD_ID_DELETE_FILE_POS, +}; +use crate::scan::FileScanTaskColumnStats; +use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, FormatVersion, Literal, ManifestEntry, + ManifestFile, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, NestedField, + PartitionSpec, PrimitiveType, Schema, SchemaRef, Struct, StructType, TableMetadata, + TableMetadataBuilder, Transform, Type, UnboundPartitionSpec, +}; +use crate::table::Table; +use crate::test_utils::test_runtime; + +pub(crate) const ROWS_PER_FIXTURE_FILE: usize = 1024; + +fn render_template(template: &str, ctx: Value) -> String { + let mut env = Environment::new(); + env.set_auto_escape_callback(|_| AutoEscape::None); + env.render_str(template, ctx).unwrap() +} + +/// Asserts every row of the `_last_updated_sequence_number` column across all +/// batches equals `expected` (or is null when `expected` is `None`), decoding +/// the logical value independent of the physical (run-end) encoding. +pub(crate) fn assert_last_updated_seq_all(batches: &[RecordBatch], expected: Option) { + use arrow_cast::cast; + use arrow_schema::DataType; + for batch in batches { + let col = batch + .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER) + .expect("_last_updated_sequence_number column should be present"); + let logical = cast(col, &DataType::Int64).unwrap(); + let values = logical.as_primitive::(); + for i in 0..values.len() { + let actual = (!values.is_null(i)).then(|| values.value(i)); + assert_eq!(actual, expected, "row {i}"); + } + } +} + +pub(crate) struct TableTestFixture { + pub(crate) table_location: String, + pub(crate) table: Table, +} + +impl TableTestFixture { + #[allow(clippy::new_without_default)] + pub(crate) fn new() -> Self { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let manifest_list1_location = table_location.join("metadata/manifests_list_1.avro"); + let manifest_list2_location = table_location.join("metadata/manifests_list_2.avro"); + let table_metadata1_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let table_metadata = { + let template_json_str = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let metadata_json = render_template(&template_json_str, context! { + table_location => &table_location, + manifest_list_1_location => &manifest_list1_location, + manifest_list_2_location => &manifest_list2_location, + table_metadata_1_location => &table_metadata1_location, + }); + serde_json::from_str::(&metadata_json).unwrap() + }; + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io.clone()) + .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) + .runtime(test_runtime()) + .build() + .unwrap(); + + Self { + table_location: table_location.to_str().unwrap().to_string(), + table, + } + } + + #[allow(clippy::new_without_default)] + pub(crate) fn new_empty() -> Self { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let table_metadata1_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let table_metadata = { + let template_json_str = fs::read_to_string(format!( + "{}/testdata/example_empty_table_metadata_v2.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let metadata_json = render_template(&template_json_str, context! { + table_location => &table_location, + table_metadata_1_location => &table_metadata1_location, + }); + serde_json::from_str::(&metadata_json).unwrap() + }; + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io.clone()) + .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) + .runtime(test_runtime()) + .build() + .unwrap(); + + Self { + table_location: table_location.to_str().unwrap().to_string(), + table, + } + } + + /// Creates a fixture with 5 snapshots chained as: + /// S1 (root) -> S2 -> S3 -> S4 -> S5 (current) + /// Useful for testing snapshot history traversal. + pub(crate) fn new_with_deep_history() -> Self { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let table_metadata1_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let table_metadata = { + let json_str = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2_deep_history.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + serde_json::from_str::(&json_str).unwrap() + }; + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io.clone()) + .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) + .runtime(test_runtime()) + .build() + .unwrap(); + + Self { + table_location: table_location.to_str().unwrap().to_string(), + table, + } + } + + pub(crate) fn new_unpartitioned() -> Self { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let manifest_list1_location = table_location.join("metadata/manifests_list_1.avro"); + let manifest_list2_location = table_location.join("metadata/manifests_list_2.avro"); + let table_metadata1_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let mut table_metadata = { + let template_json_str = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let metadata_json = render_template(&template_json_str, context! { + table_location => &table_location, + manifest_list_1_location => &manifest_list1_location, + manifest_list_2_location => &manifest_list2_location, + table_metadata_1_location => &table_metadata1_location, + }); + serde_json::from_str::(&metadata_json).unwrap() + }; + + table_metadata.default_spec = Arc::new(PartitionSpec::unpartition_spec()); + table_metadata.partition_specs.clear(); + table_metadata.default_partition_type = StructType::new(vec![]); + table_metadata + .partition_specs + .insert(0, table_metadata.default_spec.clone()); + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io.clone()) + .metadata_location(table_metadata1_location.to_str().unwrap()) + .runtime(test_runtime()) + .build() + .unwrap(); + + Self { + table_location: table_location.to_str().unwrap().to_string(), + table, + } + } + + pub(crate) fn new_with_partition_evolution() -> Self { + let table = Self::new().table; + let table_location = table.metadata().location.clone(); + + let manifest_list1_location = format!("{}/metadata/manifests_list_1.avro", table_location); + let manifest_list2_location = format!("{}/metadata/manifests_list_2.avro", table_location); + let manifest_list3_location = format!("{}/metadata/manifests_list_3.avro", table_location); + let table_metadata1_location = format!("{}/metadata/v1.json", table_location); + + let new_table_metadata = { + let template_json_str = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2_partition_evolution.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let metadata_json = render_template(&template_json_str, context! { + table_location => &table_location, + manifest_list_1_location => &manifest_list1_location, + manifest_list_2_location => &manifest_list2_location, + manifest_list_3_location => &manifest_list3_location, + table_metadata_1_location => &table_metadata1_location, + }); + Arc::new(serde_json::from_str::(&metadata_json).unwrap()) + }; + + Self { + table_location, + table: table.with_metadata(new_table_metadata), + } + } + + /// Creates a fixture with a column added to its schema after its last + /// write, so its current snapshot was written with an older schema than + /// its current one. + pub(crate) fn new_with_column_added_after_last_write() -> Self { + let fixture = Self::new(); + let metadata = fixture.table.metadata(); + + let mut fields = metadata.current_schema().as_struct().fields().to_vec(); + // Beyond the fixture's field ids 1..=8. + fields + .push(NestedField::optional(100, "added", Type::Primitive(PrimitiveType::Long)).into()); + let new_table_metadata = TableMetadataBuilder::new_from_metadata(metadata.clone(), None) + .add_schema(Schema::builder().with_fields(fields).build().unwrap()) + .unwrap() + .set_current_schema(-1) + .unwrap() + .build() + .unwrap() + .metadata; + + Self { + table: fixture.table.with_metadata(Arc::new(new_table_metadata)), + ..fixture + } + } + + pub(crate) fn next_manifest_file(&self) -> OutputFile { + self.table + .file_io() + .new_output(format!( + "{}/metadata/manifest_{}.avro", + self.table_location, + Uuid::new_v4() + )) + .unwrap() + } + + pub(crate) async fn setup_manifest_files(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let parent_snapshot = current_snapshot + .parent_snapshot(self.table.metadata()) + .unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // Write the data files first, then use the file size in the manifest entries + let parquet_file_size = self.write_parquet_data_files(); + + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + writer + .add_delete_entry( + ManifestEntry::builder() + .status(ManifestStatus::Deleted) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/2.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(200))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + writer + .add_existing_entry( + ManifestEntry::builder() + .status(ManifestStatus::Existing) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/3.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes a v3 data manifest with a manifest-level `first_row_id` of 42, + /// so live entries inherit a per-file `first_row_id` on read. Upgrades the + /// table to v3 first, so the manifest list is read as v3. + pub(crate) async fn setup_v3_manifest_files(&mut self) { + let metadata = TableMetadataBuilder::new_from_metadata( + self.table.metadata().clone(), + self.table.metadata_location().map(str::to_string), + ) + .upgrade_format_version(FormatVersion::V3) + .unwrap() + .build() + .unwrap() + .metadata; + self.table = Table::builder() + .metadata(metadata) + .identifier(self.table.identifier().clone()) + .file_io(self.table.file_io().clone()) + .metadata_location(self.table.metadata_location().unwrap().to_string()) + .runtime(test_runtime()) + .build() + .unwrap(); + + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + let parquet_file_size = self.write_parquet_data_files(); + + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v3_data(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v3( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + Some(42), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + pub(crate) async fn setup_manifest_files_with_partition_evolution(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let parent_snapshot = current_snapshot + .parent_snapshot(self.table.metadata()) + .unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // Write the data files first, then use the file size in the manifest entries + let parquet_file_size = self.write_parquet_data_files(); + + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(1) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([ + Some(Literal::long(100)), + Some(Literal::string("apa")), + Some(Literal::int(27)), + ])) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + writer + .add_delete_entry( + ManifestEntry::builder() + .status(ManifestStatus::Deleted) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(1) + .content(DataContentType::Data) + .file_path(format!("{}/2.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([ + Some(Literal::long(200)), + Some(Literal::string("ice")), + Some(Literal::int(5)), + ])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + writer + .add_existing_entry( + ManifestEntry::builder() + .status(ManifestStatus::Existing) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(1) + .content(DataContentType::Data) + .file_path(format!("{}/3.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([ + Some(Literal::long(300)), + Some(Literal::string("apa")), + Some(Literal::int(19)), + ])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + async fn write_unpromoted_data_manifest( + &self, + current_schema: &Schema, + partition: Struct, + ) -> (SchemaRef, ManifestFile) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let parent_snapshot = current_snapshot + .parent_snapshot(self.table.metadata()) + .unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // The schema the data manifest was written under: the current schema with the + // partition source `x` still `int`. + let mut fields = current_schema.as_struct().fields().to_vec(); + fields[0] = NestedField::required(1, "x", PrimitiveType::Int).into(); + let written_schema = Arc::new( + Schema::builder() + .with_schema_id(0) + .with_fields(fields) + .build() + .unwrap(), + ); + + // Data manifest under the old (int) schema; an Existing entry from the parent + // snapshot so its data sequence number is below the equality delete's. + let data_writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + written_schema.clone(), + current_partition_spec.as_ref().clone(), + ); + let mut data_writer = match self.table.metadata().format_version() { + FormatVersion::V1 => data_writer.build_v1(), + FormatVersion::V2 => data_writer.build_v2_data(), + FormatVersion::V3 => data_writer.build_v3_data(), + }; + data_writer + .add_existing_entry( + ManifestEntry::builder() + .status(ManifestStatus::Existing) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(current_partition_spec.spec_id()) + .content(DataContentType::Data) + .file_path(format!("{}/x7.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(partition) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_manifest = data_writer.write_manifest_file().await.unwrap(); + + (written_schema, data_manifest) + } + + /// Writes manifests for int-partitioned data and an equality delete. + /// `delete_before_promotion` selects an int delete partition; otherwise it is long. + pub(crate) async fn setup_promoted_partition_with_eq_delete( + &mut self, + delete_before_promotion: bool, + ) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + let (written_schema, data_manifest) = self + .write_unpromoted_data_manifest( + ¤t_schema, + Struct::from_iter([Some(Literal::int(7))]), + ) + .await; + + // Equality-delete manifest, Added in the current snapshot so its sequence + // number is above the data file's. Written under the old `int` schema or the + // current `long` schema depending on `delete_before_promotion`. + let (delete_schema, delete_partition) = if delete_before_promotion { + (written_schema, Struct::from_iter([Some(Literal::int(7))])) + } else { + ( + current_schema.clone(), + Struct::from_iter([Some(Literal::long(7))]), + ) + }; + let mut delete_writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + delete_schema, + current_partition_spec.as_ref().clone(), + ) + .build_v2_deletes(); + delete_writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::EqualityDeletes) + .equality_ids(Some(vec![2])) + .file_path(format!("{}/eq-delete.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(delete_partition) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let delete_manifest = delete_writer.write_manifest_file().await.unwrap(); + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_manifest, delete_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes a v3 data manifest under an older schema where the identity partition + /// source `x` (id 1) is still `int`, plus a v3 deletion-vector manifest under the + /// current `long` schema whose DV references the data file with a `long` partition. + /// The reader-side widening must make the data file's `int` partition agree with + /// the DV's `long` partition so the DV partition check passes and the DV attaches; + /// on main the check errors. Parquet/Puffin files are not written (`plan_files` + /// reads only manifests). + pub(crate) async fn setup_promoted_partition_with_dv(&mut self) { + let metadata = TableMetadataBuilder::new_from_metadata( + self.table.metadata().clone(), + self.table.metadata_location().map(str::to_string), + ) + .upgrade_format_version(FormatVersion::V3) + .unwrap() + .build() + .unwrap() + .metadata; + self.table = Table::builder() + .metadata(metadata) + .identifier(self.table.identifier().clone()) + .file_io(self.table.file_io().clone()) + .metadata_location(self.table.metadata_location().unwrap().to_string()) + .runtime(test_runtime()) + .build() + .unwrap(); + + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + let data_file_path = format!("{}/x7.parquet", &self.table_location); + let (_, data_manifest) = self + .write_unpromoted_data_manifest( + ¤t_schema, + Struct::from_iter([Some(Literal::int(7))]), + ) + .await; + + let mut dv_writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v3_deletes(); + dv_writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::PositionDeletes) + .file_path(format!("{}/dv.puffin", &self.table_location)) + .file_format(DataFileFormat::Puffin) + .file_size_in_bytes(100) + .record_count(1) + .referenced_data_file(Some(data_file_path)) + .content_offset(Some(0)) + .content_size_in_bytes(Some(40)) + .partition(Struct::from_iter([Some(Literal::long(7))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let dv_manifest = dv_writer.write_manifest_file().await.unwrap(); + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v3( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + Some(42), + ); + manifest_list_write + .add_manifests(vec![data_manifest, dv_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes a two-field spec `identity(x)`, `identity(d date)` where the data manifest + /// was written with `x` still `int` (so the current `long` schema widens it), then + /// drops the date source `d`. A dropped `date` is stored as an `int`, so if it were + /// widened like a dropped numeric the data file's partition would become `long` and + /// diverge from the equality delete's, detaching it. Parquet files are not written + /// (`plan_files` reads only manifests). + pub(crate) async fn setup_promoted_partition_with_eq_delete_and_dropped_date(&mut self) { + // Evolve to the two-field spec with the date source still present. + let mut fields = self + .table + .metadata() + .current_schema() + .as_struct() + .fields() + .to_vec(); + fields.push(NestedField::optional(9, "d", PrimitiveType::Date).into()); + let date_schema = Schema::builder().with_fields(fields).build().unwrap(); + let date_spec = PartitionSpec::builder(date_schema.clone()) + .add_partition_field("x", "x", Transform::Identity) + .unwrap() + .add_partition_field("d", "d", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let metadata = TableMetadataBuilder::new_from_metadata(self.table.metadata().clone(), None) + .add_current_schema(date_schema) + .unwrap() + .add_default_partition_spec(date_spec.into_unbound()) + .unwrap() + .build() + .unwrap() + .metadata; + self.table = self.table.clone().with_metadata(Arc::new(metadata)); + + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = self.table.metadata().current_schema().clone(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + let (_, data_manifest) = self + .write_unpromoted_data_manifest( + ¤t_schema, + Struct::from_iter([Some(Literal::int(7)), Some(Literal::date(20_000))]), + ) + .await; + + // Equality-delete manifest, Added in the current snapshot (sequence number above + // the data file's) under the current `long` schema. + let mut delete_writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_deletes(); + delete_writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(current_partition_spec.spec_id()) + .content(DataContentType::EqualityDeletes) + .equality_ids(Some(vec![2])) + .file_path(format!("{}/eq-delete.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(Struct::from_iter([ + Some(Literal::long(7)), + Some(Literal::date(20_000)), + ])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let delete_manifest = delete_writer.write_manifest_file().await.unwrap(); + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_manifest, delete_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + + // Drop the date source `d`: the manifests keep the two-field spec while the + // current schema loses `d`, so the spec resolves `d` to `unknown`. + self.drop_partition_source(9); + } + + /// Drops a source column after making the default partition spec unpartitioned. + pub(crate) fn drop_partition_source(&mut self, source_id: i32) { + let current_schema = self.table.metadata().current_schema(); + let evolved_schema = Schema::builder() + .with_fields( + current_schema + .as_struct() + .fields() + .iter() + .filter(|field| field.id != source_id) + .cloned(), + ) + .with_identifier_field_ids( + current_schema + .identifier_field_ids() + .filter(|id| *id != source_id), + ) + .build() + .unwrap(); + let metadata = TableMetadataBuilder::new_from_metadata(self.table.metadata().clone(), None) + .add_default_partition_spec(UnboundPartitionSpec::builder().build()) + .unwrap() + .add_current_schema(evolved_schema) + .unwrap() + .build() + .unwrap() + .metadata; + self.table = self.table.clone().with_metadata(Arc::new(metadata)); + } + + /// Writes identical Parquet data files (1.parquet, 2.parquet, 3.parquet) + /// and returns the file size in bytes. + fn write_parquet_data_files(&self) -> u64 { + fs::create_dir_all(&self.table_location).unwrap(); + + let schema = { + let fields = vec![ + arrow_schema::Field::new("x", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]), + ), + arrow_schema::Field::new("y", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())]), + ), + arrow_schema::Field::new("z", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())]), + ), + arrow_schema::Field::new("a", arrow_schema::DataType::Utf8, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "4".to_string())]), + ), + arrow_schema::Field::new("dbl", arrow_schema::DataType::Float64, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "5".to_string(), + )])), + arrow_schema::Field::new("i32", arrow_schema::DataType::Int32, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "6".to_string(), + )])), + arrow_schema::Field::new("i64", arrow_schema::DataType::Int64, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "7".to_string(), + )])), + arrow_schema::Field::new("bool", arrow_schema::DataType::Boolean, false) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "8".to_string(), + )])), + ]; + Arc::new(arrow_schema::Schema::new(fields)) + }; + // x: [1, 1, 1, 1, ...] + let col1 = + Arc::new(Int64Array::from_iter_values(vec![1; ROWS_PER_FIXTURE_FILE])) as ArrayRef; + + let mut values = vec![2; 512]; + values.append(vec![3; 200].as_mut()); + values.append(vec![4; 300].as_mut()); + values.append(vec![5; 12].as_mut()); + + // y: [2, 2, 2, 2, ..., 3, 3, 3, 3, ..., 4, 4, 4, 4, ..., 5, 5, 5, 5] + let col2 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; + + let mut values = vec![3; 512]; + values.append(vec![4; 512].as_mut()); + + // z: [3, 3, 3, 3, ..., 4, 4, 4, 4] + let col3 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; + + // a: ["Apache", "Apache", "Apache", ..., "Iceberg", "Iceberg", "Iceberg"] + let mut values = vec!["Apache"; 512]; + values.append(vec!["Iceberg"; 512].as_mut()); + let col4 = Arc::new(StringArray::from_iter_values(values)) as ArrayRef; + + // dbl: + let mut values = vec![100.0f64; 512]; + values.append(vec![150.0f64; 12].as_mut()); + values.append(vec![200.0f64; 500].as_mut()); + let col5 = Arc::new(Float64Array::from_iter_values(values)) as ArrayRef; + + // i32: + let mut values = vec![100i32; 512]; + values.append(vec![150i32; 12].as_mut()); + values.append(vec![200i32; 500].as_mut()); + let col6 = Arc::new(Int32Array::from_iter_values(values)) as ArrayRef; + + // i64: + let mut values = vec![100i64; 512]; + values.append(vec![150i64; 12].as_mut()); + values.append(vec![200i64; 500].as_mut()); + let col7 = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef; + + // bool: + let mut values = vec![false; 512]; + values.append(vec![true; 512].as_mut()); + let values: BooleanArray = values.into(); + let col8 = Arc::new(values) as ArrayRef; + + let to_write = RecordBatch::try_new(schema.clone(), vec![ + col1, col2, col3, col4, col5, col6, col7, col8, + ]) + .unwrap(); + + // Write the Parquet files + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + + for n in 1..=3 { + let file = File::create(format!("{}/{}.parquet", &self.table_location, n)).unwrap(); + let mut writer = + ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap(); + + writer.write(&to_write).expect("Writing batch"); + + // writer must be closed to write footer + writer.close().unwrap(); + } + + fs::metadata(format!("{}/1.parquet", &self.table_location)) + .unwrap() + .len() + } + + pub(crate) async fn setup_unpartitioned_manifest_files(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let parent_snapshot = current_snapshot + .parent_snapshot(self.table.metadata()) + .unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = Arc::new(PartitionSpec::unpartition_spec()); + + // Write the data files first, then use the file size in the manifest entries + let parquet_file_size = self.write_parquet_data_files(); + + // Write data files using an empty partition for unpartitioned tables. + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + + // Create an empty partition value. + let empty_partition = Struct::empty(); + + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/1.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(empty_partition.clone()) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + + writer + .add_delete_entry( + ManifestEntry::builder() + .status(ManifestStatus::Deleted) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/2.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(empty_partition.clone()) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + + writer + .add_existing_entry( + ManifestEntry::builder() + .status(ManifestStatus::Existing) + .snapshot_id(parent_snapshot.snapshot_id()) + .sequence_number(parent_snapshot.sequence_number()) + .file_sequence_number(parent_snapshot.sequence_number()) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/3.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(empty_partition.clone()) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_file_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + pub(crate) async fn setup_deadlock_manifests(&mut self) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let _parent_snapshot = current_snapshot + .parent_snapshot(self.table.metadata()) + .unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // 1. Write DATA manifest with MULTIPLE entries to fill buffer + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + + // Add 10 data entries + for i in 0..10 { + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/{}.parquet", &self.table_location, i)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + } + let data_manifest = writer.write_manifest_file().await.unwrap(); + + // 2. Write DELETE manifest + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_deletes(); + + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::PositionDeletes) + .file_path(format!("{}/del.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let delete_manifest = writer.write_manifest_file().await.unwrap(); + + // Write to manifest list - DATA FIRST then DELETE + // This order is crucial for reproduction + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(vec![data_manifest, delete_manifest].into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Sets up a single data file `mrg.parquet` with three 100-row row groups + /// (column `x` = 1000..1300, so row position `p` carries `x = 1000 + p`) and + /// registers it in the current snapshot. When `delete_positions` is non-empty, + /// also writes a positional delete file targeting those file-absolute positions + /// and registers it in a delete manifest. + /// + /// Used to exercise the `_pos` metadata column through the real `TableScan` + /// planning path across row-group boundaries and (optionally) positional deletes. + pub(crate) async fn setup_multi_row_group_manifest(&mut self, delete_positions: &[i64]) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + + // The table's spec 0 is identity on `x`, so give the data and delete files a + // fixed partition value. Filter tests deliberately filter on `y` (a + // non-partition column) so pruning is driven by Parquet row-group statistics + // rather than partition values. + let partition = Struct::from_iter([Some(Literal::long(1000))]); + + let (data_file_path, data_file_size) = self.write_multi_row_group_data_file(); + + let mut data_writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + data_writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(data_file_path.clone()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(data_file_size) + .record_count(300) + .partition(partition.clone()) + .key_metadata(None) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let data_manifest = data_writer.write_manifest_file().await.unwrap(); + + let mut manifests = vec![data_manifest]; + + if !delete_positions.is_empty() { + let (del_path, del_size) = + self.write_positional_delete_file(&data_file_path, delete_positions); + + let mut delete_writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_deletes(); + delete_writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::PositionDeletes) + .file_path(del_path) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(del_size) + .record_count(delete_positions.len() as u64) + .partition(partition.clone()) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + manifests.push(delete_writer.write_manifest_file().await.unwrap()); + } + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(manifests.into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes a manifest with three live "Added" data-file entries (partitioned on `x` + /// = 100, 200, 300), each with the given `sort_order_id` set on its `DataFile` + /// (`None` leaves the field unset). Used to test how `sort_order_id` resolution + /// against the table's sort orders flows into each entry's `FileScanTask`. + pub(crate) async fn setup_manifest_files_with_sort_order_ids( + &mut self, + sort_order_ids: [Option; 4], + ) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_schema = current_snapshot.schema(self.table.metadata()).unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + let parquet_file_size = self.write_parquet_data_files(); + + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + current_schema.clone(), + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + + for (i, sort_order_id) in sort_order_ids.into_iter().enumerate() { + let mut data_file_builder = DataFileBuilder::default(); + data_file_builder + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/{}.parquet", &self.table_location, i + 1)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long( + 100 * (i as i64 + 1), + ))])); + if let Some(id) = sort_order_id { + data_file_builder.sort_order_id(id); + } + let data_file = data_file_builder.build().unwrap(); + + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(data_file) + .build(), + ) + .unwrap(); + } + + let data_file_manifest = writer.write_manifest_file().await.unwrap(); + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(std::iter::once(data_file_manifest)) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes one manifest per element of `manifests`, in order. Each holds one live "Added" + /// data-file entry per element of its statistics (`1.parquet`, `2.parquet`, ..., numbered + /// across manifests and partitioned on `x` = 100, 200, ...), carrying that element's column + /// sizes, value, null and NaN counts and bounds. A manifest is written with its schema, or + /// the current snapshot's schema if `None`. + pub(crate) async fn setup_manifest_files_with_column_stats( + &mut self, + manifests: &[(Option, Vec)], + ) { + let current_snapshot = self.table.metadata().current_snapshot().unwrap(); + let current_partition_spec = self.table.metadata().default_partition_spec(); + let parquet_file_size = self.write_parquet_data_files(); + + let mut manifest_files = vec![]; + let mut file_number = 0; + for (manifest_schema, column_stats) in manifests { + let schema = match manifest_schema { + Some(schema) => schema.clone(), + None => current_snapshot.schema(self.table.metadata()).unwrap(), + }; + let mut writer = ManifestWriterBuilder::new( + self.next_manifest_file(), + Some(current_snapshot.snapshot_id()), + schema, + current_partition_spec.as_ref().clone(), + ) + .build_v2_data(); + + for stats in column_stats { + file_number += 1; + let data_file = DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{}/{file_number}.parquet", &self.table_location)) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(parquet_file_size) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100 * file_number))])) + .column_sizes(stats.column_sizes().clone()) + .value_counts(stats.value_counts().clone()) + .null_value_counts(stats.null_value_counts().clone()) + .nan_value_counts(stats.nan_value_counts().clone()) + .lower_bounds(stats.lower_bounds().clone()) + .upper_bounds(stats.upper_bounds().clone()) + .build() + .unwrap(); + + writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(data_file) + .build(), + ) + .unwrap(); + } + + manifest_files.push(writer.write_manifest_file().await.unwrap()); + } + + let manifest_list_writer = self + .table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(); + let mut manifest_list_write = ManifestListWriter::v2( + manifest_list_writer, + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_write + .add_manifests(manifest_files.into_iter()) + .unwrap(); + manifest_list_write.close().await.unwrap(); + } + + /// Writes `mrg.parquet` with three 100-row row groups. Columns `x` (field + /// id `1`) and `y` (field id `2`) both run 1000..1300, so row position `p` + /// carries `x = y = 1000 + p`. Returns `(path, file_size_in_bytes)`. + fn write_multi_row_group_data_file(&self) -> (String, u64) { + fs::create_dir_all(&self.table_location).unwrap(); + + let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![ + arrow_schema::Field::new("x", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]), + ), + arrow_schema::Field::new("y", arrow_schema::DataType::Int64, false).with_metadata( + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())]), + ), + ])); + + let path = format!("{}/mrg.parquet", &self.table_location); + let max_row_group_row_count = 100; + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_max_row_group_row_count(Some(max_row_group_row_count)) + .build(); + + let file = File::create(&path).unwrap(); + let mut writer = ArrowWriter::try_new(file, arrow_schema.clone(), Some(props)).unwrap(); + for group in 0..3i64 { + let base = 1000 + group * max_row_group_row_count as i64; + let col = Arc::new(Int64Array::from_iter_values( + base..base + max_row_group_row_count as i64, + )) as ArrayRef; + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![col.clone(), col]).unwrap(); + writer.write(&batch).unwrap(); + } + writer.close().unwrap(); + + let size = fs::metadata(&path).unwrap().len(); + (path, size) + } + + /// Writes a positional delete file targeting `positions` in `data_path`. + /// Returns `(path, file_size_in_bytes)`. + fn write_positional_delete_file(&self, data_path: &str, positions: &[i64]) -> (String, u64) { + let del_schema = Arc::new(arrow_schema::Schema::new(vec![ + arrow_schema::Field::new( + RESERVED_COL_NAME_DELETE_FILE_PATH, + arrow_schema::DataType::Utf8, + false, + ) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_DELETE_FILE_PATH.to_string(), // 2147483546 + )])), + arrow_schema::Field::new( + RESERVED_COL_NAME_DELETE_FILE_POS, + arrow_schema::DataType::Int64, + false, + ) + .with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + RESERVED_FIELD_ID_DELETE_FILE_POS.to_string(), // 2147483545 + )])), + ])); + + let batch = RecordBatch::try_new(del_schema.clone(), vec![ + Arc::new(StringArray::from_iter_values(std::iter::repeat_n( + data_path.to_string(), + positions.len(), + ))) as ArrayRef, + Arc::new(Int64Array::from_iter_values(positions.iter().copied())) as ArrayRef, + ]) + .unwrap(); + + let path = format!("{}/pos-del.parquet", &self.table_location); + let props = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .build(); + let file = File::create(&path).unwrap(); + let mut writer = ArrowWriter::try_new(file, del_schema, Some(props)).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let size = fs::metadata(&path).unwrap().len(); + (path, size) + } +} diff --git a/crates/iceberg/src/test_utils/transaction.rs b/crates/iceberg/src/test_utils/transaction.rs new file mode 100644 index 0000000000..292848359b --- /dev/null +++ b/crates/iceberg/src/test_utils/transaction.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::error::Result; +use crate::table::Table; +use crate::transaction::{ActionCommit, TransactionAction}; + +/// Runs a single commit attempt of an action with fresh state. +pub(crate) async fn commit_with_fresh_state( + action: A, + table: &Table, +) -> Result { + let mut state = action.new_state(); + action.commit(&mut state, table).await +} diff --git a/crates/iceberg/src/transaction/action.rs b/crates/iceberg/src/transaction/action.rs index aa0a05d0d9..46868c3a25 100644 --- a/crates/iceberg/src/transaction/action.rs +++ b/crates/iceberg/src/transaction/action.rs @@ -16,7 +16,6 @@ // under the License. use std::mem::take; -use std::sync::Arc; use as_any::AsAny; use async_trait::async_trait; @@ -25,28 +24,174 @@ use crate::table::Table; use crate::transaction::Transaction; use crate::{Result, TableRequirement, TableUpdate}; -/// A boxed, thread-safe reference to a `TransactionAction`. -pub(crate) type BoxedTransactionAction = Arc; +/// A boxed entry pairing a transaction action with its retry-persistent state. +pub(crate) type TransactionActionEntry = Box; /// A trait representing an atomic action that can be part of a transaction. /// /// Implementors of this trait define how a specific action is committed to a table. /// Each action is responsible for generating the updates and requirements needed /// to modify the table metadata. +/// +/// An action's intent is immutable once applied to a transaction. Retry-persistent +/// state lives in the associated [`TransactionAction::State`], which survives replay +/// attempts of one logical execution but is never shared between executions. #[async_trait] -pub(crate) trait TransactionAction: AsAny + Sync + Send { +pub(crate) trait TransactionAction: Clone + Send + Sync + 'static { + /// Retry-persistent state exclusively owned by one logical execution of this + /// action. Stateless actions implement [`StatelessAction`] instead, which + /// supplies `State = ()`. + type State: Send + Sync + 'static; + + /// Creates fresh state for one logical execution. + /// + /// This is infallible and table-independent; table-dependent initialization + /// happens during [`TransactionAction::commit`]. + fn new_state(&self) -> Self::State; + /// Commits this action against the provided table and returns the resulting updates. /// NOTE: This function is intended for internal use only and should not be called directly by users. /// + /// One replay attempt: the action (intent) is borrowed immutably, its execution + /// state mutably, and the table reflects the current transaction-local base. + /// /// # Arguments /// + /// * `state` - The retry-persistent state for this logical execution. /// * `table` - The current state of the table this action should apply to. /// /// # Returns /// /// An `ActionCommit` containing table updates and table requirements, /// or an error if the commit fails. - async fn commit(self: Arc, table: &Table) -> Result; + async fn commit(&self, state: &mut Self::State, table: &Table) -> Result; + + /// Best-effort cleanup after the transaction reaches a terminal result. + /// + /// Consumes the action and its state: ownership makes any further use + /// of this action after cleanup a compile error. + /// Cleanup must not change the already-determined transaction result. + /// + /// See [`CommitStatus`] for what each terminal status allows cleanup to do. + // TODO: invoke this from `Transaction::commit` once terminal status + // classification is wired up (stateful transaction RFC, milestone 3). + #[allow(dead_code)] + async fn cleanup(self, state: Self::State, table: &Table, status: CommitStatus); +} + +/// Convenience trait for actions that carry no per-attempt state. +/// +/// Implementing `StatelessAction` yields a [`TransactionAction`] impl with +/// `State = ()`, a no-op [`TransactionAction::new_state`], a no-op +/// [`TransactionAction::cleanup`], and a [`TransactionAction::commit`] that +/// forwards to [`StatelessAction::commit`]. Actions that need retry-persistent +/// state (e.g. a persistent snapshot producer) implement `TransactionAction` +/// directly instead. +#[async_trait] +pub(crate) trait StatelessAction: Clone + Send + Sync + 'static { + /// One commit attempt against the current transaction-local table. + /// + /// Same contract as [`TransactionAction::commit`], minus the state parameter. + async fn commit(&self, table: &Table) -> Result; +} + +#[async_trait] +impl TransactionAction for T { + type State = (); + + fn new_state(&self) {} + + async fn commit(&self, _state: &mut (), table: &Table) -> Result { + StatelessAction::commit(self, table).await + } + + async fn cleanup(self, _state: (), _table: &Table, _status: CommitStatus) {} +} + +/// Classification of a transaction's terminal result, consumed by +/// [`TransactionAction::cleanup`] to determine what is safe to delete. +/// +/// The transaction/catalog layer determines this classification; actions +/// consume it rather than independently interpreting catalog errors. +// TODO: produce this classification in `Transaction::commit` once terminal +// cleanup is wired up (stateful transaction RFC, milestone 3). +#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CommitStatus { + /// The catalog confirmed that the commit was applied. Cleanup may remove + /// owned artifacts not retained by the committed result. + Committed, + /// The transaction definitively did not commit and will not retry. + /// Owned artifacts are deletable. + Failed, + /// The commit request was submitted, but its outcome could not be + /// resolved: the catalog may or may not have applied it. Cleanup must + /// delete nothing, and the transaction must not be retried. + /// + /// Example: every action executed and validated successfully, but the + /// connection failed while awaiting the catalog's response to + /// `update_table`. + Unknown, +} + +/// An entry in a transaction, pairing an action's immutable intent with the +/// retry-persistent state of one logical execution. +/// +/// The pairing is preserved by construction: the entry is created with fresh +/// state and owns both exclusively, so terminal cleanup can consume them together. +pub(crate) struct ActionEntry { + action: A, + state: A::State, +} + +impl ActionEntry { + fn new(action: A) -> Self { + let state = action.new_state(); + Self { action, state } + } + + /// The action (intent) held by this entry. + #[allow(dead_code)] + pub(crate) fn action(&self) -> &A { + &self.action + } +} + +/// Object-safe adapter over [`ActionEntry`], allowing a transaction to store +/// heterogeneous entries while keeping each action paired with its own state. +#[async_trait] +pub(crate) trait DynActionEntry: AsAny + Send + Sync { + /// One replay attempt against the current transaction-local table. + async fn commit(&mut self, table: &Table) -> Result; + + /// Best-effort terminal cleanup, consuming the entry. + // TODO: invoke from `Transaction::commit` once terminal status + // classification is wired up (stateful transaction RFC, milestone 3). + #[allow(dead_code)] + async fn cleanup(self: Box, table: &Table, status: CommitStatus); + + /// Creates a new entry for a new logical execution of the same action: + /// the action (intent) is cloned, and it is paired with fresh state from + /// [`TransactionAction::new_state`]. Accrued retry state is never carried + /// over to the new entry. + #[allow(dead_code)] + fn clone_with_new_state(&self) -> TransactionActionEntry; +} + +#[async_trait] +impl DynActionEntry for ActionEntry { + async fn commit(&mut self, table: &Table) -> Result { + self.action.commit(&mut self.state, table).await + } + + async fn cleanup(self: Box, table: &Table, status: CommitStatus) { + let ActionEntry { action, state } = *self; + action.cleanup(state, table, status).await + } + + fn clone_with_new_state(&self) -> TransactionActionEntry { + Box::new(ActionEntry::new(self.action.clone())) + } } /// A helper trait for applying a `TransactionAction` to a `Transaction`. @@ -66,10 +211,10 @@ pub trait ApplyTransactionAction { fn apply(self, tx: Transaction) -> Result; } -impl ApplyTransactionAction for T { +impl ApplyTransactionAction for T { fn apply(self, mut tx: Transaction) -> Result where Self: Sized { - tx.actions.push(Arc::new(self)); + tx.actions.push(Box::new(ActionEntry::new(self))); Ok(tx) } } @@ -106,23 +251,31 @@ impl ActionCommit { #[cfg(test)] mod tests { use std::str::FromStr; - use std::sync::Arc; use as_any::Downcast; use async_trait::async_trait; use uuid::Uuid; use crate::table::Table; + use crate::test_utils::commit_with_fresh_state; use crate::transaction::Transaction; - use crate::transaction::action::{ActionCommit, ApplyTransactionAction, TransactionAction}; + use crate::transaction::action::{ + ActionCommit, ActionEntry, ApplyTransactionAction, CommitStatus, DynActionEntry, + StatelessAction, TransactionAction, + }; use crate::transaction::tests::make_v2_table; use crate::{Result, TableRequirement, TableUpdate}; + #[derive(Clone)] struct TestAction; #[async_trait] impl TransactionAction for TestAction { - async fn commit(self: Arc, _table: &Table) -> Result { + type State = (); + + fn new_state(&self) -> Self::State {} + + async fn commit(&self, _state: &mut (), _table: &Table) -> Result { Ok(ActionCommit::new( vec![TableUpdate::SetLocation { location: String::from("s3://bucket/prefix/table/"), @@ -132,6 +285,42 @@ mod tests { }], )) } + + async fn cleanup(self, _state: (), _table: &Table, _status: CommitStatus) {} + } + + #[derive(Clone)] + struct CountingAction; + + #[async_trait] + impl TransactionAction for CountingAction { + type State = u32; + + fn new_state(&self) -> Self::State { + 0 + } + + async fn commit(&self, state: &mut u32, _table: &Table) -> Result { + *state += 1; + Ok(ActionCommit::new(vec![], vec![])) + } + + async fn cleanup(self, _state: u32, _table: &Table, _status: CommitStatus) {} + } + + #[derive(Clone)] + struct StatelessTestAction; + + #[async_trait] + impl StatelessAction for StatelessTestAction { + async fn commit(&self, _table: &Table) -> Result { + Ok(ActionCommit::new( + vec![TableUpdate::SetLocation { + location: String::from("s3://bucket/prefix/stateless/"), + }], + vec![], + )) + } } #[tokio::test] @@ -139,7 +328,7 @@ mod tests { let table = make_v2_table(); let action = TestAction; - let mut action_commit = Arc::new(action).commit(&table).await.unwrap(); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); let updates = action_commit.take_updates(); let requirements = action_commit.take_requirements(); @@ -159,14 +348,69 @@ mod tests { let tx = Transaction::new(&table); let updated_tx = action.apply(tx).unwrap(); - // There should be one action in the transaction now + // There should be one action entry in the transaction now assert_eq!(updated_tx.actions.len(), 1); (*updated_tx.actions[0]) - .downcast_ref::() + .downcast_ref::>() .expect("TestAction was not applied to Transaction!"); } + #[tokio::test] + async fn test_entry_retains_state_across_commits() { + let table = make_v2_table(); + let mut tx = CountingAction.apply(Transaction::new(&table)).unwrap(); + + let entry = &mut tx.actions[0]; + entry.commit(&table).await.unwrap(); + entry.commit(&table).await.unwrap(); + + let entry = (**entry) + .downcast_ref::>() + .unwrap(); + assert_eq!(entry.state, 2); + } + + #[tokio::test] + async fn test_clone_with_new_state_resets_state() { + let table = make_v2_table(); + let mut tx = CountingAction.apply(Transaction::new(&table)).unwrap(); + tx.actions[0].commit(&table).await.unwrap(); + + let cloned = tx.actions[0].clone_with_new_state(); + + let original = (*tx.actions[0]) + .downcast_ref::>() + .unwrap(); + let cloned = (*cloned) + .downcast_ref::>() + .unwrap(); + assert_eq!(original.state, 1); + assert_eq!(cloned.state, 0); + } + + #[tokio::test] + async fn test_stateless_action_blanket_impl() { + let table = make_v2_table(); + + // commit goes through the blanket TransactionAction impl + let mut commit = commit_with_fresh_state(StatelessTestAction, &table) + .await + .unwrap(); + assert_eq!(commit.take_updates(), vec![TableUpdate::SetLocation { + location: String::from("s3://bucket/prefix/stateless/"), + }]); + + // ApplyTransactionAction is available through the blanket impl too + let tx = StatelessTestAction.apply(Transaction::new(&table)).unwrap(); + assert_eq!(tx.actions.len(), 1); + + // and the erased cleanup forwarding chain compiles and runs + let entry: Box> = + Box::new(ActionEntry::new(StatelessTestAction)); + entry.cleanup(&table, CommitStatus::Committed).await; + } + #[test] fn test_action_commit() { // Create dummy updates and requirements diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index f248543df2..a60157496e 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -15,28 +15,31 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; -use std::sync::Arc; +use std::collections::{HashMap, HashSet}; use async_trait::async_trait; use uuid::Uuid; -use crate::error::Result; -use crate::spec::{DataFile, ManifestEntry, ManifestFile, Operation}; +use crate::error::{Result, invalid_data}; +use crate::spec::{DataContentType, DataFile}; use crate::table::Table; -use crate::transaction::snapshot::{ - DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, -}; +use crate::transaction::action::CommitStatus; +use crate::transaction::snapshot::simple::{AppendIntent, SimpleSnapshotProducer}; +use crate::transaction::snapshot::validate_files_not_live; use crate::transaction::{ActionCommit, TransactionAction}; +use crate::{Error, ErrorKind}; /// FastAppendAction is a transaction action for fast append data files to the table. +#[derive(Clone)] pub struct FastAppendAction { check_duplicate: bool, - // below are properties used to create SnapshotProducer when commit + /// Overrides the commit UUID of the producer created in `new_state`. commit_uuid: Option, - key_metadata: Option>, snapshot_properties: HashMap, + /// Free of duplicate paths; see [`Self::add_data_files`]. added_data_files: Vec, + /// Paths already in `added_data_files`. + added_paths: HashSet, } impl FastAppendAction { @@ -44,9 +47,9 @@ impl FastAppendAction { Self { check_duplicate: true, commit_uuid: None, - key_metadata: None, snapshot_properties: HashMap::default(), added_data_files: vec![], + added_paths: HashSet::new(), } } @@ -57,8 +60,16 @@ impl FastAppendAction { } /// Add data files to the snapshot. + /// + /// A path added more than once keeps its first occurrence, so a manifest + /// never references the same file twice. This is independent of the + /// `check_duplicate`-gated check against files already in the table. pub fn add_data_files(mut self, data_files: impl IntoIterator) -> Self { - self.added_data_files.extend(data_files); + for data_file in data_files { + if self.added_paths.insert(data_file.file_path.clone()) { + self.added_data_files.push(data_file); + } + } self } @@ -68,102 +79,503 @@ impl FastAppendAction { self } - /// Set key metadata for manifest files. - pub fn set_key_metadata(mut self, key_metadata: Vec) -> Self { - self.key_metadata = Some(key_metadata); - self - } - /// Set snapshot summary properties. pub fn set_snapshot_properties(mut self, snapshot_properties: HashMap) -> Self { self.snapshot_properties = snapshot_properties; self } + + /// Checks the parts of this action that do not depend on the table. + fn validate(&self) -> Result<()> { + // TODO: Allowing snapshot property setup with no added data files is a workaround. + // We should clean it up after all necessary actions are supported. + // For details, please refer to https://github.com/apache/iceberg-rust/issues/1548 + if self.added_data_files.is_empty() && self.snapshot_properties.is_empty() { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "No added data files or added snapshot properties found when write a manifest file", + )); + } + if let Some(data_file) = self + .added_data_files + .iter() + .find(|f| f.content_type() != DataContentType::Data) + { + return Err(invalid_data!( + "Only data content type is allowed for fast append, got {:?} for {}", + data_file.content_type(), + data_file.file_path() + )); + } + Ok(()) + } } #[async_trait] impl TransactionAction for FastAppendAction { - async fn commit(self: Arc, table: &Table) -> Result { - let snapshot_producer = SnapshotProducer::new( - table, - self.commit_uuid.unwrap_or_else(Uuid::now_v7), - self.key_metadata.clone(), - self.snapshot_properties.clone(), - self.added_data_files.clone(), - ); + type State = SimpleSnapshotProducer; - // validate added files - snapshot_producer.validate_added_data_files(&self.added_data_files)?; + fn new_state(&self) -> SimpleSnapshotProducer { + SimpleSnapshotProducer::new(AppendIntent { + commit_uuid: self.commit_uuid, + snapshot_properties: self.snapshot_properties.clone(), + data_files: self.added_data_files.clone(), + }) + } - // Checks duplicate files + async fn commit( + &self, + state: &mut SimpleSnapshotProducer, + table: &Table, + ) -> Result { + self.validate()?; if self.check_duplicate { - snapshot_producer - .validate_duplicate_files(&self.added_data_files) - .await?; + validate_files_not_live(table, &self.added_data_files).await?; } + state.apply(table).await + } - snapshot_producer - .commit(FastAppendOperation, DefaultManifestProcess) - .await + async fn cleanup(self, _state: SimpleSnapshotProducer, _table: &Table, _status: CommitStatus) { + // TODO(#3417): remove the producer's uncommitted artifacts once terminal + // cleanup is wired up. } } -struct FastAppendOperation; +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::fs; + use std::sync::Arc; + use std::sync::atomic::{AtomicU32, Ordering}; -impl SnapshotProduceOperation for FastAppendOperation { - fn operation(&self) -> Operation { - Operation::Append - } + use minijinja::{AutoEscape, Environment, Value, context}; + use tempfile::TempDir; + use uuid::Uuid; - async fn delete_entries( - &self, - _snapshot_produce: &SnapshotProducer<'_>, - ) -> Result> { - Ok(vec![]) + use crate::catalog::MockCatalog; + use crate::encryption::kms::MemoryKeyManagementClient; + use crate::encryption::{SensitiveBytes, StandardKeyMetadata}; + use crate::io::FileIO; + use crate::spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, + ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, SnapshotRef, + Struct, TableMetadata, TableProperties, + }; + use crate::table::Table; + use crate::test_utils::{commit_with_fresh_state, make_encrypted_table, test_runtime}; + use crate::transaction::tests::{ + make_v1_table, make_v2_minimal_table, make_v3_minimal_table, with_bucket_partition_spec, + with_retry_properties, + }; + use crate::transaction::{ApplyTransactionAction, FastAppendAction, Transaction}; + use crate::{ErrorKind, TableCommit, TableIdent, TableRequirement, TableUpdate}; + + fn render_template(template: &str, ctx: Value) -> String { + let mut env = Environment::new(); + env.set_auto_escape_callback(|_| AutoEscape::None); + env.render_str(template, ctx).unwrap() } - async fn existing_manifest( - &self, - snapshot_produce: &SnapshotProducer<'_>, - ) -> Result> { - let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { - return Ok(vec![]); + /// Builds a table whose current snapshot's manifest list contains a data manifest + /// followed by a delete-only manifest (one entry with `ManifestStatus::Deleted`, + /// so `deleted_files_count > 0` while `added_files_count == existing_files_count == 0`). + /// + /// Returns the table plus the `manifest_path` of the delete-only manifest so callers + /// can assert whether a subsequent append carries it forward. + async fn make_table_with_delete_only_manifest() -> (Table, TempDir, String) { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let manifest_list_location = table_location.join("metadata/manifests_list_1.avro"); + let table_metadata_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let template = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + // The template has two snapshots; point the current one at our manifest list. + let metadata_json = render_template(&template, context! { + table_location => &table_location, + manifest_list_1_location => &manifest_list_location, + manifest_list_2_location => &manifest_list_location, + table_metadata_1_location => &table_metadata_location, + }); + let table_metadata = serde_json::from_str::(&metadata_json).unwrap(); + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io) + .metadata_location(table_metadata_location.to_str().unwrap()) + .runtime(test_runtime()) + .build() + .unwrap(); + + let current_snapshot = table.metadata().current_snapshot().unwrap(); + let schema = current_snapshot.schema(table.metadata()).unwrap(); + let partition_spec = table.metadata().default_partition_spec(); + + let next_manifest_file = |location: &str| { + table + .file_io() + .new_output(format!( + "{}/metadata/manifest_{}.avro", + location, + Uuid::new_v4() + )) + .unwrap() }; + let table_location_str = table_location.to_str().unwrap().to_string(); - let manifest_list = snapshot - .load_manifest_list( - snapshot_produce.table.file_io(), - &snapshot_produce.table.metadata_ref(), + // Data manifest: one Added data file. + let mut data_writer = ManifestWriterBuilder::new( + next_manifest_file(&table_location_str), + Some(current_snapshot.snapshot_id()), + schema.clone(), + partition_spec.as_ref().clone(), + ) + .build_v2_data(); + data_writer + .add_entry( + ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{table_location_str}/data.parquet")) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .build() + .unwrap(), + ) + .build(), ) - .await?; + .unwrap(); + let data_manifest = data_writer.write_manifest_file().await.unwrap(); + + // Delete-only manifest: a single Deleted entry, nothing added or existing. + let mut delete_writer = ManifestWriterBuilder::new( + next_manifest_file(&table_location_str), + Some(current_snapshot.snapshot_id()), + schema.clone(), + partition_spec.as_ref().clone(), + ) + .build_v2_data(); + delete_writer + .add_delete_entry( + ManifestEntry::builder() + .status(ManifestStatus::Deleted) + .sequence_number(0) + .file_sequence_number(0) + .data_file( + DataFileBuilder::default() + .partition_spec_id(0) + .content(DataContentType::Data) + .file_path(format!("{table_location_str}/removed.parquet")) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .build() + .unwrap(), + ) + .build(), + ) + .unwrap(); + let delete_manifest = delete_writer.write_manifest_file().await.unwrap(); + let delete_manifest_path = delete_manifest.manifest_path.clone(); + + // Sanity: the delete manifest really is delete-only. + assert!(delete_manifest.has_deleted_files()); + assert!(!delete_manifest.has_added_files()); + assert!(!delete_manifest.has_existing_files()); + + let mut manifest_list_writer = ManifestListWriter::v2( + table + .file_io() + .new_output(current_snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(), + current_snapshot.snapshot_id(), + current_snapshot.parent_snapshot_id(), + current_snapshot.sequence_number(), + ); + manifest_list_writer + .add_manifests(vec![data_manifest, delete_manifest].into_iter()) + .unwrap(); + manifest_list_writer.close().await.unwrap(); - Ok(manifest_list + (table, tmp_dir, delete_manifest_path) + } + + /// Regression test for #2148: a `fast_append` must carry delete-only manifests + /// forward into the new snapshot. Dropping them lets the files they mark as + /// removed reappear as live data on the next append. + #[tokio::test] + async fn test_fast_append_preserves_delete_only_manifest() { + let (table, _tmp_dir, delete_manifest_path) = make_table_with_delete_only_manifest().await; + + // Append a new data file via the public transaction API. + let new_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(format!("{}/appended.parquet", table.metadata().location())) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(100))])) + .build() + .unwrap(); + + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![new_file]); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { + SnapshotRef::new(snapshot.clone()) + } else { + unreachable!("first update of a fast append should be AddSnapshot") + }; + + let manifest_list = table + .manifest_list_reader(&new_snapshot) + .load() + .await + .unwrap(); + + assert!( + manifest_list + .entries() + .iter() + .any(|m| m.manifest_path == delete_manifest_path), + "delete-only manifest {delete_manifest_path} was dropped from the new snapshot's \ + manifest list; the files it removed would reappear as live data" + ); + } + + /// Load the data files written by a single-manifest fast-append commit. + async fn committed_data_files(table: &Table, updates: &[TableUpdate]) -> Vec { + let TableUpdate::AddSnapshot { snapshot } = &updates[0] else { + unreachable!("first update is always AddSnapshot") + }; + let manifest_list = table + .manifest_list_reader(&SnapshotRef::new(snapshot.clone())) + .load() + .await + .unwrap(); + assert_eq!(1, manifest_list.entries().len()); + table + .manifest_reader() + .read(&manifest_list.entries()[0]) + .await + .unwrap() .entries() .iter() - .filter(|entry| entry.has_added_files() || entry.has_existing_files()) - .cloned() - .collect()) + .map(|entry| entry.data_file().clone()) + .collect() } -} -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; + #[tokio::test] + async fn test_fast_append_writes_encrypted_manifest() { + let table = make_encrypted_table().await; + assert!( + table.encryption_manager().is_some(), + "fixture table should have an EncryptionManager" + ); - use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, Struct, - }; - use crate::transaction::tests::make_v2_minimal_table; - use crate::transaction::{Transaction, TransactionAction}; - use crate::{TableRequirement, TableUpdate}; + let new_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("memory:///table/data/00000.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .partition(Struct::empty()) + .record_count(100) + .file_size_in_bytes(4096) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap(); + + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![new_file]); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_snapshot: SnapshotRef = updates + .iter() + .find_map(|u| match u { + TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot.clone())), + _ => None, + }) + .expect("a fast append should emit an AddSnapshot update"); + + let manifest_list_key_metadata = table + .encryption_manager() + .unwrap() + .decrypt_manifest_list_key_metadata(new_snapshot.encryption_key_id().unwrap()) + .await + .unwrap(); + let manifest_list_size = table + .file_io() + .new_input(new_snapshot.manifest_list()) + .unwrap() + .metadata() + .await + .unwrap() + .size; + assert_eq!( + manifest_list_key_metadata.file_length(), + Some(manifest_list_size) + ); + + let manifest_list = table + .manifest_list_reader(&new_snapshot) + .load() + .await + .unwrap(); + let manifest_file = manifest_list + .entries() + .iter() + .find(|m| m.added_files_count.unwrap_or(0) > 0) + .expect("new snapshot should carry the appended data manifest"); + + // The manifest list entry must carry decodable key metadata. + let key_metadata_bytes = manifest_file + .key_metadata + .as_ref() + .expect("encrypted manifest must record key metadata"); + let key_metadata = StandardKeyMetadata::decode(key_metadata_bytes) + .expect("recorded key metadata must decode as StandardKeyMetadata"); + let manifest_size = table + .file_io() + .new_input(&manifest_file.manifest_path) + .unwrap() + .metadata() + .await + .unwrap() + .size; + assert_eq!(key_metadata.file_length(), Some(manifest_size)); + assert_eq!(manifest_file.manifest_length, manifest_size as i64); + + // The reader self-decrypts using the recorded key metadata and must + // recover the entry we appended. Because the read goes through the + // decryption path, this succeeding also proves the bytes + // on disk were genuinely encrypted (not silently written as plaintext). + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + assert_eq!(manifest.entries().len(), 1); + assert_eq!( + manifest.entries()[0].data_file().file_path(), + "memory:///table/data/00000.parquet" + ); + } + + #[tokio::test] + async fn test_fast_append_rejects_delete_files() { + let table = make_v2_minimal_table(); + let delete_file = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path("test/1-deletes.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .equality_ids(Some(vec![1])) + .build() + .unwrap(); + + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![delete_file]); + let err = commit_with_fresh_state(action, &table) + .await + .err() + .expect("delete files must be rejected"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } #[tokio::test] async fn test_empty_data_append_action() { let table = make_v2_minimal_table(); let tx = Transaction::new(&table); let action = tx.fast_append().add_data_files(vec![]); - assert!(Arc::new(action).commit(&table).await.is_err()); + assert!(commit_with_fresh_state(action, &table).await.is_err()); + } + + /// A `fast_append` must write the manifest list and the manifest + /// files under the `write.metadata.path` prefix when configured, + /// rather than the default `/metadata` directory. + #[tokio::test] + async fn test_fast_append_honors_write_metadata_path() { + let base = make_v2_minimal_table(); + let metadata_root = format!("{}/custom-meta", base.metadata().location()); + let metadata = base + .metadata() + .clone() + .into_builder(None) + .set_properties(HashMap::from([( + "write.metadata.path".to_string(), + metadata_root.clone(), + )])) + .unwrap() + .build() + .unwrap() + .metadata; + let table = base.with_metadata(Arc::new(metadata)); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(format!("{}/data/1.parquet", table.metadata().location())) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap(); + + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![data_file]); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { + SnapshotRef::new(snapshot.clone()) + } else { + unreachable!("first update of a fast append should be AddSnapshot") + }; + + let prefix = format!("{metadata_root}/"); + + // Manifest list + assert!( + new_snapshot.manifest_list().starts_with(prefix.as_str()), + "manifest list {} not under configured write.metadata.path {metadata_root}", + new_snapshot.manifest_list() + ); + + // Manifest files + let manifest_list = table + .manifest_list_reader(&new_snapshot) + .load() + .await + .unwrap(); + assert!( + !manifest_list.entries().is_empty(), + "expected at least one manifest entry" + ); + for entry in manifest_list.entries() { + assert!( + entry.manifest_path.starts_with(prefix.as_str()), + "manifest {} not under configured write.metadata.path {metadata_root}", + entry.manifest_path + ); + } } #[tokio::test] @@ -189,7 +601,7 @@ mod tests { .fast_append() .set_snapshot_properties(snapshot_properties) .add_data_files(vec![data_file]); - let mut action_commit = Arc::new(action).commit(&table).await.unwrap(); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); let updates = action_commit.take_updates(); // Check customized properties is contained in snapshot summary properties. @@ -208,6 +620,200 @@ mod tests { ); } + /// See `testdata/manifests_lists/README.md`. + const FIXTURE_MASTER_KEY_ID: &str = "master-1"; + const FIXTURE_MASTER_KEY_BYTES: [u8; 16] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, + 0x0f, + ]; + + async fn make_v3_encrypted_table() -> Table { + let json = fs::read_to_string(format!( + "{}/testdata/table_metadata/TableMetadataV3ValidEncryption.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let metadata = serde_json::from_str::(&json).unwrap(); + + let kms = MemoryKeyManagementClient::new(); + kms.add_master_key_bytes( + FIXTURE_MASTER_KEY_ID, + SensitiveBytes::new(FIXTURE_MASTER_KEY_BYTES), + ) + .unwrap(); + + let file_io = FileIO::new_with_memory(); + + let manifest_list_bytes = fs::read(format!( + "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + let parent_manifest_list = metadata.current_snapshot().unwrap().manifest_list(); + file_io + .new_output(parent_manifest_list) + .unwrap() + .write(manifest_list_bytes.into()) + .await + .unwrap(); + + Table::builder() + .metadata(metadata) + .metadata_location("memory:///table/metadata/v1.json") + .identifier(TableIdent::from_strs(["ns1", "enc"]).unwrap()) + .file_io(file_io) + .kms_client(Arc::new(kms)) + .runtime(test_runtime()) + .build() + .unwrap() + } + + #[tokio::test] + async fn test_commit_with_encryption_adds_keys_and_records_snapshot_key_id() { + let table = make_v3_encrypted_table().await; + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("test/1.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::empty()) + .build() + .unwrap(); + + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![data_file]); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let added_key_ids: Vec = updates + .iter() + .filter_map(|u| match u { + TableUpdate::AddEncryptionKey { encryption_key } => { + Some(encryption_key.key_id().to_string()) + } + _ => None, + }) + .collect(); + assert!(!added_key_ids.is_empty(), "got {updates:?}"); + + // Encryption keys are added before the snapshot, so it isn't updates[0] here. + let new_snapshot = updates + .iter() + .find_map(|u| match u { + TableUpdate::AddSnapshot { snapshot } => Some(snapshot), + _ => None, + }) + .expect("commit should add a snapshot"); + + let snapshot_key_id = new_snapshot + .encryption_key_id() + .expect("encrypted snapshot should record its manifest-list key id"); + assert!( + added_key_ids.iter().any(|id| id == snapshot_key_id), + "snapshot key id {snapshot_key_id} not in added keys {added_key_ids:?}" + ); + + let new_snapshot_ref: SnapshotRef = Arc::new(new_snapshot.clone()); + let manifest_list = table + .manifest_list_reader(&new_snapshot_ref) + .load() + .await + .expect("newly written encrypted manifest list should decrypt and parse"); + assert_eq!( + manifest_list.entries().len(), + 1, + "append should record exactly the one new data manifest" + ); + } + + #[tokio::test] + async fn test_snapshot_properties_cannot_override_computed_metrics() { + // A user-supplied snapshot property must not shadow a computed metric key + // such as `added-data-files`. Matching iceberg-java, the computed value + // wins, so the summary reflects the real count and a bad value can neither + // corrupt the summary nor panic total computation (see #2184-adjacent fix). + let table = make_v2_minimal_table(); + let tx = Transaction::new(&table); + + let mut snapshot_properties = HashMap::new(); + // Both a benign-but-wrong value and a non-integer value collide with + // computed metric keys; neither should reach the final summary. + snapshot_properties.insert("added-data-files".to_string(), "9999".to_string()); + snapshot_properties.insert("added-records".to_string(), "not-a-number".to_string()); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("test/1.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap(); + + let action = tx + .fast_append() + .set_snapshot_properties(snapshot_properties) + .add_data_files(vec![data_file]); + // Must not panic during total computation. + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { + snapshot + } else { + unreachable!() + }; + let props = &new_snapshot.summary().additional_properties; + + // Computed metric wins over the user's colliding values. + assert_eq!( + props.get("added-data-files").unwrap(), + "1", + "computed added-data-files must override the user-supplied value" + ); + assert_eq!( + props.get("added-records").unwrap(), + "1", + "computed added-records must override the user-supplied non-integer value" + ); + } + + #[tokio::test] + async fn test_snapshot_summary_records_writer_identity() { + let table = make_v2_minimal_table(); + let tx = Transaction::new(&table); + + let snapshot_properties = HashMap::from([ + ("key".to_string(), "val".to_string()), + ("iceberg-version".to_string(), "caller".to_string()), + ]); + + let action = tx + .fast_append() + .set_snapshot_properties(snapshot_properties); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { + snapshot + } else { + unreachable!() + }; + let props = &new_snapshot.summary().additional_properties; + let version = env!("CARGO_PKG_VERSION"); + + assert_eq!( + props.get("iceberg-version").unwrap(), + &format!("Apache Iceberg Rust {version}") + ); + assert_eq!(props.get("key").unwrap(), "val"); + } + #[tokio::test] async fn test_append_snapshot_properties() { let table = make_v2_minimal_table(); @@ -219,7 +825,7 @@ mod tests { let action = tx .fast_append() .set_snapshot_properties(snapshot_properties); - let mut action_commit = Arc::new(action).commit(&table).await.unwrap(); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); let updates = action_commit.take_updates(); // Check customized properties is contained in snapshot summary properties. @@ -258,7 +864,65 @@ mod tests { let action = action.add_data_files(vec![data_file.clone()]); - assert!(Arc::new(action).commit(&table).await.is_err()); + assert!(commit_with_fresh_state(action, &table).await.is_err()); + } + + #[tokio::test] + async fn test_fast_append_dedupes_intra_batch_duplicate_paths() { + let table = make_v2_minimal_table(); + let tx = Transaction::new(&table); + + let make_file = |size: u64, records: u64| { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("test/dup.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(size) + .record_count(records) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(1))])) + .build() + .unwrap() + }; + + // Same path three times, across two `add_data_files` calls: the manifest + // keeps a single entry, the first one. + let action = tx + .fast_append() + .add_data_files(vec![make_file(100, 10), make_file(200, 20)]) + .add_data_files(vec![make_file(300, 30)]); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let files = committed_data_files(&table, &action_commit.take_updates()).await; + assert_eq!(1, files.len()); + assert_eq!(100, files[0].file_size_in_bytes()); + } + + #[tokio::test] + async fn test_fast_append_dedupes_regardless_of_check_duplicate_flag() { + let table = make_v2_minimal_table(); + let tx = Transaction::new(&table); + + let make_file = || { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("test/dup.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(10) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(1))])) + .build() + .unwrap() + }; + + // `check_duplicate` only gates the cross-snapshot check; intra-batch dedupe runs regardless. + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![make_file(), make_file()]); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let files = committed_data_files(&table, &action_commit.take_updates()).await; + assert_eq!(1, files.len()); } #[tokio::test] @@ -279,7 +943,7 @@ mod tests { .unwrap(); let action = action.add_data_files(vec![data_file.clone()]); - let mut action_commit = Arc::new(action).commit(&table).await.unwrap(); + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); let updates = action_commit.take_updates(); let requirements = action_commit.take_requirements(); @@ -301,13 +965,14 @@ mod tests { ); // check manifest list - let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { - snapshot + let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { + SnapshotRef::new(snapshot.clone()) } else { unreachable!() }; - let manifest_list = new_snapshot - .load_manifest_list(table.file_io(), table.metadata()) + let manifest_list = table + .manifest_list_reader(&new_snapshot) + .load() .await .unwrap(); assert_eq!(1, manifest_list.entries().len()); @@ -317,8 +982,9 @@ mod tests { ); // check manifest - let manifest = manifest_list.entries()[0] - .load_manifest(table.file_io()) + let manifest = table + .manifest_reader() + .read(&manifest_list.entries()[0]) .await .unwrap(); assert_eq!(1, manifest.entries().len()); @@ -335,4 +1001,312 @@ mod tests { ); assert_eq!(data_file, *manifest.entries()[0].data_file()); } + + #[tokio::test] + async fn test_fast_append_groups_files_by_partition_spec() { + for table in [ + make_v1_table(), + make_v2_minimal_table(), + make_v3_minimal_table(), + ] { + let metadata = table + .metadata() + .clone() + .into_builder(None) + .set_properties(HashMap::from([( + TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT.to_string(), + "2".to_string(), + )])) + .unwrap() + .build() + .unwrap() + .metadata; + let table = table.with_metadata(Arc::new(metadata)); + let table = with_bucket_partition_spec(table).with_metadata_location( + "s3://bucket/test/location/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json".to_string(), + ); + let old_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("test/old.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(0) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap(); + let mut other_old_file = old_file.clone(); + other_old_file.file_path = "test/other-old.parquet".to_string(); + let mut new_file = old_file.clone(); + new_file.file_path = "test/new.parquet".to_string(); + new_file.partition_spec_id = table.metadata().default_partition_spec_id(); + new_file.partition = + Struct::from_iter([Some(Literal::long(300)), Some(Literal::int(3))]); + + let action = Transaction::new(&table).fast_append().add_data_files(vec![ + new_file.clone(), + old_file.clone(), + other_old_file.clone(), + ]); + let mut commit = commit_with_fresh_state(action, &table).await.unwrap(); + let table = TableCommit::builder() + .ident(table.identifier().clone()) + .updates(commit.take_updates()) + .requirements(commit.take_requirements()) + .build() + .apply(table) + .unwrap(); + let snapshot = table.metadata().current_snapshot().unwrap(); + let list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let mut files_by_spec = Vec::new(); + for manifest_file in list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + assert_eq!( + manifest.metadata().partition_spec(), + table + .metadata() + .partition_spec_by_id(manifest_file.partition_spec_id) + .unwrap() + .as_ref() + ); + let files = manifest + .entries() + .iter() + .map(|entry| entry.data_file().clone()) + .collect::>(); + files_by_spec.push((manifest_file.partition_spec_id, files)); + } + let mut expected = vec![ + (0, vec![old_file.clone(), other_old_file]), + (new_file.partition_spec_id, vec![new_file.clone()]), + ]; + if let Some(mut row_id) = snapshot.first_row_id() { + for (_, files) in &mut expected { + for file in files { + file.first_row_id = Some(row_id.try_into().unwrap()); + row_id += file.record_count; + } + } + } + assert_eq!(files_by_spec, expected); + + let expected_summary = HashMap::from([ + ("added-data-files".to_string(), "3".to_string()), + ("added-records".to_string(), "3".to_string()), + ("added-files-size".to_string(), "300".to_string()), + ("changed-partition-count".to_string(), "2".to_string()), + ("total-data-files".to_string(), "3".to_string()), + ("total-delete-files".to_string(), "0".to_string()), + ("total-records".to_string(), "3".to_string()), + ("total-files-size".to_string(), "300".to_string()), + ("total-position-deletes".to_string(), "0".to_string()), + ("total-equality-deletes".to_string(), "0".to_string()), + ( + "partitions.x=300".to_string(), + "added-data-files=2,added-files-size=200,added-records=2".to_string(), + ), + ( + "partitions.x=300/y_bucket=3".to_string(), + "added-data-files=1,added-files-size=100,added-records=1".to_string(), + ), + ( + "iceberg-version".to_string(), + format!("Apache Iceberg Rust {}", env!("CARGO_PKG_VERSION")), + ), + ]); + let mut actual_summary = snapshot.summary().additional_properties.clone(); + for (key, value) in &mut actual_summary { + if key.starts_with("partitions.") { + let mut metrics = value.split(',').collect::>(); + metrics.sort_unstable(); + *value = metrics.join(","); + } + } + assert_eq!(actual_summary, expected_summary); + } + } + + #[tokio::test] + async fn test_fast_append_validates_non_default_partition_type() { + let table = with_bucket_partition_spec(make_v2_minimal_table()); + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("test/invalid.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(0) + .partition(Struct::from_iter([ + Some(Literal::long(300)), + Some(Literal::int(3)), + ])) + .build() + .unwrap(); + let action = Transaction::new(&table) + .fast_append() + .add_data_files(vec![data_file]); + let err = commit_with_fresh_state(action, &table) + .await + .err() + .expect("partition must match the file's spec"); + assert_eq!( + (err.kind(), err.to_string()), + ( + ErrorKind::DataInvalid, + "DataInvalid, context: { file_path: test/invalid.parquet, partition_spec_id: 0 } => Partition value is not compatible with partition type".to_string() + ) + ); + } + + /// Two writers append concurrently. Writer A starts from S0, but writer B + /// commits S1 first, so A's first attempt fails the `RefSnapshotIdMatch` + /// requirement and A rebases onto S1 to commit S2. + /// + /// The producer state survives the retry: A's manifest is written once and + /// reused, while parent, sequence number, manifest list and summary are + /// recomputed against S1 (see #3418). + #[tokio::test] + async fn test_fast_append_retry_rebases_onto_concurrent_commit() { + let (s1, s2) = rebase_onto_concurrent_commit(make_v2_minimal_table()).await; + assert_eq!(s1.first_row_id(), None); + assert_eq!(s2.first_row_id(), None); + } + + /// On a v3 table the retry must also move the row range: S2's first row id + /// is S1's `next_row_id`, not S0's. + #[tokio::test] + async fn test_fast_append_retry_rebases_row_range_onto_concurrent_commit() { + let (s1, s2) = rebase_onto_concurrent_commit(make_v3_minimal_table()).await; + + // B's single-row file takes row 0 in S1, so A's file starts at row 1. + assert_eq!(s1.row_range(), Some((0, 1))); + assert_eq!(s2.row_range(), Some((1, 1))); + } + + /// Runs the concurrent-append rebase scenario on `s0` and returns the + /// snapshots committed by writer B (S1) and writer A (S2). + async fn rebase_onto_concurrent_commit(s0: Table) -> (SnapshotRef, SnapshotRef) { + // `TableCommit::apply` derives the next metadata location from the + // current one, so it needs a well-formed name. + let s0 = with_retry_properties(s0, "3").with_metadata_location( + "memory:///table/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json" + .to_string(), + ); + let data_file = |path: &str| { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(s0.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap() + }; + + // Writer B commits S1 on top of S0. + let mut b_commit = commit_with_fresh_state( + FastAppendAction::new().add_data_files(vec![data_file("test/b.parquet")]), + &s0, + ) + .await + .unwrap(); + let s1 = TableCommit::builder() + .ident(s0.identifier().clone()) + .updates(b_commit.take_updates()) + .requirements(b_commit.take_requirements()) + .build() + .apply(s0.clone()) + .unwrap(); + let s1_snapshot = s1.metadata().current_snapshot().unwrap().clone(); + + // The catalog serves S0 to A's first attempt and S1 afterwards; every + // update is validated against S1, so the first attempt's + // `RefSnapshotIdMatch { snapshot_id: None }` fails with a real conflict. + let mut mock_catalog = MockCatalog::new(); + let loads = AtomicU32::new(0); + let (load_s0, load_s1) = (s0.clone(), s1.clone()); + mock_catalog + .expect_load_table() + .times(2) + .returning_st(move |_| { + let table = if loads.fetch_add(1, Ordering::SeqCst) == 0 { + load_s0.clone() + } else { + load_s1.clone() + }; + Box::pin(async move { Ok(table) }) + }); + let apply_s1 = s1.clone(); + mock_catalog + .expect_update_table() + .times(2) + .returning_st(move |commit| { + let s1 = apply_s1.clone(); + Box::pin(async move { commit.apply(s1) }) + }); + + // Writer A starts from S0. + let a_uuid = Uuid::now_v7(); + let tx = Transaction::new(&s0); + let tx = tx + .fast_append() + .set_commit_uuid(a_uuid) + .add_data_files(vec![data_file("test/a.parquet")]) + .apply(tx) + .unwrap(); + let s2 = tx.commit(&mock_catalog).await.unwrap(); + let s2_snapshot = s2.metadata().current_snapshot().unwrap().clone(); + + // S2 is rebased onto S1, not S0. + assert_eq!( + s2_snapshot.parent_snapshot_id(), + Some(s1_snapshot.snapshot_id()) + ); + assert_eq!( + s2_snapshot.sequence_number(), + s1_snapshot.sequence_number() + 1 + ); + // The committed manifest list is A's second attempt. + assert!( + s2_snapshot + .manifest_list() + .ends_with(&format!("-1-{a_uuid}.avro")), + "unexpected manifest list path {}", + s2_snapshot.manifest_list() + ); + + // S2 carries B's manifest forward and adds A's. A's manifest is the one + // written during the first attempt (counter 0), reused after rebase. + let manifest_list = s2.manifest_list_reader(&s2_snapshot).load().await.unwrap(); + assert_eq!(manifest_list.entries().len(), 2); + let b_manifest = manifest_list + .entries() + .iter() + .find(|m| m.added_snapshot_id == s1_snapshot.snapshot_id()) + .expect("S2 should carry B's manifest forward"); + let a_manifest = manifest_list + .entries() + .iter() + .find(|m| m.added_snapshot_id == s2_snapshot.snapshot_id()) + .expect("S2 should add A's manifest"); + assert!( + a_manifest + .manifest_path + .ends_with(&format!("{a_uuid}-m0.avro")), + "unexpected manifest path {}", + a_manifest.manifest_path + ); + assert_eq!(a_manifest.sequence_number, s2_snapshot.sequence_number()); + assert_eq!(b_manifest.sequence_number, s1_snapshot.sequence_number()); + + // Summary totals are computed against S1, not S0. + let summary = &s2_snapshot.summary().additional_properties; + assert_eq!(summary.get("added-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-data-files").unwrap(), "2"); + assert_eq!(summary.get("total-records").unwrap(), "2"); + + (s1_snapshot, s2_snapshot) + } } diff --git a/crates/iceberg/src/transaction/expire_snapshots.rs b/crates/iceberg/src/transaction/expire_snapshots.rs new file mode 100644 index 0000000000..13813fd7fa --- /dev/null +++ b/crates/iceberg/src/transaction/expire_snapshots.rs @@ -0,0 +1,1197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashSet; + +use async_trait::async_trait; +use chrono::Utc; + +use crate::error::invalid_data; +use crate::spec::{ + MAIN_BRANCH, SnapshotReference, SnapshotRetention, TableMetadata, TableProperties, +}; +use crate::table::Table; +use crate::transaction::action::{ActionCommit, StatelessAction}; +use crate::{Error, Result, TableRequirement, TableUpdate}; + +/// A transaction action that removes snapshots from table metadata. +/// +/// This only rewrites metadata; the now-unreferenced data and metadata files are left untouched. +/// Physical file cleanup is the responsibility of a higher-level maintenance operation built on +/// top of this action. +/// +/// Selection follows Java `RemoveSnapshots`: +/// - Explicit ids ([`expire_snapshot_ids`](Self::expire_snapshot_ids)) and age-based expiry are +/// combined: a snapshot is expired if it is named explicitly *or* selected by age. +/// - Age-based expiry always runs. The cutoff is [`expire_older_than_ms`](Self::expire_older_than_ms) +/// when set, a per-branch `max_snapshot_age_ms` for that branch, otherwise +/// `now - history.expire.max-snapshot-age-ms` (default 5 days), matching Java's constructor default. +/// - Expiry is computed per branch along each branch's ancestry: each branch keeps its most recent +/// [`retain_last`](Self::retain_last) snapshots — defaulting to `history.expire.min-snapshots-to-keep`, +/// with a per-ref `min_snapshots_to_keep` overriding both — plus any ancestor newer than the cutoff, +/// so a shared ancestor reachable from a retained branch is never expired. +/// - Refs are aged out first: a non-`main` branch or tag whose head is older than its +/// `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is removed, and snapshots only +/// that ref retained then become expirable. +/// - Heads of retained refs (including the current snapshot) are never expired, and naming one +/// explicitly is an error, since +/// [`remove_snapshots`](crate::spec::TableMetadataBuilder::remove_snapshots) would otherwise +/// drop the ref silently. +#[derive(Clone)] +pub struct ExpireSnapshotsAction { + explicit_ids_to_remove: Vec, + older_than_ms: Option, + retain_last: Option, +} + +impl ExpireSnapshotsAction { + pub(crate) fn new() -> Self { + Self { + explicit_ids_to_remove: vec![], + older_than_ms: None, + retain_last: None, + } + } + + /// Expire these snapshot ids in addition to any age-based selection. + /// + /// Age-based expiry runs by default (see the type-level docs), so a call that only names ids + /// still expires snapshots older than `history.expire.max-snapshot-age-ms`. Pin + /// [`expire_older_than_ms`](Self::expire_older_than_ms) to a very old timestamp to expire by id + /// alone. + /// + /// Ids accumulate across calls (like [`add_data_files`](crate::transaction::Transaction::fast_append)). + /// An id that is still referenced by a branch or tag cannot be expired and causes commits to fail. + pub fn expire_snapshot_ids(mut self, snapshot_ids: impl IntoIterator) -> Self { + self.explicit_ids_to_remove.extend(snapshot_ids); + self + } + + /// Expire snapshots whose timestamp is strictly older than `older_than_ms`. + pub fn expire_older_than_ms(mut self, older_than_ms: i64) -> Self { + self.older_than_ms = Some(older_than_ms); + self + } + + /// Keep at least the `retain_last` most recent snapshots of each branch when expiring by age + /// (defaults to the table's `history.expire.min-snapshots-to-keep`, must be at least 1). + /// + /// This only bounds the age cutoff; it does not protect snapshots named via + /// [`expire_snapshot_ids`](Self::expire_snapshot_ids). Setting it to 0 makes commit fail. + pub fn retain_last(mut self, retain_last: usize) -> Self { + self.retain_last = Some(retain_last); + self + } + + /// Resolves the snapshots and refs to remove, following Java `RemoveSnapshots.internalApply`. + fn plan(&self, table: &Table, properties: &TableProperties<'_>) -> Result { + // Matches Java `RemoveSnapshots.retainLast`, which requires at least one snapshot. + if self.retain_last == Some(0) { + return Err(invalid_data!( + "Number of snapshots to retain must be at least 1" + )); + } + + let metadata = table.metadata(); + let now = Utc::now().timestamp_millis(); + // When a knob is not set explicitly, fall back to the table's `history.expire.*` properties, + // matching Java `RemoveSnapshots`' constructor. With the default `max-snapshot-age-ms` (5 + // days) the age path always runs, so even an explicit-id-only call applies the default cutoff. + let default_cutoff = match self.older_than_ms { + Some(older_than_ms) => older_than_ms, + None => now.saturating_sub(properties.max_snapshot_age_ms()?), + }; + let default_min_to_keep = match self.retain_last { + Some(retain_last) => retain_last, + None => properties.min_snapshots_to_keep()?, + }; + + // Ref aging: `main` is always kept; any other ref whose head is older than its + // `max_ref_age_ms` (defaulting to `history.expire.max-ref-age-ms`) is dropped, like Java's + // `computeRetainedRefs`. + let default_max_ref_age_ms = properties.max_ref_age_ms()?; + let mut removed_ref_names: Vec = vec![]; + let mut retained_refs: Vec<&SnapshotReference> = vec![]; + for (ref_name, snapshot_ref) in &metadata.refs { + if ref_name == MAIN_BRANCH + || !Self::ref_aged_out(metadata, snapshot_ref, now, default_max_ref_age_ms) + { + retained_refs.push(snapshot_ref); + } else { + removed_ref_names.push(ref_name.clone()); + } + } + + // Heads of retained refs (plus the current snapshot) are never expired; naming one + // explicitly is an error, since `remove_snapshots` would otherwise drop the ref silently. + let mut ref_head_ids: HashSet = retained_refs.iter().map(|r| r.snapshot_id).collect(); + if let Some(current_id) = metadata.current_snapshot_id() { + ref_head_ids.insert(current_id); + } + + let existing_ids: HashSet = metadata.snapshots().map(|s| s.snapshot_id()).collect(); + let mut expiring_ids: HashSet = HashSet::new(); + for id in &self.explicit_ids_to_remove { + if ref_head_ids.contains(id) { + return Err(Self::reference_error(metadata, *id)); + } + if existing_ids.contains(id) { + expiring_ids.insert(*id); + } + } + + // Per-branch retention: keep each branch's most recent `min_to_keep` ancestors plus any + // newer than the branch cutoff. The current snapshot is treated as a branch (default policy) + // so its lineage is protected even when there is no explicit `main` ref. + let mut retained_ids = ref_head_ids.clone(); + let mut referenced_ids = ref_head_ids.clone(); + let mut branches: Vec<(i64, usize, i64)> = vec![]; + for snapshot_ref in &retained_refs { + match &snapshot_ref.retention { + SnapshotRetention::Branch { + min_snapshots_to_keep, + max_snapshot_age_ms, + .. + } => { + let min_to_keep = + min_snapshots_to_keep.map_or(default_min_to_keep, |m| m as usize); + let cutoff = + max_snapshot_age_ms.map_or(default_cutoff, |age| now.saturating_sub(age)); + branches.push((snapshot_ref.snapshot_id, min_to_keep, cutoff)); + } + SnapshotRetention::Tag { .. } => { + referenced_ids.insert(snapshot_ref.snapshot_id); + } + } + } + if let Some(current_id) = metadata.current_snapshot_id() + && !branches + .iter() + .any(|(head_id, _, _)| *head_id == current_id) + { + branches.push((current_id, default_min_to_keep, default_cutoff)); + } + for (head_id, min_to_keep, cutoff) in branches { + Self::retain_branch( + metadata, + head_id, + min_to_keep, + cutoff, + &mut retained_ids, + &mut referenced_ids, + ); + } + + // Unreferenced snapshots newer than the default cutoff are kept (Java's + // `unreferencedSnapshotsToRetain`); everything else not retained is expired. + for snapshot in metadata.snapshots() { + let id = snapshot.snapshot_id(); + if !referenced_ids.contains(&id) && snapshot.timestamp_ms() >= default_cutoff { + retained_ids.insert(id); + } + } + for snapshot in metadata.snapshots() { + if !retained_ids.contains(&snapshot.snapshot_id()) { + expiring_ids.insert(snapshot.snapshot_id()); + } + } + + let mut ids_to_remove: Vec = expiring_ids.into_iter().collect(); + ids_to_remove.sort_unstable(); + removed_ref_names.sort(); + Ok(ExpirePlan { + ids_to_remove, + refs_to_remove: removed_ref_names, + }) + } + + /// Whether a non-main ref should be dropped because its head is older than its `max_ref_age_ms`, + /// defaulting to `default_max_ref_age_ms` (`history.expire.max-ref-age-ms`) when the ref sets no + /// window of its own. The default `i64::MAX` effectively never ages a ref out. + fn ref_aged_out( + metadata: &TableMetadata, + snapshot_ref: &SnapshotReference, + now: i64, + default_max_ref_age_ms: i64, + ) -> bool { + let max_ref_age_ms = match snapshot_ref.retention { + SnapshotRetention::Branch { max_ref_age_ms, .. } + | SnapshotRetention::Tag { max_ref_age_ms } => max_ref_age_ms, + } + .unwrap_or(default_max_ref_age_ms); + match metadata.snapshot_by_id(snapshot_ref.snapshot_id) { + Some(snapshot) => now.saturating_sub(snapshot.timestamp_ms()) > max_ref_age_ms, + None => false, + } + } + + /// Walks a branch's ancestry (Java's `computeBranchSnapshotsToRetain`), retaining each ancestor + /// while fewer than `min` are kept or it is newer than `cutoff`, and recording every ancestor as + /// referenced. Ancestry timestamps decrease monotonically, so this needs no early break. + fn retain_branch( + metadata: &TableMetadata, + head_id: i64, + min_to_keep: usize, + cutoff: i64, + retained_ids: &mut HashSet, + referenced_ids: &mut HashSet, + ) { + let mut kept_count = 0usize; + for ancestor_id in Self::ancestors(metadata, head_id) { + referenced_ids.insert(ancestor_id); + let timestamp = metadata + .snapshot_by_id(ancestor_id) + .map_or(i64::MIN, |snapshot| snapshot.timestamp_ms()); + if kept_count < min_to_keep || timestamp >= cutoff { + retained_ids.insert(ancestor_id); + kept_count += 1; + } + } + } + + /// Iterates a snapshot and its ancestors, newest first, following `parent_snapshot_id`. + fn ancestors(metadata: &TableMetadata, head_id: i64) -> impl Iterator + '_ { + let mut next_id = Some(head_id); + std::iter::from_fn(move || { + let id = next_id?; + next_id = metadata + .snapshot_by_id(id) + .and_then(|snapshot| snapshot.parent_snapshot_id()); + Some(id) + }) + } + + fn reference_error(metadata: &TableMetadata, snapshot_id: i64) -> Error { + if metadata.current_snapshot_id() == Some(snapshot_id) { + return invalid_data!("Cannot expire the current snapshot"); + } + let ref_names: Vec<&str> = metadata + .refs + .iter() + .filter(|(_, snapshot_ref)| snapshot_ref.snapshot_id == snapshot_id) + .map(|(ref_name, _)| ref_name.as_str()) + .collect(); + invalid_data!("Cannot expire snapshot {snapshot_id}: still referenced by {ref_names:?}") + } +} + +/// Snapshots and refs an [`ExpireSnapshotsAction`] resolves to remove. +struct ExpirePlan { + ids_to_remove: Vec, + refs_to_remove: Vec, +} + +#[async_trait] +impl StatelessAction for ExpireSnapshotsAction { + async fn commit(&self, table: &Table) -> Result { + let metadata = table.metadata(); + let properties = metadata.table_properties(); + + // Expiring metadata defeats a user's explicit decision to disable GC (Java refuses too). + if !properties.gc_enabled()? { + return Err(invalid_data!( + "Cannot expire snapshots: gc.enabled is false" + )); + } + + let plan = self.plan(table, &properties)?; + + if plan.ids_to_remove.is_empty() && plan.refs_to_remove.is_empty() { + return Ok(ActionCommit::new(vec![], vec![])); + } + + // Drop aged-out refs first, then the snapshots no ref retains anymore. + let mut updates: Vec = plan + .refs_to_remove + .into_iter() + .map(|ref_name| TableUpdate::RemoveSnapshotRef { ref_name }) + .collect(); + + // Drop statistics metadata for expired snapshots. + // This only updates metadata; puffin files are cleaned up separately. + let mut stats_updates: Vec = vec![]; + for &snapshot_id in &plan.ids_to_remove { + stats_updates.extend( + metadata + .statistics_for_snapshot(snapshot_id) + .is_some() + .then_some(TableUpdate::RemoveStatistics { snapshot_id }), + ); + stats_updates.extend( + metadata + .partition_statistics_for_snapshot(snapshot_id) + .is_some() + .then_some(TableUpdate::RemovePartitionStatistics { snapshot_id }), + ); + } + + if !plan.ids_to_remove.is_empty() { + updates.push(TableUpdate::RemoveSnapshots { + snapshot_ids: plan.ids_to_remove, + }); + } + updates.extend(stats_updates); + + // The ref assertion closes the race where a concurrent writer advances `main` between + // selection and commit, which could orphan a snapshot whose parent we are about to remove. + Ok(ActionCommit::new(updates, vec![ + TableRequirement::UuidMatch { + uuid: metadata.uuid(), + }, + TableRequirement::RefSnapshotIdMatch { + r#ref: MAIN_BRANCH.to_string(), + snapshot_id: metadata.current_snapshot_id(), + }, + ])) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use chrono::Utc; + + use crate::spec::{ + MAIN_BRANCH, Operation, PartitionStatisticsFile, Snapshot, SnapshotReference, + SnapshotRetention, StatisticsFile, Summary, + }; + use crate::table::Table; + use crate::test_utils::commit_with_fresh_state; + use crate::transaction::Transaction; + use crate::transaction::action::ApplyTransactionAction; + use crate::transaction::expire_snapshots::ExpireSnapshotsAction; + use crate::transaction::tests::{make_v2_minimal_table, make_v2_table}; + use crate::{TableRequirement, TableUpdate}; + + // `make_v2_table` carries an older snapshot (ts 1515100955770) and a current + // snapshot (ts 1555100955770). + const OLD_SNAPSHOT: i64 = 3051729675574597004; + const CURRENT_SNAPSHOT: i64 = 3055729675574597004; + // Well after the minimal table's last-updated-ms, so synthetic snapshots pass timestamp checks. + const TS: i64 = 1_700_000_000_000; + // A cutoff at the epoch makes age-based expiry a no-op (every snapshot is newer), isolating the + // explicit-id behavior under test now that the default cutoff (`now - 5 days`) always runs. + const NO_AGE_EXPIRY: i64 = 0; + + fn action() -> ExpireSnapshotsAction { + ExpireSnapshotsAction::new() + } + + async fn removed_ids(action: ExpireSnapshotsAction) -> Vec { + expired(&make_v2_table(), action).await + } + + async fn updates_of(table: &Table, action: ExpireSnapshotsAction) -> Vec { + commit_with_fresh_state(action, table) + .await + .unwrap() + .take_updates() + } + + async fn expired(table: &Table, action: ExpireSnapshotsAction) -> Vec { + updates_of(table, action) + .await + .into_iter() + .find_map(|update| match update { + TableUpdate::RemoveSnapshots { snapshot_ids } => Some(snapshot_ids), + _ => None, + }) + .unwrap_or_default() + } + + fn removed_refs(updates: &[TableUpdate]) -> Vec { + let mut refs: Vec = updates + .iter() + .filter_map(|update| match update { + TableUpdate::RemoveSnapshotRef { ref_name } => Some(ref_name.clone()), + _ => None, + }) + .collect(); + refs.sort(); + refs + } + + fn snapshot(id: i64, parent: Option, sequence_number: i64, timestamp_ms: i64) -> Snapshot { + Snapshot::builder() + .with_snapshot_id(id) + .with_parent_snapshot_id(parent) + .with_sequence_number(sequence_number) + .with_timestamp_ms(timestamp_ms) + .with_schema_id(0) + .with_manifest_list(format!("/snap-{id}.avro")) + .with_summary(Summary { + operation: Operation::Append, + additional_properties: HashMap::new(), + }) + .build() + } + + fn branch(snapshot_id: i64, min_snapshots_to_keep: Option) -> SnapshotReference { + branch_with(snapshot_id, min_snapshots_to_keep, None, None) + } + + fn branch_with( + snapshot_id: i64, + min_snapshots_to_keep: Option, + max_snapshot_age_ms: Option, + max_ref_age_ms: Option, + ) -> SnapshotReference { + SnapshotReference { + snapshot_id, + retention: SnapshotRetention::Branch { + min_snapshots_to_keep, + max_snapshot_age_ms, + max_ref_age_ms, + }, + } + } + + fn tag(snapshot_id: i64, max_ref_age_ms: Option) -> SnapshotReference { + SnapshotReference { + snapshot_id, + retention: SnapshotRetention::Tag { max_ref_age_ms }, + } + } + + /// Builds a table from synthetic snapshots and refs on top of an empty base. + fn table_with(snapshots: Vec, refs: Vec<(&str, SnapshotReference)>) -> Table { + table_with_props(snapshots, refs, HashMap::new()) + } + + /// Like [`table_with`], but also seeds table properties (e.g. `history.expire.*` defaults). + fn table_with_props( + snapshots: Vec, + refs: Vec<(&str, SnapshotReference)>, + properties: HashMap, + ) -> Table { + let base = make_v2_minimal_table(); + let mut builder = base + .metadata() + .clone() + .into_builder(None) + .set_properties(properties) + .unwrap(); + for snapshot in snapshots { + builder = builder.add_snapshot(snapshot).unwrap(); + } + for (name, reference) in refs { + builder = builder.set_ref(name, reference).unwrap(); + } + base.with_metadata(Arc::new(builder.build().unwrap().metadata)) + } + + /// Like [`table_with`], but also attaches statistics and partition-statistics files. Reuses + /// [`table_with`] for the snapshot/ref wiring so the two can't drift, then layers stats on top. + fn table_with_stats( + snapshots: Vec, + refs: Vec<(&str, SnapshotReference)>, + statistics: Vec, + partition_statistics: Vec, + ) -> Table { + let table = table_with(snapshots, refs); + let mut builder = table.metadata().clone().into_builder(None); + for stats in statistics { + builder = builder.set_statistics(stats); + } + for stats in partition_statistics { + builder = builder.set_partition_statistics(stats); + } + table.with_metadata(Arc::new(builder.build().unwrap().metadata)) + } + + fn stats_file(snapshot_id: i64) -> StatisticsFile { + StatisticsFile { + snapshot_id, + statistics_path: format!("/stats-{snapshot_id}.puffin"), + file_size_in_bytes: 1, + file_footer_size_in_bytes: 1, + key_metadata: None, + blob_metadata: vec![], + } + } + + fn partition_stats_file(snapshot_id: i64) -> PartitionStatisticsFile { + PartitionStatisticsFile { + snapshot_id, + statistics_path: format!("/partition-stats-{snapshot_id}.puffin"), + file_size_in_bytes: 1, + } + } + + fn removed_statistics(updates: &[TableUpdate]) -> Vec { + updates + .iter() + .filter_map(|update| match update { + TableUpdate::RemoveStatistics { snapshot_id } => Some(*snapshot_id), + _ => None, + }) + .collect() + } + + fn removed_partition_statistics(updates: &[TableUpdate]) -> Vec { + updates + .iter() + .filter_map(|update| match update { + TableUpdate::RemovePartitionStatistics { snapshot_id } => Some(*snapshot_id), + _ => None, + }) + .collect() + } + + #[tokio::test] + async fn test_expire_explicit_snapshot_id() { + assert_eq!( + removed_ids( + action() + .expire_snapshot_ids(vec![OLD_SNAPSHOT]) + .expire_older_than_ms(NO_AGE_EXPIRY) + ) + .await, + vec![OLD_SNAPSHOT] + ); + } + + #[tokio::test] + async fn test_explicit_unknown_id_is_ignored() { + assert!( + removed_ids( + action() + .expire_snapshot_ids(vec![42]) + .expire_older_than_ms(NO_AGE_EXPIRY) + ) + .await + .is_empty() + ); + } + + #[tokio::test] + async fn test_cannot_expire_current_snapshot() { + let table = make_v2_table(); + let action = action().expire_snapshot_ids(vec![CURRENT_SNAPSHOT]); + assert!(commit_with_fresh_state(action, &table).await.is_err()); + } + + /// `make_v2_table` with a tag pointing at the older snapshot. + fn table_with_tag_on_old() -> Table { + let table = make_v2_table(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .set_ref("history-tag", SnapshotReference { + snapshot_id: OLD_SNAPSHOT, + retention: SnapshotRetention::Tag { + max_ref_age_ms: None, + }, + }) + .unwrap() + .build() + .unwrap() + .metadata; + table.with_metadata(Arc::new(metadata)) + } + + #[tokio::test] + async fn test_cannot_expire_tagged_snapshot_explicitly() { + let table = table_with_tag_on_old(); + let action = action().expire_snapshot_ids(vec![OLD_SNAPSHOT]); + assert!(commit_with_fresh_state(action, &table).await.is_err()); + } + + #[tokio::test] + async fn test_age_expiry_skips_tagged_snapshot() { + let table = table_with_tag_on_old(); + let mut commit = commit_with_fresh_state(action().expire_older_than_ms(i64::MAX), &table) + .await + .unwrap(); + // Both snapshots are referenced (current + tag), so nothing is expired. + assert!(commit.take_updates().is_empty()); + } + + #[tokio::test] + async fn test_retain_last_default_expires_older_non_current() { + assert_eq!( + removed_ids(action().expire_older_than_ms(i64::MAX)).await, + vec![OLD_SNAPSHOT] + ); + } + + #[tokio::test] + async fn test_retain_last_noop_when_enough_retained() { + assert!(removed_ids(action().retain_last(5)).await.is_empty()); + } + + #[tokio::test] + async fn test_older_than_excludes_newer_snapshots() { + // Threshold older than every snapshot -> nothing qualifies. + assert!( + removed_ids(action().expire_older_than_ms(1)) + .await + .is_empty() + ); + } + + #[tokio::test] + async fn test_apply_registers_action() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + let tx = tx + .expire_snapshots() + .expire_snapshot_ids(vec![OLD_SNAPSHOT]) + .apply(tx) + .unwrap(); + assert_eq!(tx.actions.len(), 1); + } + + #[tokio::test] + async fn test_per_branch_retention_protects_shared_ancestor() { + // main: 1 -> 2 -> 3 ; branch `b`: 1 -> 2 -> 4. Snapshot 2 is a shared ancestor of both + // branches but is not a ref head. + let table = table_with( + vec![ + snapshot(1, None, 35, TS + 1), + snapshot(2, Some(1), 36, TS + 2), + snapshot(3, Some(2), 37, TS + 3), + snapshot(4, Some(2), 38, TS + 4), + ], + vec![(MAIN_BRANCH, branch(3, None)), ("b", branch(4, None))], + ); + + // A global "newest 2" would expire 2 and orphan branch `b`; per-branch retention keeps it. + let removed = expired( + &table, + action().retain_last(2).expire_older_than_ms(i64::MAX), + ) + .await; + assert_eq!(removed, vec![1]); + } + + #[tokio::test] + async fn test_per_ref_min_snapshots_to_keep_overrides_retain_last() { + let table = table_with( + vec![ + snapshot(1, None, 35, TS + 1), + snapshot(2, Some(1), 36, TS + 2), + snapshot(3, Some(2), 37, TS + 3), + ], + vec![(MAIN_BRANCH, branch(3, Some(3)))], + ); + + // The branch's own min_snapshots_to_keep=3 wins over the action's retain_last(1). + let removed = expired( + &table, + action().retain_last(1).expire_older_than_ms(i64::MAX), + ) + .await; + assert!(removed.is_empty()); + } + + #[tokio::test] + async fn test_explicit_and_age_combine() { + let table = table_with( + vec![ + snapshot(1, None, 35, TS + 1), + snapshot(2, Some(1), 36, TS + 2), + snapshot(3, Some(2), 37, TS + 3), + snapshot(4, Some(3), 38, TS + 4), + ], + vec![(MAIN_BRANCH, branch(4, None))], + ); + + // Age expires 1 and 2 (older than the cutoff, beyond retain_last). 3 is newer than the + // cutoff so age keeps it, but it is named explicitly, so all three are expired. + let removed = expired( + &table, + action() + .retain_last(1) + .expire_older_than_ms(TS + 3) + .expire_snapshot_ids(vec![3]), + ) + .await; + assert_eq!(removed, vec![1, 2, 3]); + } + + #[tokio::test] + async fn test_expire_snapshot_ids_accumulates() { + let table = table_with( + vec![ + snapshot(1, None, 35, TS + 1), + snapshot(2, Some(1), 36, TS + 2), + snapshot(3, Some(2), 37, TS + 3), + ], + vec![(MAIN_BRANCH, branch(3, None))], + ); + + // Two separate calls both take effect (age expiry pinned off to isolate accumulation). + let removed = expired( + &table, + action() + .expire_snapshot_ids(vec![1]) + .expire_snapshot_ids(vec![2]) + .expire_older_than_ms(NO_AGE_EXPIRY), + ) + .await; + assert_eq!(removed, vec![1, 2]); + } + + #[tokio::test] + async fn test_gc_disabled_errors() { + let table = make_v2_table(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .set_properties(HashMap::from([( + "gc.enabled".to_string(), + "false".to_string(), + )])) + .unwrap() + .build() + .unwrap() + .metadata; + let table = table.with_metadata(Arc::new(metadata)); + + let action = action().expire_snapshot_ids(vec![OLD_SNAPSHOT]); + assert!(commit_with_fresh_state(action, &table).await.is_err()); + } + + #[tokio::test] + async fn test_commit_asserts_main_ref() { + let table = make_v2_table(); + let mut commit = + commit_with_fresh_state(action().expire_snapshot_ids(vec![OLD_SNAPSHOT]), &table) + .await + .unwrap(); + assert!( + commit + .take_requirements() + .iter() + .any(|requirement| matches!( + requirement, + TableRequirement::RefSnapshotIdMatch { r#ref, snapshot_id } + if r#ref == MAIN_BRANCH && *snapshot_id == Some(CURRENT_SNAPSHOT) + )) + ); + } + + #[tokio::test] + async fn test_ref_aging_drops_old_tag_and_expires_its_snapshot() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // main: 2 (recent). Isolated snapshot 1 (old) is only kept alive by `old-tag`, whose own + // age (10 days) exceeds its max-ref-age of 1 day. + let table = table_with( + vec![ + snapshot(1, None, 35, now - 10 * day_ms), + snapshot(2, None, 36, now - 1000), + ], + // Set the tag before main so the builder's last-updated bookkeeping stays monotonic. + vec![ + ("old-tag", tag(1, Some(day_ms))), + (MAIN_BRANCH, branch(2, None)), + ], + ); + + // An explicit cutoff lets the freed snapshot 1 actually expire once the tag is gone. + let updates = updates_of(&table, action().expire_older_than_ms(now - 5 * day_ms)).await; + // The tag is dropped, and snapshot 1 (now unreferenced and old) is expired with it. + assert_eq!(removed_refs(&updates), vec!["old-tag".to_string()]); + assert!(updates.iter().any( + |u| matches!(u, TableUpdate::RemoveSnapshots { snapshot_ids } if snapshot_ids == &[1]) + )); + } + + #[tokio::test] + async fn test_ref_aging_keeps_recent_tag() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + let table = table_with( + vec![ + snapshot(1, None, 35, now - 1000), + snapshot(2, None, 36, now - 500), + ], + vec![ + ("fresh-tag", tag(1, Some(day_ms))), + (MAIN_BRANCH, branch(2, None)), + ], + ); + + let updates = updates_of(&table, action()).await; + // The tag is younger than its max-ref-age, so neither it nor its snapshot is removed. + assert!(removed_refs(&updates).is_empty()); + assert_eq!(expired(&table, action()).await, Vec::::new()); + } + + #[tokio::test] + async fn test_per_ref_max_snapshot_age_overrides_default() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // main: 1 (3 days old) -> 2 (1 day old) -> 3 (recent), with a 2-day per-ref window. + let table = table_with( + vec![ + snapshot(1, None, 35, now - 3 * day_ms), + snapshot(2, Some(1), 36, now - day_ms), + snapshot(3, Some(2), 37, now - 1000), + ], + vec![(MAIN_BRANCH, branch_with(3, Some(1), Some(2 * day_ms), None))], + ); + + // With no explicit cutoff nothing would expire, but main's 2-day window expires snapshot 1. + let removed = expired(&table, action()).await; + assert_eq!(removed, vec![1]); + } + + #[tokio::test] + async fn test_ref_aging_drops_old_branch() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // A stale non-main branch (head 1, 10 days old) past its 1-day max-ref-age. + let table = table_with( + vec![ + snapshot(1, None, 35, now - 10 * day_ms), + snapshot(2, None, 36, now - 1000), + ], + vec![ + ("stale", branch_with(1, None, None, Some(day_ms))), + (MAIN_BRANCH, branch(2, None)), + ], + ); + + let updates = updates_of(&table, action().expire_older_than_ms(now - 5 * day_ms)).await; + // The stale branch is dropped and snapshot 1 (now unreferenced and old) is expired. + assert_eq!(removed_refs(&updates), vec!["stale".to_string()]); + assert!(updates.iter().any( + |u| matches!(u, TableUpdate::RemoveSnapshots { snapshot_ids } if snapshot_ids == &[1]) + )); + } + + #[tokio::test] + async fn test_unreferenced_snapshots_retained_only_while_young() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // Snapshot 1 is the main head; 2 and 3 are orphans (no ref, not ancestors). Add oldest + // first so the builder's timestamp bookkeeping stays monotonic. + let table = table_with( + vec![ + snapshot(3, None, 35, now - 10 * day_ms), // old orphan + snapshot(1, None, 36, now - 1000), // main head + snapshot(2, None, 37, now - 1000), // young orphan + ], + vec![(MAIN_BRANCH, branch(1, None))], + ); + + // The young orphan (2) is kept; only the old orphan (3) is expired. + let removed = expired(&table, action().expire_older_than_ms(now - 5 * day_ms)).await; + assert_eq!(removed, vec![3]); + } + + #[tokio::test] + async fn test_retain_last_zero_errors() { + let table = make_v2_table(); + assert!( + commit_with_fresh_state(action().retain_last(0), &table) + .await + .is_err() + ); + } + + #[tokio::test] + async fn test_cannot_expire_branch_head_explicitly() { + let table = table_with( + vec![snapshot(1, None, 35, TS + 1), snapshot(2, None, 36, TS + 2)], + vec![(MAIN_BRANCH, branch(1, None)), ("branch", branch(2, None))], + ); + // 2 is the head of a non-main branch, so it cannot be expired explicitly. + let action = action().expire_snapshot_ids(vec![2]); + assert!(commit_with_fresh_state(action, &table).await.is_err()); + } + + #[tokio::test] + async fn test_tag_does_not_protect_its_ancestry() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // Chain 1 -> 2 -> 3 with main rewound to 1 and a tag on 3. Snapshot 2 (the tag target's + // parent) is reachable from no ref's retained set, so it is expired. + let table = table_with( + vec![ + snapshot(1, None, 35, now - 10 * day_ms), + snapshot(2, Some(1), 36, now - 8 * day_ms), + snapshot(3, Some(2), 37, now - 1000), + ], + vec![(MAIN_BRANCH, branch(1, None)), ("tag", tag(3, None))], + ); + + let removed = expired(&table, action().expire_older_than_ms(now - 5 * day_ms)).await; + assert_eq!(removed, vec![2]); + } + + #[tokio::test] + async fn test_branch_protects_its_ancestry() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // Same topology as the tag case, but a branch (keeping its whole history) replaces the tag. + // The branch's parent snapshot 2 is now reachable history and is not expired. + let table = table_with( + vec![ + snapshot(1, None, 35, now - 10 * day_ms), + snapshot(2, Some(1), 36, now - 8 * day_ms), + snapshot(3, Some(2), 37, now - 1000), + ], + vec![ + (MAIN_BRANCH, branch(1, None)), + ("branch", branch_with(3, None, Some(i64::MAX), None)), + ], + ); + + let removed = expired(&table, action().expire_older_than_ms(now - 5 * day_ms)).await; + assert!(removed.is_empty()); + } + + #[tokio::test] + async fn test_per_branch_max_snapshot_age_differs_across_branches() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // main (1 -> 2) keeps 5 days; branch `keep` (3 -> 4) keeps 60 days. Snapshots 1 and 3 are + // both 30 days old, but only main's short window expires its ancestor. + let table = table_with( + vec![ + snapshot(1, None, 35, now - 30 * day_ms), + snapshot(3, None, 36, now - 30 * day_ms), + snapshot(2, Some(1), 37, now - 2 * day_ms), + snapshot(4, Some(3), 38, now - 2 * day_ms), + ], + vec![ + (MAIN_BRANCH, branch_with(2, None, Some(5 * day_ms), None)), + ("keep", branch_with(4, None, Some(60 * day_ms), None)), + ], + ); + + // main's 5-day window expires its old ancestor 1; keep's 60-day window retains its old + // ancestor 3. + let removed = expired(&table, action()).await; + assert_eq!(removed, vec![1]); + } + + #[tokio::test] + async fn test_default_cutoff_expires_snapshots_older_than_max_age() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + let table = table_with( + vec![ + snapshot(1, None, 35, now - 10 * day_ms), // older than the default 5-day cutoff + snapshot(2, Some(1), 36, now - 1000), // recent + ], + vec![(MAIN_BRANCH, branch(2, None))], + ); + + // No explicit cutoff: defaults to now - history.expire.max-snapshot-age-ms (5 days). + let removed = expired(&table, action()).await; + assert_eq!(removed, vec![1]); + } + + #[tokio::test] + async fn test_min_snapshots_to_keep_property_is_the_default_floor() { + let table = table_with_props( + vec![ + snapshot(1, None, 35, TS + 1), + snapshot(2, Some(1), 36, TS + 2), + snapshot(3, Some(2), 37, TS + 3), + ], + vec![(MAIN_BRANCH, branch(3, None))], + HashMap::from([( + "history.expire.min-snapshots-to-keep".to_string(), + "3".to_string(), + )]), + ); + + // The snapshots predate the default cutoff, but the table's min-snapshots-to-keep=3 keeps + // the whole chain. + let removed = expired(&table, action()).await; + assert!(removed.is_empty()); + } + + #[tokio::test] + async fn test_max_snapshot_age_ms_property_sets_the_cutoff() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // Snapshot 1 is 2 days old: the built-in 5-day default would keep it, but the table's + // 1-day history.expire.max-snapshot-age-ms expires it. This pins the cutoff to the property + // value rather than the hardcoded default. + let table = table_with_props( + vec![ + snapshot(1, None, 35, now - 2 * day_ms), + snapshot(2, Some(1), 36, now - 1000), + ], + vec![(MAIN_BRANCH, branch(2, None))], + HashMap::from([( + "history.expire.max-snapshot-age-ms".to_string(), + day_ms.to_string(), + )]), + ); + + let removed = expired(&table, action()).await; + assert_eq!(removed, vec![1]); + } + + #[tokio::test] + async fn test_max_ref_age_ms_property_ages_out_ref_without_its_own_window() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // `old-tag` sets no max_ref_age_ms of its own, so it ages against the table's + // history.expire.max-ref-age-ms (1 day); its head is 10 days old, so the tag is dropped. + // Set the tag before main so the builder's last-updated bookkeeping stays monotonic. + let table = table_with_props( + vec![ + snapshot(1, None, 35, now - 10 * day_ms), + snapshot(2, None, 36, now - 1000), + ], + vec![("old-tag", tag(1, None)), (MAIN_BRANCH, branch(2, None))], + HashMap::from([( + "history.expire.max-ref-age-ms".to_string(), + day_ms.to_string(), + )]), + ); + + let updates = updates_of(&table, action()).await; + assert_eq!(removed_refs(&updates), vec!["old-tag".to_string()]); + // Once the tag is gone, snapshot 1 (unreferenced and well past the default cutoff) expires. + assert!(updates.iter().any( + |u| matches!(u, TableUpdate::RemoveSnapshots { snapshot_ids } if snapshot_ids == &[1]) + )); + } + + #[tokio::test] + async fn test_expiring_snapshot_drops_its_statistics() { + // main: 1 -> 2, both carrying statistics. retain_last(1) keeps only the head (2). + let table = table_with_stats( + vec![ + snapshot(1, None, 35, TS + 1), + snapshot(2, Some(1), 36, TS + 2), + ], + vec![(MAIN_BRANCH, branch(2, None))], + vec![stats_file(1), stats_file(2)], + vec![partition_stats_file(1), partition_stats_file(2)], + ); + + let updates = updates_of( + &table, + action().retain_last(1).expire_older_than_ms(i64::MAX), + ) + .await; + + // Snapshot 1 expires, so its stats entries are dropped; the retained head 2 keeps its stats. + assert_eq!(removed_statistics(&updates), vec![1]); + assert_eq!(removed_partition_statistics(&updates), vec![1]); + } + + #[tokio::test] + async fn test_expiring_snapshot_without_statistics_emits_no_removal() { + // Same expiry, but no statistics attached to any snapshot. + let table = table_with( + vec![ + snapshot(1, None, 35, TS + 1), + snapshot(2, Some(1), 36, TS + 2), + ], + vec![(MAIN_BRANCH, branch(2, None))], + ); + + let updates = updates_of( + &table, + action().retain_last(1).expire_older_than_ms(i64::MAX), + ) + .await; + + assert!(removed_statistics(&updates).is_empty()); + assert!(removed_partition_statistics(&updates).is_empty()); + } + + #[tokio::test] + async fn test_only_present_statistics_variant_is_removed() { + // Snapshot 1 has statistics but no partition statistics. + let table = table_with_stats( + vec![ + snapshot(1, None, 35, TS + 1), + snapshot(2, Some(1), 36, TS + 2), + ], + vec![(MAIN_BRANCH, branch(2, None))], + vec![stats_file(1)], + vec![], + ); + + let updates = updates_of( + &table, + action().retain_last(1).expire_older_than_ms(i64::MAX), + ) + .await; + + assert_eq!(removed_statistics(&updates), vec![1]); + assert!(removed_partition_statistics(&updates).is_empty()); + } + + #[tokio::test] + async fn test_ref_aging_expiry_drops_statistics() { + let now = Utc::now().timestamp_millis(); + let day_ms = 24 * 60 * 60 * 1000; + // Snapshot 1 is kept alive only by `old-tag`; once the tag ages out (1-day max-ref-age vs a + // 10-day-old head) snapshot 1 becomes expirable and its statistics go with it. This reaches + // the stats loop through the ref-aging branch of `plan()`, distinct from the retain/age path. + let table = table_with_stats( + vec![ + snapshot(1, None, 35, now - 10 * day_ms), + snapshot(2, None, 36, now - 1000), + ], + vec![ + ("old-tag", tag(1, Some(day_ms))), + (MAIN_BRANCH, branch(2, None)), + ], + vec![stats_file(1)], + vec![partition_stats_file(1)], + ); + + let updates = updates_of(&table, action().expire_older_than_ms(now - 5 * day_ms)).await; + assert_eq!(removed_refs(&updates), vec!["old-tag".to_string()]); + assert_eq!(removed_statistics(&updates), vec![1]); + assert_eq!(removed_partition_statistics(&updates), vec![1]); + } + + #[tokio::test] + async fn test_multiple_expired_snapshots_drop_their_statistics() { + // Chain 1 -> 2 -> 3 on main; retain_last(1) keeps only head 3, expiring 1 and 2, both + // carrying statistics. + let table = table_with_stats( + vec![ + snapshot(1, None, 35, TS + 1), + snapshot(2, Some(1), 36, TS + 2), + snapshot(3, Some(2), 37, TS + 3), + ], + vec![(MAIN_BRANCH, branch(3, None))], + vec![stats_file(1), stats_file(2)], + vec![partition_stats_file(1), partition_stats_file(2)], + ); + + let updates = updates_of( + &table, + action().retain_last(1).expire_older_than_ms(i64::MAX), + ) + .await; + + assert_eq!(removed_statistics(&updates), vec![1, 2]); + assert_eq!(removed_partition_statistics(&updates), vec![1, 2]); + } +} diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 06549a95c5..2018af2fc1 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -52,14 +52,14 @@ /// that allows users to apply a transaction action to a `Transaction`. mod action; -use std::collections::HashMap; - pub use action::*; mod append; +mod expire_snapshots; mod snapshot; mod sort_order; mod update_location; mod update_properties; +mod update_schema; mod update_statistics; mod upgrade_format_version; @@ -67,29 +67,26 @@ use std::sync::Arc; use std::time::Duration; use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder, RetryableWithContext}; +pub use update_schema::AddColumn; use crate::error::Result; -use crate::spec::{ - PROPERTY_COMMIT_MAX_RETRY_WAIT_MS, PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, - PROPERTY_COMMIT_MIN_RETRY_WAIT_MS, PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, - PROPERTY_COMMIT_NUM_RETRIES, PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, - PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS, PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, -}; +use crate::spec::TableProperties; use crate::table::Table; -use crate::transaction::action::BoxedTransactionAction; -use crate::transaction::append::FastAppendAction; -use crate::transaction::sort_order::ReplaceSortOrderAction; -use crate::transaction::update_location::UpdateLocationAction; -use crate::transaction::update_properties::UpdatePropertiesAction; -use crate::transaction::update_statistics::UpdateStatisticsAction; -use crate::transaction::upgrade_format_version::UpgradeFormatVersionAction; +use crate::transaction::action::TransactionActionEntry; +pub use crate::transaction::append::FastAppendAction; +pub use crate::transaction::expire_snapshots::ExpireSnapshotsAction; +pub use crate::transaction::sort_order::ReplaceSortOrderAction; +pub use crate::transaction::update_location::UpdateLocationAction; +pub use crate::transaction::update_properties::UpdatePropertiesAction; +pub use crate::transaction::update_schema::UpdateSchemaAction; +pub use crate::transaction::update_statistics::UpdateStatisticsAction; +pub use crate::transaction::upgrade_format_version::UpgradeFormatVersionAction; use crate::{Catalog, Error, ErrorKind, TableCommit, TableRequirement, TableUpdate}; /// Table transaction. -#[derive(Clone)] pub struct Transaction { table: Table, - actions: Vec, + actions: Vec, } impl Transaction { @@ -143,6 +140,11 @@ impl Transaction { UpdatePropertiesAction::new() } + /// Creates an update schema action. + pub fn update_schema(&self) -> UpdateSchemaAction { + UpdateSchemaAction::new() + } + /// Creates a fast append action. pub fn fast_append(&self) -> FastAppendAction { FastAppendAction::new() @@ -163,14 +165,32 @@ impl Transaction { UpdateStatisticsAction::new() } + /// Expire snapshots from the table metadata. + pub fn expire_snapshots(&self) -> ExpireSnapshotsAction { + ExpireSnapshotsAction::new() + } + /// Commit transaction. + /// + /// Returns `ErrorKind::PreconditionFailed` if the table is read-only + /// (`Table::readonly()`), even when the transaction has no actions. pub async fn commit(self, catalog: &dyn Catalog) -> Result
{ + if self.table.readonly() { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "Cannot modify a read-only table", + ) + .with_context("table", self.table.identifier().to_string())); + } + if self.actions.is_empty() { // nothing to commit return Ok(self.table); } - let backoff = Self::build_backoff(self.table.metadata().properties())?; + let table_props = self.table.metadata().table_properties(); + + let backoff = Self::build_backoff(table_props)?; let tx = self; (|mut tx: Transaction| async { @@ -185,54 +205,16 @@ impl Transaction { .1 } - fn build_backoff(props: &HashMap) -> Result { - let min_delay = match props.get(PROPERTY_COMMIT_MIN_RETRY_WAIT_MS) { - Some(value_str) => value_str.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Invalid value for commit.retry.min-wait-ms", - ) - .with_source(e) - })?, - None => PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT, - }; - let max_delay = match props.get(PROPERTY_COMMIT_MAX_RETRY_WAIT_MS) { - Some(value_str) => value_str.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Invalid value for commit.retry.max-wait-ms", - ) - .with_source(e) - })?, - None => PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT, - }; - let total_delay = match props.get(PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS) { - Some(value_str) => value_str.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Invalid value for commit.retry.total-timeout-ms", - ) - .with_source(e) - })?, - None => PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT, - }; - let max_times = match props.get(PROPERTY_COMMIT_NUM_RETRIES) { - Some(value_str) => value_str.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Invalid value for commit.retry.num-retries", - ) - .with_source(e) - })?, - None => PROPERTY_COMMIT_NUM_RETRIES_DEFAULT, - }; - + fn build_backoff(props: TableProperties<'_>) -> Result { Ok(ExponentialBuilder::new() - .with_min_delay(Duration::from_millis(min_delay)) - .with_max_delay(Duration::from_millis(max_delay)) - .with_total_delay(Some(Duration::from_millis(total_delay))) - .with_max_times(max_times) + .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms()?)) + .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms()?)) + .with_total_delay(Some(Duration::from_millis( + props.commit_total_retry_timeout_ms()?, + ))) + .with_max_times(props.commit_num_retries()?) .with_factor(2.0) + .with_jitter() .build()) } @@ -250,8 +232,8 @@ impl Transaction { let mut existing_updates: Vec = vec![]; let mut existing_requirements: Vec = vec![]; - for action in &self.actions { - let action_commit = Arc::clone(action).commit(¤t_table).await?; + for entry in self.actions.iter_mut() { + let action_commit = entry.commit(¤t_table).await?; // apply action commit to current_table current_table = Self::apply( current_table, @@ -280,11 +262,16 @@ mod tests { use std::sync::atomic::{AtomicU32, Ordering}; use crate::catalog::MockCatalog; - use crate::io::FileIOBuilder; - use crate::spec::TableMetadata; - use crate::table::Table; + use crate::io::FileIO; + use crate::memory::tests::new_memory_catalog; + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Literal, PartitionSpec, Struct, + TableMetadata, TableProperties, Transform, + }; + use crate::table::{StaticTable, Table}; + use crate::test_utils::{make_encrypted_table, test_runtime}; use crate::transaction::{ApplyTransactionAction, Transaction}; - use crate::{Error, ErrorKind, TableIdent}; + use crate::{Catalog, Error, ErrorKind, TableCreation, TableIdent}; pub fn make_v1_table() -> Table { let file = File::open(format!( @@ -298,9 +285,10 @@ mod tests { Table::builder() .metadata(resp) - .metadata_location("s3://bucket/test/location/metadata/v1.json".to_string()) + .metadata_location("s3://bucket/test/location/metadata/v1.json") .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .file_io(FileIOBuilder::new("memory").build().unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) .build() .unwrap() } @@ -317,9 +305,10 @@ mod tests { Table::builder() .metadata(resp) - .metadata_location("s3://bucket/test/location/metadata/v1.json".to_string()) + .metadata_location("s3://bucket/test/location/metadata/v1.json") .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .file_io(FileIOBuilder::new("memory").build().unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) .build() .unwrap() } @@ -336,17 +325,107 @@ mod tests { Table::builder() .metadata(resp) - .metadata_location("s3://bucket/test/location/metadata/v1.json".to_string()) + .metadata_location("s3://bucket/test/location/metadata/v1.json") .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) - .file_io(FileIOBuilder::new("memory").build().unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) + .build() + .unwrap() + } + + pub(crate) fn make_v3_minimal_table() -> Table { + let file = File::open(format!( + "{}/testdata/table_metadata/{}", + env!("CARGO_MANIFEST_DIR"), + "TableMetadataV3ValidMinimal.json" + )) + .unwrap(); + let reader = BufReader::new(file); + let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap(); + + Table::builder() + .metadata(resp) + .metadata_location("s3://bucket/test/location/metadata/v1.json") + .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) + .build() + .unwrap() + } + + pub(super) fn with_bucket_partition_spec(table: Table) -> Table { + let spec = PartitionSpec::builder(table.metadata().current_schema().clone()) + .add_partition_field("x", "x", Transform::Identity) + .unwrap() + .add_partition_field("y", "y_bucket", Transform::Bucket(16)) + .unwrap() .build() .unwrap() + .into_unbound(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .add_default_partition_spec(spec) + .unwrap() + .build() + .unwrap() + .metadata; + table.with_metadata(Arc::new(metadata)) + } + + async fn make_v2_minimal_readonly_table() -> Table { + let table = make_v2_minimal_table(); + StaticTable::from_metadata( + table.metadata().clone(), + table.identifier().clone(), + FileIO::new_with_memory(), + ) + .await + .unwrap() + .into_table() + } + + pub(crate) async fn make_v3_minimal_table_in_catalog(catalog: &impl Catalog) -> Table { + let table_ident = + TableIdent::from_strs([format!("ns1-{}", uuid::Uuid::new_v4()), "test1".to_string()]) + .unwrap(); + + catalog + .create_namespace(table_ident.namespace(), HashMap::new()) + .await + .unwrap(); + + let file = File::open(format!( + "{}/testdata/table_metadata/{}", + env!("CARGO_MANIFEST_DIR"), + "TableMetadataV3ValidMinimal.json" + )) + .unwrap(); + let reader = BufReader::new(file); + let base_metadata = serde_json::from_reader::<_, TableMetadata>(reader).unwrap(); + + let table_creation = TableCreation::builder() + .schema((**base_metadata.current_schema()).clone()) + .partition_spec((**base_metadata.default_partition_spec()).clone()) + .sort_order((**base_metadata.default_sort_order()).clone()) + .name(table_ident.name().to_string()) + .format_version(crate::spec::FormatVersion::V3) + .build(); + + catalog + .create_table(table_ident.namespace(), table_creation) + .await + .unwrap() } /// Helper function to create a test table with retry properties - fn setup_test_table(num_retries: &str) -> Table { - let table = make_v2_table(); + pub(super) fn setup_test_table(num_retries: &str) -> Table { + with_retry_properties(make_v2_table(), num_retries) + } + /// Sets fast commit retry properties on `table`. + pub(super) fn with_retry_properties(table: Table, num_retries: &str) -> Table { // Set retry properties let mut props = HashMap::new(); props.insert("commit.retry.min-wait-ms".to_string(), "10".to_string()); @@ -509,4 +588,224 @@ mod tests { assert!(err.retryable(), "Error should be retryable"); } } + + #[test] + fn test_commit_backoff_is_jittered() { + let table = setup_test_table("4"); + let delays = || { + Transaction::build_backoff(table.metadata().table_properties()) + .unwrap() + .collect::>() + }; + + assert_ne!(delays(), delays()); + } + + #[tokio::test] + async fn test_transaction_snapshot_summary() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let mut file_seq = 0u32; + let mut append_file = |table: &Table, record_count: u64, file_size: u64| { + file_seq += 1; + let file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(format!("test/{file_seq}.parquet")) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(file_size) + .record_count(record_count) + .partition(Struct::from_iter([Some(Literal::long(1))])) + .partition_spec_id(0) + .build() + .unwrap(); + let tx = Transaction::new(table); + tx.fast_append() + .add_data_files(vec![file]) + .apply(tx) + .unwrap() + }; + + let table = append_file(&table, /*record_count=*/ 10, /*file_size=*/ 100) + .commit(&catalog) + .await + .unwrap(); + let table = append_file(&table, /*record_count=*/ 20, /*file_size=*/ 200) + .commit(&catalog) + .await + .unwrap(); + + let summary = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + + assert_eq!(summary.get("total-records").unwrap(), "30"); + assert_eq!(summary.get("total-data-files").unwrap(), "2"); + assert_eq!(summary.get("total-files-size").unwrap(), "300"); + } + + #[tokio::test] + async fn test_commit_to_encrypted_table() { + let table = make_encrypted_table().await.with_metadata_location( + "memory:///table/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json" + .to_string(), + ); + let refreshed_table = table.clone(); + let update_table = table.clone(); + let mut mock_catalog = MockCatalog::new(); + mock_catalog + .expect_load_table() + .times(1) + .returning_st(move |_| { + let refreshed_table = refreshed_table.clone(); + Box::pin(async move { Ok(refreshed_table) }) + }); + mock_catalog + .expect_update_table() + .times(1) + .returning_st(move |commit| { + let update_table = update_table.clone(); + Box::pin(async move { commit.apply(update_table) }) + }); + + let tx = Transaction::new(&table); + let tx = tx + .update_table_properties() + .set("test.key".to_string(), "test.value".to_string()) + .apply(tx) + .unwrap(); + + let updated_table = tx.commit(&mock_catalog).await.unwrap(); + + assert_eq!( + updated_table + .metadata() + .properties() + .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID) + .map(String::as_str), + Some("master-1") + ); + assert_eq!( + updated_table + .metadata() + .properties() + .get("test.key") + .map(String::as_str), + Some("test.value") + ); + assert!(updated_table.encryption_manager().is_some()); + } + + #[tokio::test] + async fn test_commit_readonly_table_fails() { + let table = make_v2_minimal_readonly_table().await; + let catalog = MockCatalog::new(); + + // make sure even empty transactions fail + let err = Transaction::new(&table).commit(&catalog).await.unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + assert_eq!(err.message(), "Cannot modify a read-only table"); + assert!(!err.retryable(), "Error should not be retryable"); + } + + #[tokio::test] + async fn test_commit_readonly_table_with_actions_fails() { + let table = make_v2_minimal_readonly_table().await; + let catalog = MockCatalog::new(); + + let err = create_test_transaction(&table) + .commit(&catalog) + .await + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + } + + #[tokio::test] + async fn test_commit_empty_transaction_is_noop() { + let table = make_v2_minimal_table(); + let catalog = MockCatalog::new(); + + let result = Transaction::new(&table).commit(&catalog).await.unwrap(); + + assert_eq!(result.metadata(), table.metadata()); + } +} + +#[cfg(test)] +mod test_row_lineage { + use crate::memory::tests::new_memory_catalog; + use crate::spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, Struct, + }; + use crate::transaction::tests::make_v3_minimal_table_in_catalog; + use crate::transaction::{ApplyTransactionAction, Transaction}; + + #[tokio::test] + async fn test_fast_append_with_row_lineage() { + // Helper function to create a data file with specified number of rows + fn file_with_rows(record_count: u64) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(format!("test/{record_count}.parquet")) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(record_count) + .partition(Struct::from_iter([Some(Literal::long(0))])) + .partition_spec_id(0) + .build() + .unwrap() + } + let catalog = new_memory_catalog().await; + + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + // Check initial state - next_row_id should be 0 + assert_eq!(table.metadata().next_row_id(), 0); + + // First fast append with 30 rows + let tx = Transaction::new(&table); + let data_file_30 = file_with_rows(30); + let action = tx.fast_append().add_data_files(vec![data_file_30]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Check snapshot and table state after first append + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.first_row_id(), Some(0)); + assert_eq!(table.metadata().next_row_id(), 30); + + // Check written manifest for first_row_id + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + + assert_eq!(manifest_list.entries().len(), 1); + let manifest_file = &manifest_list.entries()[0]; + assert_eq!(manifest_file.first_row_id, Some(0)); + + // Second fast append with 17 and 11 rows + let tx = Transaction::new(&table); + let data_file_17 = file_with_rows(17); + let data_file_11 = file_with_rows(11); + let action = tx + .fast_append() + .add_data_files(vec![data_file_17, data_file_11]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Check snapshot and table state after second append + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.first_row_id(), Some(30)); + assert_eq!(table.metadata().next_row_id(), 30 + 17 + 11); + + // Check written manifest for first_row_id + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + assert_eq!(manifest_list.entries().len(), 2); + let manifest_file = &manifest_list.entries()[1]; + assert_eq!(manifest_file.first_row_id, Some(30)); + } } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs deleted file mode 100644 index 48dc2b5b90..0000000000 --- a/crates/iceberg/src/transaction/snapshot.rs +++ /dev/null @@ -1,451 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::{HashMap, HashSet}; -use std::future::Future; -use std::ops::RangeFrom; - -use uuid::Uuid; - -use crate::error::Result; -use crate::spec::{ - DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, - ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, - PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT, PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT, - Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, StructType, - Summary, update_snapshot_summaries, -}; -use crate::table::Table; -use crate::transaction::ActionCommit; -use crate::{Error, ErrorKind, TableRequirement, TableUpdate}; - -const META_ROOT_PATH: &str = "metadata"; - -pub(crate) trait SnapshotProduceOperation: Send + Sync { - fn operation(&self) -> Operation; - #[allow(unused)] - fn delete_entries( - &self, - snapshot_produce: &SnapshotProducer, - ) -> impl Future>> + Send; - fn existing_manifest( - &self, - snapshot_produce: &SnapshotProducer<'_>, - ) -> impl Future>> + Send; -} - -pub(crate) struct DefaultManifestProcess; - -impl ManifestProcess for DefaultManifestProcess { - fn process_manifests( - &self, - _snapshot_produce: &SnapshotProducer<'_>, - manifests: Vec, - ) -> Vec { - manifests - } -} - -pub(crate) trait ManifestProcess: Send + Sync { - fn process_manifests( - &self, - snapshot_produce: &SnapshotProducer<'_>, - manifests: Vec, - ) -> Vec; -} - -pub(crate) struct SnapshotProducer<'a> { - pub(crate) table: &'a Table, - snapshot_id: i64, - commit_uuid: Uuid, - key_metadata: Option>, - snapshot_properties: HashMap, - added_data_files: Vec, - // A counter used to generate unique manifest file names. - // It starts from 0 and increments for each new manifest file. - // Note: This counter is limited to the range of (0..u64::MAX). - manifest_counter: RangeFrom, -} - -impl<'a> SnapshotProducer<'a> { - pub(crate) fn new( - table: &'a Table, - commit_uuid: Uuid, - key_metadata: Option>, - snapshot_properties: HashMap, - added_data_files: Vec, - ) -> Self { - Self { - table, - snapshot_id: Self::generate_unique_snapshot_id(table), - commit_uuid, - key_metadata, - snapshot_properties, - added_data_files, - manifest_counter: (0..), - } - } - - pub(crate) fn validate_added_data_files(&self, added_data_files: &[DataFile]) -> Result<()> { - for data_file in added_data_files { - if data_file.content_type() != crate::spec::DataContentType::Data { - return Err(Error::new( - ErrorKind::DataInvalid, - "Only data content type is allowed for fast append", - )); - } - // Check if the data file partition spec id matches the table default partition spec id. - if self.table.metadata().default_partition_spec_id() != data_file.partition_spec_id { - return Err(Error::new( - ErrorKind::DataInvalid, - "Data file partition spec id does not match table default partition spec id", - )); - } - Self::validate_partition_value( - data_file.partition(), - self.table.metadata().default_partition_type(), - )?; - } - - Ok(()) - } - - pub(crate) async fn validate_duplicate_files( - &self, - added_data_files: &[DataFile], - ) -> Result<()> { - let new_files: HashSet<&str> = added_data_files - .iter() - .map(|df| df.file_path.as_str()) - .collect(); - - let mut referenced_files = Vec::new(); - if let Some(current_snapshot) = self.table.metadata().current_snapshot() { - let manifest_list = current_snapshot - .load_manifest_list(self.table.file_io(), &self.table.metadata_ref()) - .await?; - for manifest_list_entry in manifest_list.entries() { - let manifest = manifest_list_entry - .load_manifest(self.table.file_io()) - .await?; - for entry in manifest.entries() { - let file_path = entry.file_path(); - if new_files.contains(file_path) && entry.is_alive() { - referenced_files.push(file_path.to_string()); - } - } - } - } - - if !referenced_files.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot add files that are already referenced by table, files: {}", - referenced_files.join(", ") - ), - )); - } - - Ok(()) - } - - fn generate_unique_snapshot_id(table: &Table) -> i64 { - let generate_random_id = || -> i64 { - let (lhs, rhs) = Uuid::new_v4().as_u64_pair(); - let snapshot_id = (lhs ^ rhs) as i64; - if snapshot_id < 0 { - -snapshot_id - } else { - snapshot_id - } - }; - let mut snapshot_id = generate_random_id(); - - while table - .metadata() - .snapshots() - .any(|s| s.snapshot_id() == snapshot_id) - { - snapshot_id = generate_random_id(); - } - snapshot_id - } - - fn new_manifest_writer(&mut self, content: ManifestContentType) -> Result { - let new_manifest_path = format!( - "{}/{}/{}-m{}.{}", - self.table.metadata().location(), - META_ROOT_PATH, - self.commit_uuid, - self.manifest_counter.next().unwrap(), - DataFileFormat::Avro - ); - let output_file = self.table.file_io().new_output(new_manifest_path)?; - let builder = ManifestWriterBuilder::new( - output_file, - Some(self.snapshot_id), - self.key_metadata.clone(), - self.table.metadata().current_schema().clone(), - self.table - .metadata() - .default_partition_spec() - .as_ref() - .clone(), - ); - if self.table.metadata().format_version() == FormatVersion::V1 { - Ok(builder.build_v1()) - } else { - match content { - ManifestContentType::Data => Ok(builder.build_v2_data()), - ManifestContentType::Deletes => Ok(builder.build_v2_deletes()), - } - } - } - - // Check if the partition value is compatible with the partition type. - fn validate_partition_value( - partition_value: &Struct, - partition_type: &StructType, - ) -> Result<()> { - if partition_value.fields().len() != partition_type.fields().len() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Partition value is not compatible with partition type", - )); - } - - for (value, field) in partition_value.fields().iter().zip(partition_type.fields()) { - let field = field.field_type.as_primitive_type().ok_or_else(|| { - Error::new( - ErrorKind::Unexpected, - "Partition field should only be primitive type.", - ) - })?; - if let Some(value) = value { - if !field.compatible(&value.as_primitive_literal().unwrap()) { - return Err(Error::new( - ErrorKind::DataInvalid, - "Partition value is not compatible partition type", - )); - } - } - } - Ok(()) - } - - // Write manifest file for added data files and return the ManifestFile for ManifestList. - async fn write_added_manifest(&mut self) -> Result { - let added_data_files = std::mem::take(&mut self.added_data_files); - if added_data_files.is_empty() { - return Err(Error::new( - ErrorKind::PreconditionFailed, - "No added data files found when write an added manifest file", - )); - } - - let snapshot_id = self.snapshot_id; - let format_version = self.table.metadata().format_version(); - let manifest_entries = added_data_files.into_iter().map(|data_file| { - let builder = ManifestEntry::builder() - .status(crate::spec::ManifestStatus::Added) - .data_file(data_file); - if format_version == FormatVersion::V1 { - builder.snapshot_id(snapshot_id).build() - } else { - // For format version > 1, we set the snapshot id at the inherited time to avoid rewrite the manifest file when - // commit failed. - builder.build() - } - }); - let mut writer = self.new_manifest_writer(ManifestContentType::Data)?; - for entry in manifest_entries { - writer.add_entry(entry)?; - } - writer.write_manifest_file().await - } - - async fn manifest_file( - &mut self, - snapshot_produce_operation: &OP, - manifest_process: &MP, - ) -> Result> { - // Assert current snapshot producer contains new content to add to new snapshot. - // - // TODO: Allowing snapshot property setup with no added data files is a workaround. - // We should clean it up after all necessary actions are supported. - // For details, please refer to https://github.com/apache/iceberg-rust/issues/1548 - if self.added_data_files.is_empty() && self.snapshot_properties.is_empty() { - return Err(Error::new( - ErrorKind::PreconditionFailed, - "No added data files or added snapshot properties found when write a manifest file", - )); - } - - let existing_manifests = snapshot_produce_operation.existing_manifest(self).await?; - let mut manifest_files = existing_manifests; - - // Process added entries. - if !self.added_data_files.is_empty() { - let added_manifest = self.write_added_manifest().await?; - manifest_files.push(added_manifest); - } - - // # TODO - // Support process delete entries. - - let manifest_files = manifest_process.process_manifests(self, manifest_files); - Ok(manifest_files) - } - - // Returns a `Summary` of the current snapshot - fn summary( - &self, - snapshot_produce_operation: &OP, - ) -> Result { - let mut summary_collector = SnapshotSummaryCollector::default(); - let table_metadata = self.table.metadata_ref(); - - let partition_summary_limit = if let Some(limit) = table_metadata - .properties() - .get(PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT) - { - if let Ok(limit) = limit.parse::() { - limit - } else { - PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT - } - } else { - PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT - }; - - summary_collector.set_partition_summary_limit(partition_summary_limit); - - for data_file in &self.added_data_files { - summary_collector.add_file( - data_file, - table_metadata.current_schema().clone(), - table_metadata.default_partition_spec().clone(), - ); - } - - let previous_snapshot = table_metadata - .snapshot_by_id(self.snapshot_id) - .and_then(|snapshot| snapshot.parent_snapshot_id()) - .and_then(|parent_id| table_metadata.snapshot_by_id(parent_id)); - - let mut additional_properties = summary_collector.build(); - additional_properties.extend(self.snapshot_properties.clone()); - - let summary = Summary { - operation: snapshot_produce_operation.operation(), - additional_properties, - }; - - update_snapshot_summaries( - summary, - previous_snapshot.map(|s| s.summary()), - snapshot_produce_operation.operation() == Operation::Overwrite, - ) - } - - fn generate_manifest_list_file_path(&self, attempt: i64) -> String { - format!( - "{}/{}/snap-{}-{}-{}.{}", - self.table.metadata().location(), - META_ROOT_PATH, - self.snapshot_id, - attempt, - self.commit_uuid, - DataFileFormat::Avro - ) - } - - /// Finished building the action and return the [`ActionCommit`] to the transaction. - pub(crate) async fn commit( - mut self, - snapshot_produce_operation: OP, - process: MP, - ) -> Result { - let new_manifests = self - .manifest_file(&snapshot_produce_operation, &process) - .await?; - let next_seq_num = self.table.metadata().next_sequence_number(); - - let summary = self.summary(&snapshot_produce_operation).map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) - })?; - - let manifest_list_path = self.generate_manifest_list_file_path(0); - - let mut manifest_list_writer = match self.table.metadata().format_version() { - FormatVersion::V1 => ManifestListWriter::v1( - self.table - .file_io() - .new_output(manifest_list_path.clone())?, - self.snapshot_id, - self.table.metadata().current_snapshot_id(), - ), - FormatVersion::V2 => ManifestListWriter::v2( - self.table - .file_io() - .new_output(manifest_list_path.clone())?, - self.snapshot_id, - self.table.metadata().current_snapshot_id(), - next_seq_num, - ), - }; - manifest_list_writer.add_manifests(new_manifests.into_iter())?; - manifest_list_writer.close().await?; - - let commit_ts = chrono::Utc::now().timestamp_millis(); - let new_snapshot = Snapshot::builder() - .with_manifest_list(manifest_list_path) - .with_snapshot_id(self.snapshot_id) - .with_parent_snapshot_id(self.table.metadata().current_snapshot_id()) - .with_sequence_number(next_seq_num) - .with_summary(summary) - .with_schema_id(self.table.metadata().current_schema_id()) - .with_timestamp_ms(commit_ts) - .build(); - - let updates = vec![ - TableUpdate::AddSnapshot { - snapshot: new_snapshot, - }, - TableUpdate::SetSnapshotRef { - ref_name: MAIN_BRANCH.to_string(), - reference: SnapshotReference::new( - self.snapshot_id, - SnapshotRetention::branch(None, None, None), - ), - }, - ]; - - let requirements = vec![ - TableRequirement::UuidMatch { - uuid: self.table.metadata().uuid(), - }, - TableRequirement::RefSnapshotIdMatch { - r#ref: MAIN_BRANCH.to_string(), - snapshot_id: self.table.metadata().current_snapshot_id(), - }, - ]; - - Ok(ActionCommit::new(updates, requirements)) - } -} diff --git a/crates/iceberg/src/transaction/snapshot/mod.rs b/crates/iceberg/src/transaction/snapshot/mod.rs new file mode 100644 index 0000000000..bd6c1276ad --- /dev/null +++ b/crates/iceberg/src/transaction/snapshot/mod.rs @@ -0,0 +1,758 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Snapshot production for transaction actions. +//! +//! Producers (see [`simple`]) own retry-persistent state: identity, reusable +//! artifacts, and the action's intent. The functions in this module are the +//! stateless building blocks they share: they translate explicit inputs into +//! Iceberg metadata and files, never select operation policy, and never cache +//! across calls. + +pub(crate) mod simple; + +use std::collections::{HashMap, HashSet}; + +use futures::TryStreamExt; +use futures::stream::FuturesUnordered; +use uuid::Uuid; + +use crate::error::{Result, invalid_data}; +use crate::spec::{ + DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestFile, + ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, PartitionSpecRef, + SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, + StructType, Summary, TableProperties, set_iceberg_version, update_snapshot_summaries, +}; +use crate::table::Table; +use crate::transaction::ActionCommit; +use crate::{Error, ErrorKind, TableRequirement, TableUpdate}; + +/// Generates a positive snapshot id not already present in `table`. +pub(crate) fn generate_snapshot_id(table: &Table) -> i64 { + loop { + let (lhs, rhs) = Uuid::new_v4().as_u64_pair(); + let snapshot_id = (lhs ^ rhs) as i64 & i64::MAX; + if table.metadata().snapshot_by_id(snapshot_id).is_none() { + return snapshot_id; + } + } +} + +/// Returns the metadata path for a producer-owned manifest. +/// +/// `counter` must be unique per `commit_uuid` for the lifetime of the producer, +/// so that no attempt overwrites a manifest written by an earlier one. +pub(crate) fn manifest_path(table: &Table, commit_uuid: Uuid, counter: u64) -> Result { + Ok(format!( + "{}/{}-m{}.{}", + table.metadata().metadata_location()?, + commit_uuid, + counter, + DataFileFormat::Avro + )) +} + +/// Returns the metadata path for one attempt's manifest list. +/// +/// Each attempt gets a distinct `attempt` number so the path is write-once. +pub(crate) fn manifest_list_path( + table: &Table, + snapshot_id: i64, + attempt: u64, + commit_uuid: Uuid, +) -> Result { + Ok(format!( + "{}/snap-{}-{}-{}.{}", + table.metadata().metadata_location()?, + snapshot_id, + attempt, + commit_uuid, + DataFileFormat::Avro + )) +} + +/// Creates a manifest writer at `path`, honoring the table's format version and +/// encryption configuration, under the given schema and partition spec. +pub(crate) fn new_manifest_writer( + table: &Table, + path: String, + snapshot_id: i64, + content: ManifestContentType, + schema: SchemaRef, + partition_spec: PartitionSpecRef, +) -> Result { + let output_file = table.file_io().new_output(path)?; + let partition_spec = partition_spec.as_ref().clone(); + + let builder = if let Some(em) = table.encryption_manager() { + ManifestWriterBuilder::new_from_encrypted( + em.encrypt(output_file), + Some(snapshot_id), + schema, + partition_spec, + )? + } else { + ManifestWriterBuilder::new(output_file, Some(snapshot_id), schema, partition_spec) + }; + + match table.metadata().format_version() { + FormatVersion::V1 => Ok(builder.build_v1()), + FormatVersion::V2 => match content { + ManifestContentType::Data => Ok(builder.build_v2_data()), + ManifestContentType::Deletes => Ok(builder.build_v2_deletes()), + }, + FormatVersion::V3 => match content { + ManifestContentType::Data => Ok(builder.build_v3_data()), + ManifestContentType::Deletes => Ok(builder.build_v3_deletes()), + }, + } +} + +/// Checks that a partition value is compatible with the partition type. +pub(crate) fn validate_partition_value( + partition_value: &Struct, + partition_type: &StructType, +) -> Result<()> { + if partition_value.fields().len() != partition_type.fields().len() { + return Err(invalid_data!( + "Partition value is not compatible with partition type" + )); + } + + for (value, field) in partition_value.fields().iter().zip(partition_type.fields()) { + let field = field.field_type.as_primitive_type().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "Partition field should only be primitive type.", + ) + })?; + if let Some(value) = value + && !field.compatible(&value.as_primitive_literal().unwrap()) + { + return Err(invalid_data!( + "Partition value is not compatible partition type" + )); + } + } + Ok(()) +} + +/// Returns the current snapshot's manifests that a new snapshot carries forward. +/// +/// Delete-only manifests are kept too: they record which files were removed and +/// must persist across snapshots until `expire_snapshots` cleans them up. +/// Dropping them lets the removed files reappear as live data (see #2148). +pub(crate) async fn current_manifests(table: &Table) -> Result> { + let Some(snapshot) = table.metadata().current_snapshot() else { + return Ok(vec![]); + }; + + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + + Ok(manifest_list + .consume_entries() + .into_iter() + .filter(|entry| { + entry.has_added_files() || entry.has_existing_files() || entry.has_deleted_files() + }) + .collect()) +} + +/// Fails if any of `data_files` is already live in the table's current snapshot. +/// +/// Must run against each attempt's base, since a concurrent commit may have +/// added one of these files. Manifest reads go through the table's object +/// cache, so manifests unchanged by a rebase are not re-read. +pub(crate) async fn validate_files_not_live(table: &Table, data_files: &[DataFile]) -> Result<()> { + let Some(current_snapshot) = table.metadata().current_snapshot() else { + return Ok(()); + }; + + let new_files: HashSet<&str> = data_files.iter().map(|df| df.file_path()).collect(); + if new_files.is_empty() { + return Ok(()); + } + + let manifest_list = table.manifest_list_reader(current_snapshot).load().await?; + let runtime = table.runtime(); + let object_cache = table.object_cache(); + + let new_files_ref = &new_files; + let referenced_files: Vec = manifest_list + .consume_entries() + .into_iter() + .map(|entry| { + let object_cache = object_cache.clone(); + runtime + .io() + .spawn(async move { object_cache.get_manifest(&entry).await }) + }) + .collect::>() + .try_fold(Vec::new(), |mut acc, manifest| async move { + acc.extend( + manifest? + .entries() + .iter() + .filter(|e| new_files_ref.contains(e.file_path()) && e.is_alive()) + .map(|e| e.file_path().to_string()), + ); + Ok(acc) + }) + .await?; + + if !referenced_files.is_empty() { + return Err(invalid_data!( + "Cannot add files that are already referenced by table, files: {}", + referenced_files.join(", ") + )); + } + + Ok(()) +} + +/// Builds a snapshot summary from an explicit set of file changes. +/// +/// User-supplied `properties` are applied first, then the computed metrics +/// overwrite any colliding keys. This matches iceberg-java +/// (`SnapshotProducer.summary`), where computed `added-*`/`total-*` values are +/// written after user properties so a user cannot shadow them with a bad (or +/// merely wrong) value that would corrupt the snapshot summary. +fn build_summary( + table: &Table, + operation: Operation, + mut properties: HashMap, + added_files: &[DataFile], + removed_files: &[DataFile], + truncate_full_table: bool, +) -> Result { + let metadata = table.metadata(); + let mut collector = SnapshotSummaryCollector::default(); + + collector.set_partition_summary_limit( + metadata + .properties() + .get(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT) + .and_then(|limit| limit.parse().ok()) + .unwrap_or(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT), + ); + + for data_file in added_files { + let partition_spec = metadata + .partition_spec_by_id(data_file.partition_spec_id) + .ok_or_else(|| { + invalid_data!("Unknown partition spec id {}", data_file.partition_spec_id) + })?; + collector.add_file( + data_file, + metadata.current_schema().clone(), + partition_spec.clone(), + ); + } + + let mut schemas_by_spec = HashMap::new(); + for data_file in removed_files { + let partition_spec = metadata + .partition_spec_by_id(data_file.partition_spec_id) + .ok_or_else(|| { + invalid_data!("Unknown partition spec id {}", data_file.partition_spec_id) + })?; + // A dropped source column keeps its type, such as a decimal's scale, only in an + // older schema. + let schema = schemas_by_spec + .entry(data_file.partition_spec_id) + .or_insert_with(|| { + metadata + .schemas_iter() + .filter(|schema| { + let mut source_ids = partition_spec.fields().iter().map(|f| f.source_id); + source_ids.all(|id| schema.field_by_id(id).is_some()) + }) + .max_by_key(|schema| schema.schema_id()) + .unwrap_or(metadata.current_schema()) + .clone() + }); + collector.remove_file(data_file, schema.clone(), partition_spec.clone()); + } + + properties.extend(collector.build()); + + let mut summary = update_snapshot_summaries( + Summary { + operation, + additional_properties: properties, + }, + metadata.current_snapshot().map(|s| s.summary()), + truncate_full_table, + )?; + // Recorded last so it also overrides a user-supplied `iceberg-version`. + set_iceberg_version(&mut summary.additional_properties); + Ok(summary) +} + +/// Everything one attempt needs to turn a resulting manifest set into a snapshot. +/// +/// The producer resolves identity (`snapshot_id`, `manifest_list_path`) and the +/// manifest set; the attempt-sensitive values (parent, sequence number, row-id +/// allocation) are read from `table` by [`write_snapshot_commit`]. +pub(crate) struct SnapshotCommitInput<'a> { + pub snapshot_id: i64, + pub manifest_list_path: String, + pub operation: Operation, + pub properties: HashMap, + pub added_files: &'a [DataFile], + /// Must match the data files marked Deleted by this commit, including their metrics. + pub removed_files: &'a [DataFile], + pub manifests: Vec, + /// Whether the summary resets table totals (a full-table overwrite). + pub truncate_full_table: bool, +} + +/// Writes the attempt's manifest list, assembles the snapshot, and returns the +/// resulting updates and requirements as an [`ActionCommit`]. +pub(crate) async fn write_snapshot_commit( + table: &Table, + input: SnapshotCommitInput<'_>, +) -> Result { + let SnapshotCommitInput { + snapshot_id, + manifest_list_path, + operation, + properties, + added_files, + removed_files, + manifests, + truncate_full_table, + } = input; + + debug_assert_eq!( + manifests + .iter() + .filter(|m| m.added_snapshot_id == snapshot_id && m.content == ManifestContentType::Data) + .filter_map(|m| m.deleted_files_count) + .map(u64::from) + .sum::(), + removed_files.len() as u64, + "deleted files must match this commit's manifest entries" + ); + + let metadata = table.metadata(); + let parent_snapshot_id = metadata.current_snapshot_id(); + let next_seq_num = metadata.next_sequence_number(); + let first_row_id = metadata.next_row_id(); + + let summary = build_summary( + table, + operation, + properties, + added_files, + removed_files, + truncate_full_table, + ) + .map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) + })?; + + let raw_output = table.file_io().new_output(manifest_list_path.clone())?; + let (writer, encrypted_output) = match table.encryption_manager() { + Some(em) => { + let encrypted_output = em.encrypt(raw_output); + ( + encrypted_output.writer().await?, + Some((em.clone(), encrypted_output)), + ) + } + None => (raw_output.writer().await?, None), + }; + + let mut manifest_list_writer = match metadata.format_version() { + FormatVersion::V1 => ManifestListWriter::v1(writer, snapshot_id, parent_snapshot_id), + FormatVersion::V2 => { + ManifestListWriter::v2(writer, snapshot_id, parent_snapshot_id, next_seq_num) + } + FormatVersion::V3 => ManifestListWriter::v3( + writer, + snapshot_id, + parent_snapshot_id, + next_seq_num, + Some(first_row_id), + ), + }; + + manifest_list_writer.add_manifests(manifests.into_iter())?; + let writer_next_row_id = manifest_list_writer.next_row_id(); + let file_metadata = manifest_list_writer.close().await?; + + let encryption_key_id = match encrypted_output { + Some((em, encrypted_output)) => Some( + em.encrypt_manifest_list_key_metadata( + &encrypted_output.key_metadata_with_saved_file_metadata(&file_metadata), + ) + .await?, + ), + None => None, + }; + + let snapshot = Snapshot::builder() + .with_manifest_list(manifest_list_path) + .with_snapshot_id(snapshot_id) + .with_parent_snapshot_id(parent_snapshot_id) + .with_sequence_number(next_seq_num) + .with_summary(summary) + .with_schema_id(metadata.current_schema_id()) + .with_encryption_key_id(encryption_key_id) + .with_timestamp_ms(chrono::Utc::now().timestamp_millis()); + + let snapshot = match writer_next_row_id { + Some(next_row_id) => snapshot + .with_row_range(first_row_id, next_row_id - first_row_id) + .build(), + None => snapshot.build(), + }; + + let mut updates: Vec = table + .encryption_manager() + .map(|em| { + em.with_encryption_keys(|keys| { + keys.values() + .filter(|k| metadata.encryption_key(k.key_id()).is_none()) + .map(|k| TableUpdate::AddEncryptionKey { + encryption_key: k.clone(), + }) + .collect() + }) + }) + .unwrap_or_default(); + + // `SetSnapshotRef` replaces the whole reference, so keep the retention of `main`, as Java does. + let main_retention = match metadata.refs.get(MAIN_BRANCH) { + Some(main) if main.is_branch() => main.retention.clone(), + _ => SnapshotRetention::branch(None, None, None), + }; + + updates.extend([ + TableUpdate::AddSnapshot { snapshot }, + TableUpdate::SetSnapshotRef { + ref_name: MAIN_BRANCH.to_string(), + reference: SnapshotReference::new(snapshot_id, main_retention), + }, + ]); + + let requirements = vec![ + TableRequirement::UuidMatch { + uuid: metadata.uuid(), + }, + TableRequirement::RefSnapshotIdMatch { + r#ref: MAIN_BRANCH.to_string(), + snapshot_id: parent_snapshot_id, + }, + ]; + + Ok(ActionCommit::new(updates, requirements)) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use uuid::Uuid; + + use super::simple::{AppendIntent, SimpleSnapshotProducer}; + use super::*; + use crate::ErrorKind; + use crate::spec::{ + DataContentType, DataFileBuilder, Literal, Manifest, NestedField, PartitionSpec, + PrimitiveType, Schema, SortOrder, TableMetadataBuilder, Transform, Type, + }; + use crate::transaction::Transaction; + use crate::transaction::tests::make_v2_minimal_table; + + fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(record_count) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap() + } + + fn deleted_file(table: &Table) -> DataFile { + data_file(table, "test/deleted.parquet", 2) + } + + /// Commits `data_files` to `table` through a fresh producer, returning the + /// table with that snapshot as current. + async fn append_to(table: &Table, data_files: Vec) -> Table { + let mut producer = SimpleSnapshotProducer::new(AppendIntent { + commit_uuid: None, + snapshot_properties: HashMap::new(), + data_files, + }); + let updates = producer.apply(table).await.unwrap().take_updates(); + Transaction::update_table_metadata(table.clone(), &updates).unwrap() + } + + #[tokio::test] + async fn test_commit_keeps_main_branch_retention() { + let table = make_v2_minimal_table(); + let table = append_to(&table, vec![data_file(&table, "test/1.parquet", 1)]).await; + let retention = + SnapshotRetention::branch(Some(9), Some(3_456_000_000), Some(2_592_000_000)); + let main = SnapshotReference::new( + table.metadata().current_snapshot_id().unwrap(), + retention.clone(), + ); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .set_ref(MAIN_BRANCH, main) + .unwrap() + .build() + .unwrap() + .metadata; + let table = table.with_metadata(Arc::new(metadata)); + + let table = append_to(&table, vec![data_file(&table, "test/2.parquet", 1)]).await; + + assert_eq!(table.metadata().refs[MAIN_BRANCH].retention, retention); + } + + #[tokio::test] + async fn test_validate_files_not_live_sees_files_added_by_rebase() { + // No current snapshot: nothing is live, so nothing is a duplicate. + let table = make_v2_minimal_table(); + let files = vec![data_file(&table, "test/dup.parquet", 1)]; + validate_files_not_live(&table, &files).await.unwrap(); + + // A concurrent writer commits the same file; validating against that + // base must detect it. + let rebased = append_to(&table, files.clone()).await; + let err = validate_files_not_live(&rebased, &files) + .await + .expect_err("file committed concurrently must be detected"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("test/dup.parquet")); + + // A different file is not flagged. + validate_files_not_live(&rebased, &[data_file(&table, "test/other.parquet", 1)]) + .await + .unwrap(); + } + + #[test] + fn test_metadata_paths_carry_identity_and_attempt() { + let table = make_v2_minimal_table(); + let uuid = Uuid::nil(); + + assert!( + manifest_path(&table, uuid, 3) + .unwrap() + .ends_with("00000000-0000-0000-0000-000000000000-m3.avro") + ); + assert!( + manifest_list_path(&table, 7, 2, uuid) + .unwrap() + .ends_with("snap-7-2-00000000-0000-0000-0000-000000000000.avro") + ); + } + + #[test] + fn test_summary_counts_deleted_data_files() { + let table = make_v2_minimal_table(); + let deleted = deleted_file(&table); + + let summary = build_summary( + &table, + Operation::Delete, + HashMap::new(), + &[], + &[deleted], + false, + ) + .unwrap(); + let properties = &summary.additional_properties; + + assert_eq!(properties.get("deleted-data-files").unwrap(), "1"); + assert_eq!(properties.get("deleted-records").unwrap(), "2"); + assert_eq!(properties.get("removed-files-size").unwrap(), "100"); + } + + #[test] + fn test_summary_keeps_decimal_scale_of_dropped_partition_source() { + let decimal = PrimitiveType::Decimal { + precision: 9, + scale: 2, + }; + let d = Arc::new(NestedField::required(1, "d", Type::Primitive(decimal))); + let x = |ty| Arc::new(NestedField::required(2, "x", Type::Primitive(ty))); + let schema = |fields: Vec<_>| Schema::builder().with_fields(fields).build().unwrap(); + let base = schema(vec![d.clone(), x(PrimitiveType::Int)]); + let spec = PartitionSpec::builder(base.clone()) + .add_partition_field("d", "d", Transform::Identity) + .unwrap() + .add_partition_field("x", "x", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let table = make_v2_minimal_table(); + let metadata = TableMetadataBuilder::new( + base, + spec, + SortOrder::unsorted_order(), + table.metadata().location().to_string(), + FormatVersion::V2, + HashMap::from([( + TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT.to_string(), + "10".to_string(), + )]), + ) + .unwrap() + .add_current_schema(schema(vec![d, x(PrimitiveType::Long)])) + .unwrap() + .add_default_partition_spec(Default::default()) + .unwrap() + .add_current_schema(schema(vec![x(PrimitiveType::Long)])) + .unwrap() + .build() + .unwrap() + .metadata; + let table = table.with_metadata(Arc::new(metadata)); + let deleted = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("test/deleted.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(2) + .partition_spec_id(0) + .partition(Struct::from_iter([ + Some(Literal::decimal(123)), + Some(Literal::int(5)), + ])) + .build() + .unwrap(); + + let summary = build_summary( + &table, + Operation::Delete, + HashMap::new(), + &[], + &[deleted], + false, + ) + .unwrap(); + + assert!( + summary + .additional_properties + .contains_key("partitions.d=1.23/x=5") + ); + } + + #[cfg(debug_assertions)] + #[tokio::test] + #[should_panic(expected = "deleted files must match this commit's manifest entries")] + async fn test_commit_rejects_unrecorded_deletions() { + let table = make_v2_minimal_table(); + let deleted = deleted_file(&table); + let snapshot_id = generate_snapshot_id(&table); + + write_snapshot_commit(&table, SnapshotCommitInput { + snapshot_id, + manifest_list_path: manifest_list_path(&table, snapshot_id, 0, Uuid::now_v7()).unwrap(), + operation: Operation::Delete, + properties: HashMap::new(), + added_files: &[], + removed_files: &[deleted], + manifests: vec![], + truncate_full_table: false, + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_new_manifest_writer_uses_the_given_schema_and_spec() { + let table = make_v2_minimal_table(); + let schema = Arc::new( + Schema::builder() + .with_schema_id(7) + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .with_spec_id(3) + .add_partition_field("a", "a", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + + let commit_uuid = Uuid::now_v7(); + let snapshot_id = generate_snapshot_id(&table); + let first = new_manifest_writer( + &table, + manifest_path(&table, commit_uuid, 0).unwrap(), + snapshot_id, + ManifestContentType::Data, + schema.clone(), + partition_spec.clone(), + ) + .unwrap() + .write_manifest_file() + .await + .unwrap(); + let second = new_manifest_writer( + &table, + manifest_path(&table, commit_uuid, 1).unwrap(), + snapshot_id, + ManifestContentType::Data, + schema.clone(), + partition_spec, + ) + .unwrap() + .write_manifest_file() + .await + .unwrap(); + + assert_ne!(first.manifest_path, second.manifest_path); + + let bytes = table + .file_io() + .new_input(&first.manifest_path) + .unwrap() + .read() + .await + .unwrap(); + let metadata = Manifest::parse_avro(&bytes).unwrap().into_parts().1; + + // Not the table's defaults, which are schema 0 and spec 0. + assert_eq!(metadata.schema_id(), 7); + assert_eq!(metadata.schema().as_ref(), schema.as_ref()); + assert_eq!(metadata.partition_spec().spec_id(), 3); + assert_eq!(first.partition_spec_id, 3); + } +} diff --git a/crates/iceberg/src/transaction/snapshot/simple.rs b/crates/iceberg/src/transaction/snapshot/simple.rs new file mode 100644 index 0000000000..91766e1a87 --- /dev/null +++ b/crates/iceberg/src/transaction/snapshot/simple.rs @@ -0,0 +1,614 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Persistent producer for append-shaped snapshot actions. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use uuid::Uuid; + +use super::{ + SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, + manifest_path, new_manifest_writer, validate_partition_value, write_snapshot_commit, +}; +use crate::error::{Result, invalid_data}; +use crate::spec::{ + DataFile, FormatVersion, ManifestContentType, ManifestEntry, ManifestFile, ManifestStatus, + Operation, PartitionSpecRef, +}; +use crate::table::Table; +use crate::transaction::ActionCommit; + +/// What an append-shaped action asks the producer to do. +/// +/// Fixed for one logical execution; the producer derives everything else from +/// it and the table of the current attempt. +pub(crate) struct AppendIntent { + /// Overrides the generated commit UUID. + pub commit_uuid: Option, + pub snapshot_properties: HashMap, + /// Data files to add. The action validates them before each attempt. + pub data_files: Vec, +} + +/// Retry-persistent state for an append-shaped action: added data files go +/// into new manifests, and the current snapshot's manifests are carried +/// forward untouched (no filtering, no merging). +/// +/// One producer lives for one logical execution of an action. It is driven +/// once per attempt via [`SimpleSnapshotProducer::apply`]: logical identity +/// (commit UUID, snapshot id) is fixed on first use, the manifests written for +/// the added files are reused across attempts while their semantic inputs are +/// unchanged, and every attempt writes a fresh manifest list under a distinct +/// path. +pub(crate) struct SimpleSnapshotProducer { + intent: AppendIntent, + /// Fixed when the producer is created, so every attempt shares one UUID. + commit_uuid: Uuid, + /// Resolved against the table on the first attempt, then stable. + snapshot_id: Option, + /// Incremented per attempt; distinguishes manifest-list paths. + attempt: u64, + /// Numbers this producer's manifests. Never reset, so reused manifests keep + /// their paths and new ones never collide with earlier attempts. + manifest_counter: u64, + /// Manifests written for the added files, reused across attempts. + added_manifests: Option, + /// Metadata files written by this producer (manifests and every attempt's + /// manifest list). + // TODO(#3417): cleanup is not implemented yet; terminal cleanup consumes + // this set. + #[allow(dead_code)] + owned_artifacts: HashSet, +} + +/// The manifests written for one append's added files. +/// +/// Reusable while the table's format version is unchanged. Per spec, sequence +/// number inheritance does not apply to v1 manifests, so one written before a +/// concurrent upgrade cannot carry the v2 sequence number a retry assigns and +/// must be rewritten. The schema is not a guard: the manifest's embedded schema +/// describes the files it was written for, which do not change. +struct AddedManifests { + manifests: Vec, + format_version: FormatVersion, +} + +struct DataFileGroup<'a> { + partition_spec: PartitionSpecRef, + data_files: Vec<&'a DataFile>, +} + +impl SimpleSnapshotProducer { + pub(crate) fn new(intent: AppendIntent) -> Self { + Self { + commit_uuid: intent.commit_uuid.unwrap_or_else(Uuid::now_v7), + intent, + snapshot_id: None, + attempt: 0, + manifest_counter: 0, + added_manifests: None, + owned_artifacts: HashSet::new(), + } + } + + /// Produces the resulting manifest set for the current + /// table and writes this attempt's snapshot. + pub(crate) async fn apply(&mut self, table: &Table) -> Result { + let snapshot_id = self.snapshot_id(table); + + let mut manifests = current_manifests(table).await?; + if !self.intent.data_files.is_empty() { + manifests.extend(self.added_manifests(table, snapshot_id).await?); + } + + let manifest_list_path = self.next_manifest_list_path(table, snapshot_id)?; + + write_snapshot_commit(table, SnapshotCommitInput { + snapshot_id, + manifest_list_path, + operation: Operation::Append, + properties: self.intent.snapshot_properties.clone(), + added_files: &self.intent.data_files, + removed_files: &[], + manifests, + truncate_full_table: false, + }) + .await + } + + fn snapshot_id(&mut self, table: &Table) -> i64 { + *self + .snapshot_id + .get_or_insert_with(|| generate_snapshot_id(table)) + } + + fn next_manifest_list_path(&mut self, table: &Table, snapshot_id: i64) -> Result { + let path = manifest_list_path(table, snapshot_id, self.attempt, self.commit_uuid)?; + self.attempt += 1; + self.owned_artifacts.insert(path.clone()); + Ok(path) + } + + /// Returns the manifests for the added files, writing them only if no + /// reusable set exists for the current format version. + async fn added_manifests( + &mut self, + table: &Table, + snapshot_id: i64, + ) -> Result> { + let format_version = table.metadata().format_version(); + + if let Some(cached) = &self.added_manifests + && cached.format_version == format_version + { + for manifest in &cached.manifests { + if table + .metadata() + .partition_spec_by_id(manifest.partition_spec_id) + .is_none() + { + return Err(invalid_data!( + "Cannot find partition spec {} for cached manifest {}", + manifest.partition_spec_id, + manifest.manifest_path + )); + } + } + return Ok(cached.manifests.clone()); + } + + let data_files_by_spec = validated_added_data_files(table, &self.intent.data_files)?; + + // TODO(#3499): roll a group over at the target manifest size, and + // write the resulting manifests concurrently. + let mut manifests = Vec::with_capacity(data_files_by_spec.len()); + for group in data_files_by_spec { + let path = manifest_path(table, self.commit_uuid, self.manifest_counter)?; + self.manifest_counter += 1; + // Record ownership before writing: a partial file is an orphan to clean up too. + self.owned_artifacts.insert(path.clone()); + manifests.push( + write_added_manifest( + table, + path, + snapshot_id, + group.partition_spec, + &group.data_files, + ) + .await?, + ); + } + + self.added_manifests = Some(AddedManifests { + manifests: manifests.clone(), + format_version, + }); + Ok(manifests) + } +} + +fn validated_added_data_files<'a>( + table: &Table, + data_files: &'a [DataFile], +) -> Result>> { + let metadata = table.metadata(); + // BTreeMap keeps added manifests in deterministic partition spec order. + let mut data_files_by_spec: BTreeMap> = BTreeMap::new(); + for data_file in data_files { + data_files_by_spec + .entry(data_file.partition_spec_id) + .or_default() + .push(data_file); + } + + data_files_by_spec + .into_iter() + .map(|(spec_id, data_files)| { + let partition_spec = metadata.partition_spec_by_id(spec_id).ok_or_else(|| { + invalid_data!( + "Cannot find partition spec {} for data file {}", + spec_id, + data_files[0].file_path() + ) + })?; + let partition_type = partition_spec.partition_type(metadata.current_schema())?; + for data_file in &data_files { + validate_partition_value(data_file.partition(), &partition_type).map_err( + |err| { + err.with_context("file_path", data_file.file_path()) + .with_context("partition_spec_id", spec_id.to_string()) + }, + )?; + } + Ok(DataFileGroup { + partition_spec: partition_spec.clone(), + data_files, + }) + }) + .collect() +} + +/// Writes one data manifest at `path` for `data_files`, all of which must +/// belong to `partition_spec`, and returns its manifest-list entry. +async fn write_added_manifest( + table: &Table, + path: String, + snapshot_id: i64, + partition_spec: PartitionSpecRef, + data_files: &[&DataFile], +) -> Result { + let metadata = table.metadata(); + let mut writer = new_manifest_writer( + table, + path, + snapshot_id, + ManifestContentType::Data, + metadata.current_schema().clone(), + partition_spec, + )?; + let format_version = metadata.format_version(); + for data_file in data_files { + let builder = ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file((*data_file).clone()); + let entry = if format_version == FormatVersion::V1 { + builder.snapshot_id(snapshot_id).build() + } else { + // For format version > 1, the snapshot id is inherited from the + // manifest list, so a rebase does not require rewriting the manifest. + builder.build() + }; + writer.add_entry(entry)?; + } + writer.write_manifest_file().await +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use uuid::Uuid; + + use super::{AppendIntent, SimpleSnapshotProducer}; + use crate::spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, FormatVersion, Literal, + NestedField, PrimitiveType, SnapshotRef, Struct, Type, UnboundPartitionSpec, + }; + use crate::table::Table; + use crate::transaction::tests::{ + make_v1_table, make_v2_minimal_table, with_bucket_partition_spec, + }; + use crate::{ErrorKind, TableUpdate}; + + fn data_file(table: &Table, path: &str) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap() + } + + fn producer(data_files: Vec) -> SimpleSnapshotProducer { + producer_with_uuid(None, data_files) + } + + fn producer_with_uuid( + commit_uuid: Option, + data_files: Vec, + ) -> SimpleSnapshotProducer { + SimpleSnapshotProducer::new(AppendIntent { + commit_uuid, + snapshot_properties: HashMap::new(), + data_files, + }) + } + + fn added_snapshot(commit: &mut crate::transaction::ActionCommit) -> SnapshotRef { + commit + .take_updates() + .into_iter() + .find_map(|u| match u { + TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot)), + _ => None, + }) + .expect("append should add a snapshot") + } + + /// The minimal table has no current snapshot, so `current_manifests` is + /// empty and the only manifest in the result is the one we wrote. + #[tokio::test] + async fn test_retry_reuses_identity_and_added_manifest() { + let table = make_v2_minimal_table(); + let requested_uuid = Uuid::now_v7(); + let mut producer = producer_with_uuid(Some(requested_uuid), vec![data_file( + &table, + "test/1.parquet", + )]); + + let mut first = producer.apply(&table).await.unwrap(); + let mut second = producer.apply(&table).await.unwrap(); + + let first_snapshot = added_snapshot(&mut first); + let second_snapshot = added_snapshot(&mut second); + + // Identity is stable across attempts. + assert_eq!(producer.commit_uuid, requested_uuid); + assert_eq!(first_snapshot.snapshot_id(), second_snapshot.snapshot_id()); + assert_eq!(producer.snapshot_id, Some(first_snapshot.snapshot_id())); + + // Each attempt writes its own manifest list under a distinct path. + assert_ne!( + first_snapshot.manifest_list(), + second_snapshot.manifest_list() + ); + assert!(first_snapshot.manifest_list().contains("-0-")); + assert!(second_snapshot.manifest_list().contains("-1-")); + + // Both manifest lists point at the same, single added manifest. + let first_list = table + .manifest_list_reader(&first_snapshot) + .load() + .await + .unwrap(); + let second_list = table + .manifest_list_reader(&second_snapshot) + .load() + .await + .unwrap(); + assert_eq!(first_list.entries().len(), 1); + assert_eq!(second_list.entries().len(), 1); + assert_eq!( + first_list.entries()[0].manifest_path, + second_list.entries()[0].manifest_path + ); + assert_eq!(producer.manifest_counter, 1); + + // One manifest plus two manifest lists are owned. + assert_eq!(producer.owned_artifacts.len(), 3); + } + + /// The manifest's embedded schema still describes the same files after a + /// concurrent schema change, so the manifest stays valid (as in Java). + #[tokio::test] + async fn test_added_manifest_is_reused_across_schema_change() { + let table = make_v2_minimal_table(); + let mut producer = producer(vec![data_file(&table, "test/1.parquet")]); + + let mut first = producer.apply(&table).await.unwrap(); + let first_snapshot = added_snapshot(&mut first); + + // Rebase onto a table whose current schema changed. + let new_schema = table + .metadata() + .current_schema() + .as_ref() + .clone() + .into_builder() + .with_fields([ + NestedField::optional(100, "added", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .add_current_schema(new_schema) + .unwrap() + .build() + .unwrap() + .metadata; + let rebased = table.clone().with_metadata(Arc::new(metadata)); + + let mut second = producer.apply(&rebased).await.unwrap(); + let second_snapshot = added_snapshot(&mut second); + + let first_list = table + .manifest_list_reader(&first_snapshot) + .load() + .await + .unwrap(); + let second_list = rebased + .manifest_list_reader(&second_snapshot) + .load() + .await + .unwrap(); + assert_eq!( + first_list.entries()[0].manifest_path, + second_list.entries()[0].manifest_path + ); + assert_eq!(producer.manifest_counter, 1); + } + + #[tokio::test] + async fn test_append_uses_file_spec_after_default_spec_changes() { + let table = make_v2_minimal_table(); + let mut state = producer(vec![data_file(&table, "test/1.parquet")]); + let first_snapshot = added_snapshot(&mut state.apply(&table).await.unwrap()); + let first_list = table + .manifest_list_reader(&first_snapshot) + .load() + .await + .unwrap(); + + let rebased = with_bucket_partition_spec(table); + let retried_snapshot = added_snapshot(&mut state.apply(&rebased).await.unwrap()); + let retried_list = rebased + .manifest_list_reader(&retried_snapshot) + .load() + .await + .unwrap(); + assert_eq!(first_list.entries(), retried_list.entries()); + assert_eq!(retried_snapshot.summary(), first_snapshot.summary()); + } + + #[tokio::test] + async fn test_append_rejects_unknown_partition_spec() { + let table = make_v2_minimal_table(); + let mut file = data_file(&table, "test/unknown.parquet"); + file.partition_spec_id = 123; + let mut state = producer(vec![data_file(&table, "test/valid.parquet"), file]); + let err = state + .apply(&table) + .await + .err() + .expect("unknown partition spec must be rejected"); + assert_eq!( + (err.kind(), err.message()), + ( + ErrorKind::DataInvalid, + "Cannot find partition spec 123 for data file test/unknown.parquet" + ) + ); + assert_eq!(state.manifest_counter, 0); + } + + #[tokio::test] + async fn test_retry_rejects_removed_partition_spec() { + let table = make_v2_minimal_table(); + let old_file = data_file(&table, "test/old.parquet"); + let table = with_bucket_partition_spec(table); + let mut new_file = data_file(&table, "test/new.parquet"); + new_file.partition = Struct::from_iter([Some(Literal::long(300)), Some(Literal::int(3))]); + let mut state = producer(vec![new_file, old_file]); + state.apply(&table).await.unwrap(); + let removed_manifest = &state.added_manifests.as_ref().unwrap().manifests[1]; + let expected_message = format!( + "Cannot find partition spec {} for cached manifest {}", + removed_manifest.partition_spec_id, removed_manifest.manifest_path + ); + + let metadata = table + .metadata() + .clone() + .into_builder(None) + .set_default_partition_spec(0) + .unwrap() + .remove_partition_specs(&[table.metadata().default_partition_spec_id()]) + .unwrap() + .build() + .unwrap() + .metadata; + let rebased = table.with_metadata(Arc::new(metadata)); + let err = state + .apply(&rebased) + .await + .err() + .expect("cached manifest's spec must still exist"); + assert_eq!( + (err.kind(), err.message()), + (ErrorKind::DataInvalid, expected_message.as_str()) + ); + } + + /// Per spec, a v1 manifest's entries always read with sequence number 0, so + /// a v1 manifest written before a concurrent format upgrade cannot carry the + /// sequence number the retry's v2 manifest list assigns. The upgrade must + /// invalidate the cached manifest. + #[tokio::test] + async fn test_added_manifests_are_rewritten_when_format_version_changes() { + let table = make_v1_table(); + let partitioned_file = data_file(&table, "test/1.parquet"); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .add_default_partition_spec(UnboundPartitionSpec::builder().build()) + .unwrap() + .build() + .unwrap() + .metadata; + let table = table.with_metadata(Arc::new(metadata)); + let mut unpartitioned_file = partitioned_file.clone(); + unpartitioned_file.file_path = "test/2.parquet".to_string(); + unpartitioned_file.partition_spec_id = table.metadata().default_partition_spec_id(); + unpartitioned_file.partition = Struct::empty(); + let mut producer = producer(vec![unpartitioned_file.clone(), partitioned_file.clone()]); + + let mut first = producer.apply(&table).await.unwrap(); + let first_snapshot = added_snapshot(&mut first); + + // A concurrent commit upgraded the table; schema and spec are unchanged. + let metadata = table + .metadata() + .clone() + .into_builder(None) + .upgrade_format_version(FormatVersion::V2) + .unwrap() + .build() + .unwrap() + .metadata; + let upgraded = table.clone().with_metadata(Arc::new(metadata)); + assert_eq!( + upgraded.metadata().current_schema_id(), + table.metadata().current_schema_id() + ); + + let mut second = producer.apply(&upgraded).await.unwrap(); + let second_snapshot = added_snapshot(&mut second); + + let first_list = table + .manifest_list_reader(&first_snapshot) + .load() + .await + .unwrap(); + let second_list = upgraded + .manifest_list_reader(&second_snapshot) + .load() + .await + .unwrap(); + let expected_spec_ids = vec![0, unpartitioned_file.partition_spec_id]; + for list in [&first_list, &second_list] { + let actual_spec_ids = list + .entries() + .iter() + .map(|manifest| manifest.partition_spec_id) + .collect::>(); + assert_eq!(actual_spec_ids, expected_spec_ids); + } + assert_eq!(producer.manifest_counter, 4); + + let mut actual_files = Vec::new(); + for (first_added, second_added) in first_list.entries().iter().zip(second_list.entries()) { + assert_ne!(first_added.manifest_path, second_added.manifest_path); + let manifest = upgraded.manifest_reader().read(second_added).await.unwrap(); + let files = manifest + .entries() + .iter() + .map(|entry| (entry.sequence_number(), entry.data_file().clone())) + .collect::>(); + actual_files.push((second_added.partition_spec_id, files)); + } + let sequence_number = Some(second_snapshot.sequence_number()); + let expected_files = vec![ + (0, vec![(sequence_number, partitioned_file)]), + (unpartitioned_file.partition_spec_id, vec![( + sequence_number, + unpartitioned_file, + )]), + ]; + assert_eq!(actual_files, expected_files); + assert_ne!(second_snapshot.sequence_number(), 0); + } +} diff --git a/crates/iceberg/src/transaction/sort_order.rs b/crates/iceberg/src/transaction/sort_order.rs index dfa1328c09..4543a1e2cc 100644 --- a/crates/iceberg/src/transaction/sort_order.rs +++ b/crates/iceberg/src/transaction/sort_order.rs @@ -15,15 +15,13 @@ // specific language governing permissions and limitations // under the License. -use std::sync::Arc; - use async_trait::async_trait; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::{NullOrder, SchemaRef, SortDirection, SortField, SortOrder, Transform}; use crate::table::Table; -use crate::transaction::{ActionCommit, TransactionAction}; -use crate::{Error, ErrorKind, TableRequirement, TableUpdate}; +use crate::transaction::{ActionCommit, StatelessAction}; +use crate::{TableRequirement, TableUpdate}; /// Represents a sort field whose construction and validation are deferred until commit time. /// This avoids the need to pass a `Table` reference into methods like `asc` or `desc` when @@ -31,22 +29,26 @@ use crate::{Error, ErrorKind, TableRequirement, TableUpdate}; #[derive(Debug, PartialEq, Eq, Clone)] struct PendingSortField { name: String, + transform: Transform, direction: SortDirection, null_order: NullOrder, } impl PendingSortField { fn to_sort_field(&self, schema: &SchemaRef) -> Result { - let field_id = schema.field_id_by_name(self.name.as_str()).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!("Cannot find field {} in table schema", self.name), - ) - })?; + let field_id = schema + .field_id_by_name(self.name.as_str()) + .ok_or_else(|| invalid_data!("Cannot find field {} in table schema", self.name))?; + + // This action intentionally refuses to author all-null (void) sort keys; + // this policy does not restrict reading or preserving existing metadata. + if matches!(self.transform, Transform::Unknown | Transform::Void) { + return Err(invalid_data!("Cannot sort by transform {}", self.transform)); + } Ok(SortField::builder() .source_id(field_id) - .transform(Transform::Identity) + .transform(self.transform) .direction(self.direction) .null_order(self.null_order) .build()) @@ -54,35 +56,77 @@ impl PendingSortField { } /// Transaction action for replacing sort order. +#[derive(Clone)] pub struct ReplaceSortOrderAction { pending_sort_fields: Vec, } impl ReplaceSortOrderAction { - pub fn new() -> Self { + pub(crate) fn new() -> Self { ReplaceSortOrderAction { pending_sort_fields: vec![], } } - /// Adds a field for sorting in ascending order. + /// Adds a field for sorting in ascending order, sorting by the column's raw value + /// (an identity transform). To sort by a transform of the column instead (e.g. + /// `bucket[N]`, `year`, `truncate[W]`), use [`Self::asc_with_transform`]. pub fn asc(self, name: &str, null_order: NullOrder) -> Self { - self.add_sort_field(name, SortDirection::Ascending, null_order) + self.asc_with_transform(name, Transform::Identity, null_order) } - /// Adds a field for sorting in descending order. + /// Adds a field for sorting in descending order, sorting by the column's raw value + /// (an identity transform). To sort by a transform of the column instead (e.g. + /// `bucket[N]`, `year`, `truncate[W]`), use [`Self::desc_with_transform`]. pub fn desc(self, name: &str, null_order: NullOrder) -> Self { - self.add_sort_field(name, SortDirection::Descending, null_order) + self.desc_with_transform(name, Transform::Identity, null_order) + } + + /// Adds a field for sorting in ascending order by a transform of the column's value + /// (e.g. `Transform::Bucket(16)`, `Transform::Year`, `Transform::Truncate(4)`). + /// + /// Whether the transform is valid for the column's type is checked at commit time, + /// once the table schema is available (mirroring Java's `SortOrder.Builder.build()`). + /// `Transform::Unknown` and `Transform::Void` are rejected at commit time with + /// [`ErrorKind::DataInvalid`](crate::ErrorKind::DataInvalid). Rejecting `Void` is an + /// intentional authoring policy for this action: an all-null sort key adds no + /// ordering information. + /// + /// Note: `Term` is currently a plain column reference. Once it becomes + /// transform-carrying (#2665), sort-order declaration is expected to converge on + /// Term-based `asc`/`desc` (as in Java's `SortOrderBuilder`), at which point the + /// `_with_transform` variants can be deprecated in its favor. + pub fn asc_with_transform( + self, + name: &str, + transform: Transform, + null_order: NullOrder, + ) -> Self { + self.add_sort_field(name, transform, SortDirection::Ascending, null_order) + } + + /// Adds a field for sorting in descending order by a transform of the column's value. + /// + /// See [`Self::asc_with_transform`] for validation and future API direction. + pub fn desc_with_transform( + self, + name: &str, + transform: Transform, + null_order: NullOrder, + ) -> Self { + self.add_sort_field(name, transform, SortDirection::Descending, null_order) } fn add_sort_field( mut self, name: &str, + transform: Transform, sort_direction: SortDirection, null_order: NullOrder, ) -> Self { self.pending_sort_fields.push(PendingSortField { name: name.to_string(), + transform, direction: sort_direction, null_order, }); @@ -91,15 +135,9 @@ impl ReplaceSortOrderAction { } } -impl Default for ReplaceSortOrderAction { - fn default() -> Self { - Self::new() - } -} - #[async_trait] -impl TransactionAction for ReplaceSortOrderAction { - async fn commit(self: Arc, table: &Table) -> Result { +impl StatelessAction for ReplaceSortOrderAction { + async fn commit(&self, table: &Table) -> Result { let current_schema = table.metadata().current_schema(); let sort_fields: Result> = self .pending_sort_fields @@ -135,10 +173,15 @@ impl TransactionAction for ReplaceSortOrderAction { mod tests { use as_any::Downcast; - use crate::spec::{NullOrder, SortDirection}; + use crate::catalog::Catalog; + use crate::memory::tests::new_memory_catalog; + use crate::spec::{NullOrder, SortDirection, Transform}; + use crate::test_utils::commit_with_fresh_state; + use crate::transaction::action::ActionEntry; use crate::transaction::sort_order::{PendingSortField, ReplaceSortOrderAction}; - use crate::transaction::tests::make_v2_table; + use crate::transaction::tests::{make_v2_table, make_v3_minimal_table_in_catalog}; use crate::transaction::{ApplyTransactionAction, Transaction}; + use crate::{ErrorKind, TableUpdate}; #[test] fn test_replace_sort_order() { @@ -153,20 +196,147 @@ mod tests { .unwrap(); let replace_sort_order = (*tx.actions[0]) - .downcast_ref::() + .downcast_ref::>() + .unwrap() + .action(); + + assert_eq!(replace_sort_order.pending_sort_fields, vec![ + PendingSortField { + name: String::from("x"), + transform: Transform::Identity, + direction: SortDirection::Ascending, + null_order: NullOrder::First, + }, + PendingSortField { + name: String::from("y"), + transform: Transform::Identity, + direction: SortDirection::Descending, + null_order: NullOrder::Last, + } + ]); + } + + #[test] + fn test_replace_sort_order_with_transform() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + let replace_sort_order = tx.replace_sort_order(); + + let tx = replace_sort_order + .asc_with_transform("x", Transform::Bucket(16), NullOrder::First) + .desc_with_transform("y", Transform::Truncate(4), NullOrder::Last) + .apply(tx) .unwrap(); + let replace_sort_order = (*tx.actions[0]) + .downcast_ref::>() + .unwrap() + .action(); + assert_eq!(replace_sort_order.pending_sort_fields, vec![ PendingSortField { name: String::from("x"), + transform: Transform::Bucket(16), direction: SortDirection::Ascending, null_order: NullOrder::First, }, PendingSortField { name: String::from("y"), + transform: Transform::Truncate(4), direction: SortDirection::Descending, null_order: NullOrder::Last, } ]); } + + #[tokio::test] + async fn test_replace_sort_order_with_transform_commits() { + let table = make_v2_table(); + let action = ReplaceSortOrderAction::new().asc_with_transform( + "x", + Transform::Bucket(16), + NullOrder::First, + ); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + assert_eq!(updates.len(), 2); + let TableUpdate::AddSortOrder { sort_order } = &updates[0] else { + panic!("expected AddSortOrder, got {:?}", updates[0]); + }; + assert_eq!(sort_order.fields.len(), 1); + assert_eq!(sort_order.fields[0].transform, Transform::Bucket(16)); + } + + #[tokio::test] + async fn test_replace_sort_order_rejects_incompatible_transform() { + let table = make_v2_table(); + // `x` is a `long` column; `year` only accepts date/timestamp types. + let action = ReplaceSortOrderAction::new().asc_with_transform( + "x", + Transform::Year, + NullOrder::First, + ); + + let err = commit_with_fresh_state(action, &table) + .await + .err() + .expect("year transform on a long column should be rejected"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_replace_sort_order_rejects_unknown_and_void_transforms() { + let table = make_v2_table(); + for transform in [Transform::Unknown, Transform::Void] { + for action in [ + ReplaceSortOrderAction::new().asc_with_transform("x", transform, NullOrder::First), + ReplaceSortOrderAction::new().desc_with_transform("x", transform, NullOrder::Last), + ] { + let err = commit_with_fresh_state(action, &table) + .await + .err() + .expect("unknown and void sort transforms should be rejected"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!( + err.message(), + format!("Cannot sort by transform {transform}") + ); + } + } + } + + #[tokio::test] + async fn test_sort_order_transform_survives_metadata_json_round_trip() { + // Commit a transform-based sort order through a real catalog: the memory + // catalog serializes the updated table metadata to a metadata.json file + // (`TableMetadata::write_to`), and `load_table` reads that file back and + // parses it (`TableMetadata::read_from`). This exercises the full + // JSON round-trip of the transform (e.g. `"bucket[16]"`), not just the + // in-memory `TableUpdate`. + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let tx = Transaction::new(&table); + let tx = tx + .replace_sort_order() + .asc_with_transform("x", Transform::Bucket(16), NullOrder::First) + .desc_with_transform("y", Transform::Truncate(4), NullOrder::Last) + .apply(tx) + .unwrap(); + let committed = tx.commit(&catalog).await.unwrap(); + + // Reload from the catalog: this parses the metadata.json written above. + let reloaded = catalog.load_table(committed.identifier()).await.unwrap(); + let sort_order = reloaded.metadata().default_sort_order(); + + assert_eq!(sort_order.fields.len(), 2); + assert_eq!(sort_order.fields[0].transform, Transform::Bucket(16)); + assert_eq!(sort_order.fields[0].direction, SortDirection::Ascending); + assert_eq!(sort_order.fields[0].null_order, NullOrder::First); + assert_eq!(sort_order.fields[1].transform, Transform::Truncate(4)); + assert_eq!(sort_order.fields[1].direction, SortDirection::Descending); + assert_eq!(sort_order.fields[1].null_order, NullOrder::Last); + } } diff --git a/crates/iceberg/src/transaction/update_location.rs b/crates/iceberg/src/transaction/update_location.rs index 0c32c75355..147bd6062d 100644 --- a/crates/iceberg/src/transaction/update_location.rs +++ b/crates/iceberg/src/transaction/update_location.rs @@ -15,26 +15,26 @@ // specific language governing permissions and limitations // under the License. -use std::sync::Arc; - use async_trait::async_trait; +use crate::error::invalid_data; use crate::table::Table; -use crate::transaction::action::{ActionCommit, TransactionAction}; -use crate::{Error, ErrorKind, Result, TableUpdate}; +use crate::transaction::action::{ActionCommit, StatelessAction}; +use crate::{Result, TableUpdate}; /// A transaction action that sets or updates the location of a table. /// /// This action is used to explicitly set a new metadata location during a transaction, /// typically as part of advanced commit or recovery flows. The location is optional until -/// explicitly set via [`set_location`]. +/// explicitly set via [`UpdateLocationAction::set_location`]. +#[derive(Clone)] pub struct UpdateLocationAction { location: Option, } impl UpdateLocationAction { /// Creates a new [`UpdateLocationAction`] with no location set. - pub fn new() -> Self { + pub(crate) fn new() -> Self { UpdateLocationAction { location: None } } @@ -53,22 +53,17 @@ impl UpdateLocationAction { } } -impl Default for UpdateLocationAction { - fn default() -> Self { - Self::new() - } -} - #[async_trait] -impl TransactionAction for UpdateLocationAction { - async fn commit(self: Arc, _table: &Table) -> Result { +impl StatelessAction for UpdateLocationAction { + async fn commit(&self, _table: &Table) -> Result { let updates: Vec; - if let Some(location) = self.location.clone() { - updates = vec![TableUpdate::SetLocation { location }]; + if let Some(location) = self.location.as_deref() { + updates = vec![TableUpdate::SetLocation { + location: location.to_owned(), + }]; } else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Location is not set for UpdateLocationAction!", + return Err(invalid_data!( + "Location is not set for UpdateLocationAction!" )); } @@ -81,7 +76,7 @@ mod tests { use as_any::Downcast; use crate::transaction::Transaction; - use crate::transaction::action::ApplyTransactionAction; + use crate::transaction::action::{ActionEntry, ApplyTransactionAction}; use crate::transaction::tests::make_v2_table; use crate::transaction::update_location::UpdateLocationAction; @@ -98,8 +93,9 @@ mod tests { assert_eq!(tx.actions.len(), 1); let action = (*tx.actions[0]) - .downcast_ref::() - .unwrap(); + .downcast_ref::>() + .unwrap() + .action(); assert_eq!( action.location, diff --git a/crates/iceberg/src/transaction/update_properties.rs b/crates/iceberg/src/transaction/update_properties.rs index 825d6bec8d..3050805d50 100644 --- a/crates/iceberg/src/transaction/update_properties.rs +++ b/crates/iceberg/src/transaction/update_properties.rs @@ -16,12 +16,11 @@ // under the License. use std::collections::{HashMap, HashSet}; -use std::sync::Arc; use async_trait::async_trait; use crate::table::Table; -use crate::transaction::action::{ActionCommit, TransactionAction}; +use crate::transaction::action::{ActionCommit, StatelessAction}; use crate::{Error, ErrorKind, Result, TableUpdate}; /// A transactional action that updates or removes table properties @@ -30,6 +29,7 @@ use crate::{Error, ErrorKind, Result, TableUpdate}; /// properties during a transaction. It supports setting new values for existing keys /// or adding new keys, as well as removing existing keys. Each key can only be updated /// or removed in a single action, not both. +#[derive(Clone)] pub struct UpdatePropertiesAction { updates: HashMap, removals: HashSet, @@ -37,7 +37,7 @@ pub struct UpdatePropertiesAction { impl UpdatePropertiesAction { /// Creates a new [`UpdatePropertiesAction`] with no updates or removals. - pub fn new() -> Self { + pub(crate) fn new() -> Self { UpdatePropertiesAction { updates: HashMap::default(), removals: HashSet::default(), @@ -74,23 +74,14 @@ impl UpdatePropertiesAction { } } -impl Default for UpdatePropertiesAction { - fn default() -> Self { - Self::new() - } -} - #[async_trait] -impl TransactionAction for UpdatePropertiesAction { - async fn commit(self: Arc, _table: &Table) -> Result { +impl StatelessAction for UpdatePropertiesAction { + async fn commit(&self, _table: &Table) -> Result { if let Some(overlapping_key) = self.removals.iter().find(|k| self.updates.contains_key(*k)) { return Err(Error::new( ErrorKind::PreconditionFailed, - format!( - "Key {} is present in both removal set and update set", - overlapping_key - ), + format!("Key {overlapping_key} is present in both removal set and update set"), )); } @@ -114,7 +105,7 @@ mod tests { use as_any::Downcast; use crate::transaction::Transaction; - use crate::transaction::action::ApplyTransactionAction; + use crate::transaction::action::{ActionEntry, ApplyTransactionAction}; use crate::transaction::tests::make_v2_table; use crate::transaction::update_properties::UpdatePropertiesAction; @@ -132,8 +123,9 @@ mod tests { assert_eq!(tx.actions.len(), 1); let action = (*tx.actions[0]) - .downcast_ref::() - .unwrap(); + .downcast_ref::>() + .unwrap() + .action(); assert_eq!( action.updates, HashMap::from([("a".to_string(), "b".to_string())]) diff --git a/crates/iceberg/src/transaction/update_schema.rs b/crates/iceberg/src/transaction/update_schema.rs new file mode 100644 index 0000000000..b603d0e5f7 --- /dev/null +++ b/crates/iceberg/src/transaction/update_schema.rs @@ -0,0 +1,1551 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use typed_builder::TypedBuilder; + +use crate::spec::{ + LIST_FIELD_NAME, ListType, Literal, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, + MappedField, NameMapping, NestedField, NestedFieldRef, SCHEMA_NAME_DELIMITER, Schema, + StructType, TableProperties, Type, +}; +use crate::table::Table; +use crate::transaction::action::{ActionCommit, StatelessAction}; +use crate::{Error, ErrorKind, Result, TableRequirement, TableUpdate}; + +// Default ID for a new column. This will be re-assigned to a fresh ID at commit time. +const DEFAULT_FIELD_ID: i32 = 0; + +/// Declarative specification for adding a column in an [`UpdateSchemaAction`]. +/// +/// Use helper constructors such as [`AddColumn::optional`] and [`AddColumn::required`], +/// optionally combined with the builder's `parent` and `doc` setters via +/// [`AddColumn::builder`], then pass the value to [`UpdateSchemaAction::add_column`]. +#[derive(Clone, TypedBuilder)] +pub struct AddColumn { + #[builder(default = None, setter(strip_option, into))] + parent: Option, + #[builder(setter(into))] + name: String, + #[builder(default = false)] + required: bool, + field_type: Type, + #[builder(default = None, setter(strip_option, into))] + doc: Option, + #[builder(default = None, setter(strip_option))] + initial_default: Option, + #[builder(default = None, setter(strip_option))] + write_default: Option, +} + +impl AddColumn { + /// Create a root-level optional column specification. + pub fn optional(name: impl ToString, field_type: Type) -> Self { + Self::builder() + .name(name.to_string()) + .field_type(field_type) + .required(false) + .build() + } + + /// Create a root-level required column specification. + pub fn required(name: impl ToString, field_type: Type, initial_default: Literal) -> Self { + Self::builder() + .name(name.to_string()) + .field_type(field_type) + .required(true) + .initial_default(initial_default.clone()) + .write_default(initial_default) + .build() + } + + fn to_nested_field(&self) -> NestedFieldRef { + let mut field = NestedField::new( + DEFAULT_FIELD_ID, + self.name.clone(), + self.field_type.clone(), + self.required, + ); + + field.doc = self.doc.clone(); + field.initial_default = self.initial_default.clone(); + field.write_default = self.write_default.clone(); + Arc::new(field) + } +} + +/// Schema evolution API modeled after the Java `SchemaUpdate` implementation. +/// +/// This action accumulates schema modifications (column additions and deletions) +/// via builder methods. At commit time, it validates all operations against the +/// current table schema, auto-assigns field IDs from `table.metadata().last_column_id()`, +/// builds a new schema, and emits `AddSchema` + `SetCurrentSchema` updates with a +/// `CurrentSchemaIdMatch` requirement. +/// +/// # Example +/// +/// ```ignore +/// let tx = Transaction::new(&table); +/// let action = tx.update_schema() +/// .add_column(AddColumn::optional("new_col", Type::Primitive(PrimitiveType::Int))) +/// .add_column( +/// AddColumn::optional("email", Type::Primitive(PrimitiveType::String)) +/// .with_parent("person") +/// ) +/// .delete_column("old_col"); +/// let tx = action.apply(tx).unwrap(); +/// let table = tx.commit(&catalog).await.unwrap(); +/// ``` +#[derive(Clone)] +pub struct UpdateSchemaAction { + additions: Vec, + deletes: Vec, +} + +impl UpdateSchemaAction { + /// Creates a new empty `UpdateSchemaAction`. + pub(crate) fn new() -> Self { + Self { + additions: Vec::new(), + deletes: Vec::new(), + } + } + + // --- Root-level additions --- + + /// Add a column to the table schema. + /// + /// To add a root-level column, leave `AddColumn::parent` as `None`. + /// For nested additions, set a parent path. + /// If the parent resolves to a map/list, the column is added to map value/list element. + pub fn add_column(mut self, add_column: AddColumn) -> Self { + self.additions.push(add_column); + self + } + + // --- Other builder methods --- + + /// Record a column deletion by name. + /// + /// At commit time, the column must exist in the current schema. + pub fn delete_column(mut self, name: impl ToString) -> Self { + self.deletes.push(name.to_string()); + self + } +} + +// --------------------------------------------------------------------------- +// ID assignment helpers +// --------------------------------------------------------------------------- + +/// Recursively assign fresh field IDs to a `NestedField` and all its nested sub-fields. +/// +/// This follows the same recursive pattern as `ReassignFieldIds::reassign_ids_visit_type` +/// from `crate::spec::schema::id_reassigner`, but operates on new fields with placeholder +/// IDs rather than reassigning an existing schema. `ReassignFieldIds` cannot be used +/// directly here because it rejects duplicate old IDs (all new fields share placeholder +/// ID `DEFAULT_FIELD_ID`). +fn assign_fresh_ids(field: &NestedField, next_id: &mut i32) -> NestedFieldRef { + *next_id += 1; + let new_id = *next_id; + let new_type = assign_fresh_ids_to_type(&field.field_type, next_id); + + Arc::new(NestedField { + id: new_id, + name: field.name.clone(), + required: field.required, + field_type: Box::new(new_type), + doc: field.doc.clone(), + initial_default: field.initial_default.clone(), + write_default: field.write_default.clone(), + }) +} + +/// Recursively assign fresh field IDs to all nested fields within a `Type`. +fn assign_fresh_ids_to_type(field_type: &Type, next_id: &mut i32) -> Type { + match field_type { + Type::Primitive(_) => field_type.clone(), + // Variant carries no nested fields, so there is nothing to reassign + // (matches id_reassigner.rs). + Type::Variant(v) => Type::Variant(*v), + Type::Struct(struct_type) => { + let new_fields: Vec = struct_type + .fields() + .iter() + .map(|f| assign_fresh_ids(f, next_id)) + .collect(); + Type::Struct(StructType::new(new_fields)) + } + Type::List(list_type) => { + let new_element = assign_fresh_ids(&list_type.element_field, next_id); + Type::List(ListType { + element_field: new_element, + }) + } + Type::Map(map_type) => { + let new_key = assign_fresh_ids(&map_type.key_field, next_id); + let new_value = assign_fresh_ids(&map_type.value_field, next_id); + Type::Map(MapType { + key_field: new_key, + value_field: new_value, + }) + } + } +} + +// --------------------------------------------------------------------------- +// Parent path resolution +// --------------------------------------------------------------------------- + +/// Resolve a parent path to the target struct's parent field ID and a reference +/// to its `StructType`. +/// +/// If the parent is a map, navigates to the value field. If a list, navigates to +/// the element field. The final target must be a struct type. +fn resolve_parent_target<'a>( + base_schema: &'a Schema, + parent: &str, +) -> Result<(i32, &'a StructType)> { + base_schema + .field_by_name(parent) + .ok_or_else(|| { + Error::new( + ErrorKind::PreconditionFailed, + format!("Cannot add column: parent '{parent}' not found"), + ) + }) + .and_then(|parent_field| match parent_field.field_type.as_ref() { + Type::Struct(s) => Ok((parent_field.id, s)), + Type::Map(m) => match m.value_field.field_type.as_ref() { + Type::Struct(s) => Ok((m.value_field.id, s)), + _ => Err(Error::new( + ErrorKind::PreconditionFailed, + format!("Cannot add column: map value of '{parent}' is not a struct"), + )), + }, + Type::List(l) => match l.element_field.field_type.as_ref() { + Type::Struct(s) => Ok((l.element_field.id, s)), + _ => Err(Error::new( + ErrorKind::PreconditionFailed, + format!("Cannot add column: list element of '{parent}' is not a struct"), + )), + }, + _ => Err(Error::new( + ErrorKind::PreconditionFailed, + format!("Cannot add column: parent '{parent}' is not a struct, map, or list"), + )), + }) +} + +// --------------------------------------------------------------------------- +// Schema tree rebuild +// --------------------------------------------------------------------------- + +/// Rebuild a slice of fields, applying deletions and additions at every level, +/// plus any additions keyed by `parent_id` (`None` represents the table root). +fn rebuild_fields( + fields: &[NestedFieldRef], + adds: &HashMap, Vec>, + delete_ids: &HashSet, + parent_id: Option, +) -> Vec { + fields + .iter() + .filter(|f| !delete_ids.contains(&f.id)) + .map(|f| rebuild_field(f, adds, delete_ids)) + .chain(adds.get(&parent_id).into_iter().flatten().cloned()) + .collect() +} + +/// Recursively rebuild a single field. If the field (or any descendant) is a struct +/// that has pending additions, those additions are appended to the struct's fields. +/// Fields whose IDs appear in `delete_ids` are filtered out at every struct level. +fn rebuild_field( + field: &NestedFieldRef, + adds: &HashMap, Vec>, + delete_ids: &HashSet, +) -> NestedFieldRef { + match field.field_type.as_ref() { + Type::Primitive(_) | Type::Variant(_) => field.clone(), + Type::Struct(s) => { + let new_fields = rebuild_fields(s.fields(), adds, delete_ids, Some(field.id)); + Arc::new(NestedField { + id: field.id, + name: field.name.clone(), + required: field.required, + field_type: Box::new(Type::Struct(StructType::new(new_fields))), + doc: field.doc.clone(), + initial_default: field.initial_default.clone(), + write_default: field.write_default.clone(), + }) + } + Type::List(l) => { + let new_element = rebuild_field(&l.element_field, adds, delete_ids); + Arc::new(NestedField { + id: field.id, + name: field.name.clone(), + required: field.required, + field_type: Box::new(Type::List(ListType { + element_field: new_element, + })), + doc: field.doc.clone(), + initial_default: field.initial_default.clone(), + write_default: field.write_default.clone(), + }) + } + Type::Map(m) => { + let new_key = rebuild_field(&m.key_field, adds, delete_ids); + let new_value = rebuild_field(&m.value_field, adds, delete_ids); + Arc::new(NestedField { + id: field.id, + name: field.name.clone(), + required: field.required, + field_type: Box::new(Type::Map(MapType { + key_field: new_key, + value_field: new_value, + })), + doc: field.doc.clone(), + initial_default: field.initial_default.clone(), + write_default: field.write_default.clone(), + }) + } + } +} + +// --------------------------------------------------------------------------- +// Name mapping update +// --------------------------------------------------------------------------- + +/// Add entries for added columns to a name mapping, as Java's `MappingUtil.update` does. +/// +/// Each added field gets an entry, with entries for its nested fields, appended under its +/// parent's entry. An existing entry in the same struct loses the added name, so a dropped +/// column no longer claims it. Entries of deleted columns are kept. +fn add_columns_to_name_mapping( + mapping: &NameMapping, + adds: &HashMap, Vec>, +) -> NameMapping { + let root_adds = adds.get(&None).map_or(&[][..], Vec::as_slice); + NameMapping::new(add_columns_to_mapped_fields( + mapping.fields().iter(), + root_adds, + adds, + )) +} + +fn add_columns_to_mapped_fields<'a>( + fields: impl Iterator, + added: &[NestedFieldRef], + adds: &HashMap, Vec>, +) -> Vec { + let added_ids: HashMap<&str, i32> = added + .iter() + .map(|add| (add.name.as_str(), add.id)) + .collect(); + fields + .map(|field| { + let nested_added = field + .field_id() + .and_then(|id| adds.get(&Some(id))) + .map_or(&[][..], Vec::as_slice); + let nested = add_columns_to_mapped_fields( + field.fields().iter().map(Arc::as_ref), + nested_added, + adds, + ); + let names = field + .names() + .iter() + .filter(|name| { + added_ids + .get(name.as_str()) + .is_none_or(|id| Some(*id) == field.field_id()) + }) + .cloned() + .collect(); + MappedField::new(field.field_id(), names, nested) + }) + .chain(added.iter().map(|add| create_mapped_field(add, &add.name))) + .collect() +} + +/// Create a mapping entry for a field and its nested fields, as Java's `MappingUtil.create`. +fn create_mapped_field(field: &NestedField, name: &str) -> MappedField { + let nested = match field.field_type.as_ref() { + Type::Primitive(_) | Type::Variant(_) => vec![], + Type::Struct(s) => s + .fields() + .iter() + .map(|f| create_mapped_field(f, &f.name)) + .collect(), + Type::List(l) => vec![create_mapped_field(&l.element_field, LIST_FIELD_NAME)], + Type::Map(m) => vec![ + create_mapped_field(&m.key_field, MAP_KEY_FIELD_NAME), + create_mapped_field(&m.value_field, MAP_VALUE_FIELD_NAME), + ], + }; + MappedField::new(Some(field.id), vec![name.to_string()], nested) +} + +// --------------------------------------------------------------------------- +// StatelessAction implementation +// --------------------------------------------------------------------------- + +#[async_trait] +impl StatelessAction for UpdateSchemaAction { + async fn commit(&self, table: &Table) -> Result { + let base_schema = table.metadata().current_schema(); + let mut last_column_id = table.metadata().last_column_id(); + + // --- 1. Validate deletes --- + let delete_ids = self + .deletes + .iter() + .map(|name: &String| { + base_schema + .field_by_name(name) + .ok_or_else(|| { + Error::new( + ErrorKind::PreconditionFailed, + format!("Cannot delete missing column: {name}"), + ) + }) + .and_then(|field| { + match base_schema + .identifier_field_ids() + .find(|id| *id == field.id) + { + Some(_) => Err(Error::new( + ErrorKind::PreconditionFailed, + format!("Cannot delete identifier field: {name}"), + )), + None => Ok(field.id), + } + }) + }) + .collect::>>()?; + + // --- 2. Resolve parents, validate additions, assign IDs, and group by parent ID --- + // We assign IDs inline (before grouping) to preserve the caller's insertion order, + // since HashMap iteration order is non-deterministic. + let mut additions_by_parent: HashMap, Vec> = HashMap::new(); + + for add in &self.additions { + let pending_field = add.to_nested_field(); + + // Check that name does not contain `SCHEMA_NAME_DELIMITER`. + if pending_field.name.contains(SCHEMA_NAME_DELIMITER) { + return Err(Error::new( + ErrorKind::PreconditionFailed, + format!( + "Cannot add column with ambiguous name: {}. Use `AddColumn::with_parent` to add a column to a nested struct.", + pending_field.name + ), + )); + } + + // Required columns without an initial default need allow_incompatible_changes. + if pending_field.required && pending_field.initial_default.is_none() { + return Err(Error::new( + ErrorKind::PreconditionFailed, + format!( + "Incompatible change: cannot add required column without an initial default: {}", + pending_field.name + ), + )); + } + + let parent_id = match &add.parent { + None => { + // Root-level: check name conflict against root-level fields. + if let Some(existing) = base_schema.field_by_name(&pending_field.name) + && !delete_ids.contains(&existing.id) + { + return Err(Error::new( + ErrorKind::PreconditionFailed, + format!( + "Cannot add column, name already exists: {}", + pending_field.name + ), + )); + } + None + } + Some(parent_path) => { + // Nested: resolve parent, check name conflict within parent struct. + let (resolved_parent_id, parent_struct) = + resolve_parent_target(base_schema, parent_path)?; + + if parent_struct.fields().iter().any(|f| { + f.name == pending_field.name + && !delete_ids.contains(&f.id) + && !delete_ids.contains(&resolved_parent_id) + }) { + return Err(Error::new( + ErrorKind::PreconditionFailed, + format!( + "Cannot add column, name already exists in '{}': {}", + parent_path, pending_field.name + ), + )); + } + + Some(resolved_parent_id) + } + }; + + // Assign fresh IDs immediately, preserving insertion order. + let field = assign_fresh_ids(&pending_field, &mut last_column_id); + + additions_by_parent + .entry(parent_id) + .or_default() + .push(field); + } + + // --- 4. Rebuild the schema tree with additions and deletions --- + let new_fields = rebuild_fields( + base_schema.as_struct().fields(), + &additions_by_parent, + &delete_ids, + None, + ); + + // --- 5. Build the new schema --- + let schema = Schema::builder() + .with_fields(new_fields) + .with_identifier_field_ids(base_schema.identifier_field_ids()) + .build()?; + + // Additions under a deleted ancestor are not in the new schema and their IDs can be + // reassigned later, so they must not get name mapping entries. + additions_by_parent + .retain(|parent_id, _| parent_id.is_none_or(|id| schema.field_by_id(id).is_some())); + + let mut updates = vec![ + TableUpdate::AddSchema { schema }, + TableUpdate::SetCurrentSchema { schema_id: -1 }, + ]; + + // --- 6. Add the new columns to the name mapping, if the table has one --- + if !additions_by_parent.is_empty() + && let Some(mapping_json) = table + .metadata() + .properties() + .get(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING) + { + // Like Java, a mapping that can't be updated is logged and does not fail the update. + match mapping_json.parse::().and_then(|mapping| { + serde_json::to_string(&add_columns_to_name_mapping(&mapping, &additions_by_parent)) + .map_err(Error::from) + }) { + Ok(updated_json) => updates.push(TableUpdate::SetProperties { + updates: HashMap::from([( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(), + updated_json, + )]), + }), + Err(error) => { + tracing::warn!("Failed to update name mapping {mapping_json}: {error}") + } + } + } + + let requirements = vec![TableRequirement::CurrentSchemaIdMatch { + current_schema_id: base_schema.schema_id(), + }]; + + Ok(ActionCommit::new(updates, requirements)) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::io::BufReader; + use std::sync::Arc; + + use as_any::Downcast; + + use crate::spec::{ + DEFAULT_SCHEMA_ID, ListType, Literal, MapType, MappedField, NameMapping, NestedField, + PrimitiveType, StructType, TableMetadata, TableProperties, Type, VariantType, + }; + use crate::table::Table; + use crate::test_utils::commit_with_fresh_state; + use crate::transaction::Transaction; + use crate::transaction::action::{ActionEntry, ApplyTransactionAction}; + use crate::transaction::tests::make_v2_table; + use crate::transaction::update_schema::{AddColumn, DEFAULT_FIELD_ID, UpdateSchemaAction}; + use crate::{ErrorKind, TableIdent, TableRequirement, TableUpdate}; + + // The V2 test table has: + // last_column_id: 3 + // current schema (id=1): x(1, req, long), y(2, req, long), z(3, req, long) + // identifier_field_ids: [1, 2] + + /// Build a V2 test table that includes nested types: + /// + /// last_column_id: 14 + /// current schema (id=0): + /// x(1, req, long) -- identifier + /// y(2, req, long) -- identifier + /// z(3, req, long) + /// person(4, opt, struct) + /// name(5, opt, string) + /// age(6, req, int) + /// tags(7, opt, list) + /// element(8, req, struct) + /// key(9, opt, string) + /// value(10, opt, string) + /// props(11, opt, map) + /// key(12, req, string) + /// value(13, req, struct) + /// data(14, opt, string) + fn make_v2_table_with_nested() -> Table { + let json = r#"{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c2", + "location": "s3://bucket/test/location", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 14, + "current-schema-id": 0, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"}, + {"id": 2, "name": "y", "required": true, "type": "long"}, + {"id": 3, "name": "z", "required": true, "type": "long"}, + {"id": 4, "name": "person", "required": false, "type": { + "type": "struct", + "fields": [ + {"id": 5, "name": "name", "required": false, "type": "string"}, + {"id": 6, "name": "age", "required": true, "type": "int"} + ] + }}, + {"id": 7, "name": "tags", "required": false, "type": { + "type": "list", + "element-id": 8, + "element": { + "type": "struct", + "fields": [ + {"id": 9, "name": "key", "required": false, "type": "string"}, + {"id": 10, "name": "value", "required": false, "type": "string"} + ] + }, + "element-required": true + }}, + {"id": 11, "name": "props", "required": false, "type": { + "type": "map", + "key-id": 12, + "key": "string", + "value-id": 13, + "value": { + "type": "struct", + "fields": [ + {"id": 14, "name": "data", "required": false, "type": "string"} + ] + }, + "value-required": true + }} + ] + } + ], + "default-spec-id": 0, + "partition-specs": [ + {"spec-id": 0, "fields": []} + ], + "last-partition-id": 999, + "default-sort-order-id": 0, + "sort-orders": [ + {"order-id": 0, "fields": []} + ], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [] + }"#; + + let reader = BufReader::new(json.as_bytes()); + let metadata = serde_json::from_reader::<_, TableMetadata>(reader).unwrap(); + + Table::builder() + .metadata(metadata) + .metadata_location("s3://bucket/test/location/metadata/v1.json".to_string()) + .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap()) + .file_io(crate::io::FileIO::new_with_memory()) + .runtime(crate::test_utils::test_runtime()) + .build() + .unwrap() + } + + // ----------------------------------------------------------------------- + // Existing root-level tests + // ----------------------------------------------------------------------- + + #[test] + fn test_assign_fresh_ids_variant() { + // Variant carries no sub-fields, so fresh-id assignment only renames the field + // itself and leaves the type untouched. + let mut next_id = 10; + let field = NestedField::optional(1, "data", Type::Variant(VariantType)); + let assigned = super::assign_fresh_ids(&field, &mut next_id); + + assert_eq!(assigned.id, 11); + assert_eq!(*assigned.field_type, Type::Variant(VariantType)); + assert_eq!(next_id, 11); + } + + #[tokio::test] + async fn test_add_column() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + + let action = tx.update_schema().add_column(AddColumn::optional( + "new_col", + Type::Primitive(PrimitiveType::Int), + )); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + let requirements = action_commit.take_requirements(); + + assert_eq!(updates.len(), 2); + + // Extract the new schema from the AddSchema update. + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + let expected_schema = table + .metadata() + .current_schema() + .as_ref() + .clone() + .into_builder() + .with_schema_id(DEFAULT_SCHEMA_ID) + .with_fields([ + NestedField::optional(4, "new_col", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + assert_eq!(new_schema, &expected_schema); + + assert_eq!(updates[1], TableUpdate::SetCurrentSchema { schema_id: -1 }); + + // Verify requirement. + assert_eq!(requirements.len(), 1); + assert_eq!(requirements[0], TableRequirement::CurrentSchemaIdMatch { + current_schema_id: table.metadata().current_schema().schema_id() + }); + } + + #[tokio::test] + async fn test_add_column_with_doc() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + + let action = tx.update_schema().add_column( + AddColumn::builder() + .name("documented_col") + .field_type(Type::Primitive(PrimitiveType::String)) + .doc("A documented column") + .build(), + ); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + let field = new_schema + .field_by_name("documented_col") + .expect("documented_col should exist"); + assert_eq!(field.id, 4); + assert!(!field.required); + assert_eq!(field.doc.as_deref(), Some("A documented column")); + } + + #[tokio::test] + async fn test_add_required_column_with_initial_default() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + + let action = tx.update_schema().add_column(AddColumn::required( + "req_col", + Type::Primitive(PrimitiveType::Int), + Literal::int(0), + )); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + let field = new_schema + .field_by_name("req_col") + .expect("req_col should exist"); + assert_eq!(field.id, 4); + assert!(field.required); + assert_eq!(field.initial_default, Some(Literal::int(0))); + assert_eq!(field.write_default, Some(Literal::int(0))); + } + + #[tokio::test] + async fn test_add_column_name_conflict_fails() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + + // "x" already exists in the V2 test schema. + let action = tx.update_schema().add_column(AddColumn::optional( + "x", + Type::Primitive(PrimitiveType::Int), + )); + + let result = commit_with_fresh_state(action, &table).await; + let err = match result { + Err(e) => e, + Ok(_) => panic!("should reject adding a column with an existing name"), + }; + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + assert!( + err.message().contains("already exists"), + "error should mention name conflict, got: {}", + err.message() + ); + } + + #[tokio::test] + async fn test_delete_column() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + + // z is not an identifier field, so we can delete it. + let action = tx.update_schema().delete_column("z"); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + assert!( + new_schema.field_by_name("z").is_none(), + "z should be deleted" + ); + assert!(new_schema.field_by_name("x").is_some()); + assert!(new_schema.field_by_name("y").is_some()); + } + + #[tokio::test] + async fn test_delete_missing_column_fails() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + + let action = tx.update_schema().delete_column("nonexistent"); + + let result = commit_with_fresh_state(action, &table).await; + let err = match result { + Err(e) => e, + Ok(_) => panic!("should reject deleting a non-existent column"), + }; + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + assert!( + err.message().contains("nonexistent"), + "error should mention the missing column, got: {}", + err.message() + ); + } + + #[tokio::test] + async fn test_add_and_delete_combined() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + + // Delete z, add a new column. + let action = tx + .update_schema() + .delete_column("z") + .add_column(AddColumn::optional( + "w", + Type::Primitive(PrimitiveType::Boolean), + )); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + assert!( + new_schema.field_by_name("z").is_none(), + "z should be deleted" + ); + let w = new_schema.field_by_name("w").expect("w should exist"); + assert_eq!(w.id, 4); + assert!(!w.required); + } + + #[tokio::test] + async fn test_delete_and_readd_same_name() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + + // Delete z, then add a new column named z -- should succeed. + let action = tx + .update_schema() + .delete_column("z") + .add_column(AddColumn::optional( + "z", + Type::Primitive(PrimitiveType::Boolean), + )); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + let z = new_schema + .field_by_name("z") + .expect("z should exist with new type"); + assert_eq!(z.id, 4); // new ID, not the old 3 + assert_eq!(*z.field_type, Type::Primitive(PrimitiveType::Boolean)); + } + + #[test] + fn test_apply() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + + let tx = tx + .update_schema() + .add_column(AddColumn::optional( + "new_col", + Type::Primitive(PrimitiveType::Int), + )) + .apply(tx) + .unwrap(); + + assert_eq!(tx.actions.len(), 1); + (*tx.actions[0]) + .downcast_ref::>() + .expect("UpdateSchemaAction was not applied to Transaction!"); + } + + // ----------------------------------------------------------------------- + // Nested add tests + // ----------------------------------------------------------------------- + + #[tokio::test] + async fn test_add_column_to_struct() { + let table = make_v2_table_with_nested(); + let tx = Transaction::new(&table); + + // Add "email" to the "person" struct. + let action = tx.update_schema().add_column( + AddColumn::builder() + .name("email") + .field_type(Type::Primitive(PrimitiveType::String)) + .parent("person") + .build(), + ); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + // "email" should be nested under "person" with ID = last_column_id + 1 = 15. + let email = new_schema + .field_by_name("person.email") + .expect("person.email should exist"); + assert_eq!(email.id, 15); + assert!(!email.required); + assert_eq!(*email.field_type, Type::Primitive(PrimitiveType::String)); + + // Original nested fields should still be there. + assert!(new_schema.field_by_name("person.name").is_some()); + assert!(new_schema.field_by_name("person.age").is_some()); + } + + #[tokio::test] + async fn test_add_column_to_struct_with_doc() { + let table = make_v2_table_with_nested(); + let tx = Transaction::new(&table); + + let action = tx.update_schema().add_column( + AddColumn::builder() + .name("phone") + .field_type(Type::Primitive(PrimitiveType::String)) + .parent("person") + .doc("Phone number") + .build(), + ); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + let phone = new_schema + .field_by_name("person.phone") + .expect("person.phone should exist"); + assert_eq!(phone.id, 15); + assert_eq!(phone.doc.as_deref(), Some("Phone number")); + } + + #[tokio::test] + async fn test_add_column_to_list_element_struct() { + let table = make_v2_table_with_nested(); + let tx = Transaction::new(&table); + + // "tags" is a list. Adding to the list navigates to its + // element struct automatically. + let action = tx.update_schema().add_column( + AddColumn::builder() + .name("score") + .field_type(Type::Primitive(PrimitiveType::Double)) + .parent("tags") + .build(), + ); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + // The list element struct should now contain "score". + let score = new_schema + .field_by_name("tags.element.score") + .expect("tags.element.score should exist"); + assert_eq!(score.id, 15); + assert!(!score.required); + + // Existing fields preserved. + assert!(new_schema.field_by_name("tags.element.key").is_some()); + assert!(new_schema.field_by_name("tags.element.value").is_some()); + } + + #[tokio::test] + async fn test_add_column_to_map_value_struct() { + let table = make_v2_table_with_nested(); + let tx = Transaction::new(&table); + + // "props" is a map. Adding to the map navigates to its + // value struct automatically. + let action = tx.update_schema().add_column( + AddColumn::builder() + .name("version") + .field_type(Type::Primitive(PrimitiveType::Int)) + .parent("props") + .build(), + ); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + let version = new_schema + .field_by_name("props.value.version") + .expect("props.value.version should exist"); + assert_eq!(version.id, 15); + + // Existing map value fields preserved. + assert!(new_schema.field_by_name("props.value.data").is_some()); + } + + #[tokio::test] + async fn test_add_column_to_nonexistent_parent_fails() { + let table = make_v2_table_with_nested(); + let tx = Transaction::new(&table); + + let action = tx.update_schema().add_column( + AddColumn::builder() + .name("col") + .field_type(Type::Primitive(PrimitiveType::Int)) + .parent("nonexistent") + .build(), + ); + + let err = match commit_with_fresh_state(action, &table).await { + Err(e) => e, + Ok(_) => panic!("should reject adding to a nonexistent parent"), + }; + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + assert!( + err.message().contains("nonexistent"), + "error should mention the missing parent, got: {}", + err.message() + ); + } + + #[tokio::test] + async fn test_add_column_to_primitive_parent_fails() { + let table = make_v2_table_with_nested(); + let tx = Transaction::new(&table); + + // "x" is a primitive (long), not a struct. + let action = tx.update_schema().add_column( + AddColumn::builder() + .name("col") + .field_type(Type::Primitive(PrimitiveType::Int)) + .parent("x") + .build(), + ); + + let err = match commit_with_fresh_state(action, &table).await { + Err(e) => e, + Ok(_) => panic!("should reject adding to a primitive parent"), + }; + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + assert!( + err.message().contains("not a struct"), + "error should mention type mismatch, got: {}", + err.message() + ); + } + + #[tokio::test] + async fn test_add_column_to_nested_name_conflict_fails() { + let table = make_v2_table_with_nested(); + let tx = Transaction::new(&table); + + // "name" already exists in the "person" struct. + let action = tx.update_schema().add_column( + AddColumn::builder() + .name("name") + .field_type(Type::Primitive(PrimitiveType::String)) + .parent("person") + .build(), + ); + + let err = match commit_with_fresh_state(action, &table).await { + Err(e) => e, + Ok(_) => panic!("should reject adding a column with conflicting name"), + }; + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + assert!( + err.message().contains("already exists"), + "error should mention name conflict, got: {}", + err.message() + ); + } + + #[tokio::test] + async fn test_root_and_nested_add_combined() { + let table = make_v2_table_with_nested(); + let tx = Transaction::new(&table); + + // Add a root column and a nested column in the same action. + let action = tx + .update_schema() + .add_column(AddColumn::optional( + "root_col", + Type::Primitive(PrimitiveType::Boolean), + )) + .add_column( + AddColumn::builder() + .name("email") + .field_type(Type::Primitive(PrimitiveType::String)) + .parent("person") + .build(), + ); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + // Root column gets the first fresh ID. + let root_col = new_schema + .field_by_name("root_col") + .expect("root_col should exist"); + assert_eq!(root_col.id, 15); + + // Nested column gets the next ID. + let email = new_schema + .field_by_name("person.email") + .expect("person.email should exist"); + assert_eq!(email.id, 16); + } + + #[tokio::test] + async fn test_add_nested_struct_type_with_fresh_ids() { + // Adding a new column whose TYPE contains nested fields (e.g. a struct column). All sub-fields must receive + // fresh IDs, not placeholder `DEFAULT_FIELD_ID`. + let table = make_v2_table(); + let tx = Transaction::new(&table); + + let action = tx.update_schema().add_column(AddColumn::optional( + "address", + Type::Struct(StructType::new(vec![ + NestedField::optional( + DEFAULT_FIELD_ID, + "street", + Type::Primitive(PrimitiveType::String), + ) + .into(), + NestedField::optional( + DEFAULT_FIELD_ID, + "city", + Type::Primitive(PrimitiveType::String), + ) + .into(), + ])), + )); + + let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_schema = match &updates[0] { + TableUpdate::AddSchema { schema } => schema, + other => panic!("expected AddSchema, got {other:?}"), + }; + + // "address" gets ID 4 (last_column_id=3, +1). + let address = new_schema + .field_by_name("address") + .expect("address should exist"); + assert_eq!(address.id, 4); + + // Sub-fields get IDs 5 and 6. + let street = new_schema + .field_by_name("address.street") + .expect("address.street should exist"); + assert_eq!(street.id, 5); + + let city = new_schema + .field_by_name("address.city") + .expect("address.city should exist"); + assert_eq!(city.id, 6); + } + + // ----------------------------------------------------------------------- + // Name mapping tests + // ----------------------------------------------------------------------- + + fn with_name_mapping_property(table: Table, mapping_json: String) -> Table { + let metadata = table + .metadata() + .clone() + .into_builder(None) + .set_properties(HashMap::from([( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(), + mapping_json, + )])) + .unwrap() + .build() + .unwrap() + .metadata; + table.with_metadata(Arc::new(metadata)) + } + + fn mapped(id: i32, name: &str, fields: Vec) -> MappedField { + MappedField::new(Some(id), vec![name.to_string()], fields) + } + + fn name_mapping_property_update(mapping: &NameMapping) -> TableUpdate { + TableUpdate::SetProperties { + updates: HashMap::from([( + TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(), + serde_json::to_string(mapping).unwrap(), + )]), + } + } + + fn v2_table_name_mapping() -> NameMapping { + NameMapping::new(vec![ + mapped(1, "x", vec![]), + mapped(2, "y", vec![]), + mapped(3, "z", vec![]), + ]) + } + + fn nested_table_name_mapping() -> NameMapping { + NameMapping::new(vec![ + mapped(1, "x", vec![]), + mapped(2, "y", vec![]), + mapped(3, "z", vec![]), + mapped(4, "person", vec![ + mapped(5, "name", vec![]), + mapped(6, "age", vec![]), + ]), + mapped(7, "tags", vec![mapped(8, "element", vec![ + mapped(9, "key", vec![]), + mapped(10, "value", vec![]), + ])]), + mapped(11, "props", vec![ + mapped(12, "key", vec![]), + mapped(13, "value", vec![mapped(14, "data", vec![])]), + ]), + ]) + } + + #[tokio::test] + async fn test_add_root_column_adds_name_mapping_entry() { + let table = with_name_mapping_property( + make_v2_table(), + serde_json::to_string(&v2_table_name_mapping()).unwrap(), + ); + let action = Transaction::new(&table) + .update_schema() + .add_column(AddColumn::optional( + "new_col", + Type::Primitive(PrimitiveType::Int), + )); + + let updates = commit_with_fresh_state(action, &table) + .await + .unwrap() + .take_updates(); + + let expected = NameMapping::new(vec![ + mapped(1, "x", vec![]), + mapped(2, "y", vec![]), + mapped(3, "z", vec![]), + mapped(4, "new_col", vec![]), + ]); + assert_eq!(updates[2..], [name_mapping_property_update(&expected)]); + } + + #[tokio::test] + async fn test_add_nested_columns_adds_name_mapping_entries_under_parents() { + let mapping = nested_table_name_mapping(); + let table = with_name_mapping_property( + make_v2_table_with_nested(), + serde_json::to_string(&mapping).unwrap(), + ); + let action = Transaction::new(&table) + .update_schema() + .add_column( + AddColumn::builder() + .name("email") + .field_type(Type::Primitive(PrimitiveType::String)) + .parent("person") + .build(), + ) + .add_column( + AddColumn::builder() + .name("version") + .field_type(Type::Primitive(PrimitiveType::Int)) + .parent("props") + .build(), + ) + .add_column(AddColumn::optional( + "attrs", + Type::Map(MapType::new( + NestedField::map_key_element( + DEFAULT_FIELD_ID, + Type::Primitive(PrimitiveType::String), + ) + .into(), + NestedField::map_value_element( + DEFAULT_FIELD_ID, + Type::Primitive(PrimitiveType::Long), + false, + ) + .into(), + )), + )) + .add_column(AddColumn::optional( + "points", + Type::List(ListType::new( + NestedField::list_element( + DEFAULT_FIELD_ID, + Type::Struct(StructType::new(vec![ + NestedField::optional( + DEFAULT_FIELD_ID, + "px", + Type::Primitive(PrimitiveType::Double), + ) + .into(), + ])), + true, + ) + .into(), + )), + )); + + let updates = commit_with_fresh_state(action, &table) + .await + .unwrap() + .take_updates(); + + let expected = NameMapping::new(vec![ + mapped(1, "x", vec![]), + mapped(2, "y", vec![]), + mapped(3, "z", vec![]), + mapped(4, "person", vec![ + mapped(5, "name", vec![]), + mapped(6, "age", vec![]), + mapped(15, "email", vec![]), + ]), + mapped(7, "tags", vec![mapped(8, "element", vec![ + mapped(9, "key", vec![]), + mapped(10, "value", vec![]), + ])]), + mapped(11, "props", vec![ + mapped(12, "key", vec![]), + mapped(13, "value", vec![ + mapped(14, "data", vec![]), + mapped(16, "version", vec![]), + ]), + ]), + mapped(17, "attrs", vec![ + mapped(18, "key", vec![]), + mapped(19, "value", vec![]), + ]), + mapped(20, "points", vec![mapped(21, "element", vec![mapped( + 22, + "px", + vec![], + )])]), + ]); + assert_eq!(updates[2..], [name_mapping_property_update(&expected)]); + } + + #[tokio::test] + async fn test_add_column_under_deleted_parent_adds_no_name_mapping_entry() { + let table = with_name_mapping_property( + make_v2_table_with_nested(), + serde_json::to_string(&nested_table_name_mapping()).unwrap(), + ); + let action = Transaction::new(&table) + .update_schema() + .delete_column("person") + .add_column( + AddColumn::builder() + .name("name") + .field_type(Type::Primitive(PrimitiveType::String)) + .parent("person") + .build(), + ) + .add_column(AddColumn::optional( + "new_col", + Type::Primitive(PrimitiveType::Int), + )); + + let updates = commit_with_fresh_state(action, &table) + .await + .unwrap() + .take_updates(); + + let mut expected_fields = nested_table_name_mapping().fields().to_vec(); + expected_fields.push(mapped(16, "new_col", vec![])); + assert_eq!(updates[2..], [name_mapping_property_update( + &NameMapping::new(expected_fields) + )]); + } + + #[tokio::test] + async fn test_readd_deleted_column_moves_name_to_new_name_mapping_entry() { + let table = with_name_mapping_property( + make_v2_table(), + serde_json::to_string(&v2_table_name_mapping()).unwrap(), + ); + let action = Transaction::new(&table) + .update_schema() + .delete_column("z") + .add_column(AddColumn::optional( + "z", + Type::Primitive(PrimitiveType::Boolean), + )); + + let updates = commit_with_fresh_state(action, &table) + .await + .unwrap() + .take_updates(); + + let expected = NameMapping::new(vec![ + mapped(1, "x", vec![]), + mapped(2, "y", vec![]), + MappedField::new(Some(3), vec![], vec![]), + mapped(4, "z", vec![]), + ]); + assert_eq!(updates[2..], [name_mapping_property_update(&expected)]); + } + + #[tokio::test] + async fn test_delete_column_leaves_name_mapping_unchanged() { + let table = with_name_mapping_property( + make_v2_table(), + serde_json::to_string(&v2_table_name_mapping()).unwrap(), + ); + let action = Transaction::new(&table).update_schema().delete_column("z"); + + let updates = commit_with_fresh_state(action, &table) + .await + .unwrap() + .take_updates(); + + assert_eq!(updates.len(), 2); + } + + #[tokio::test] + async fn test_add_column_with_invalid_name_mapping_leaves_it_unchanged() { + let table = with_name_mapping_property(make_v2_table(), "not json".to_string()); + let action = Transaction::new(&table) + .update_schema() + .add_column(AddColumn::optional( + "new_col", + Type::Primitive(PrimitiveType::Int), + )); + + let updates = commit_with_fresh_state(action, &table) + .await + .unwrap() + .take_updates(); + + assert_eq!(updates.len(), 2); + } +} diff --git a/crates/iceberg/src/transaction/update_statistics.rs b/crates/iceberg/src/transaction/update_statistics.rs index dfce95d898..f42765ed4a 100644 --- a/crates/iceberg/src/transaction/update_statistics.rs +++ b/crates/iceberg/src/transaction/update_statistics.rs @@ -16,22 +16,22 @@ // under the License. use std::collections::HashMap; -use std::sync::Arc; use async_trait::async_trait; use crate::spec::StatisticsFile; use crate::table::Table; -use crate::transaction::{ActionCommit, TransactionAction}; +use crate::transaction::{ActionCommit, StatelessAction}; use crate::{Result, TableUpdate}; /// A transactional action for updating statistics files in a table +#[derive(Clone)] pub struct UpdateStatisticsAction { statistics_to_set: HashMap>, } impl UpdateStatisticsAction { - pub fn new() -> Self { + pub(crate) fn new() -> Self { Self { statistics_to_set: HashMap::default(), } @@ -68,15 +68,9 @@ impl UpdateStatisticsAction { } } -impl Default for UpdateStatisticsAction { - fn default() -> Self { - Self::new() - } -} - #[async_trait] -impl TransactionAction for UpdateStatisticsAction { - async fn commit(self: Arc, _table: &Table) -> Result { +impl StatelessAction for UpdateStatisticsAction { + async fn commit(&self, _table: &Table) -> Result { let mut updates: Vec = vec![]; self.statistics_to_set @@ -104,6 +98,7 @@ mod tests { use as_any::Downcast; use crate::spec::{BlobMetadata, StatisticsFile}; + use crate::transaction::action::ActionEntry; use crate::transaction::tests::make_v2_table; use crate::transaction::update_statistics::UpdateStatisticsAction; use crate::transaction::{ApplyTransactionAction, Transaction}; @@ -153,8 +148,9 @@ mod tests { .unwrap(); let action = (*tx.actions[0]) - .downcast_ref::() - .unwrap(); + .downcast_ref::>() + .unwrap() + .action(); assert!( action .statistics_to_set @@ -194,8 +190,9 @@ mod tests { .unwrap(); let action = (*tx.actions[0]) - .downcast_ref::() - .unwrap(); + .downcast_ref::>() + .unwrap() + .action(); // Verify that the statistics file is set correctly assert_eq!( @@ -217,8 +214,9 @@ mod tests { let tx = tx.update_statistics().apply(tx).unwrap(); let action = (*tx.actions[0]) - .downcast_ref::() - .unwrap(); + .downcast_ref::>() + .unwrap() + .action(); // Verify that no statistics are set assert!(action.statistics_to_set.is_empty()); diff --git a/crates/iceberg/src/transaction/upgrade_format_version.rs b/crates/iceberg/src/transaction/upgrade_format_version.rs index ff15926d00..c65f955c64 100644 --- a/crates/iceberg/src/transaction/upgrade_format_version.rs +++ b/crates/iceberg/src/transaction/upgrade_format_version.rs @@ -15,28 +15,28 @@ // specific language governing permissions and limitations // under the License. -use std::sync::Arc; - use async_trait::async_trait; +use crate::Result; use crate::TableUpdate::UpgradeFormatVersion; +use crate::error::invalid_data; use crate::spec::FormatVersion; use crate::table::Table; -use crate::transaction::action::{ActionCommit, TransactionAction}; -use crate::{Error, ErrorKind, Result}; +use crate::transaction::action::{ActionCommit, StatelessAction}; /// A transaction action to upgrade a table's format version. /// /// This action is used within a transaction to indicate that the /// table's format version should be upgraded to a specified version. -/// The location remains optional until explicitly set via [`set_format_version`]. +/// The location remains optional until explicitly set via [`UpgradeFormatVersionAction::set_format_version`]. +#[derive(Clone)] pub struct UpgradeFormatVersionAction { format_version: Option, } impl UpgradeFormatVersionAction { /// Creates a new `UpgradeFormatVersionAction` with no version set. - pub fn new() -> Self { + pub(crate) fn new() -> Self { UpgradeFormatVersionAction { format_version: None, } @@ -57,20 +57,11 @@ impl UpgradeFormatVersionAction { } } -impl Default for UpgradeFormatVersionAction { - fn default() -> Self { - Self::new() - } -} - #[async_trait] -impl TransactionAction for UpgradeFormatVersionAction { - async fn commit(self: Arc, _table: &Table) -> Result { +impl StatelessAction for UpgradeFormatVersionAction { + async fn commit(&self, _table: &Table) -> Result { let format_version = self.format_version.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "FormatVersion is not set for UpgradeFormatVersionAction!", - ) + invalid_data!("FormatVersion is not set for UpgradeFormatVersionAction!") })?; Ok(ActionCommit::new( @@ -86,7 +77,7 @@ mod tests { use crate::spec::FormatVersion; use crate::transaction::Transaction; - use crate::transaction::action::ApplyTransactionAction; + use crate::transaction::action::{ActionEntry, ApplyTransactionAction}; use crate::transaction::upgrade_format_version::UpgradeFormatVersionAction; #[test] @@ -102,8 +93,9 @@ mod tests { assert_eq!(tx.actions.len(), 1); let action = (*tx.actions[0]) - .downcast_ref::() - .unwrap(); + .downcast_ref::>() + .unwrap() + .action(); assert_eq!(action.format_version, Some(FormatVersion::V2)); } diff --git a/crates/iceberg/src/transform/bucket.rs b/crates/iceberg/src/transform/bucket.rs index 8807fb1f79..33911eb076 100644 --- a/crates/iceberg/src/transform/bucket.rs +++ b/crates/iceberg/src/transform/bucket.rs @@ -23,6 +23,8 @@ use arrow_schema::{DataType, TimeUnit}; use super::TransformFunction; use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType}; +const NANOSECONDS_PER_MICROSECOND: i64 = 1_000; + #[derive(Debug)] pub struct Bucket { mod_n: u32, @@ -78,12 +80,26 @@ impl Bucket { /// ref: https://iceberg.apache.org/spec/#appendix-b-32-bit-hash-requirements #[inline] fn hash_decimal(v: i128) -> i32 { + if v == 0 { + return Self::hash_bytes(&[0]); + } + let bytes = v.to_be_bytes(); - if let Some(start) = bytes.iter().position(|&x| x != 0) { - Self::hash_bytes(&bytes[start..]) + let start = if v > 0 { + // Positive: skip 0x00 unless next byte would appear negative + bytes + .windows(2) + .position(|w| w[0] != 0x00 || w[1] & 0x80 != 0) + .unwrap_or(15) } else { - Self::hash_bytes(&[0]) - } + // Negative: skip 0xFF only if next byte stays negative + bytes + .windows(2) + .position(|w| w[0] != 0xFF || w[1] & 0x80 == 0) + .unwrap_or(15) + }; + + Self::hash_bytes(&bytes[start..]) } /// def bucket_N(x) = (murmur3_x86_32_hash(x) & Integer.MAX_VALUE) % N @@ -132,6 +148,11 @@ impl Bucket { fn bucket_bytes(&self, v: &[u8]) -> i32 { self.bucket_n(Self::hash_bytes(v)) } + + #[inline] + fn nanos_to_micros(v: i64) -> i64 { + v.div_euclid(NANOSECONDS_PER_MICROSECOND) + } } impl TransformFunction for Bucket { @@ -171,12 +192,12 @@ impl TransformFunction for Bucket { .as_any() .downcast_ref::() .unwrap() - .unary(|v| self.bucket_time(v / 1000)), + .unary(|v| self.bucket_time(Self::nanos_to_micros(v))), DataType::Timestamp(TimeUnit::Nanosecond, _) => input .as_any() .downcast_ref::() .unwrap() - .unary(|v| self.bucket_timestamp(v / 1000)), + .unary(|v| self.bucket_timestamp(Self::nanos_to_micros(v))), DataType::Utf8 => arrow_array::Int32Array::from_iter( input .as_any() @@ -240,10 +261,10 @@ impl TransformFunction for Bucket { (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => self.bucket_timestamp(*v), (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(v)) => self.bucket_timestamp(*v), (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(v)) => { - self.bucket_timestamp(*v / 1000) + self.bucket_timestamp(Self::nanos_to_micros(*v)) } (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(v)) => { - self.bucket_timestamp(*v / 1000) + self.bucket_timestamp(Self::nanos_to_micros(*v)) } (PrimitiveType::String, PrimitiveLiteral::String(v)) => self.bucket_str(v.as_str()), (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(v)) => { @@ -276,11 +297,12 @@ mod test { use crate::Result; use crate::expr::PredicateOperator; use crate::spec::PrimitiveType::{ - Binary, Date, Decimal, Fixed, Int, Long, String as StringType, Time, Timestamp, + Binary, Date, Decimal, Fixed, Geometry, Int, Long, String as StringType, Time, Timestamp, TimestampNs, Timestamptz, TimestamptzNs, Uuid, }; use crate::spec::Type::{Primitive, Struct}; - use crate::spec::{Datum, NestedField, PrimitiveType, StructType, Transform, Type}; + use crate::spec::decimal_utils::decimal_new; + use crate::spec::{Datum, GeometryType, NestedField, PrimitiveType, StructType, Transform}; use crate::transform::TransformFunction; use crate::transform::test::{TestProjectionFixture, TestTransformFixture}; @@ -319,6 +341,7 @@ mod test { (Primitive(Timestamptz), Some(Primitive(Int))), (Primitive(TimestampNs), Some(Primitive(Int))), (Primitive(TimestamptzNs), Some(Primitive(Int))), + (Primitive(Geometry(GeometryType::default())), None), ( Struct(StructType::new(vec![ NestedField::optional(1, "a", Primitive(Timestamp)).into(), @@ -339,7 +362,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Bucket(10), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Uuid)), + NestedField::required(1, "value", Primitive(Uuid)), ); fixture.assert_projection( @@ -399,11 +422,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Bucket(10), "name", - NestedField::required( - 1, - "value", - Type::Primitive(PrimitiveType::Fixed(value.len() as u64)), - ), + NestedField::required(1, "value", Primitive(Fixed(value.len() as u64))), ); fixture.assert_projection( @@ -466,7 +485,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Bucket(10), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::String)), + NestedField::required(1, "value", Primitive(PrimitiveType::String)), ); fixture.assert_projection( @@ -530,7 +549,7 @@ mod test { NestedField::required( 1, "value", - Type::Primitive(PrimitiveType::Decimal { + Primitive(Decimal { precision: 9, scale: 2, }), @@ -602,7 +621,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Bucket(10), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Long)), + NestedField::required(1, "value", Primitive(Long)), ); fixture.assert_projection( @@ -662,7 +681,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Bucket(10), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Int)), + NestedField::required(1, "value", Primitive(Int)), ); fixture.assert_projection( @@ -790,6 +809,27 @@ mod test { ); } + #[test] + fn test_hash_decimal_with_negative_value() { + // Test cases from GitHub issue #1981 + assert_eq!(Bucket::hash_decimal(1), -463810133); + assert_eq!(Bucket::hash_decimal(-1), -43192051); + + // Additional test cases for edge case values + assert_eq!(Bucket::hash_decimal(0), Bucket::hash_decimal(0)); + assert_eq!(Bucket::hash_decimal(127), Bucket::hash_decimal(127)); + assert_eq!(Bucket::hash_decimal(-128), Bucket::hash_decimal(-128)); + + // Test minimum representation is used + // -1 should hash as [0xFF] not [0xFF, 0xFF, ..., 0xFF] + // 128 should hash as [0x00, 0x80] not [0x00, 0x00, ..., 0x80] + assert_eq!(Bucket::hash_decimal(128), Bucket::hash_bytes(&[0x00, 0x80])); + assert_eq!( + Bucket::hash_decimal(-129), + Bucket::hash_bytes(&[0xFF, 0x7F]) + ); + } + #[test] fn test_int_literal() { let bucket = Bucket::new(10); @@ -813,7 +853,7 @@ mod test { let bucket = Bucket::new(10); assert_eq!( bucket - .transform_literal(&Datum::decimal(1420).unwrap()) + .transform_literal(&Datum::decimal(decimal_new(1420, 0)).unwrap()) .unwrap() .unwrap(), Datum::int(9) @@ -967,4 +1007,46 @@ mod test { assert_eq!(micro_result.value(0), nano_result.value(0)); } + + #[test] + fn test_nanos_to_micros_uses_floor_division() { + assert_eq!(Bucket::nanos_to_micros(i64::MIN), -9_223_372_036_854_776); + assert_eq!(Bucket::nanos_to_micros(1_001), 1); + assert_eq!(Bucket::nanos_to_micros(1_000), 1); + assert_eq!(Bucket::nanos_to_micros(0), 0); + assert_eq!(Bucket::nanos_to_micros(-1_000), -1); + assert_eq!(Bucket::nanos_to_micros(-1_001), -2); + assert_eq!(Bucket::nanos_to_micros(i64::MAX), 9_223_372_036_854_775); + } + + #[test] + fn test_negative_timestamp_nanos_round_down_to_micros() { + let bucket = Bucket::new(100); + let micros_value = -1_510_871_468_000_001; + let nanos_value = micros_value * 1000 + 999; + + let expected = bucket + .transform_literal(&Datum::timestamp_micros(micros_value)) + .unwrap(); + assert_eq!( + bucket + .transform_literal(&Datum::timestamp_nanos(nanos_value)) + .unwrap(), + expected + ); + assert_eq!( + bucket + .transform_literal(&Datum::timestamptz_nanos(nanos_value)) + .unwrap(), + expected + ); + + let micro_array = TimestampMicrosecondArray::from_iter_values([micros_value]); + let nano_array = TimestampNanosecondArray::from_iter_values([nanos_value]); + let micro_result = bucket.transform(Arc::new(micro_array)).unwrap(); + let nano_result = bucket.transform(Arc::new(nano_array)).unwrap(); + let micro_result = micro_result.as_any().downcast_ref::().unwrap(); + let nano_result = nano_result.as_any().downcast_ref::().unwrap(); + assert_eq!(micro_result, nano_result); + } } diff --git a/crates/iceberg/src/transform/identity.rs b/crates/iceberg/src/transform/identity.rs index dd096337ac..d2d9cc4762 100644 --- a/crates/iceberg/src/transform/identity.rs +++ b/crates/iceberg/src/transform/identity.rs @@ -37,11 +37,11 @@ impl TransformFunction for Identity { #[cfg(test)] mod test { use crate::spec::PrimitiveType::{ - Binary, Date, Decimal, Fixed, Int, Long, String as StringType, Time, Timestamp, + Binary, Date, Decimal, Fixed, Geometry, Int, Long, String as StringType, Time, Timestamp, TimestampNs, Timestamptz, TimestamptzNs, Uuid, }; use crate::spec::Type::{Primitive, Struct}; - use crate::spec::{NestedField, StructType, Transform}; + use crate::spec::{GeometryType, NestedField, StructType, Transform}; use crate::transform::test::TestTransformFixture; #[test] @@ -83,6 +83,7 @@ mod test { (Primitive(Timestamptz), Some(Primitive(Timestamptz))), (Primitive(TimestampNs), Some(Primitive(TimestampNs))), (Primitive(TimestamptzNs), Some(Primitive(TimestamptzNs))), + (Primitive(Geometry(GeometryType::default())), None), ( Struct(StructType::new(vec![ NestedField::optional(1, "a", Primitive(Timestamp)).into(), diff --git a/crates/iceberg/src/transform/mod.rs b/crates/iceberg/src/transform/mod.rs index 4cc0d1fe8f..220e5c2602 100644 --- a/crates/iceberg/src/transform/mod.rs +++ b/crates/iceberg/src/transform/mod.rs @@ -17,6 +17,8 @@ //! Transform function used to compute partition values. +use std::fmt::Debug; + use arrow_array::ArrayRef; use crate::spec::{Datum, Transform}; @@ -29,7 +31,7 @@ mod truncate; mod void; /// TransformFunction is a trait that defines the interface for all transform functions. -pub trait TransformFunction: Send + Sync { +pub trait TransformFunction: Send + Sync + Debug { /// transform will take an input array and transform it into a new array. /// The implementation of this function will need to check and downcast the input to specific /// type. @@ -42,7 +44,7 @@ pub trait TransformFunction: Send + Sync { self.transform_literal(input)?.ok_or_else(|| { Error::new( ErrorKind::Unexpected, - format!("Returns 'None' for literal {}", input), + format!("Returns 'None' for literal {input}"), ) }) } @@ -62,8 +64,8 @@ pub fn create_transform_function(transform: &Transform) -> Result Ok(Box::new(temporal::Hour {})), Transform::Bucket(mod_n) => Ok(Box::new(bucket::Bucket::new(*mod_n))), Transform::Truncate(width) => Ok(Box::new(truncate::Truncate::new(*width))), - Transform::Unknown => Err(crate::error::Error::new( - crate::ErrorKind::FeatureUnsupported, + Transform::Unknown => Err(Error::new( + ErrorKind::FeatureUnsupported, "Transform Unknown is not implemented", )), } @@ -169,10 +171,7 @@ mod test { assert_eq!( satisfies_order_of, &trans.satisfies_order_of(other_trans), - "Failed to check satisfies order {}, {}, {}", - trans, - other_trans, - satisfies_order_of + "Failed to check satisfies order {trans}, {other_trans}, {satisfies_order_of}" ); } @@ -180,8 +179,7 @@ mod test { let actual = trans.result_type(input_type).ok(); assert_eq!( result_type, &actual, - "type mismatch at index {}, input: {}, expected: {:?}, actual: {:?}", - i, input_type, result_type, actual + "type mismatch at index {i}, input: {input_type}, expected: {result_type:?}, actual: {actual:?}" ); } } diff --git a/crates/iceberg/src/transform/temporal.rs b/crates/iceberg/src/transform/temporal.rs index 3b333d7bad..02ac2b4b93 100644 --- a/crates/iceberg/src/transform/temporal.rs +++ b/crates/iceberg/src/transform/temporal.rs @@ -24,9 +24,10 @@ use arrow_array::{ Array, ArrayRef, Date32Array, Int32Array, TimestampMicrosecondArray, TimestampNanosecondArray, }; use arrow_schema::{DataType, TimeUnit}; -use chrono::{DateTime, Datelike, Duration}; +use chrono::{DateTime, Datelike}; use super::TransformFunction; +use crate::error::invalid_data; use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType}; use crate::{Error, ErrorKind, Result}; @@ -36,10 +37,10 @@ const MICROSECONDS_PER_HOUR: i64 = 3_600_000_000; const NANOSECONDS_PER_HOUR: i64 = 3_600_000_000_000; /// Year of unix epoch. const UNIX_EPOCH_YEAR: i32 = 1970; -/// One second in micros. -const MICROS_PER_SECOND: i64 = 1_000_000; -/// One second in nanos. -const NANOS_PER_SECOND: i64 = 1_000_000_000; +/// Microseconds in one day. +const MICROSECONDS_PER_DAY: i64 = 86_400_000_000; +/// Nanoseconds in one day. +const NANOSECONDS_PER_DAY: i64 = 86_400_000_000_000; /// Extract a date or timestamp year, as years from 1970 #[derive(Debug)] @@ -49,12 +50,7 @@ impl Year { #[inline] fn timestamp_to_year_micros(timestamp: i64) -> Result { Ok(DateTime::from_timestamp_micros(timestamp) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Fail to convert timestamp to date in year transform", - ) - })? + .ok_or_else(|| invalid_data!("Fail to convert timestamp to date in year transform"))? .year() - UNIX_EPOCH_YEAR) } @@ -78,10 +74,10 @@ impl TransformFunction for Year { )) } - fn transform_literal(&self, input: &crate::spec::Datum) -> Result> { + fn transform_literal(&self, input: &Datum) -> Result> { let val = match (input.data_type(), input.literal()) { (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => { - Date32Type::to_naive_date(*v).year() - UNIX_EPOCH_YEAR + Date32Type::to_naive_date_opt(*v).unwrap().year() - UNIX_EPOCH_YEAR } (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { Self::timestamp_to_year_micros(*v)? @@ -96,8 +92,8 @@ impl TransformFunction for Year { Self::timestamp_to_year_nanos(*v)? } _ => { - return Err(crate::Error::new( - crate::ErrorKind::FeatureUnsupported, + return Err(Error::new( + ErrorKind::FeatureUnsupported, format!( "Unsupported data type for year transform: {:?}", input.data_type() @@ -120,12 +116,8 @@ impl Month { // unix epoch date: 1970-01-01 // if date > unix epoch date, delta month = (aa - 1) + 12 * (aaaa-1970) // if date < unix epoch date, delta month = (12 - (aa - 1)) + 12 * (1970-aaaa-1) - let date = DateTime::from_timestamp_micros(timestamp).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - "Fail to convert timestamp to date in month transform", - ) - })?; + let date = DateTime::from_timestamp_micros(timestamp) + .ok_or_else(|| invalid_data!("Fail to convert timestamp to date in month transform"))?; let unix_epoch_date = DateTime::from_timestamp_micros(0) .expect("0 timestamp from unix epoch should be valid"); if date > unix_epoch_date { @@ -175,11 +167,11 @@ impl TransformFunction for Month { )) } - fn transform_literal(&self, input: &crate::spec::Datum) -> Result> { + fn transform_literal(&self, input: &Datum) -> Result> { let val = match (input.data_type(), input.literal()) { (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => { - (Date32Type::to_naive_date(*v).year() - UNIX_EPOCH_YEAR) * 12 - + Date32Type::to_naive_date(*v).month0() as i32 + (Date32Type::to_naive_date_opt(*v).unwrap().year() - UNIX_EPOCH_YEAR) * 12 + + Date32Type::to_naive_date_opt(*v).unwrap().month0() as i32 } (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { Self::timestamp_to_month_micros(*v)? @@ -194,8 +186,8 @@ impl TransformFunction for Month { Self::timestamp_to_month_nanos(*v)? } _ => { - return Err(crate::Error::new( - crate::ErrorKind::FeatureUnsupported, + return Err(Error::new( + ErrorKind::FeatureUnsupported, format!( "Unsupported data type for month transform: {:?}", input.data_type() @@ -213,62 +205,13 @@ pub struct Day; impl Day { #[inline] - fn day_timestamp_micro(v: i64) -> Result { - let secs = v / MICROS_PER_SECOND; - - let (nanos, offset) = if v >= 0 { - let nanos = (v.rem_euclid(MICROS_PER_SECOND) * 1_000) as u32; - let offset = 0i64; - (nanos, offset) - } else { - let v = v + 1; - let nanos = (v.rem_euclid(MICROS_PER_SECOND) * 1_000) as u32; - let offset = 1i64; - (nanos, offset) - }; - - let delta = Duration::new(secs, nanos).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Failed to create 'TimeDelta' from seconds {} and nanos {}", - secs, nanos - ), - ) - })?; - - let days = (delta.num_days() - offset) as i32; - - Ok(days) + fn day_timestamp_micro(v: i64) -> i32 { + v.div_euclid(MICROSECONDS_PER_DAY) as i32 } - fn day_timestamp_nano(v: i64) -> Result { - let secs = v / NANOS_PER_SECOND; - - let (nanos, offset) = if v >= 0 { - let nanos = (v.rem_euclid(NANOS_PER_SECOND)) as u32; - let offset = 0i64; - (nanos, offset) - } else { - let v = v + 1; - let nanos = (v.rem_euclid(NANOS_PER_SECOND)) as u32; - let offset = 1i64; - (nanos, offset) - }; - - let delta = Duration::new(secs, nanos).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Failed to create 'TimeDelta' from seconds {} and nanos {}", - secs, nanos - ), - ) - })?; - - let days = (delta.num_days() - offset) as i32; - - Ok(days) + #[inline] + fn day_timestamp_nano(v: i64) -> i32 { + v.div_euclid(NANOSECONDS_PER_DAY) as i32 } } @@ -279,12 +222,12 @@ impl TransformFunction for Day { .as_any() .downcast_ref::() .unwrap() - .try_unary(|v| -> Result { Self::day_timestamp_micro(v) })?, + .unary(|v| -> i32 { Self::day_timestamp_micro(v) }), DataType::Timestamp(TimeUnit::Nanosecond, _) => input .as_any() .downcast_ref::() .unwrap() - .try_unary(|v| -> Result { Self::day_timestamp_nano(v) })?, + .unary(|v| -> i32 { Self::day_timestamp_nano(v) }), DataType::Date32 => input .as_any() .downcast_ref::() @@ -303,22 +246,20 @@ impl TransformFunction for Day { Ok(Arc::new(res)) } - fn transform_literal(&self, input: &crate::spec::Datum) -> Result> { + fn transform_literal(&self, input: &Datum) -> Result> { let val = match (input.data_type(), input.literal()) { (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => *v, - (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => Self::day_timestamp_micro(*v)?, + (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => Self::day_timestamp_micro(*v), (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(v)) => { - Self::day_timestamp_micro(*v)? - } - (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(v)) => { - Self::day_timestamp_nano(*v)? + Self::day_timestamp_micro(*v) } + (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(v)) => Self::day_timestamp_nano(*v), (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(v)) => { - Self::day_timestamp_nano(*v)? + Self::day_timestamp_nano(*v) } _ => { - return Err(crate::Error::new( - crate::ErrorKind::FeatureUnsupported, + return Err(Error::new( + ErrorKind::FeatureUnsupported, format!( "Unsupported data type for day transform: {:?}", input.data_type() @@ -354,9 +295,14 @@ impl TransformFunction for Hour { .downcast_ref::() .unwrap() .unary(|v| -> i32 { Self::hour_timestamp_micro(v) }), + DataType::Timestamp(TimeUnit::Nanosecond, _) => input + .as_any() + .downcast_ref::() + .unwrap() + .unary(|v| -> i32 { Self::hour_timestamp_nano(v) }), _ => { - return Err(crate::Error::new( - crate::ErrorKind::FeatureUnsupported, + return Err(Error::new( + ErrorKind::FeatureUnsupported, format!( "Unsupported data type for hour transform: {:?}", input.data_type() @@ -367,7 +313,7 @@ impl TransformFunction for Hour { Ok(Arc::new(res)) } - fn transform_literal(&self, input: &crate::spec::Datum) -> Result> { + fn transform_literal(&self, input: &Datum) -> Result> { let val = match (input.data_type(), input.literal()) { (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => Self::hour_timestamp_micro(*v), (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(v)) => { @@ -380,8 +326,8 @@ impl TransformFunction for Hour { Self::hour_timestamp_nano(*v) } _ => { - return Err(crate::Error::new( - crate::ErrorKind::FeatureUnsupported, + return Err(Error::new( + ErrorKind::FeatureUnsupported, format!( "Unsupported data type for hour transform: {:?}", input.data_type() @@ -397,7 +343,9 @@ impl TransformFunction for Hour { mod test { use std::sync::Arc; - use arrow_array::{ArrayRef, Date32Array, Int32Array, TimestampMicrosecondArray}; + use arrow_array::{ + ArrayRef, Date32Array, Int32Array, TimestampMicrosecondArray, TimestampNanosecondArray, + }; use chrono::{NaiveDate, NaiveDateTime}; use crate::Result; @@ -407,7 +355,7 @@ mod test { TimestampNs, Timestamptz, TimestamptzNs, Uuid, }; use crate::spec::Type::{Primitive, Struct}; - use crate::spec::{Datum, NestedField, PrimitiveType, StructType, Transform, Type}; + use crate::spec::{Datum, NestedField, StructType, Transform}; use crate::transform::test::{TestProjectionFixture, TestTransformFixture}; use crate::transform::{BoxedTransformFunction, TransformFunction}; @@ -617,7 +565,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Hour, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -691,7 +639,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Hour, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -763,7 +711,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Year, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -835,7 +783,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Year, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -907,7 +855,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Month, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -979,7 +927,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Month, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -1050,7 +998,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Month, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -1122,7 +1070,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Month, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -1196,7 +1144,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Day, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -1260,6 +1208,46 @@ mod test { Ok(()) } + #[test] + fn test_projection_timestamp_types_day_negative() -> Result<()> { + // 1969-12-30T23:59:59.5 + let micros = -86_400_500_000; + for (field_type, value) in [ + (Timestamp, Datum::timestamp_micros(micros)), + (Timestamptz, Datum::timestamptz_micros(micros)), + (TimestampNs, Datum::timestamp_nanos(micros * 1_000)), + (TimestamptzNs, Datum::timestamptz_nanos(micros * 1_000)), + ] { + let fixture = TestProjectionFixture::new( + Transform::Day, + "name", + NestedField::required(1, "value", Primitive(field_type)), + ); + + fixture.assert_projection( + &fixture.binary_predicate(PredicateOperator::LessThan, value.clone()), + Some("name <= 1969-12-31"), + )?; + + fixture.assert_projection( + &fixture.binary_predicate(PredicateOperator::LessThanOrEq, value.clone()), + Some("name <= 1969-12-31"), + )?; + + fixture.assert_projection( + &fixture.binary_predicate(PredicateOperator::Eq, value.clone()), + Some("name IN (1969-12-31, 1969-12-30)"), + )?; + + fixture.assert_projection( + &fixture.set_predicate(PredicateOperator::In, vec![value]), + Some("name IN (1969-12-31, 1969-12-30)"), + )?; + } + + Ok(()) + } + #[test] fn test_projection_timestamp_day_upper_bound() -> Result<()> { // 17501 @@ -1270,7 +1258,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Day, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -1344,7 +1332,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Day, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -1418,7 +1406,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Day, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -1492,7 +1480,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Day, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Timestamp)), + NestedField::required(1, "value", Primitive(Timestamp)), ); fixture.assert_projection( @@ -1566,7 +1554,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Day, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -1634,7 +1622,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Day, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -1702,7 +1690,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Month, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -1770,7 +1758,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Month, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -1838,7 +1826,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Month, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -1906,7 +1894,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Month, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -1974,7 +1962,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Month, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -2041,7 +2029,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Year, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -2109,7 +2097,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Year, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -2177,7 +2165,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Year, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -2245,7 +2233,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Year, "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Date)), + NestedField::required(1, "value", Primitive(Date)), ); fixture.assert_projection( @@ -2663,6 +2651,48 @@ mod test { test_timestamp_ns_and_tz_transform("2017-12-01T10:30:42.123000", &day, Datum::date(17501)); } + #[test] + fn test_transform_days_pre_epoch() { + let day = Box::new(super::Day) as BoxedTransformFunction; + let expected = [-1, -2, -2, -2, -2, -366, -365]; + + let micros = vec![ + -500_000, // 1969-12-31T23:59:59.500000 + -86_400_000_001, // 1969-12-30T23:59:59.999999 + -86_400_000_002, // 1969-12-30T23:59:59.999998 + -86_400_500_000, // 1969-12-30T23:59:59.500000 + -86_401_000_000, // 1969-12-30T23:59:59.000000 + -31_536_000_500_000, // 1968-12-31T23:59:59.500000 + -31_535_999_000_001, // 1969-01-01T00:00:00.999999, Iceberg Java gives -366 + ]; + let res = day + .transform(Arc::new(TimestampMicrosecondArray::from(micros.clone()))) + .unwrap(); + let res = res.as_any().downcast_ref::().unwrap(); + assert_eq!(res.values(), &expected); + for (v, d) in micros.into_iter().zip(expected) { + test_timestamp_and_tz_transform_using_i64(v, &day, Datum::date(d)); + } + + let nanos = vec![ + -500_000_000, // 1969-12-31T23:59:59.500000000 + -86_400_000_000_001, // 1969-12-30T23:59:59.999999999 + -86_400_000_000_002, // 1969-12-30T23:59:59.999999998 + -86_400_500_000_000, // 1969-12-30T23:59:59.500000000 + -86_401_000_000_000, // 1969-12-30T23:59:59.000000000 + -31_536_000_500_000_000, // 1968-12-31T23:59:59.500000000 + -31_535_999_000_000_001, // 1969-01-01T00:00:00.999999999, Iceberg Java gives -366 + ]; + let res = day + .transform(Arc::new(TimestampNanosecondArray::from(nanos.clone()))) + .unwrap(); + let res = res.as_any().downcast_ref::().unwrap(); + assert_eq!(res.values(), &expected); + for (v, d) in nanos.into_iter().zip(expected) { + test_timestamp_ns_and_tz_transform_using_i64(v, &day, Datum::date(d)); + } + } + #[test] fn test_transform_hours() { let hour = super::Hour; @@ -2720,6 +2750,19 @@ mod test { assert_eq!(res.value(2), expect_hour[2]); assert_eq!(res.value(3), expect_hour[3]); assert_eq!(res.value(4), -1); + + // Test TimestampNanosecond with and without timezone + let timestamp_nanos = vec![0, super::NANOSECONDS_PER_HOUR, -1]; + let date_arrays = [ + Arc::new(TimestampNanosecondArray::from(timestamp_nanos.clone())) as ArrayRef, + Arc::new(TimestampNanosecondArray::from(timestamp_nanos).with_timezone_utc()) + as ArrayRef, + ]; + for date_array in date_arrays { + let res = hour.transform(date_array).unwrap(); + let res = res.as_any().downcast_ref::().unwrap(); + assert_eq!(res.values(), &[0, 1, -1]); + } } #[test] diff --git a/crates/iceberg/src/transform/truncate.rs b/crates/iceberg/src/transform/truncate.rs index d712df518e..3efb09258a 100644 --- a/crates/iceberg/src/transform/truncate.rs +++ b/crates/iceberg/src/transform/truncate.rs @@ -22,7 +22,8 @@ use arrow_schema::DataType; use super::TransformFunction; use crate::Error; -use crate::spec::{Datum, PrimitiveLiteral}; +use crate::error::invalid_data; +use crate::spec::{Datum, PrimitiveLiteral, PrimitiveType}; #[derive(Debug)] pub struct Truncate { @@ -49,17 +50,24 @@ impl Truncate { #[inline] fn truncate_i32(v: i32, width: i32) -> i32 { - v - v.rem_euclid(width) + // The spec requires a non-negative remainder, which `rem_euclid` computes without + // overflow. Only the subtraction can overflow, when the result is below `i32::MIN`. + // The spec doesn't cover that case, so wrap as Java's `TruncateUtil` does. + // The wrapped result isn't order-preserving, which truncate projection assumes. + v.wrapping_sub(v.rem_euclid(width)) } #[inline] fn truncate_i64(v: i64, width: i64) -> i64 { - v - (((v % width) + width) % width) + // See `truncate_i32`. + v.wrapping_sub(v.rem_euclid(width)) } #[inline] fn truncate_decimal_i128(v: i128, width: i128) -> i128 { - v - (((v % width) + width) % width) + // Only mantissas outside the 38-digit decimal range can wrap. Java's `truncateDecimal` + // uses `BigInteger`, so there is no Java result to match and the wrap only avoids a panic. + v.wrapping_sub(v.rem_euclid(width)) } } @@ -68,10 +76,7 @@ impl TransformFunction for Truncate { match input.data_type() { DataType::Int32 => { let width: i32 = self.width.try_into().map_err(|_| { - Error::new( - crate::ErrorKind::DataInvalid, - "width is failed to convert to i32 when truncate Int32Array", - ) + invalid_data!("width is failed to convert to i32 when truncate Int32Array") })?; let res: arrow_array::Int32Array = input .as_any() @@ -136,7 +141,7 @@ impl TransformFunction for Truncate { ); Ok(Arc::new(res)) } - _ => Err(crate::Error::new( + _ => Err(Error::new( crate::ErrorKind::FeatureUnsupported, format!( "Unsupported data type for truncate transform: {:?}", @@ -150,10 +155,7 @@ impl TransformFunction for Truncate { match input.literal() { PrimitiveLiteral::Int(v) => Ok(Some({ let width: i32 = self.width.try_into().map_err(|_| { - Error::new( - crate::ErrorKind::DataInvalid, - "width is failed to convert to i32 when truncate Int32Array", - ) + invalid_data!("width is failed to convert to i32 when truncate Int32Array") })?; Datum::int(Self::truncate_i32(*v, width)) })), @@ -163,13 +165,21 @@ impl TransformFunction for Truncate { })), PrimitiveLiteral::Int128(v) => Ok(Some({ let width = self.width as i128; - Datum::decimal(Self::truncate_decimal_i128(*v, width))? + input.with_decimal_mantissa(Self::truncate_decimal_i128(*v, width))? })), PrimitiveLiteral::String(v) => Ok(Some({ let len = self.width as usize; Datum::string(Self::truncate_str(v, len).to_string()) })), - _ => Err(crate::Error::new( + // Fixed shares this literal variant with Binary but does not support truncate. + PrimitiveLiteral::Binary(v) if matches!(input.data_type(), PrimitiveType::Binary) => { + Ok(Some(Datum::binary( + Self::truncate_binary(v, self.width as usize) + .iter() + .copied(), + ))) + } + _ => Err(Error::new( crate::ErrorKind::FeatureUnsupported, format!( "Unsupported data type for truncate transform: {:?}", @@ -191,11 +201,13 @@ mod test { use crate::Result; use crate::expr::PredicateOperator; use crate::spec::PrimitiveType::{ - Binary, Date, Decimal, Fixed, Int, Long, String as StringType, Time, Timestamp, + Binary, Date, Decimal, Fixed, Geometry, Int, Long, String as StringType, Time, Timestamp, TimestampNs, Timestamptz, TimestamptzNs, Uuid, }; use crate::spec::Type::{Primitive, Struct}; - use crate::spec::{Datum, NestedField, PrimitiveType, StructType, Transform, Type}; + use crate::spec::{ + Datum, GeometryType, NestedField, PrimitiveLiteral, PrimitiveType, StructType, Transform, + }; use crate::transform::TransformFunction; use crate::transform::test::{TestProjectionFixture, TestTransformFixture}; @@ -238,6 +250,7 @@ mod test { (Primitive(Timestamptz), None), (Primitive(TimestampNs), None), (Primitive(TimestamptzNs), None), + (Primitive(Geometry(GeometryType::default())), None), ( Struct(StructType::new(vec![ NestedField::optional(1, "a", Primitive(Timestamp)).into(), @@ -250,6 +263,128 @@ mod test { fixture.assert_transform(trans); } + #[test] + fn test_projection_truncate_binary() -> Result<()> { + let fixture = TestProjectionFixture::new( + Transform::Truncate(3), + "name", + NestedField::required(1, "value", Primitive(Binary)), + ); + + for (value, prefix) in [ + (vec![0x00, 0xff, 0x80, 0x01], "00FF80"), + (vec![0x00, 0xff, 0x80], "00FF80"), + (vec![0x00, 0xff], "00FF"), + ] { + for (op, projected_op) in [ + (PredicateOperator::Eq, "="), + (PredicateOperator::LessThanOrEq, "<="), + ] { + fixture.assert_projection( + &fixture.binary_predicate(op, Datum::binary(value.iter().copied())), + Some(&format!("name {projected_op} {prefix}")), + )?; + } + } + + // Both source values share one truncated prefix, so the projected set has one value. + fixture.assert_projection( + &fixture.set_predicate(PredicateOperator::In, vec![ + Datum::binary([0x00, 0xff, 0x80, 0x01]), + Datum::binary([0x00, 0xff, 0x80, 0x02]), + ]), + Some("name IN (00FF80)"), + )?; + + Ok(()) + } + + #[test] + fn test_strict_projection_truncate_binary() -> Result<()> { + let transform = Transform::Truncate(3); + let fixture = TestProjectionFixture::new( + transform, + "name", + NestedField::required(1, "value", Primitive(Binary)), + ); + + for (op, expected) in [ + (PredicateOperator::LessThanOrEq, Some("name < 00FF80")), + (PredicateOperator::NotEq, Some("name != 00FF80")), + (PredicateOperator::Eq, None), + ] { + let predicate = fixture.binary_predicate(op, Datum::binary([0x00, 0xff, 0x80, 0x01])); + let result = transform.strict_project("name", &predicate)?; + assert_eq!(result.map(|p| p.to_string()).as_deref(), expected); + } + + for (op, expected) in [ + (PredicateOperator::NotIn, Some("name NOT IN (00FF80)")), + (PredicateOperator::In, None), + ] { + let predicate = fixture.set_predicate(op, vec![ + Datum::binary([0x00, 0xff, 0x80, 0x01]), + Datum::binary([0x00, 0xff, 0x80, 0x02]), + ]); + let result = transform.strict_project("name", &predicate)?; + assert_eq!(result.map(|p| p.to_string()).as_deref(), expected); + } + + Ok(()) + } + + #[test] + fn test_projection_truncate_decimal_outside_precision() -> Result<()> { + let transform = Transform::Truncate(10); + let fixture = TestProjectionFixture::new( + transform, + "name", + NestedField::required( + 1, + "value", + Primitive(Decimal { + precision: 2, + scale: 1, + }), + ), + ); + + for (value, inclusive_op, inclusive_expected, strict_op, strict_expected) in [ + ( + 99, + PredicateOperator::GreaterThan, + "name >= 10.0", + PredicateOperator::LessThanOrEq, + "name < 10.0", + ), + ( + -99, + PredicateOperator::LessThan, + "name <= -10.0", + PredicateOperator::GreaterThanOrEq, + "name > -10.0", + ), + ] { + let datum = Datum::new( + Decimal { + precision: 2, + scale: 1, + }, + PrimitiveLiteral::Int128(value), + ); + fixture.assert_projection( + &fixture.binary_predicate(inclusive_op, datum.clone()), + Some(inclusive_expected), + )?; + let result = transform + .strict_project("name", &fixture.binary_predicate(strict_op, datum))? + .unwrap(); + assert_eq!(result.to_string(), strict_expected); + } + + Ok(()) + } + #[test] fn test_projection_truncate_string_rewrite_op() -> Result<()> { let value = "abcde"; @@ -257,7 +392,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Truncate(5), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::String)), + NestedField::required(1, "value", Primitive(PrimitiveType::String)), ); fixture.assert_projection( @@ -291,7 +426,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Truncate(5), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::String)), + NestedField::required(1, "value", Primitive(PrimitiveType::String)), ); fixture.assert_projection( @@ -322,7 +457,7 @@ mod test { fixture.assert_projection( &fixture.set_predicate(PredicateOperator::In, vec![ Datum::string(value), - Datum::string(format!("{}abc", value)), + Datum::string(format!("{value}abc")), ]), Some(r#"name IN ("abcde")"#), )?; @@ -330,7 +465,7 @@ mod test { fixture.assert_projection( &fixture.set_predicate(PredicateOperator::NotIn, vec![ Datum::string(value), - Datum::string(format!("{}abc", value)), + Datum::string(format!("{value}abc")), ]), None, )?; @@ -350,7 +485,7 @@ mod test { NestedField::required( 1, "value", - Type::Primitive(PrimitiveType::Decimal { + Primitive(Decimal { precision: 9, scale: 2, }), @@ -359,7 +494,7 @@ mod test { fixture.assert_projection( &fixture.binary_predicate(PredicateOperator::LessThan, Datum::decimal_from_str(curr)?), - Some("name <= 9990"), + Some("name <= 99.90"), )?; fixture.assert_projection( @@ -367,7 +502,7 @@ mod test { PredicateOperator::LessThanOrEq, Datum::decimal_from_str(curr)?, ), - Some("name <= 9990"), + Some("name <= 99.90"), )?; fixture.assert_projection( @@ -375,12 +510,12 @@ mod test { PredicateOperator::GreaterThanOrEq, Datum::decimal_from_str(curr)?, ), - Some("name >= 9990"), + Some("name >= 99.90"), )?; fixture.assert_projection( &fixture.binary_predicate(PredicateOperator::Eq, Datum::decimal_from_str(curr)?), - Some("name = 9990"), + Some("name = 99.90"), )?; fixture.assert_projection( @@ -394,7 +529,7 @@ mod test { Datum::decimal_from_str(curr)?, Datum::decimal_from_str(next)?, ]), - Some("name IN (9890, 9990, 10090)"), + Some("name IN (99.90, 100.90, 98.90)"), )?; fixture.assert_projection( @@ -420,7 +555,7 @@ mod test { NestedField::required( 1, "value", - Type::Primitive(PrimitiveType::Decimal { + Primitive(Decimal { precision: 9, scale: 2, }), @@ -429,7 +564,7 @@ mod test { fixture.assert_projection( &fixture.binary_predicate(PredicateOperator::LessThan, Datum::decimal_from_str(curr)?), - Some("name <= 9990"), + Some("name <= 99.90"), )?; fixture.assert_projection( @@ -437,7 +572,7 @@ mod test { PredicateOperator::LessThanOrEq, Datum::decimal_from_str(curr)?, ), - Some("name <= 10000"), + Some("name <= 100.00"), )?; fixture.assert_projection( @@ -445,12 +580,12 @@ mod test { PredicateOperator::GreaterThanOrEq, Datum::decimal_from_str(curr)?, ), - Some("name >= 10000"), + Some("name >= 100.00"), )?; fixture.assert_projection( &fixture.binary_predicate(PredicateOperator::Eq, Datum::decimal_from_str(curr)?), - Some("name = 10000"), + Some("name = 100.00"), )?; fixture.assert_projection( @@ -464,7 +599,7 @@ mod test { Datum::decimal_from_str(curr)?, Datum::decimal_from_str(next)?, ]), - Some("name IN (10000, 10100, 9900)"), + Some("name IN (99.00, 100.00, 101.00)"), )?; fixture.assert_projection( @@ -485,7 +620,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Truncate(10), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Long)), + NestedField::required(1, "value", Primitive(Long)), ); fixture.assert_projection( @@ -540,7 +675,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Truncate(10), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Long)), + NestedField::required(1, "value", Primitive(Long)), ); fixture.assert_projection( @@ -595,7 +730,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Truncate(10), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Int)), + NestedField::required(1, "value", Primitive(Int)), ); fixture.assert_projection( @@ -650,7 +785,7 @@ mod test { let fixture = TestProjectionFixture::new( Transform::Truncate(10), "name", - NestedField::required(1, "value", Type::Primitive(PrimitiveType::Int)), + NestedField::required(1, "value", Primitive(Int)), ); fixture.assert_projection( @@ -698,6 +833,22 @@ mod test { Ok(()) } + #[test] + fn test_projection_truncate_integer_large_width() -> Result<()> { + let fixture = TestProjectionFixture::new( + Transform::Truncate(2_000_000_000), + "name", + NestedField::required(1, "value", Primitive(Int)), + ); + + fixture.assert_projection( + &fixture.binary_predicate(PredicateOperator::Eq, Datum::int(1_500_000_000)), + Some("name = 0"), + )?; + + Ok(()) + } + // Test case ref from: https://iceberg.apache.org/spec/#truncate-transform-details #[test] fn test_truncate_simple() { @@ -774,6 +925,32 @@ mod test { ); } + #[test] + fn test_truncate_long_min_matches_java() { + // Expected values computed with Iceberg Java's `TruncateUtil.truncateLong` + // (iceberg 1.11.0, JDK 17). + for (width, value, expected) in [ + (1000, i64::MIN, 9_223_372_036_854_775_616), // i64::MAX - 191: rem_euclid is 192 + // i64::MAX - 2_147_483_644: rem_euclid is 2_147_483_645 + (i32::MAX as u32, i64::MIN, 9_223_372_034_707_292_163), + ] { + let truncate = super::Truncate::new(width); + let input = Arc::new(Int64Array::from(vec![Some(value), None])); + let result = truncate.transform(input).unwrap(); + let expected_array = Int64Array::from(vec![Some(expected), None]); + assert_eq!( + result.as_any().downcast_ref::(), + Some(&expected_array), + "width: {width}, value: {value}" + ); + assert_eq!( + truncate.transform_literal(&Datum::long(value)).unwrap(), + Some(Datum::long(expected)), + "width: {width}, value: {value}" + ); + } + } + #[test] fn test_string_truncate() { let test1 = "イロハニホヘト"; @@ -795,6 +972,34 @@ mod test { assert_eq!(super::Truncate::truncate_str(test4, 1), test4_1_expected); } + #[test] + fn test_literal_spec_examples() -> Result<()> { + // https://iceberg.apache.org/spec/#truncate-transform-details + let cases = [ + (10, Datum::int(1), Datum::int(0)), + (10, Datum::int(-1), Datum::int(-10)), + (10, Datum::long(1), Datum::long(0)), + (10, Datum::long(-1), Datum::long(-10)), + ( + 50, + Datum::decimal_from_str("10.65")?, + Datum::decimal_from_str("10.50")?, + ), + (3, Datum::string("iceberg"), Datum::string("ice")), + ( + 3, + Datum::binary(vec![0x01, 0x02, 0x03, 0x04, 0x05]), + Datum::binary(vec![0x01, 0x02, 0x03]), + ), + ]; + + for (width, input, expected) in cases { + let result = super::Truncate::new(width).transform_literal(&input)?; + assert_eq!(result, Some(expected), "width: {width}, input: {input:?}"); + } + Ok(()) + } + #[test] fn test_literal_int() { let input = Datum::int(1); @@ -812,6 +1017,35 @@ mod test { assert_eq!(res, Datum::int(-10),); } + #[test] + fn test_truncate_int_large_width() { + // Expected values follow the spec: `v - (v % W)` with a non-negative remainder. + // Below `i32::MIN` the spec is silent, and the result wraps as Java's `TruncateUtil` does. + for (width, value, expected) in [ + (2_000_000_000, 1_500_000_000, 0), + (i32::MAX as u32, i32::MAX - 1, 0), + (1000, i32::MIN, 2_147_483_296), // i32::MAX - 351: rem_euclid is 352 + // Smallest width where `(v % w) + w` can overflow i32. + (1_073_741_825, 1_073_741_824, 0), + (i32::MAX as u32, 1, 0), + ] { + let truncate = super::Truncate::new(width); + let input = Arc::new(Int32Array::from(vec![Some(value), None])); + let result = truncate.transform(input).unwrap(); + let expected_array = Int32Array::from(vec![Some(expected), None]); + assert_eq!( + result.as_any().downcast_ref::(), + Some(&expected_array), + "width: {width}, value: {value}" + ); + assert_eq!( + truncate.transform_literal(&Datum::int(value)).unwrap(), + Some(Datum::int(expected)), + "width: {width}, value: {value}" + ); + } + } + #[test] fn test_literal_long() { let input = Datum::long(1); @@ -831,12 +1065,63 @@ mod test { #[test] fn test_decimal_literal() { - let input = Datum::decimal(1065).unwrap(); - let res = super::Truncate::new(50) - .transform_literal(&input) - .unwrap() - .unwrap(); - assert_eq!(res, Datum::decimal(1050).unwrap(),); + for (precision, scale, value, expected) in [ + (9, 0, 1065, 1050), + (9, 2, 1065, 1050), + (9, 2, -1065, -1100), + (9, 2, 1050, 1050), + (9, 2, 0, 0), + (9, 5, 1065, 1050), + (38, 2, 1065, 1050), + ] { + let data_type = Decimal { precision, scale }; + let input = Datum::new(data_type.clone(), PrimitiveLiteral::Int128(value)); + let res = super::Truncate::new(50) + .transform_literal(&input) + .unwrap() + .unwrap(); + assert_eq!( + res, + Datum::new(data_type, PrimitiveLiteral::Int128(expected)), + "input: {input:?}" + ); + } + } + + #[test] + fn test_truncate_decimal_i128_min_does_not_panic() { + // `i128::MIN` is outside the range of a 38-digit Iceberg decimal, and Java's + // `TruncateUtil.truncateDecimal` works on `BigInteger`, so there is no wrapping + // result to match. + assert_eq!( + super::Truncate::truncate_decimal_i128(i128::MIN, 1000), + 170_141_183_460_469_231_731_687_303_715_884_105_456, // i128::MAX - 271 + ); + } + + #[test] + fn test_binary_literal() { + for (value, width, expected) in [ + (vec![0x00, 0xff, 0x80, 0x01], 2, vec![0x00, 0xff]), + (vec![0x00, 0xff], 2, vec![0x00, 0xff]), + (vec![0x00, 0xff], 4, vec![0x00, 0xff]), + (vec![], 2, vec![]), + ] { + let input = Datum::binary(value); + let res = super::Truncate::new(width) + .transform_literal(&input) + .unwrap() + .unwrap(); + assert_eq!(res, Datum::binary(expected)); + } + } + + #[test] + fn test_fixed_literal_is_unsupported() { + let error = super::Truncate::new(2) + .transform_literal(&Datum::fixed(vec![0x00, 0xff, 0x80])) + .unwrap_err(); + assert_eq!(error.kind(), crate::ErrorKind::FeatureUnsupported); } #[test] diff --git a/crates/sqllogictest/src/error.rs b/crates/iceberg/src/util/location.rs similarity index 51% rename from crates/sqllogictest/src/error.rs rename to crates/iceberg/src/util/location.rs index 01eb364e2a..00681a12a2 100644 --- a/crates/sqllogictest/src/error.rs +++ b/crates/iceberg/src/util/location.rs @@ -15,32 +15,30 @@ // specific language governing permissions and limitations // under the License. -use std::fmt::{Debug, Display, Formatter}; - -pub struct Error(pub anyhow::Error); - -pub type Result = std::result::Result; - -impl Debug for Error { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } -} - -impl Display for Error { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl std::error::Error for Error { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - self.0.source() +/// Strips trailing slashes from a location, preserving a bare URI scheme root +pub(crate) fn strip_trailing_slash(path: &str) -> &str { + let mut path = path; + while !path.ends_with("://") { + let Some(stripped) = path.strip_suffix('/') else { + break; + }; + path = stripped; } + path } -impl From for Error { - fn from(value: anyhow::Error) -> Self { - Self(value) +#[test] +fn test_strip_trailing_slash() { + for (path, expected) in [ + ("s3://bucket/db/tbl", "s3://bucket/db/tbl"), + ("s3://bucket/db/tbl/", "s3://bucket/db/tbl"), + ("s3://bucket/db/tbl////", "s3://bucket/db/tbl"), + ("blobstore://", "blobstore://"), + ("blobstore:///", "blobstore://"), + ("file:///", "file://"), + ("////", ""), + ("", ""), + ] { + assert_eq!(strip_trailing_slash(path), expected); } } diff --git a/crates/iceberg/src/utils.rs b/crates/iceberg/src/util/mod.rs similarity index 81% rename from crates/iceberg/src/utils.rs rename to crates/iceberg/src/util/mod.rs index 00d3e69bd3..94da0130c4 100644 --- a/crates/iceberg/src/utils.rs +++ b/crates/iceberg/src/util/mod.rs @@ -17,6 +17,11 @@ use std::num::NonZeroUsize; +/// Utilities for working with data, metadata, database, table ...etc. locations +pub mod location; +/// Utilities for working with snapshots. +pub mod snapshot; + // Use a default value of 1 as the safest option. // See https://doc.rust-lang.org/std/thread/fn.available_parallelism.html#limitations // for more details. @@ -32,11 +37,11 @@ const DEFAULT_PARALLELISM: usize = 1; /// parallelism can change during the lifetime of an executing /// process, but this should not be called in a hot loop. pub(crate) fn available_parallelism() -> NonZeroUsize { - std::thread::available_parallelism().unwrap_or_else(|_err| { - // Failed to get the level of parallelism. - // TODO: log/trace when this fallback occurs. - - // Using a default value. + std::thread::available_parallelism().unwrap_or_else(|err| { + tracing::warn!( + error = %err, + "Failed to determine available parallelism; falling back to {DEFAULT_PARALLELISM}", + ); NonZeroUsize::new(DEFAULT_PARALLELISM).unwrap() }) } diff --git a/crates/iceberg/src/util/snapshot.rs b/crates/iceberg/src/util/snapshot.rs new file mode 100644 index 0000000000..c5f307d618 --- /dev/null +++ b/crates/iceberg/src/util/snapshot.rs @@ -0,0 +1,326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashSet; + +use crate::spec::{SnapshotRef, TableMetadataRef}; +use crate::{Error, ErrorKind, Result}; + +struct Ancestors { + next: Option, + get_snapshot: Box Option + Send>, + visited: HashSet, +} + +impl Iterator for Ancestors { + type Item = SnapshotRef; + + fn next(&mut self) -> Option { + let snapshot = self.next.take()?; + // corrupt metadata with a parent cycle must not hang the traversal + if !self.visited.insert(snapshot.snapshot_id()) { + return None; + } + self.next = snapshot + .parent_snapshot_id() + .and_then(|id| (self.get_snapshot)(id)); + Some(snapshot) + } +} + +/// Iterate starting from `snapshot_id` (inclusive) to the root snapshot. +/// +/// The iterator visits each snapshot at most once, so it terminates even on +/// corrupt metadata whose parent pointers form a cycle. +pub fn ancestors_of( + table_metadata: &TableMetadataRef, + snapshot_id: i64, +) -> impl Iterator + Send { + let initial = table_metadata.snapshot_by_id(snapshot_id).cloned(); + let table_metadata = table_metadata.clone(); + Ancestors { + next: initial, + get_snapshot: Box::new(move |id| table_metadata.snapshot_by_id(id).cloned()), + visited: HashSet::new(), + } +} + +/// Iterate starting from `latest_snapshot_id` (inclusive) to `oldest_snapshot_id` (exclusive). +/// +/// Note: if `oldest_snapshot_id` is `Some(id)` but `id` is not actually an +/// ancestor of `latest_snapshot_id`, the walk is never stopped and this yields +/// *all* ancestors of `latest_snapshot_id` down to the root. Callers that treat +/// `oldest_snapshot_id` as a lower bound must validate the lineage themselves. +pub fn ancestors_between( + table_metadata: &TableMetadataRef, + latest_snapshot_id: i64, + oldest_snapshot_id: Option, +) -> impl Iterator + Send { + ancestors_of(table_metadata, latest_snapshot_id).take_while(move |snapshot| { + oldest_snapshot_id + .map(|id| snapshot.snapshot_id() != id) + .unwrap_or(true) + }) +} + +/// Resolve the snapshot ID from the main-history entry with the greatest timestamp +/// at or before `timestamp_ms` (milliseconds since the Unix epoch), taking the first +/// entry on ties. +/// +/// This matches Java's ordering, except that an entry at `i64::MIN` can match here +/// but never matches in Java, which seeds its search with `Long.MIN_VALUE`. +/// PyIceberg takes the last qualifying log entry, so ties or out-of-order +/// timestamps can produce different results. +/// +/// Returns [`ErrorKind::DataInvalid`] if no matching history exists. +/// The returned snapshot may have expired, so +/// [`snapshot_by_id`](crate::spec::TableMetadata::snapshot_by_id) can still return `None`. +/// +/// ``` +/// # use iceberg::TableIdent; +/// # use iceberg::io::FileIO; +/// # use iceberg::table::StaticTable; +/// use iceberg::util::snapshot::snapshot_id_as_of_time; +/// # #[tokio::main] +/// # async fn main() -> iceberg::Result<()> { +/// # let location = concat!(env!("CARGO_MANIFEST_DIR"), "/testdata/example_table_metadata_v2.json"); +/// # let ident = TableIdent::from_strs(["ns", "t"])?; +/// # let table = StaticTable::from_metadata_file(location, ident, FileIO::new_with_fs()).await?; +/// let metadata = table.metadata(); +/// // The snapshot log has entries at 1515100955770 and 1555100955770. +/// let snapshot_id = snapshot_id_as_of_time(&metadata, 1_555_100_955_770)?; +/// assert_eq!(snapshot_id, 3055729675574597004); +/// assert!(snapshot_id_as_of_time(&metadata, 1_515_100_955_769).is_err()); +/// # Ok(()) +/// # } +/// ``` +pub fn snapshot_id_as_of_time(table_metadata: &TableMetadataRef, timestamp_ms: i64) -> Result { + table_metadata + .history() + .iter() + .enumerate() + .filter(|(_, entry)| entry.timestamp_ms() <= timestamp_ms) + .max_by(|(idx_a, a), (idx_b, b)| { + a.timestamp_ms() + .cmp(&b.timestamp_ms()) + // Earlier entries win ties, matching Java's timestamp selection. + .then(idx_b.cmp(idx_a)) + }) + .map(|(_, entry)| entry.snapshot_id) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("No snapshot history at or before timestamp {timestamp_ms} ms"), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spec::SnapshotLog; + use crate::test_utils::scan::TableTestFixture; + + // Five snapshots chained as: S1 (root) -> S2 -> S3 -> S4 -> S5 (current) + const S1: i64 = 3051729675574597004; + const S2: i64 = 3055729675574597004; + const S3: i64 = 3056729675574597004; + const S4: i64 = 3057729675574597004; + const S5: i64 = 3059729675574597004; + + fn metadata() -> TableMetadataRef { + let fixture = TableTestFixture::new_with_deep_history(); + std::sync::Arc::new(fixture.table.metadata().clone()) + } + + type History = [(i64, i64)]; + + fn metadata_with_history(history: &History) -> TableMetadataRef { + let mut metadata = metadata().as_ref().clone(); + metadata.snapshot_log = history + .iter() + .map(|&(timestamp_ms, snapshot_id)| SnapshotLog { + timestamp_ms, + snapshot_id, + }) + .collect(); + metadata.into() + } + + #[test] + fn test_snapshot_id_as_of_time() { + let cases: &[(&History, i64, i64)] = &[ + (&[(1000, S1), (2000, S2)], 1000, S1), + (&[(1000, S1), (2000, S2)], 1500, S1), + (&[(1000, S1), (2000, S2)], 2000, S2), + (&[(1000, S1), (2000, S2)], 3000, S2), + // Rollback records when an existing snapshot becomes current again. + (&[(1000, S1), (2000, S2), (3000, S1)], 3500, S1), + // The first entry wins on ties. + (&[(1000, S1), (2000, S2), (2000, S3)], 2000, S2), + // Clock skew means the log need not be sorted by timestamp. + (&[(1000, S1), (3000, S2), (2500, S3)], 3500, S2), + (&[(1000, S1), (3000, S2), (2500, S3)], 2600, S3), + (&[(-2000, S1), (-1000, S2)], -1500, S1), + // Java returns no match here; see the docstring. + (&[(i64::MIN, S1), (0, S2)], i64::MIN, S1), + (&[(0, S1), (i64::MAX, S2)], i64::MAX, S2), + ]; + for &(history, timestamp_ms, expected) in cases { + let metadata = metadata_with_history(history); + assert_eq!( + snapshot_id_as_of_time(&metadata, timestamp_ms).unwrap(), + expected, + "timestamp {timestamp_ms}, history {history:?}" + ); + } + } + + #[test] + fn test_snapshot_id_as_of_time_without_history_at_timestamp() { + let cases: &[(&History, i64)] = &[ + (&[], 1000), + (&[(1000, S1)], 999), + (&[(3000, S1)], 1500), // Expired prefix; S1 is still retained. + ]; + for &(history, timestamp_ms) in cases { + let metadata = metadata_with_history(history); + assert!(metadata.snapshot_by_id(S1).is_some()); + let err = snapshot_id_as_of_time(&metadata, timestamp_ms).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains(×tamp_ms.to_string())); + } + } + + #[test] + fn test_snapshot_id_as_of_time_ignores_snapshots_outside_main_history() { + let mut metadata = metadata().as_ref().clone(); + metadata.snapshot_log.truncate(2); + // Newer retained snapshots may be staged or belong only to another branch. + assert!(metadata.snapshot_by_id(S5).is_some()); + assert_eq!( + snapshot_id_as_of_time(&metadata.into(), i64::MAX).unwrap(), + S2 + ); + } + + // --- ancestors_of --- + + #[test] + fn test_ancestors_of_nonexistent_snapshot_returns_empty() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, 999).map(|s| s.snapshot_id()).collect(); + assert!(ids.is_empty()); + } + + #[test] + fn test_ancestors_of_root_returns_only_root() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, S1).map(|s| s.snapshot_id()).collect(); + assert_eq!(ids, vec![S1]); + } + + #[test] + fn test_ancestors_of_leaf_returns_full_chain() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, S5).map(|s| s.snapshot_id()).collect(); + assert_eq!(ids, vec![S5, S4, S3, S2, S1]); + } + + #[test] + fn test_ancestors_of_mid_chain_returns_partial_chain() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, S3).map(|s| s.snapshot_id()).collect(); + assert_eq!(ids, vec![S3, S2, S1]); + } + + #[test] + fn test_ancestors_of_second_snapshot() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, S2).map(|s| s.snapshot_id()).collect(); + assert_eq!(ids, vec![S2, S1]); + } + + // --- ancestors_between --- + + #[test] + fn test_ancestors_between_same_id_returns_empty() { + let meta = metadata(); + let ids: Vec = ancestors_between(&meta, S3, Some(S3)) + .map(|s| s.snapshot_id()) + .collect(); + assert!(ids.is_empty()); + } + + #[test] + fn test_ancestors_between_no_oldest_returns_all_ancestors() { + let meta = metadata(); + let ids: Vec = ancestors_between(&meta, S5, None) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S5, S4, S3, S2, S1]); + } + + #[test] + fn test_ancestors_between_excludes_oldest_snapshot() { + let meta = metadata(); + // S5 down to (but not including) S2 + let ids: Vec = ancestors_between(&meta, S5, Some(S2)) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S5, S4, S3]); + } + + #[test] + fn test_ancestors_between_adjacent_snapshots() { + let meta = metadata(); + // S3 down to (but not including) S2 — only S3 itself + let ids: Vec = ancestors_between(&meta, S3, Some(S2)) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S3]); + } + + #[test] + fn test_ancestors_between_leaf_and_root() { + let meta = metadata(); + // S5 down to (but not including) S1 + let ids: Vec = ancestors_between(&meta, S5, Some(S1)) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S5, S4, S3, S2]); + } + + #[test] + fn test_ancestors_between_nonexistent_oldest_returns_full_chain() { + let meta = metadata(); + // oldest_snapshot_id doesn't exist in the chain, so take_while never stops + let ids: Vec = ancestors_between(&meta, S5, Some(999)) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S5, S4, S3, S2, S1]); + } + + #[test] + fn test_ancestors_between_nonexistent_latest_returns_empty() { + let meta = metadata(); + let ids: Vec = ancestors_between(&meta, 999, Some(S1)) + .map(|s| s.snapshot_id()) + .collect(); + assert!(ids.is_empty()); + } +} diff --git a/crates/iceberg/src/writer/base_writer/data_file_writer.rs b/crates/iceberg/src/writer/base_writer/data_file_writer.rs index f215673df1..c7a988374e 100644 --- a/crates/iceberg/src/writer/base_writer/data_file_writer.rs +++ b/crates/iceberg/src/writer/base_writer/data_file_writer.rs @@ -18,86 +18,116 @@ //! This module provide `DataFileWriter`. use arrow_array::RecordBatch; -use itertools::Itertools; -use crate::Result; -use crate::spec::{DataContentType, DataFile, Struct}; -use crate::writer::file_writer::{FileWriter, FileWriterBuilder}; +use crate::error::invalid_data; +use crate::spec::{DataContentType, DataFile, PartitionKey}; +use crate::writer::file_writer::FileWriterBuilder; +use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; +use crate::writer::file_writer::rolling_writer::{RollingFileWriter, RollingFileWriterBuilder}; use crate::writer::{CurrentFileStatus, IcebergWriter, IcebergWriterBuilder}; +use crate::{Error, ErrorKind, Result}; /// Builder for `DataFileWriter`. -#[derive(Clone, Debug)] -pub struct DataFileWriterBuilder { - inner: B, - partition_value: Option, - partition_spec_id: i32, +#[derive(Debug)] +pub struct DataFileWriterBuilder { + inner: RollingFileWriterBuilder, } -impl DataFileWriterBuilder { - /// Create a new `DataFileWriterBuilder` using a `FileWriterBuilder`. - pub fn new(inner: B, partition_value: Option, partition_spec_id: i32) -> Self { - Self { - inner, - partition_value, - partition_spec_id, - } +impl DataFileWriterBuilder +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + /// Create a new `DataFileWriterBuilder` using a `RollingFileWriterBuilder`. + pub fn new(inner: RollingFileWriterBuilder) -> Self { + Self { inner } } } #[async_trait::async_trait] -impl IcebergWriterBuilder for DataFileWriterBuilder { - type R = DataFileWriter; - - async fn build(self) -> Result { +impl IcebergWriterBuilder for DataFileWriterBuilder +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + type R = DataFileWriter; + + async fn build(&self, partition_key: Option) -> Result { Ok(DataFileWriter { - inner_writer: Some(self.inner.clone().build().await?), - partition_value: self.partition_value.unwrap_or(Struct::empty()), - partition_spec_id: self.partition_spec_id, + inner: Some(self.inner.build()), + partition_key, }) } } /// A writer write data is within one spec/partition. #[derive(Debug)] -pub struct DataFileWriter { - inner_writer: Option, - partition_value: Struct, - partition_spec_id: i32, +pub struct DataFileWriter { + inner: Option>, + partition_key: Option, } #[async_trait::async_trait] -impl IcebergWriter for DataFileWriter { +impl IcebergWriter for DataFileWriter +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ async fn write(&mut self, batch: RecordBatch) -> Result<()> { - self.inner_writer.as_mut().unwrap().write(&batch).await + if let Some(writer) = self.inner.as_mut() { + writer.write(&self.partition_key, &batch).await + } else { + Err(Error::new( + ErrorKind::Unexpected, + "Writer is not initialized!", + )) + } } async fn close(&mut self) -> Result> { - let writer = self.inner_writer.take().unwrap(); - Ok(writer - .close() - .await? - .into_iter() - .map(|mut res| { - res.content(DataContentType::Data); - res.partition(self.partition_value.clone()); - res.partition_spec_id(self.partition_spec_id); - res.build().expect("Guaranteed to be valid") - }) - .collect_vec()) + if let Some(writer) = self.inner.take() { + writer + .close() + .await? + .into_iter() + .map(|mut res| { + res.content(DataContentType::Data); + if let Some(pk) = self.partition_key.as_ref() { + res.partition(pk.data().clone()); + res.partition_spec_id(pk.spec().spec_id()); + } + res.build() + .map_err(|e| invalid_data!("Failed to build data file: {e}")) + }) + .collect() + } else { + Err(Error::new( + ErrorKind::Unexpected, + "Data file writer has been closed.", + )) + } } } -impl CurrentFileStatus for DataFileWriter { +impl CurrentFileStatus for DataFileWriter +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ fn current_file_path(&self) -> String { - self.inner_writer.as_ref().unwrap().current_file_path() + self.inner.as_ref().unwrap().current_file_path() } fn current_row_num(&self) -> usize { - self.inner_writer.as_ref().unwrap().current_row_num() + self.inner.as_ref().unwrap().current_row_num() } fn current_written_size(&self) -> usize { - self.inner_writer.as_ref().unwrap().current_written_size() + self.inner.as_ref().unwrap().current_written_size() } } @@ -114,21 +144,23 @@ mod test { use tempfile::TempDir; use crate::Result; - use crate::io::FileIOBuilder; + use crate::io::FileIO; use crate::spec::{ - DataContentType, DataFileFormat, Literal, NestedField, PrimitiveType, Schema, Struct, Type, + DataContentType, DataFileFormat, Literal, NestedField, PartitionKey, PartitionSpec, + PrimitiveType, Schema, Struct, Type, }; use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; use crate::writer::file_writer::ParquetWriterBuilder; use crate::writer::file_writer::location_generator::{ DefaultFileNameGenerator, DefaultLocationGenerator, }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; use crate::writer::{IcebergWriter, IcebergWriterBuilder, RecordBatch}; #[tokio::test] async fn test_parquet_writer() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -143,17 +175,17 @@ mod test { ]) .build()?; - let pw = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - Arc::new(schema), - None, + let pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(schema)); + + let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + pw, file_io.clone(), location_gen, file_name_gen, ); - let mut data_file_writer = DataFileWriterBuilder::new(pw, None, 0) - .build() + let mut data_file_writer = DataFileWriterBuilder::new(rolling_file_writer_builder) + .build(None) .await .unwrap(); @@ -202,7 +234,7 @@ mod test { #[tokio::test] async fn test_parquet_writer_with_partition() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -219,22 +251,28 @@ mod test { NestedField::required(6, "name", Type::Primitive(PrimitiveType::String)).into(), ]) .build()?; + let schema_ref = Arc::new(schema); let partition_value = Struct::from_iter([Some(Literal::int(1))]); + let partition_key = PartitionKey::new( + PartitionSpec::builder(schema_ref.clone()).build()?, + schema_ref.clone(), + partition_value.clone(), + ); - let parquet_writer_builder = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - Arc::new(schema.clone()), - None, + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema_ref.clone()); + + let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, file_io.clone(), location_gen, file_name_gen, ); - let mut data_file_writer = - DataFileWriterBuilder::new(parquet_writer_builder, Some(partition_value.clone()), 0) - .build() - .await?; + let mut data_file_writer = DataFileWriterBuilder::new(rolling_file_writer_builder) + .build(Some(partition_key)) + .await?; let arrow_schema = arrow_schema::Schema::new(vec![ Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( diff --git a/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs b/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs index 765ff1cacd..f62016addc 100644 --- a/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs +++ b/crates/iceberg/src/writer/base_writer/equality_delete_writer.rs @@ -17,6 +17,7 @@ //! This module provide `EqualityDeleteWriter`. +use std::collections::HashSet; use std::sync::Arc; use arrow_array::RecordBatch; @@ -26,52 +27,92 @@ use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use crate::arrow::record_batch_projector::RecordBatchProjector; use crate::arrow::schema_to_arrow_schema; -use crate::spec::{DataFile, SchemaRef, Struct}; -use crate::writer::file_writer::{FileWriter, FileWriterBuilder}; +use crate::error::invalid_data; +use crate::spec::{DataFile, PartitionKey, Schema, SchemaRef}; +use crate::writer::file_writer::FileWriterBuilder; +use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; +use crate::writer::file_writer::rolling_writer::{RollingFileWriter, RollingFileWriterBuilder}; use crate::writer::{IcebergWriter, IcebergWriterBuilder}; use crate::{Error, ErrorKind, Result}; /// Builder for `EqualityDeleteWriter`. -#[derive(Clone, Debug)] -pub struct EqualityDeleteFileWriterBuilder { - inner: B, +#[derive(Debug)] +pub struct EqualityDeleteFileWriterBuilder< + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +> { + inner: RollingFileWriterBuilder, config: EqualityDeleteWriterConfig, } -impl EqualityDeleteFileWriterBuilder { - /// Create a new `EqualityDeleteFileWriterBuilder` using a `FileWriterBuilder`. - pub fn new(inner: B, config: EqualityDeleteWriterConfig) -> Self { +impl EqualityDeleteFileWriterBuilder +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + /// Create a new `EqualityDeleteFileWriterBuilder` using a `RollingFileWriterBuilder`. + pub fn new( + inner: RollingFileWriterBuilder, + config: EqualityDeleteWriterConfig, + ) -> Self { Self { inner, config } } } /// Config for `EqualityDeleteWriter`. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct EqualityDeleteWriterConfig { // Field ids used to determine row equality in equality delete files. equality_ids: Vec, // Projector used to project the data chunk into specific fields. projector: RecordBatchProjector, - partition_value: Struct, - partition_spec_id: i32, +} + +/// Validates equality delete field ids before constructing the writer config. +/// +/// In addition to rejecting an empty id list, this check also rejects duplicate +/// ids and ids that do not exist in the table schema. These two checks are +/// intentionally stricter than the current Java reference implementation, which +/// only guards against null/empty lists; catching duplicates and missing ids +/// here produces clearer errors and avoids generating meaningless delete files. +fn validate_equality_ids(equality_ids: &[i32], original_schema: &Schema) -> Result<()> { + if equality_ids.is_empty() { + return Err(invalid_data!( + "Equality delete field ids must not be empty." + )); + } + + let mut seen = HashSet::with_capacity(equality_ids.len()); + for id in equality_ids { + if !seen.insert(*id) { + return Err(invalid_data!("Duplicate equality delete field id: {id}") + .with_context("field_id", id.to_string())); + } + + if original_schema.field_by_id(*id).is_none() { + return Err(invalid_data!("Invalid equality delete field id: {id}") + .with_context("field_id", id.to_string())); + } + } + + Ok(()) } impl EqualityDeleteWriterConfig { - /// Create a new `DataFileWriterConfig` with equality ids. - pub fn new( - equality_ids: Vec, - original_schema: SchemaRef, - partition_value: Option, - partition_spec_id: i32, - ) -> Result { + /// Create a new `EqualityDeleteWriterConfig` with equality ids. + pub fn new(equality_ids: Vec, original_schema: SchemaRef) -> Result { + validate_equality_ids(&equality_ids, &original_schema)?; + let original_arrow_schema = Arc::new(schema_to_arrow_schema(&original_schema)?); let projector = RecordBatchProjector::new( original_arrow_schema, &equality_ids, - // The following rule comes from https://iceberg.apache.org/spec/#identifier-field-ids - // and https://iceberg.apache.org/spec/#equality-delete-files - // - The identifier field ids must be used for primitive types. - // - The identifier field ids must not be used for floating point types or nullable fields. + // Equality delete fields follow identifier-field type restrictions, + // except optional columns and columns nested under optional structs are allowed. + // Project only primitive, non-floating fields; RecordBatchProjector's traversal + // keeps fields under maps and lists unreachable. |field| { // Only primitive type is allowed to be used for identifier field ids if field.data_type().is_nested() @@ -98,8 +139,6 @@ impl EqualityDeleteWriterConfig { Ok(Self { equality_ids, projector, - partition_value: partition_value.unwrap_or(Struct::empty()), - partition_spec_id, }) } @@ -110,36 +149,48 @@ impl EqualityDeleteWriterConfig { } #[async_trait::async_trait] -impl IcebergWriterBuilder for EqualityDeleteFileWriterBuilder { - type R = EqualityDeleteFileWriter; - - async fn build(self) -> Result { +impl IcebergWriterBuilder for EqualityDeleteFileWriterBuilder +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + type R = EqualityDeleteFileWriter; + + async fn build(&self, partition_key: Option) -> Result { Ok(EqualityDeleteFileWriter { - inner_writer: Some(self.inner.clone().build().await?), - projector: self.config.projector, - equality_ids: self.config.equality_ids, - partition_value: self.config.partition_value, - partition_spec_id: self.config.partition_spec_id, + inner: Some(self.inner.build()), + projector: self.config.projector.clone(), + equality_ids: self.config.equality_ids.clone(), + partition_key, }) } } /// Writer used to write equality delete files. #[derive(Debug)] -pub struct EqualityDeleteFileWriter { - inner_writer: Option, +pub struct EqualityDeleteFileWriter< + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +> { + inner: Option>, projector: RecordBatchProjector, equality_ids: Vec, - partition_value: Struct, - partition_spec_id: i32, + partition_key: Option, } #[async_trait::async_trait] -impl IcebergWriter for EqualityDeleteFileWriter { +impl IcebergWriter for EqualityDeleteFileWriter +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ async fn write(&mut self, batch: RecordBatch) -> Result<()> { let batch = self.projector.project_batch(batch)?; - if let Some(writer) = self.inner_writer.as_mut() { - writer.write(&batch).await + if let Some(writer) = self.inner.as_mut() { + writer.write(&self.partition_key, &batch).await } else { Err(Error::new( ErrorKind::Unexpected, @@ -149,19 +200,22 @@ impl IcebergWriter for EqualityDeleteFileWriter { } async fn close(&mut self) -> Result> { - if let Some(writer) = self.inner_writer.take() { - Ok(writer + if let Some(writer) = self.inner.take() { + writer .close() .await? .into_iter() .map(|mut res| { res.content(crate::spec::DataContentType::EqualityDeletes); res.equality_ids(Some(self.equality_ids.iter().copied().collect_vec())); - res.partition(self.partition_value.clone()); - res.partition_spec_id(self.partition_spec_id); - res.build().expect("msg") + if let Some(pk) = self.partition_key.as_ref() { + res.partition(pk.data().clone()); + res.partition_spec_id(pk.spec().spec_id()); + } + res.build() + .map_err(|e| invalid_data!("Failed to build data file: {e}")) }) - .collect_vec()) + .collect() } else { Err(Error::new( ErrorKind::Unexpected, @@ -188,8 +242,9 @@ mod test { use tempfile::TempDir; use uuid::Uuid; + use crate::ErrorKind; use crate::arrow::{arrow_schema_to_schema, schema_to_arrow_schema}; - use crate::io::{FileIO, FileIOBuilder}; + use crate::io::FileIO; use crate::spec::{ DataFile, DataFileFormat, ListType, MapType, NestedField, PrimitiveType, Schema, StructType, Type, @@ -201,6 +256,7 @@ mod test { use crate::writer::file_writer::location_generator::{ DefaultFileNameGenerator, DefaultLocationGenerator, }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; use crate::writer::{IcebergWriter, IcebergWriterBuilder}; async fn check_parquet_data_file_with_equality_delete_write( @@ -268,21 +324,21 @@ mod test { assert_eq!(*data_file.null_value_counts.get(id).unwrap(), expect); } - assert_eq!(data_file.split_offsets.len(), metadata.num_row_groups()); - data_file + let split_offsets = data_file .split_offsets - .iter() - .enumerate() - .for_each(|(i, &v)| { - let expect = metadata.row_groups()[i].file_offset().unwrap(); - assert_eq!(v, expect); - }); + .as_ref() + .expect("split_offsets should be set"); + assert_eq!(split_offsets.len(), metadata.num_row_groups()); + split_offsets.iter().enumerate().for_each(|(i, &v)| { + let expect = metadata.row_groups()[i].file_offset().unwrap(); + assert_eq!(v, expect); + }); } #[tokio::test] async fn test_equality_delete_writer() -> Result<(), anyhow::Error> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -397,23 +453,24 @@ mod test { let equality_ids = vec![0_i32, 8]; let equality_config = - EqualityDeleteWriterConfig::new(equality_ids, Arc::new(schema), None, 0).unwrap(); + EqualityDeleteWriterConfig::new(equality_ids, Arc::new(schema)).unwrap(); let delete_schema = arrow_schema_to_schema(equality_config.projected_arrow_schema_ref()).unwrap(); let projector = equality_config.projector.clone(); // prepare writer - let pb = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - Arc::new(delete_schema), - None, + let pb = + ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(delete_schema)); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + pb, file_io.clone(), location_gen, file_name_gen, ); - let mut equality_delete_writer = EqualityDeleteFileWriterBuilder::new(pb, equality_config) - .build() - .await?; + let mut equality_delete_writer = + EqualityDeleteFileWriterBuilder::new(rolling_writer_builder, equality_config) + .build(None) + .await?; // write equality_delete_writer.write(to_write.clone()).await?; @@ -432,6 +489,60 @@ mod test { Ok(()) } + fn equality_id_validation_schema() -> Arc { + Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ) + } + + #[test] + fn test_equality_delete_rejects_empty_equality_ids() { + let err = + EqualityDeleteWriterConfig::new(vec![], equality_id_validation_schema()).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("Equality delete field ids must not be empty."), + "{err}" + ); + } + + #[test] + fn test_equality_delete_rejects_duplicate_equality_ids() { + let err = EqualityDeleteWriterConfig::new(vec![1, 1], equality_id_validation_schema()) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("Duplicate equality delete field id: 1"), + "{err}" + ); + assert!(err.to_string().contains("field_id: 1"), "{err}"); + } + + #[test] + fn test_equality_delete_rejects_missing_equality_id() { + let err = + EqualityDeleteWriterConfig::new(vec![99], equality_id_validation_schema()).unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("Invalid equality delete field id: 99"), + "{err}" + ); + assert!(err.to_string().contains("field_id: 99"), "{err}"); + } + #[tokio::test] async fn test_equality_delete_unreachable_column() -> Result<(), anyhow::Error> { let schema = Arc::new( @@ -499,19 +610,19 @@ mod test { .unwrap(), ); // Float and Double are not allowed to be used for equality delete - assert!(EqualityDeleteWriterConfig::new(vec![0], schema.clone(), None, 0).is_err()); - assert!(EqualityDeleteWriterConfig::new(vec![1], schema.clone(), None, 0).is_err()); + assert!(EqualityDeleteWriterConfig::new(vec![0], schema.clone()).is_err()); + assert!(EqualityDeleteWriterConfig::new(vec![1], schema.clone()).is_err()); // Struct is not allowed to be used for equality delete - assert!(EqualityDeleteWriterConfig::new(vec![3], schema.clone(), None, 0).is_err()); + assert!(EqualityDeleteWriterConfig::new(vec![3], schema.clone()).is_err()); // Nested field of struct is allowed to be used for equality delete - assert!(EqualityDeleteWriterConfig::new(vec![4], schema.clone(), None, 0).is_ok()); + assert!(EqualityDeleteWriterConfig::new(vec![4], schema.clone()).is_ok()); // Nested field of map is not allowed to be used for equality delete - assert!(EqualityDeleteWriterConfig::new(vec![7], schema.clone(), None, 0).is_err()); - assert!(EqualityDeleteWriterConfig::new(vec![8], schema.clone(), None, 0).is_err()); - assert!(EqualityDeleteWriterConfig::new(vec![9], schema.clone(), None, 0).is_err()); + assert!(EqualityDeleteWriterConfig::new(vec![7], schema.clone()).is_err()); + assert!(EqualityDeleteWriterConfig::new(vec![8], schema.clone()).is_err()); + assert!(EqualityDeleteWriterConfig::new(vec![9], schema.clone()).is_err()); // Nested field of list is not allowed to be used for equality delete - assert!(EqualityDeleteWriterConfig::new(vec![10], schema.clone(), None, 0).is_err()); - assert!(EqualityDeleteWriterConfig::new(vec![11], schema.clone(), None, 0).is_err()); + assert!(EqualityDeleteWriterConfig::new(vec![10], schema.clone()).is_err()); + assert!(EqualityDeleteWriterConfig::new(vec![11], schema.clone()).is_err()); Ok(()) } @@ -519,7 +630,7 @@ mod test { #[tokio::test] async fn test_equality_delete_with_primitive_type() -> Result<(), anyhow::Error> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -565,22 +676,22 @@ mod test { .unwrap(), ); let equality_ids = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]; - let config = - EqualityDeleteWriterConfig::new(equality_ids, schema.clone(), None, 0).unwrap(); + let config = EqualityDeleteWriterConfig::new(equality_ids, schema.clone()).unwrap(); let delete_arrow_schema = config.projected_arrow_schema_ref().clone(); let delete_schema = arrow_schema_to_schema(&delete_arrow_schema).unwrap(); - let pb = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - Arc::new(delete_schema), - None, + let pb = + ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(delete_schema)); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + pb, file_io.clone(), location_gen, file_name_gen, ); - let mut equality_delete_writer = EqualityDeleteFileWriterBuilder::new(pb, config) - .build() - .await?; + let mut equality_delete_writer = + EqualityDeleteFileWriterBuilder::new(rolling_writer_builder, config) + .build(None) + .await?; // prepare data let col0 = Arc::new(BooleanArray::from(vec![ @@ -652,7 +763,7 @@ mod test { ) .unwrap(), ) as ArrayRef; - let col13 = Arc::new(arrow_array::LargeBinaryArray::from_opt_vec(vec![ + let col13 = Arc::new(arrow_array::BinaryArray::from_opt_vec(vec![ Some(b"one"), Some(b""), Some(b"zzzz"), @@ -674,6 +785,80 @@ mod test { Ok(()) } + #[tokio::test] + async fn test_equality_delete_accepts_large_binary_input() -> Result<(), anyhow::Error> { + let temp_dir = TempDir::new().unwrap(); + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "key", Type::Primitive(PrimitiveType::Binary)).into(), + ]) + .build() + .unwrap(), + ); + let config = EqualityDeleteWriterConfig::new(vec![2], schema.clone()).unwrap(); + let delete_arrow_schema = config.projected_arrow_schema_ref().clone(); + let delete_schema = arrow_schema_to_schema(&delete_arrow_schema).unwrap(); + + let pb = + ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(delete_schema)); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + pb, + file_io.clone(), + location_gen, + file_name_gen, + ); + let mut equality_delete_writer = + EqualityDeleteFileWriterBuilder::new(rolling_writer_builder, config) + .build(None) + .await?; + + let input_schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("key", DataType::LargeBinary, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + let to_write = RecordBatch::try_new(input_schema, vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + Arc::new(arrow_array::LargeBinaryArray::from_opt_vec(vec![ + Some(b"one"), + None, + Some(b"zzzz"), + ])) as ArrayRef, + ]) + .unwrap(); + equality_delete_writer.write(to_write).await?; + let res = equality_delete_writer.close().await?; + assert_eq!(res.len(), 1); + + let expected = RecordBatch::try_new(delete_arrow_schema, vec![Arc::new( + arrow_array::BinaryArray::from_opt_vec(vec![Some(b"one"), None, Some(b"zzzz")]), + ) as ArrayRef]) + .unwrap(); + check_parquet_data_file_with_equality_delete_write( + &file_io, + &res.into_iter().next().unwrap(), + &expected, + ) + .await; + + Ok(()) + } + #[tokio::test] async fn test_equality_delete_with_nullable_field() -> Result<(), anyhow::Error> { // prepare data @@ -763,7 +948,7 @@ mod test { let to_write = RecordBatch::try_new(arrow_schema.clone(), columns).unwrap(); let equality_ids = vec![0_i32, 2, 5]; let equality_config = - EqualityDeleteWriterConfig::new(equality_ids, Arc::new(schema), None, 0).unwrap(); + EqualityDeleteWriterConfig::new(equality_ids, Arc::new(schema)).unwrap(); let projector = equality_config.projector.clone(); // check diff --git a/crates/iceberg/src/writer/base_writer/mod.rs b/crates/iceberg/src/writer/base_writer/mod.rs index 37ab97eb6d..005260dcf1 100644 --- a/crates/iceberg/src/writer/base_writer/mod.rs +++ b/crates/iceberg/src/writer/base_writer/mod.rs @@ -19,3 +19,5 @@ pub mod data_file_writer; pub mod equality_delete_writer; +pub(crate) mod position_delete_input; +pub mod position_delete_writer; diff --git a/crates/iceberg/src/writer/base_writer/position_delete_input.rs b/crates/iceberg/src/writer/base_writer/position_delete_input.rs new file mode 100644 index 0000000000..483739f685 --- /dev/null +++ b/crates/iceberg/src/writer/base_writer/position_delete_input.rs @@ -0,0 +1,673 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Accumulates position delete rows and renders them as spec-conforming [`RecordBatch`]es. +//! +//! [`PositionDeletes`] collects `(file_path, pos)` pairs and emits the two required position +//! delete columns — `file_path` (`Utf8`) and `pos` (`Int64`) — sorted by `(file_path, pos)` +//! with duplicate pairs removed, as the spec requires. Rows can be drawn either as one batch +//! ([`to_record_batch`](PositionDeletes::to_record_batch)) or, to bound peak memory over a +//! large delete set, lazily as fixed-size batches +//! ([`to_record_batches`](PositionDeletes::to_record_batches)). Either batch is exactly what +//! [`PositionDeleteFileWriter`](super::position_delete_writer::PositionDeleteFileWriter) +//! accepts, so it can be handed straight to that writer. +//! +//! Position delete files are a v2 construct. v3 tables use deletion vectors and forbid adding +//! new position delete files, so a format-version gate must be applied at the +//! transaction/commit layer before routing v3 writes here. + +// Nothing wires `PositionDeletes` into a writer yet; the write-path plumbing lands in a follow-up. +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::sync::Arc; + +use arrow_array::builder::{Int64Builder, StringBuilder}; +use arrow_array::{ArrayRef, RecordBatch}; +use roaring::RoaringTreemap; + +use crate::{Error, ErrorKind, Result}; + +/// Accumulates `(file_path, pos)` position delete rows. +/// +/// Keying paths in a [`BTreeMap`] and positions in a [`RoaringTreemap`] deduplicates repeated +/// `(file_path, pos)` pairs and, when iterated, yields the spec-required ascending +/// `(file_path, pos)` order for free, so callers may [`insert`](Self::insert) rows in any +/// order. +#[derive(Debug, Default)] +pub(crate) struct PositionDeletes { + rows: BTreeMap, +} + +impl PositionDeletes { + /// Creates an empty accumulator. + pub(crate) fn new() -> Self { + Self::default() + } + + /// Records that row `pos` of `path` is deleted. `pos` is a row position, so it is + /// non-negative by construction. + /// + /// A duplicate `(file_path, pos)` or a position exceeding `i64::MAX` is a caller bug, + /// caught by [`debug_assert!`] in debug builds so it surfaces early in debug/CI (a + /// position delete file must not list the same `(file_path, pos)` twice, and the `pos` + /// column is `Int64`). In release those assertions compile out and the old behavior + /// stands: the backing [`RoaringTreemap`] silently collapses a duplicate pair, and an + /// oversized position is still rejected as [`ErrorKind::DataInvalid`] when the batch is + /// built. + pub(crate) fn insert(&mut self, path: impl Into, pos: u64) { + debug_assert!(pos <= i64::MAX as u64, "position {pos} exceeds i64::MAX"); + let inserted = self.rows.entry(path.into()).or_default().insert(pos); + debug_assert!( + inserted, + "duplicate position delete: the same (file_path, pos) was inserted twice" + ); + } + + /// Returns whether no positions have been recorded. `insert` never leaves an empty + /// treemap behind, so an empty map means no positions. + pub(crate) fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + /// Returns the total number of recorded positions across all paths. + pub(crate) fn len(&self) -> usize { + self.rows + .values() + .map(|positions| positions.len()) + .sum::() as usize + } + + /// Renders all accumulated rows as one spec-conforming position delete [`RecordBatch`]. + /// + /// The batch has exactly the two required position delete columns, built against the + /// writer's shared [`position_delete_arrow_schema`](super::position_delete_writer::position_delete_arrow_schema) + /// so the reserved field ids stay wired in. Rows are emitted sorted by `(file_path, pos)` + /// with duplicate pairs removed. An empty accumulator yields a valid 0-row batch. + /// + /// This materializes every row at once; prefer [`to_record_batches`](Self::to_record_batches) + /// to bound peak memory over a large delete set. + pub(crate) fn to_record_batch(&self) -> Result { + // `usize::MAX` never caps a real position count, so this drains everything into one + // batch (a 0-row batch when empty). + Self::build_batch(&mut self.stream().peekable(), usize::MAX) + } + + /// Lazily renders the accumulated rows as position delete [`RecordBatch`]es of at most + /// `max_rows` rows each, so the whole delete set is never materialized at once. + /// + /// Batches carry the two required columns in ascending `(file_path, pos)` order with + /// duplicate pairs removed, split on `max_rows` boundaries; a single path whose positions + /// overflow one batch simply resumes in the next. Each batch is built on demand as the + /// iterator is polled — nothing is pre-collected — and uses the writer's shared + /// [`position_delete_arrow_schema`](super::position_delete_writer::position_delete_arrow_schema). + /// An empty accumulator yields no batches. + /// + /// `max_rows` must be greater than zero. Rather than panic, a `max_rows` of `0` makes the + /// iterator yield a single [`ErrorKind::DataInvalid`] error and then stop. + pub(crate) fn to_record_batches( + &self, + max_rows: usize, + ) -> impl Iterator> + '_ { + let mut rows = self.stream().peekable(); + let mut done = false; + std::iter::from_fn(move || { + if done { + return None; + } + if max_rows == 0 { + // Invalid, but never panic: surface it once and stop the iterator. + done = true; + return Some(Err(Error::new( + ErrorKind::DataInvalid, + "max_rows must be greater than 0", + ))); + } + // Nothing left to emit -> end the iteration. + rows.peek()?; + let batch = Self::build_batch(&mut rows, max_rows); + // A failed batch (e.g. a position overflowing `i64`) leaves its offending row + // unconsumed, so stop after the first error rather than retry it forever. + if batch.is_err() { + done = true; + } + Some(batch) + }) + } + + /// Flattens the accumulator into its rows in ascending `(file_path, pos)` order. + /// + /// The `BTreeMap` iterates paths ascending and each `RoaringTreemap` iterates positions + /// ascending, so the stream is sorted by `(file_path, pos)` and already deduplicated. + fn stream(&self) -> impl Iterator + '_ { + self.rows + .iter() + .flat_map(|(path, positions)| positions.iter().map(move |pos| (path.as_str(), pos))) + } + + /// Builds one position delete [`RecordBatch`] of at most `max_rows` rows drained from + /// `rows`, which must yield rows already in `(file_path, pos)` order. + /// + /// Consecutive rows sharing a path form a run appended to the `file_path` column in a + /// single [`StringBuilder::append_value_n`] call, keeping that column run-length friendly + /// rather than repeating the path per row. The batch is built against the writer's shared + /// Arrow projection (a cheap `Arc` clone) so the reserved field ids stay wired in. An + /// exhausted `rows` yields a valid 0-row batch. + fn build_batch<'a, I>( + rows: &mut std::iter::Peekable, + max_rows: usize, + ) -> Result + where + I: Iterator, + { + let mut path_builder = StringBuilder::new(); + let mut pos_builder = Int64Builder::new(); + + let mut taken = 0usize; + while taken < max_rows { + let Some(&(path, _)) = rows.peek() else { + break; + }; + // Consume this path's run: contiguous rows sharing `path`, up to the batch's cap. + let mut run = 0usize; + while taken < max_rows { + let Some(&(next_path, pos)) = rows.peek() else { + break; + }; + if next_path != path { + break; + } + // The `pos` column is `Int64`; positions are `u64`, so guard the top of the + // range with a checked conversion rather than a silent wrap. + let pos = i64::try_from(pos) + .map_err(|_| Error::new(ErrorKind::DataInvalid, "position exceeds i64::MAX"))?; + pos_builder.append_value(pos); + rows.next(); + run += 1; + taken += 1; + } + // One append per run of `run` rows, not one per row. + path_builder.append_value_n(path, run); + } + + let schema = super::position_delete_writer::position_delete_arrow_schema(); + RecordBatch::try_new(schema, vec![ + Arc::new(path_builder.finish()) as ArrayRef, + Arc::new(pos_builder.finish()) as ArrayRef, + ]) + .map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Failed to build position delete record batch: {e}"), + ) + }) + } +} + +#[cfg(test)] +mod test { + use arrow_array::{Array, Int64Array, StringArray}; + use arrow_schema::DataType; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use parquet::file::properties::WriterProperties; + use tempfile::TempDir; + + use super::*; + use crate::io::FileIO; + use crate::metadata_columns::{ + RESERVED_COL_NAME_DELETE_FILE_PATH, RESERVED_COL_NAME_DELETE_FILE_POS, + RESERVED_FIELD_ID_DELETE_FILE_PATH, RESERVED_FIELD_ID_DELETE_FILE_POS, + }; + use crate::spec::{DataContentType, DataFileFormat}; + use crate::writer::base_writer::position_delete_writer::{ + PositionDeleteFileWriterBuilder, position_delete_schema, + }; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + use crate::writer::{IcebergWriter, IcebergWriterBuilder}; + + /// Asserts a batch has the exact two-column position delete schema: names, Arrow types, + /// non-null flags, and the reserved field ids in the Parquet field-id metadata. + fn assert_position_delete_schema(batch: &RecordBatch) { + assert_eq!(batch.num_columns(), 2); + let schema = batch.schema(); + + let path = schema.field(0); + assert_eq!(path.name(), RESERVED_COL_NAME_DELETE_FILE_PATH); + assert_eq!(path.data_type(), &DataType::Utf8); + assert!(!path.is_nullable()); + assert_eq!( + path.metadata().get(PARQUET_FIELD_ID_META_KEY).unwrap(), + &RESERVED_FIELD_ID_DELETE_FILE_PATH.to_string() + ); + + let pos = schema.field(1); + assert_eq!(pos.name(), RESERVED_COL_NAME_DELETE_FILE_POS); + assert_eq!(pos.data_type(), &DataType::Int64); + assert!(!pos.is_nullable()); + assert_eq!( + pos.metadata().get(PARQUET_FIELD_ID_META_KEY).unwrap(), + &RESERVED_FIELD_ID_DELETE_FILE_POS.to_string() + ); + } + + /// Extracts the `(file_path, pos)` columns of a batch as parallel vectors. + fn columns(batch: &RecordBatch) -> (Vec, Vec) { + let paths = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let positions = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(paths.null_count(), 0); + assert_eq!(positions.null_count(), 0); + ( + paths.iter().map(|s| s.unwrap().to_string()).collect(), + positions.values().to_vec(), + ) + } + + #[test] + fn test_column_shape_and_field_ids() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 1); + let batch = deletes.to_record_batch().unwrap(); + assert_position_delete_schema(&batch); + } + + #[test] + fn test_unsorted_inserts_sorted_by_path_then_pos() { + let mut deletes = PositionDeletes::new(); + // Inserted out of order, across multiple paths; both the paths and the positions + // within a path must come out sorted. + deletes.insert("s3://bucket/data/f1.parquet", 2); + deletes.insert("s3://bucket/data/f0.parquet", 4); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + let batch = deletes.to_record_batch().unwrap(); + let (paths, positions) = columns(&batch); + assert_eq!(paths, vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f1.parquet", + ]); + assert_eq!(positions, vec![1, 4, 2]); + } + + // Release behavior: a duplicate `(file_path, pos)` is silently deduped. In debug builds + // that same input trips a `debug_assert!` (see `insert_duplicate_pair_debug_asserts`), so + // this test is release-only. + #[cfg(not(debug_assertions))] + #[test] + fn test_duplicate_pairs_deduped() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + assert_eq!(deletes.len(), 2); + let batch = deletes.to_record_batch().unwrap(); + let (paths, positions) = columns(&batch); + assert_eq!(batch.num_rows(), 2); + assert_eq!(paths, vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + ]); + assert_eq!(positions, vec![1, 3]); + } + + // Debug behavior: re-inserting the same `(file_path, pos)` is a caller bug that trips a + // `debug_assert!` rather than silently deduping (the release behavior asserted by + // `test_duplicate_pairs_deduped`). + #[cfg(debug_assertions)] + #[test] + #[should_panic(expected = "duplicate position delete")] + fn insert_duplicate_pair_debug_asserts() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 3); + } + + #[test] + fn test_len_and_is_empty() { + let mut deletes = PositionDeletes::new(); + assert!(deletes.is_empty()); + assert_eq!(deletes.len(), 0); + + deletes.insert("s3://bucket/data/f0.parquet", 1); + deletes.insert("s3://bucket/data/f0.parquet", 2); + deletes.insert("s3://bucket/data/f1.parquet", 9); + assert!(!deletes.is_empty()); + assert_eq!(deletes.len(), 3); + + // A duplicate pair does not change the count. This is release behavior; in debug the + // duplicate insert trips a `debug_assert!`, so the check is release-only here. + #[cfg(not(debug_assertions))] + { + deletes.insert("s3://bucket/data/f0.parquet", 1); + assert_eq!(deletes.len(), 3); + } + } + + #[test] + fn test_empty_builds_valid_zero_row_batch() { + let deletes = PositionDeletes::new(); + let batch = deletes.to_record_batch().unwrap(); + assert_eq!(batch.num_rows(), 0); + // Even with no rows the schema must be the full position delete schema. + assert_position_delete_schema(&batch); + } + + // Release behavior: a position beyond `i64::MAX` cannot fit the `Int64` column and is + // rejected as `DataInvalid` when the batch is built (the release safety net). In debug + // builds that same input trips a `debug_assert!` in `insert` up front (see + // `insert_position_above_i64_max_debug_asserts`), so this test is release-only. + #[cfg(not(debug_assertions))] + #[test] + fn test_position_above_i64_max_rejected() { + let mut deletes = PositionDeletes::new(); + // A position beyond `i64::MAX` cannot fit the `Int64` column and must be rejected + // rather than silently wrapped. + deletes.insert("s3://bucket/data/f0.parquet", i64::MAX as u64 + 1); + let err = deletes.to_record_batch().unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + // Debug behavior: a position beyond `i64::MAX` is a caller bug that trips a + // `debug_assert!` in `insert` up front, rather than being rejected only when the batch is + // built (the release behavior asserted by `test_position_above_i64_max_rejected`). + #[cfg(debug_assertions)] + #[test] + #[should_panic(expected = "exceeds i64::MAX")] + fn insert_position_above_i64_max_debug_asserts() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", i64::MAX as u64 + 1); + } + + /// Drains [`PositionDeletes::to_record_batches`] into per-batch `(paths, positions)` + /// column pairs, unwrapping each batch. + fn batched_columns(deletes: &PositionDeletes, max_rows: usize) -> Vec<(Vec, Vec)> { + deletes + .to_record_batches(max_rows) + .map(|batch| columns(&batch.unwrap())) + .collect() + } + + /// Concatenates per-batch column pairs into one flattened `(paths, positions)` pair. + fn flatten(batches: &[(Vec, Vec)]) -> (Vec, Vec) { + let mut paths = Vec::new(); + let mut positions = Vec::new(); + for (batch_paths, batch_positions) in batches { + paths.extend(batch_paths.iter().cloned()); + positions.extend(batch_positions.iter().copied()); + } + (paths, positions) + } + + #[test] + fn test_to_record_batches_splits_and_preserves_order() { + let mut deletes = PositionDeletes::new(); + // Unsorted inserts across three paths; 6 rows total. + for (path, pos) in [ + ("s3://bucket/data/f1.parquet", 5), + ("s3://bucket/data/f0.parquet", 7), + ("s3://bucket/data/f0.parquet", 1), + ("s3://bucket/data/f2.parquet", 9), + ("s3://bucket/data/f1.parquet", 2), + ("s3://bucket/data/f0.parquet", 4), + ] { + deletes.insert(path, pos); + } + + // 6 rows at 2 per batch -> exactly 3 full batches. + let batches = batched_columns(&deletes, 2); + assert_eq!(batches.len(), 3); + for (paths, positions) in &batches { + assert_eq!(paths.len(), 2); + assert_eq!(positions.len(), 2); + } + // Every streamed batch carries the full position delete schema. + for batch in deletes.to_record_batches(2) { + assert_position_delete_schema(&batch.unwrap()); + } + + // Flattened, the stream is the same rows in the same sorted order as the single batch. + let (flat_paths, flat_positions) = flatten(&batches); + let (whole_paths, whole_positions) = columns(&deletes.to_record_batch().unwrap()); + assert_eq!(flat_paths, whole_paths); + assert_eq!(flat_positions, whole_positions); + assert_eq!(whole_paths, vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f1.parquet", + "s3://bucket/data/f1.parquet", + "s3://bucket/data/f2.parquet", + ]); + assert_eq!(whole_positions, vec![1, 4, 7, 2, 5, 9]); + } + + #[test] + fn test_to_record_batches_uneven_last_batch() { + let mut deletes = PositionDeletes::new(); + for pos in 0..5 { + deletes.insert("s3://bucket/data/f0.parquet", pos); + } + + // 5 rows at 2 per batch -> two full batches and a 1-row remainder. + let sizes: Vec = deletes + .to_record_batches(2) + .map(|batch| batch.unwrap().num_rows()) + .collect(); + assert_eq!(sizes, vec![2, 2, 1]); + } + + #[test] + fn test_to_record_batches_path_spans_chunk_boundary() { + let mut deletes = PositionDeletes::new(); + // A single path with more positions than fit one batch: its run must resume across + // batch boundaries and stay correct. + for pos in 1..=5 { + deletes.insert("s3://bucket/data/f0.parquet", pos); + } + + let batches = batched_columns(&deletes, 2); + assert_eq!(batches.len(), 3); + // Same path in every batch, positions still ascending and contiguous across the split. + for (paths, _) in &batches { + assert!(paths.iter().all(|p| p == "s3://bucket/data/f0.parquet")); + } + let (paths, positions) = flatten(&batches); + assert_eq!(paths.len(), 5); + assert_eq!(positions, vec![1, 2, 3, 4, 5]); + } + + #[test] + fn test_to_record_batches_zero_max_rows_yields_error_without_panic() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + let mut it = deletes.to_record_batches(0); + // A `max_rows` of 0 is invalid, but must surface as an error rather than a panic... + let err = it.next().expect("expected one error item").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + // ...exactly once, after which the iterator is exhausted. + assert!(it.next().is_none()); + } + + // Release-only: inserting a position beyond `i64::MAX` trips a `debug_assert!` in debug + // builds, so the streaming overflow safety net can only be exercised in release. + #[cfg(not(debug_assertions))] + #[test] + fn test_to_record_batches_stops_after_overflow_error() { + let mut deletes = PositionDeletes::new(); + // A position beyond `i64::MAX` cannot fit the `Int64` column; the streaming path must + // surface the error and then stop rather than retry the unconsumed row forever. + deletes.insert("s3://bucket/data/f0.parquet", i64::MAX as u64 + 1); + + let mut it = deletes.to_record_batches(2); + let err = it.next().expect("expected one error item").unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(it.next().is_none()); + } + + #[test] + fn test_to_record_batches_empty_yields_no_batches() { + let deletes = PositionDeletes::new(); + // Empty accumulator streams nothing... + assert_eq!(deletes.to_record_batches(4).count(), 0); + // ...while the single-batch path still returns a valid 0-row batch. + let batch = deletes.to_record_batch().unwrap(); + assert_eq!(batch.num_rows(), 0); + assert_position_delete_schema(&batch); + } + + // Release-only: inserting the same `(file_path, pos)` twice trips a `debug_assert!` in + // debug builds, so the streaming dedup behavior can only be exercised in release. + #[cfg(not(debug_assertions))] + #[test] + fn test_to_record_batches_dedups_repeated_pairs() { + let mut deletes = PositionDeletes::new(); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 3); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + // The duplicate collapses, so a generous batch holds just the two distinct rows. + let batches = batched_columns(&deletes, 100); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].0, vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + ]); + assert_eq!(batches[0].1, vec![1, 3]); + } + + #[test] + fn test_to_record_batches_repeats_path_across_run() { + let mut deletes = PositionDeletes::new(); + // A run of one path followed by a second path, all within a single batch: the + // `append_value_n` run must repeat the file_path value once per position. + for pos in [1, 2, 3, 4] { + deletes.insert("s3://bucket/data/f0.parquet", pos); + } + deletes.insert("s3://bucket/data/f1.parquet", 8); + + let batches = batched_columns(&deletes, 100); + assert_eq!(batches.len(), 1); + let (paths, positions) = &batches[0]; + assert_eq!(paths, &vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f1.parquet", + ]); + assert_eq!(positions, &vec![1, 2, 3, 4, 8]); + } + + /// Wires a real [`PositionDeleteFileWriter`] over a local filesystem for the integration + /// test: the file IO, location/name generators, and the Parquet-backed rolling writer + /// configured with the [`position_delete_schema`]. + fn writer_setup( + temp_dir: &TempDir, + ) -> PositionDeleteFileWriterBuilder< + ParquetWriterBuilder, + DefaultLocationGenerator, + DefaultFileNameGenerator, + > { + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + let parquet_writer_builder = ParquetWriterBuilder::new( + WriterProperties::builder().build(), + position_delete_schema(), + ); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io, + location_gen, + file_name_gen, + ); + PositionDeleteFileWriterBuilder::new(rolling_writer_builder) + } + + #[tokio::test] + async fn test_batch_feeds_position_delete_writer() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let builder = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + let mut deletes = PositionDeletes::new(); + // Deliberately unsorted; `to_record_batch` sorts to the order the writer needs. + deletes.insert("s3://bucket/data/f1.parquet", 2); + deletes.insert("s3://bucket/data/f0.parquet", 4); + deletes.insert("s3://bucket/data/f0.parquet", 1); + + writer.write(deletes.to_record_batch()?).await?; + let data_files = writer.close().await?; + + assert_eq!(data_files.len(), 1); + let data_file = &data_files[0]; + assert_eq!(data_file.content_type(), DataContentType::PositionDeletes); + assert_eq!(data_file.record_count, deletes.len() as u64); + + Ok(()) + } + + #[tokio::test] + async fn test_streamed_batches_feed_position_delete_writer() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let builder = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + let mut deletes = PositionDeletes::new(); + // Deliberately unsorted, across paths, more rows than the batch cap so more than one + // batch is streamed to the writer. + deletes.insert("s3://bucket/data/f1.parquet", 2); + deletes.insert("s3://bucket/data/f0.parquet", 4); + deletes.insert("s3://bucket/data/f0.parquet", 1); + deletes.insert("s3://bucket/data/f1.parquet", 5); + deletes.insert("s3://bucket/data/f2.parquet", 9); + + // Each fixed-size batch is a valid position delete batch the writer accepts, and the + // batches arrive in the sorted order the writer requires. + let mut streamed = 0usize; + for batch in deletes.to_record_batches(2) { + writer.write(batch?).await?; + streamed += 1; + } + assert_eq!(streamed, 3); + + let data_files = writer.close().await?; + assert_eq!(data_files.len(), 1); + let data_file = &data_files[0]; + assert_eq!(data_file.content_type(), DataContentType::PositionDeletes); + assert_eq!(data_file.record_count, deletes.len() as u64); + + Ok(()) + } +} diff --git a/crates/iceberg/src/writer/base_writer/position_delete_writer.rs b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs new file mode 100644 index 0000000000..3b023c9c62 --- /dev/null +++ b/crates/iceberg/src/writer/base_writer/position_delete_writer.rs @@ -0,0 +1,892 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! This module provides `PositionDeleteFileWriter`. +//! +//! A position delete file has two required columns: `file_path` (`string`, field id +//! [`RESERVED_FIELD_ID_DELETE_FILE_PATH`]) and `pos` (`long`, field id +//! [`RESERVED_FIELD_ID_DELETE_FILE_POS`]). The spec also allows an optional `row` column +//! that inlines the deleted row's values; this writer does not support it yet, so batches +//! must contain only the two required columns. The writer takes batches already shaped as +//! those two columns (see [`position_delete_schema`]) and sets +//! [`DataContentType::PositionDeletes`] on the output. It does not sort its input; see +//! [`PositionDeleteFileWriter::write`]. +//! +//! Position delete files are a v2 construct. v3 replaces them with deletion vectors and +//! forbids adding new position delete files, so callers must not route v3 writes here. +//! This base writer has no format-version gate by design; that gating belongs at the +//! transaction/commit layer. + +use std::sync::Arc; + +use arrow_array::RecordBatch; +use arrow_schema::{DataType, Field}; +use once_cell::sync::Lazy; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + +use crate::error::invalid_data; +use crate::metadata_columns::{ + RESERVED_FIELD_ID_DELETE_FILE_PATH, RESERVED_FIELD_ID_DELETE_FILE_POS, delete_file_path_field, + delete_file_pos_field, +}; +use crate::spec::{DataContentType, DataFile, PartitionKey, Schema, SchemaRef}; +use crate::writer::file_writer::FileWriterBuilder; +use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; +use crate::writer::file_writer::rolling_writer::{RollingFileWriter, RollingFileWriterBuilder}; +use crate::writer::{IcebergWriter, IcebergWriterBuilder}; +use crate::{Error, ErrorKind, Result}; + +/// The canonical Iceberg schema of a position delete file: the required `file_path` +/// (`string`) and `pos` (`long`) columns with their reserved field ids. +static POSITION_DELETE_SCHEMA: Lazy = Lazy::new(|| { + Arc::new( + Schema::builder() + .with_fields(vec![ + delete_file_path_field().clone(), + delete_file_pos_field().clone(), + ]) + .build() + .expect("position delete schema is statically valid"), + ) +}); + +/// [`POSITION_DELETE_SCHEMA`] converted to Arrow, keeping the reserved field ids in each +/// field's Parquet field-id metadata. This is the crate-internal Arrow projection of a +/// position delete file, shared with +/// [`PositionDeletes`](super::position_delete_input::PositionDeletes). +static POSITION_DELETE_ARROW_SCHEMA: Lazy = Lazy::new(|| { + Arc::new( + crate::arrow::schema_to_arrow_schema(&POSITION_DELETE_SCHEMA) + .expect("position delete arrow schema is statically valid"), + ) +}); + +/// Returns the canonical Iceberg schema of a position delete file. +/// +/// Use this to build the [`ParquetWriterBuilder`](crate::writer::file_writer::ParquetWriterBuilder) +/// that backs a [`PositionDeleteFileWriter`], so the written file matches the +/// spec exactly. +pub fn position_delete_schema() -> SchemaRef { + POSITION_DELETE_SCHEMA.clone() +} + +/// Returns the canonical Arrow schema of a position delete file. +/// +/// A cheap [`Arc`] clone of the shared static, used by +/// [`PositionDeletes`](super::position_delete_input::PositionDeletes). +pub(crate) fn position_delete_arrow_schema() -> arrow_schema::SchemaRef { + POSITION_DELETE_ARROW_SCHEMA.clone() +} + +/// Reads a field's Iceberg field id from its Parquet field-id metadata. +fn field_id(field: &Field) -> Result { + field + .metadata() + .get(PARQUET_FIELD_ID_META_KEY) + .ok_or_else(|| { + invalid_data!( + "Position delete column `{}` is missing its Iceberg field id metadata.", + field.name() + ) + })? + .parse::() + .map_err(|e| { + invalid_data!( + "Position delete column `{}` has an invalid field id: {e}", + field.name() + ) + }) +} + +/// Validates that a batch is a position delete file: the `file_path` (`Utf8`) and +/// `pos` (`Int64`) columns, in order, with the two reserved field ids. Checking it +/// here gives a clear error before the batch reaches the Parquet writer. +fn validate_position_delete_batch(batch: &RecordBatch) -> Result<()> { + let fields = batch.schema_ref().fields(); + if fields.len() != 2 { + return Err(invalid_data!( + "This writer supports only the two required position delete columns (`file_path`, `pos`); \ + batches with a different column count (e.g. including the optional `row` column) are not supported. Got {} columns.", + fields.len() + )); + } + + let path = &fields[0]; + let path_id = field_id(path)?; + if path_id != RESERVED_FIELD_ID_DELETE_FILE_PATH { + return Err(invalid_data!( + "The first position delete column must be `file_path` (field id {RESERVED_FIELD_ID_DELETE_FILE_PATH}), but got field id {path_id}." + )); + } + // The canonical schema maps Iceberg `string` to `Utf8` and the file writer is + // configured with it, so a `LargeUtf8` column has to be cast to `Utf8` first. + if path.data_type() != &DataType::Utf8 { + return Err(invalid_data!( + "The position delete `file_path` column must be Utf8 (cast it first); got {:?}.", + path.data_type() + )); + } + // Required column: a nullable field could write nulls under a required schema. + if path.is_nullable() { + return Err(invalid_data!( + "The position delete `file_path` column must be required (non-nullable)." + )); + } + + let pos = &fields[1]; + let pos_id = field_id(pos)?; + if pos_id != RESERVED_FIELD_ID_DELETE_FILE_POS { + return Err(invalid_data!( + "The second position delete column must be `pos` (field id {RESERVED_FIELD_ID_DELETE_FILE_POS}), but got field id {pos_id}." + )); + } + if pos.data_type() != &DataType::Int64 { + return Err(invalid_data!( + "The position delete `pos` column must be Int64, but got {:?}.", + pos.data_type() + )); + } + if pos.is_nullable() { + return Err(invalid_data!( + "The position delete `pos` column must be required (non-nullable)." + )); + } + + Ok(()) +} + +/// Builder for [`PositionDeleteFileWriter`]. +#[derive(Debug)] +pub struct PositionDeleteFileWriterBuilder< + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +> { + inner: RollingFileWriterBuilder, +} + +impl PositionDeleteFileWriterBuilder +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + /// Create a new `PositionDeleteFileWriterBuilder` using a `RollingFileWriterBuilder`. + /// + /// The `RollingFileWriterBuilder` must be backed by a file writer configured + /// with the [`position_delete_schema`]; the per-batch validation in + /// [`PositionDeleteFileWriter::write`] guards against a mismatched batch, but + /// the caller is responsible for wiring the same schema into the file writer. + pub fn new(inner: RollingFileWriterBuilder) -> Self { + Self { inner } + } +} + +#[async_trait::async_trait] +impl IcebergWriterBuilder for PositionDeleteFileWriterBuilder +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + type R = PositionDeleteFileWriter; + + async fn build(&self, partition_key: Option) -> Result { + Ok(PositionDeleteFileWriter { + inner: Some(self.inner.build()), + partition_key, + }) + } +} + +/// Writer used to write position delete files within one spec/partition. +#[derive(Debug)] +pub struct PositionDeleteFileWriter< + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +> { + inner: Option>, + partition_key: Option, +} + +#[async_trait::async_trait] +impl IcebergWriter for PositionDeleteFileWriter +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + /// Writes a batch of `(file_path, pos)` records; the shape is validated on every + /// call. + /// + /// The writer does not sort its input. Position delete files must be sorted by + /// `file_path` then `pos`, so the caller must supply rows in that order across all + /// `write` calls; a sorting writer will remove this requirement. + async fn write(&mut self, batch: RecordBatch) -> Result<()> { + // Reject a closed writer before validating the batch. + let Some(writer) = self.inner.as_mut() else { + return Err(Error::new( + ErrorKind::Unexpected, + "Position delete writer is already closed; cannot write.", + )); + }; + validate_position_delete_batch(&batch)?; + writer.write(&self.partition_key, &batch).await + } + + async fn close(&mut self) -> Result> { + if let Some(writer) = self.inner.take() { + writer + .close() + .await? + .into_iter() + .map(|mut res| { + res.content(DataContentType::PositionDeletes); + // sort_order_id stays null, as the spec requires for position deletes. + if let Some(pk) = self.partition_key.as_ref() { + res.partition(pk.data().clone()); + res.partition_spec_id(pk.spec().spec_id()); + } + res.build() + .map_err(|e| invalid_data!("Failed to build position delete file: {e}")) + }) + .collect() + } else { + Err(Error::new( + ErrorKind::Unexpected, + "Position delete writer is already closed.", + )) + } + } +} + +#[cfg(test)] +mod test { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{Int32Array, Int64Array, LargeStringArray, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field}; + use arrow_select::concat::concat_batches; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + use parquet::file::properties::WriterProperties; + use tempfile::TempDir; + + use super::*; + use crate::io::FileIO; + use crate::metadata_columns::{ + RESERVED_COL_NAME_DELETE_FILE_PATH, RESERVED_COL_NAME_DELETE_FILE_POS, + }; + use crate::spec::{ + DataFileFormat, Literal, NestedField, PartitionSpec, PrimitiveType, Struct, Transform, Type, + }; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + + #[test] + fn test_position_delete_schema_shape() { + let schema = position_delete_schema(); + let fields = schema.as_struct().fields(); + assert_eq!(fields.len(), 2); + + assert_eq!(fields[0].id, RESERVED_FIELD_ID_DELETE_FILE_PATH); + assert_eq!(fields[0].name, RESERVED_COL_NAME_DELETE_FILE_PATH); + assert!(fields[0].required); + assert_eq!( + fields[0].field_type.as_ref(), + &Type::Primitive(PrimitiveType::String) + ); + + assert_eq!(fields[1].id, RESERVED_FIELD_ID_DELETE_FILE_POS); + assert_eq!(fields[1].name, RESERVED_COL_NAME_DELETE_FILE_POS); + assert!(fields[1].required); + assert_eq!( + fields[1].field_type.as_ref(), + &Type::Primitive(PrimitiveType::Long) + ); + + // The Arrow projection carries the reserved field ids and non-null flags. + let arrow_schema = position_delete_arrow_schema(); + assert_eq!(arrow_schema.fields().len(), 2); + assert_eq!(arrow_schema.field(0).data_type(), &DataType::Utf8); + assert_eq!(arrow_schema.field(1).data_type(), &DataType::Int64); + assert!(!arrow_schema.field(0).is_nullable()); + assert!(!arrow_schema.field(1).is_nullable()); + } + + fn position_delete_batch(paths: Vec<&str>, positions: Vec) -> RecordBatch { + RecordBatch::try_new(position_delete_arrow_schema(), vec![ + Arc::new(StringArray::from(paths)), + Arc::new(Int64Array::from(positions)), + ]) + .unwrap() + } + + /// A field carrying an explicit Iceberg field-id metadata entry. + fn field_with_id(name: &str, data_type: DataType, field_id: i32) -> Field { + Field::new(name, data_type, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + field_id.to_string(), + )])) + } + + fn writer_setup( + temp_dir: &TempDir, + ) -> ( + FileIO, + PositionDeleteFileWriterBuilder< + ParquetWriterBuilder, + DefaultLocationGenerator, + DefaultFileNameGenerator, + >, + ) { + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + let parquet_writer_builder = ParquetWriterBuilder::new( + WriterProperties::builder().build(), + position_delete_schema(), + ); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + ( + file_io, + PositionDeleteFileWriterBuilder::new(rolling_writer_builder), + ) + } + + #[tokio::test] + async fn test_position_delete_writer_round_trip() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Sorted by (file_path, pos): f0/1, f0/4, then f1/2. + let batch = position_delete_batch( + vec![ + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f0.parquet", + "s3://bucket/data/f1.parquet", + ], + vec![1, 4, 2], + ); + writer.write(batch.clone()).await?; + let data_files = writer.close().await?; + + assert_eq!(data_files.len(), 1); + let data_file = &data_files[0]; + assert_eq!(data_file.content_type(), DataContentType::PositionDeletes); + assert_eq!(data_file.file_format, DataFileFormat::Parquet); + assert_eq!(data_file.record_count, 3); + // Unpartitioned writer leaves the default (empty) partition / spec id. + assert_eq!(data_file.partition, Struct::empty()); + assert_eq!(data_file.partition_spec_id, 0); + // The rolling writer fills in file statistics. + assert!(data_file.file_size_in_bytes > 0); + + // The written Parquet file round-trips back to the exact input rows. + let read_back = read_back_single(&file_io, data_file, &batch.schema()).await; + assert_eq!(read_back, batch); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_parquet_field_ids() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + writer + .write(position_delete_batch( + vec!["s3://bucket/data/f0.parquet"], + vec![1], + )) + .await?; + let data_files = writer.close().await?; + + // The reserved field ids must survive into the written Parquet schema, not just + // the in-memory Arrow schema, so cross-engine readers resolve the columns by id. + let content = file_io + .new_input(data_files[0].file_path.clone())? + .read() + .await?; + let reader = ParquetRecordBatchReaderBuilder::try_new(content).unwrap(); + let field_ids: Vec = reader + .parquet_schema() + .columns() + .iter() + .map(|col| col.self_type().get_basic_info().id()) + .collect(); + assert_eq!(field_ids, vec![ + RESERVED_FIELD_ID_DELETE_FILE_PATH, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ]); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_multiple_writes() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + let batch1 = position_delete_batch( + vec!["s3://bucket/data/f0.parquet", "s3://bucket/data/f0.parquet"], + vec![1, 4], + ); + let batch2 = position_delete_batch( + vec!["s3://bucket/data/f1.parquet", "s3://bucket/data/f1.parquet"], + vec![2, 7], + ); + writer.write(batch1.clone()).await?; + writer.write(batch2.clone()).await?; + let data_files = writer.close().await?; + + assert_eq!(data_files.len(), 1); + assert_eq!(data_files[0].record_count, 4); + + let expected = concat_batches(&batch1.schema(), [&batch1, &batch2]).unwrap(); + let read_back = read_back_single(&file_io, &data_files[0], &batch1.schema()).await; + assert_eq!(read_back, expected); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_sets_partition() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + + // A table schema + identity partition spec with a non-default spec id, so the + // assertions distinguish real propagation from the DataFileBuilder defaults. + let table_schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build()?, + ); + let spec = PartitionSpec::builder(table_schema.clone()) + .with_spec_id(7) + .add_partition_field("id", "id", Transform::Identity)? + .build()?; + let partition_value = Struct::from_iter([Some(Literal::int(42))]); + let partition_key = PartitionKey::new(spec, table_schema.clone(), partition_value.clone()); + + let mut writer = builder.build(Some(partition_key)).await?; + writer + .write(position_delete_batch( + vec!["s3://bucket/data/f0.parquet"], + vec![1], + )) + .await?; + let data_files = writer.close().await?; + + assert_eq!(data_files.len(), 1); + let data_file = &data_files[0]; + assert_eq!(data_file.content_type(), DataContentType::PositionDeletes); + assert_eq!(data_file.partition_spec_id, 7); + assert_eq!(data_file.partition, partition_value); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_wrong_column_count() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // A batch carrying only the `file_path` column is not a position delete file. + let arrow_schema = position_delete_arrow_schema(); + let path_only = RecordBatch::try_new(Arc::new(arrow_schema.project(&[0]).unwrap()), vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + ]) + .unwrap(); + + let err = writer.write(path_only).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("only the two required position delete columns"), + "{err}" + ); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_missing_field_ids() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Correct shape and types, but plain field names without the reserved + // Iceberg field-id metadata: must be rejected. + let plain_schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new(RESERVED_COL_NAME_DELETE_FILE_PATH, DataType::Utf8, false), + Field::new(RESERVED_COL_NAME_DELETE_FILE_POS, DataType::Int64, false), + ])); + let batch = RecordBatch::try_new(plain_schema, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("field id metadata"), "{err}"); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_unparsable_field_id() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Field-id metadata present but not an integer: must be rejected. + let bad_meta = Arc::new(arrow_schema::Schema::new(vec![ + Field::new(RESERVED_COL_NAME_DELETE_FILE_PATH, DataType::Utf8, false).with_metadata( + HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "not_a_number".to_string(), + )]), + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(bad_meta, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("invalid field id"), "{err}"); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_wrong_field_id() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Right shape and types, but the two reserved field ids are swapped. + let swapped = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + ])); + let batch = RecordBatch::try_new(swapped, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be `file_path`"), "{err}"); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_bad_pos_field_id() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Correct `file_path`, but `pos` carries the wrong reserved field id — the + // pos-specific branch (not the path branch) must fire. + let wrong_pos_id = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id(RESERVED_COL_NAME_DELETE_FILE_POS, DataType::Int64, 999), + ])); + let batch = RecordBatch::try_new(wrong_pos_id, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be `pos`"), "{err}"); + + // Correct `file_path`, but `pos` is missing its field-id metadata entirely. + let missing_pos_id = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + Field::new(RESERVED_COL_NAME_DELETE_FILE_POS, DataType::Int64, false), + ])); + let batch = RecordBatch::try_new(missing_pos_id, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("field id metadata"), "{err}"); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_wrong_column_types() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Correct field ids, but `pos` is Int32 rather than Int64. + let int32_pos = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int32, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(int32_pos, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int32Array::from(vec![1_i32])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be Int64"), "{err}"); + + // Correct field ids, but `file_path` is not a string. + let int_path = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Int32, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(int_path, vec![ + Arc::new(Int32Array::from(vec![1_i32])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be Utf8"), "{err}"); + + // Correct field ids, but `file_path` is LargeUtf8 — the common shape from + // DataFusion/DuckDB/Polars, and the case the validator's comment calls out. + let large_path = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::LargeUtf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(large_path, vec![ + Arc::new(LargeStringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("must be Utf8"), "{err}"); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_rejects_nullable_columns() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Correct field ids and types, but the columns are declared nullable. A + // required schema with a nullable field could emit nulls -> malformed file. + let nullable = |name: &str, data_type: DataType, id: i32| { + Field::new(name, data_type, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + id.to_string(), + )])) + }; + + let nullable_path = Arc::new(arrow_schema::Schema::new(vec![ + nullable( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(nullable_path, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("`file_path` column must be required"), + "{err}" + ); + + let nullable_pos = Arc::new(arrow_schema::Schema::new(vec![ + field_with_id( + RESERVED_COL_NAME_DELETE_FILE_PATH, + DataType::Utf8, + RESERVED_FIELD_ID_DELETE_FILE_PATH, + ), + nullable( + RESERVED_COL_NAME_DELETE_FILE_POS, + DataType::Int64, + RESERVED_FIELD_ID_DELETE_FILE_POS, + ), + ])); + let batch = RecordBatch::try_new(nullable_pos, vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + Arc::new(Int64Array::from(vec![1_i64])), + ]) + .unwrap(); + let err = writer.write(batch).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains("`pos` column must be required"), + "{err}" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_close_without_writes() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + + // Closing a writer that never received a batch produces no data files. + let data_files = writer.close().await?; + assert!(data_files.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn test_position_delete_writer_errors_after_close() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let (_file_io, builder) = writer_setup(&temp_dir); + let mut writer = builder.build(None).await?; + writer.close().await?; + + // Both write() and a second close() report the writer is already closed. + let write_err = writer + .write(position_delete_batch( + vec!["s3://bucket/data/f0.parquet"], + vec![1], + )) + .await + .unwrap_err(); + assert_eq!(write_err.kind(), ErrorKind::Unexpected); + assert!( + write_err.to_string().contains("cannot write"), + "{write_err}" + ); + + // Even a malformed batch surfaces the closed error, not a validation error: + // the closed check runs before validation. + let arrow_schema = position_delete_arrow_schema(); + let invalid = RecordBatch::try_new(Arc::new(arrow_schema.project(&[0]).unwrap()), vec![ + Arc::new(StringArray::from(vec!["s3://bucket/data/f0.parquet"])), + ]) + .unwrap(); + let invalid_err = writer.write(invalid).await.unwrap_err(); + assert_eq!(invalid_err.kind(), ErrorKind::Unexpected); + + let close_err = writer.close().await.unwrap_err(); + assert_eq!(close_err.kind(), ErrorKind::Unexpected); + assert!( + close_err.to_string().contains("already closed"), + "{close_err}" + ); + Ok(()) + } + + async fn read_back_single( + file_io: &FileIO, + data_file: &DataFile, + schema: &arrow_schema::SchemaRef, + ) -> RecordBatch { + let input_content = file_io + .new_input(data_file.file_path.clone()) + .unwrap() + .read() + .await + .unwrap(); + let reader = ParquetRecordBatchReaderBuilder::try_new(input_content) + .unwrap() + .build() + .unwrap(); + let batches = reader.map(|b| b.unwrap()).collect::>(); + concat_batches(schema, &batches).unwrap() + } +} diff --git a/crates/iceberg/src/writer/file_writer/location_generator.rs b/crates/iceberg/src/writer/file_writer/location_generator.rs index 4a73fc148d..26207d3e1d 100644 --- a/crates/iceberg/src/writer/file_writer/location_generator.rs +++ b/crates/iceberg/src/writer/file_writer/location_generator.rs @@ -22,9 +22,10 @@ use std::sync::atomic::AtomicU64; use crate::Result; use crate::spec::{DataFileFormat, PartitionKey, TableMetadata}; +use crate::util::location::strip_trailing_slash; /// `LocationGenerator` used to generate the location of data file. -pub trait LocationGenerator: Clone + Send + 'static { +pub trait LocationGenerator: Clone + Send + Sync + 'static { /// Generate an absolute path for the given file name that includes the partition path. /// /// # Arguments @@ -41,9 +42,14 @@ pub trait LocationGenerator: Clone + Send + 'static { fn generate_location(&self, partition_key: Option<&PartitionKey>, file_name: &str) -> String; } -const WRITE_DATA_LOCATION: &str = "write.data.path"; -const WRITE_FOLDER_STORAGE_LOCATION: &str = "write.folder-storage.path"; +/// Default directory for data files const DEFAULT_DATA_DIR: &str = "/data"; +/// Number of trailing hash bits used to build the entropy directories. +const HASH_BINARY_STRING_BITS: usize = 20; +/// Length of each entropy directory. +const ENTROPY_DIR_LENGTH: usize = 4; +/// Number of entropy directories generated from the hash. +const ENTROPY_DIR_DEPTH: usize = 3; #[derive(Clone, Debug)] /// `DefaultLocationGenerator` used to generate the data dir location of data file. @@ -54,17 +60,24 @@ pub struct DefaultLocationGenerator { impl DefaultLocationGenerator { /// Create a new `DefaultLocationGenerator`. - pub fn new(table_metadata: TableMetadata) -> Result { - let table_location = table_metadata.location(); - let prop = table_metadata.properties(); - let configured_data_location = prop - .get(WRITE_DATA_LOCATION) - .or(prop.get(WRITE_FOLDER_STORAGE_LOCATION)); - let data_location = if let Some(data_location) = configured_data_location { - data_location.clone() - } else { - format!("{}{}", table_location, DEFAULT_DATA_DIR) - }; + /// Resolve the base data location from table properties, falling back to `{table_location}/data`. + /// + /// Precedence follows Java Iceberg: `write.data.path` first, then the deprecated + /// `write.folder-storage.path` properties, and finally the default + /// `{table_location}/data`. + pub fn new(table_metadata: &TableMetadata) -> Result { + let table_location = strip_trailing_slash(table_metadata.location()); + let prop = table_metadata.table_properties(); + let write_data_location = prop.write_data_location()?; + let write_folder_storage_location = prop.write_folder_storage_location()?; + let data_location = strip_trailing_slash( + write_data_location + .or(write_folder_storage_location) + .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) + .as_ref(), + ) + .to_string(); + Ok(Self { data_location }) } @@ -93,8 +106,141 @@ impl LocationGenerator for DefaultLocationGenerator { } } +/// `ObjectStorageLocationGenerator` injects hash entropy into generated file locations so that +/// files are spread across many object-store prefixes. +/// +/// Object stores such as S3 shard request throughput by key prefix, so writing every file under a +/// common `.../data/` prefix creates a throughput hotspot. This generator prepends a +/// deterministic, hashed directory tree (derived from the file name) to each location, mirroring +/// Java Iceberg's `ObjectStoreLocationProvider`. +/// +/// The behavior is controlled by these table properties: +/// * `write.data.path` / `write.object-storage.path` / `write.folder-storage.path` - the base data +/// location (checked in that order), defaulting to `{table_location}/data`. +/// * `write.object-storage.partitioned-paths` - whether partition values are included in the path +/// (defaults to `true`). +#[derive(Clone, Debug)] +pub struct ObjectStorageLocationGenerator { + storage_location: String, + /// Database/table context, only set when the storage location is outside the table location. + context: Option, + include_partition_paths: bool, +} + +impl ObjectStorageLocationGenerator { + /// Create a new `ObjectStorageLocationGenerator` from table metadata. + /// Resolve the base data location from table properties, falling back to `{table_location}/data`. + /// + /// Precedence follows Java Iceberg: `write.data.path` first, then the deprecated + /// `write.object-storage.path` and `write.folder-storage.path` properties, and finally the default + /// `{table_location}/data`. + pub fn new(table_metadata: &TableMetadata) -> Result { + let table_location = strip_trailing_slash(table_metadata.location()); + let prop = table_metadata.table_properties(); + let write_data_location = prop.write_data_location()?; + let write_object_storage_location = prop.write_object_storage_location()?; + let write_folder_storage_location = prop.write_folder_storage_location()?; + let storage_location = strip_trailing_slash( + write_data_location + .or(write_object_storage_location) + .or(write_folder_storage_location) + .unwrap_or(format!("{table_location}{DEFAULT_DATA_DIR}")) + .as_ref(), + ) + .to_string(); + + // If the storage location is within the table prefix, files are already scoped to this + // table so there is no need to add database/table context to avoid collisions. + let context = if storage_location.starts_with(table_location) { + None + } else { + Some(path_context(table_location)) + }; + + let include_partition_paths = prop.write_object_storage_partitioned_paths()?; + + Ok(Self { + storage_location, + context, + include_partition_paths, + }) + } + + /// Build the final location for a fully-formed data file name (which may already include a + /// partition path). + fn new_data_location(&self, name: &str) -> String { + let hash = self.compute_hash(name); + if let Some(context) = &self.context { + format!("{}/{}/{}/{}", self.storage_location, hash, context, name) + } else if self.include_partition_paths { + format!("{}/{}/{}", self.storage_location, hash, name) + } else { + // When partition paths are excluded, join the entropy to the file name with `-` so the + // file still lives directly under the storage location. + format!("{}/{}-{}", self.storage_location, hash, name) + } + } + + /// Compute the entropy directory tree for the given name, e.g. `0101/0110/1001/10110010`. + fn compute_hash(&self, name: &str) -> String { + let mut bytes = name.as_bytes(); + let hash_code = murmur3::murmur3_32(&mut bytes, 0).unwrap(); + let binary = format!("{:032b}", hash_code); + let hash = &binary[binary.len() - HASH_BINARY_STRING_BITS..]; + dirs_from_hash(hash) + } +} + +impl LocationGenerator for ObjectStorageLocationGenerator { + fn generate_location(&self, partition_key: Option<&PartitionKey>, file_name: &str) -> String { + let name = + if self.include_partition_paths && !PartitionKey::is_effectively_none(partition_key) { + format!("{}/{}", partition_key.unwrap().to_path(), file_name) + } else { + file_name.to_string() + }; + self.new_data_location(&name) + } +} + +/// Derive the `{parent}/{name}` context from a table location, mirroring Hadoop's +/// `Path.getParent().getName()` / `Path.getName()`. +fn path_context(table_location: &str) -> String { + let mut segments = table_location.rsplit('/').filter(|s| !s.is_empty()); + let name = segments.next().unwrap_or(""); + match segments.next() { + Some(parent) => format!("{parent}/{name}"), + None => name.to_string(), + } +} + +/// Divide a binary hash string into directories for optimized listing/orphan removal. +/// +/// With `ENTROPY_DIR_DEPTH = 3` and `ENTROPY_DIR_LENGTH = 4`, the 20-bit hash +/// `10011001100110011001` becomes `1001/1001/1001/10011001`. +fn dirs_from_hash(hash: &str) -> String { + let mut result = String::new(); + + let mut i = 0; + while i < ENTROPY_DIR_DEPTH * ENTROPY_DIR_LENGTH { + if i > 0 { + result.push('/'); + } + let end = (i + ENTROPY_DIR_LENGTH).min(hash.len()); + result.push_str(&hash[i..end]); + i += ENTROPY_DIR_LENGTH; + } + + if hash.len() > ENTROPY_DIR_DEPTH * ENTROPY_DIR_LENGTH { + result.push('/'); + result.push_str(&hash[ENTROPY_DIR_DEPTH * ENTROPY_DIR_LENGTH..]); + } + + result +} + /// `FileNameGeneratorTrait` used to generate file name for data file. The file name can be passed to `LocationGenerator` to generate the location of the file. -pub trait FileNameGenerator: Clone + Send + 'static { +pub trait FileNameGenerator: Clone + Send + Sync + 'static { /// Generate a file name. fn generate_file_name(&self) -> String; } @@ -114,7 +260,7 @@ impl DefaultFileNameGenerator { /// Create a new `FileNameGenerator`. pub fn new(prefix: String, suffix: Option, format: DataFileFormat) -> Self { let suffix = if let Some(suffix) = suffix { - format!("-{}", suffix) + format!("-{suffix}") } else { "".to_string() }; @@ -150,40 +296,15 @@ pub(crate) mod test { use super::LocationGenerator; use crate::spec::{ FormatVersion, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Schema, - Struct, StructType, TableMetadata, Transform, Type, + Struct, StructType, TableMetadata, TableProperties, Transform, Type, }; use crate::writer::file_writer::location_generator::{ - DefaultLocationGenerator, FileNameGenerator, WRITE_DATA_LOCATION, - WRITE_FOLDER_STORAGE_LOCATION, + DefaultLocationGenerator, FileNameGenerator, ObjectStorageLocationGenerator, }; #[test] fn test_default_location_generate() { - let mut table_metadata = TableMetadata { - format_version: FormatVersion::V2, - table_uuid: Uuid::parse_str("fb072c92-a02b-11e9-ae9c-1bb7bc9eca94").unwrap(), - location: "s3://data.db/table".to_string(), - last_updated_ms: 1515100955770, - last_column_id: 1, - schemas: HashMap::new(), - current_schema_id: 1, - partition_specs: HashMap::new(), - default_spec: PartitionSpec::unpartition_spec().into(), - default_partition_type: StructType::new(vec![]), - last_partition_id: 1000, - default_sort_order_id: 0, - sort_orders: HashMap::from_iter(vec![]), - snapshots: HashMap::default(), - current_snapshot_id: None, - last_sequence_number: 1, - properties: HashMap::new(), - snapshot_log: Vec::new(), - metadata_log: vec![], - refs: HashMap::new(), - statistics: HashMap::new(), - partition_statistics: HashMap::new(), - encryption_keys: HashMap::new(), - }; + let mut table_metadata = table_metadata_with("s3://data.db/table", HashMap::new()); let file_name_generator = super::DefaultFileNameGenerator::new( "part".to_string(), @@ -192,19 +313,17 @@ pub(crate) mod test { ); // test default data location - let location_generator = - super::DefaultLocationGenerator::new(table_metadata.clone()).unwrap(); + let location_generator = DefaultLocationGenerator::new(&table_metadata).unwrap(); let location = location_generator.generate_location(None, &file_name_generator.generate_file_name()); assert_eq!(location, "s3://data.db/table/data/part-00000-test.parquet"); // test custom data location table_metadata.properties.insert( - WRITE_FOLDER_STORAGE_LOCATION.to_string(), + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION.to_string(), "s3://data.db/table/data_1".to_string(), ); - let location_generator = - super::DefaultLocationGenerator::new(table_metadata.clone()).unwrap(); + let location_generator = DefaultLocationGenerator::new(&table_metadata).unwrap(); let location = location_generator.generate_location(None, &file_name_generator.generate_file_name()); assert_eq!( @@ -213,11 +332,10 @@ pub(crate) mod test { ); table_metadata.properties.insert( - WRITE_DATA_LOCATION.to_string(), + TableProperties::PROPERTY_WRITE_DATA_LOCATION.to_string(), "s3://data.db/table/data_2".to_string(), ); - let location_generator = - super::DefaultLocationGenerator::new(table_metadata.clone()).unwrap(); + let location_generator = DefaultLocationGenerator::new(&table_metadata).unwrap(); let location = location_generator.generate_location(None, &file_name_generator.generate_file_name()); assert_eq!( @@ -226,12 +344,11 @@ pub(crate) mod test { ); table_metadata.properties.insert( - WRITE_DATA_LOCATION.to_string(), + TableProperties::PROPERTY_WRITE_DATA_LOCATION.to_string(), // invalid table location "s3://data.db/data_3".to_string(), ); - let location_generator = - super::DefaultLocationGenerator::new(table_metadata.clone()).unwrap(); + let location_generator = DefaultLocationGenerator::new(&table_metadata).unwrap(); let location = location_generator.generate_location(None, &file_name_generator.generate_file_name()); assert_eq!(location, "s3://data.db/data_3/part-00003-test.parquet"); @@ -273,10 +390,231 @@ pub(crate) mod test { assert_eq!(location, "/base/path/id=42/name=alice/data-00000.parquet"); // Create a table metadata for DefaultLocationGenerator - let table_metadata = TableMetadata { + let table_metadata = table_metadata_with("s3://data.db/table", HashMap::new()); + + // Test with DefaultLocationGenerator + let default_location_gen = DefaultLocationGenerator::new(&table_metadata).unwrap(); + let location = default_location_gen.generate_location(Some(&partition_key), file_name); + assert_eq!( + location, + "s3://data.db/table/data/id=42/name=alice/data-00000.parquet" + ); + } + + #[test] + fn test_location_generate_with_special_characters_partition() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "data#1", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("data#1", "data#1", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition_data = Struct::from_iter([Some(Literal::string("val#1"))]); + let partition_key = PartitionKey::new(partition_spec, schema, partition_data); + + let table_metadata = table_metadata_with("s3://data.db/table", HashMap::new()); + let location_gen = DefaultLocationGenerator::new(&table_metadata).unwrap(); + let location = location_gen.generate_location(Some(&partition_key), "test.parquet"); + + assert_eq!( + location, + "s3://data.db/table/data/data%231=val%231/test.parquet" + ); + } + + // This test is ported from https://github.com/apache/iceberg/blob/apache-iceberg-1.11.0/core/src/test/java/org/apache/iceberg/TestLocationProvider.java#L303 + #[test] + fn test_object_storage_hash_injection() { + // Golden vectors ported from Java's TestLocationProvider#testHashInjection, verifying that + // the murmur3 entropy directories match Java Iceberg exactly. + let table_metadata = table_metadata_with("s3://data.db/table", HashMap::new()); + let location_gen = ObjectStorageLocationGenerator::new(&table_metadata).unwrap(); + + for (file_name, expected) in [ + ("a", "s3://data.db/table/data/0101/0110/1001/10110010/a"), + ("b", "s3://data.db/table/data/1110/0111/1110/00000011/b"), + ("c", "s3://data.db/table/data/0010/1101/0110/01011111/c"), + ("d", "s3://data.db/table/data/1001/0001/0100/01110011/d"), + ] { + assert_eq!(location_gen.generate_location(None, file_name), expected); + } + } + + #[test] + fn test_object_storage_include_partition_paths() { + // With partitioned paths enabled (the default), the partition path is part of the hashed + // name and appears after the entropy directories. + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "a", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .add_partition_field("a", "a_trunc_3", Transform::Truncate(3)) + .unwrap() + .build() + .unwrap(); + let partition_data = + Struct::from_iter([Some(Literal::int(0)), Some(Literal::string("apa"))]); + let partition_key = PartitionKey::new(partition_spec, schema, partition_data); + + let table_metadata = table_metadata_with("s3://data.db/table", HashMap::new()); + let location_gen = ObjectStorageLocationGenerator::new(&table_metadata).unwrap(); + let location = location_gen.generate_location(Some(&partition_key), "test.parquet"); + + assert!( + location.starts_with("s3://data.db/table/data/"), + "unexpected location: {location}" + ); + // The partition path is included + assert!( + location.ends_with("/id=0/a_trunc_3=apa/test.parquet"), + "unexpected location: {location}" + ); + } + + // This test is ported from https://github.com/apache/iceberg/blob/apache-iceberg-1.11.0/core/src/test/java/org/apache/iceberg/TestLocationProvider.java#L267 + #[test] + fn test_object_storage_include_partition_paths_with_special_characters() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "data#1", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("data#1", "data#1", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition_data = Struct::from_iter([Some(Literal::string("val#1"))]); + let partition_key = PartitionKey::new(partition_spec, schema, partition_data); + + let table_metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), + "true".to_string(), + )]), + ); + let location_gen = ObjectStorageLocationGenerator::new(&table_metadata).unwrap(); + let location = location_gen.generate_location(Some(&partition_key), "test.parquet"); + + assert_eq!( + location, + "s3://data.db/table/data/0000/1011/0110/00001000/data%231=val%231/test.parquet" + ); + } + + // This test is ported from https://github.com/apache/iceberg/blob/apache-iceberg-1.11.0/core/src/test/java/org/apache/iceberg/TestLocationProvider.java#L285 + #[test] + fn test_object_storage_exclude_partition_paths() { + // Golden vector ported from Java's TestLocationProvider#testExcludePartitionInPath. With + // partitioned paths disabled, the partition value is dropped and the last entropy dir is + // joined to the file name with `-`. + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("id", "id", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let partition_data = Struct::from_iter([Some(Literal::int(0))]); + let partition_key = PartitionKey::new(partition_spec, schema, partition_data); + + let table_metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([( + TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(), + "false".to_string(), + )]), + ); + let location_gen = ObjectStorageLocationGenerator::new(&table_metadata).unwrap(); + let location = location_gen.generate_location(Some(&partition_key), "test.parquet"); + + assert_eq!( + location, + "s3://data.db/table/data/0110/1010/0011/11101000-test.parquet" + ); + } + + #[test] + fn test_object_storage_context_when_data_location_outside_table() { + // When the data location is outside the table location, the database/table context is + // injected after the entropy directories to avoid cross-table collisions. + let table_metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([( + TableProperties::PROPERTY_WRITE_DATA_LOCATION.to_string(), + "s3://custom-bucket/objects".to_string(), + )]), + ); + let location_gen = ObjectStorageLocationGenerator::new(&table_metadata).unwrap(); + let location = location_gen.generate_location(None, "a"); + + // Entropy for "a" is 0101/0110/1001/10110010, then the "data.db/table" context, then file. + assert_eq!( + location, + "s3://custom-bucket/objects/0101/0110/1001/10110010/data.db/table/a" + ); + } + + #[test] + fn test_object_storage_data_location_precedence() { + // write.data.path takes precedence over the deprecated folder-storage fallback. + let table_metadata = table_metadata_with( + "s3://data.db/table", + HashMap::from([ + ( + TableProperties::PROPERTY_WRITE_DATA_LOCATION.to_string(), + "s3://data.db/table/data_primary".to_string(), + ), + ( + TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION.to_string(), + "s3://data.db/table/data_legacy".to_string(), + ), + ]), + ); + let location_gen = ObjectStorageLocationGenerator::new(&table_metadata).unwrap(); + let location = location_gen.generate_location(None, "a"); + + assert_eq!( + location, + "s3://data.db/table/data_primary/0101/0110/1001/10110010/a" + ); + } + + /// Build a minimal `TableMetadata` for location generator tests. + fn table_metadata_with(location: &str, properties: HashMap) -> TableMetadata { + TableMetadata { format_version: FormatVersion::V2, table_uuid: Uuid::parse_str("fb072c92-a02b-11e9-ae9c-1bb7bc9eca94").unwrap(), - location: "s3://data.db/table".to_string(), + location: location.to_string(), last_updated_ms: 1515100955770, last_column_id: 2, schemas: HashMap::new(), @@ -290,21 +628,14 @@ pub(crate) mod test { snapshots: HashMap::default(), current_snapshot_id: None, last_sequence_number: 1, - properties: HashMap::new(), + properties, snapshot_log: Vec::new(), metadata_log: vec![], refs: HashMap::new(), statistics: HashMap::new(), partition_statistics: HashMap::new(), encryption_keys: HashMap::new(), - }; - - // Test with DefaultLocationGenerator - let default_location_gen = super::DefaultLocationGenerator::new(table_metadata).unwrap(); - let location = default_location_gen.generate_location(Some(&partition_key), file_name); - assert_eq!( - location, - "s3://data.db/table/data/id=42/name=alice/data-00000.parquet" - ); + next_row_id: 0, + } } } diff --git a/crates/iceberg/src/writer/file_writer/mod.rs b/crates/iceberg/src/writer/file_writer/mod.rs index 2a5a735534..101919f5b3 100644 --- a/crates/iceberg/src/writer/file_writer/mod.rs +++ b/crates/iceberg/src/writer/file_writer/mod.rs @@ -27,6 +27,8 @@ use crate::spec::DataFileBuilder; mod parquet_writer; pub use parquet_writer::{ParquetWriter, ParquetWriterBuilder}; +use crate::io::OutputFile; + pub mod location_generator; /// Module providing writers that can automatically roll over to new files based on size thresholds. pub mod rolling_writer; @@ -34,11 +36,11 @@ pub mod rolling_writer; type DefaultOutput = Vec; /// File writer builder trait. -pub trait FileWriterBuilder: Send + Clone + 'static { +pub trait FileWriterBuilder: Clone + Send + Sync + 'static { /// The associated file writer type. type R: FileWriter; /// Build file writer. - fn build(self) -> impl Future> + Send; + fn build(&self, output_file: OutputFile) -> impl Future> + Send; } /// File writer focus on writing record batch to different physical file format.(Such as parquet. orc) diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 0a8a095ea8..d1a021d8e1 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -27,24 +27,28 @@ use itertools::Itertools; use parquet::arrow::AsyncArrowWriter; use parquet::arrow::async_reader::AsyncFileReader; use parquet::arrow::async_writer::AsyncFileWriter as ArrowAsyncFileWriter; -use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader}; -use parquet::file::properties::WriterProperties; +use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; +use parquet::encryption::encrypt::FileEncryptionProperties; +use parquet::file::metadata::{ColumnChunkMetaData, ParquetMetaData}; +use parquet::file::properties::{CdcOptions, WriterProperties}; use parquet::file::statistics::Statistics; -use parquet::format::FileMetaData; -use parquet::thrift::{TCompactOutputProtocol, TSerializable}; -use thrift::protocol::TOutputProtocol; +use parquet::geospatial::bounding_box::BoundingBox; +use parquet_geospatial::interval::{Interval, IntervalTrait, WraparoundInterval}; -use super::location_generator::{FileNameGenerator, LocationGenerator}; use super::{FileWriter, FileWriterBuilder}; use crate::arrow::{ ArrowFileReader, DEFAULT_MAP_FIELD_NAME, FieldMatchMode, NanValueCountVisitor, get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum, }; +use crate::compression::CompressionCodec; +use crate::encryption::{EncryptionManager, StandardKeyMetadata}; +use crate::error::invalid_data; use crate::io::{FileIO, FileWrite, OutputFile}; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MapType, - NestedFieldRef, PartitionKey, PartitionSpec, PrimitiveType, Schema, SchemaRef, SchemaVisitor, - Struct, StructType, TableMetadata, Type, visit_schema, + NestedFieldRef, PartitionSpec, PrimitiveLiteral, PrimitiveType, Schema, SchemaRef, + SchemaVisitor, Struct, StructType, TableMetadata, TableProperties, Type, VariantType, + visit_schema, }; use crate::transform::create_transform_function; use crate::writer::{CurrentFileStatus, DataFile}; @@ -52,79 +56,130 @@ use crate::{Error, ErrorKind, Result}; /// ParquetWriterBuilder is used to builder a [`ParquetWriter`] #[derive(Clone, Debug)] -pub struct ParquetWriterBuilder { +pub struct ParquetWriterBuilder { props: WriterProperties, schema: SchemaRef, - partition_key: Option, match_mode: FieldMatchMode, - - file_io: FileIO, - location_generator: T, - file_name_generator: F, + encryption_manager: Option>, } -impl ParquetWriterBuilder { +impl ParquetWriterBuilder { /// Create a new `ParquetWriterBuilder` /// To construct the write result, the schema should contain the `PARQUET_FIELD_ID_META_KEY` metadata for each field. - pub fn new( - props: WriterProperties, - schema: SchemaRef, - partition_key: Option, - file_io: FileIO, - location_generator: T, - file_name_generator: F, - ) -> Self { - Self::new_with_match_mode( - props, - schema, - partition_key, - FieldMatchMode::Id, - file_io, - location_generator, - file_name_generator, - ) + /// + /// When writing into an existing Iceberg table, prefer + /// [`Self::from_table_properties`], which derives `WriterProperties` from + /// the table's `write.parquet.*` properties. + pub fn new(props: WriterProperties, schema: SchemaRef) -> Self { + Self::new_with_match_mode(props, schema, FieldMatchMode::Id) } /// Create a new `ParquetWriterBuilder` with custom match mode pub fn new_with_match_mode( props: WriterProperties, schema: SchemaRef, - partition_key: Option, match_mode: FieldMatchMode, - file_io: FileIO, - location_generator: T, - file_name_generator: F, ) -> Self { Self { props, schema, - partition_key, match_mode, - file_io, - location_generator, - file_name_generator, + encryption_manager: None, } } + + /// Build a `ParquetWriterBuilder` from Iceberg table properties and a + /// schema, translating `write.parquet.*` settings into `WriterProperties` + /// instead of using parquet-rs defaults. + pub fn from_table_properties( + table_props: &TableProperties<'_>, + schema: SchemaRef, + ) -> Result { + let cdc = if table_props.cdc_enabled()? { + Some(CdcOptions { + min_chunk_size: table_props.cdc_min_chunk_size()?, + max_chunk_size: table_props.cdc_max_chunk_size()?, + norm_level: table_props.cdc_norm_level()?, + }) + } else { + None + }; + let compression = parquet_compression(table_props.parquet_compression_codec()?)?; + let props = WriterProperties::builder() + .set_content_defined_chunking(cdc) + .set_compression(compression) + .set_max_row_group_bytes(Some(table_props.parquet_row_group_size_bytes()?)) + .set_data_page_size_limit(table_props.parquet_page_size_bytes()?) + .set_data_page_row_count_limit(table_props.parquet_page_row_limit()?) + .set_dictionary_page_size_limit(table_props.parquet_dict_size_bytes()?) + .build(); + Ok(Self::new_with_match_mode(props, schema, FieldMatchMode::Id)) + } + + /// Set the field match mode used to map Arrow fields to Iceberg fields. + /// + /// Defaults to [`FieldMatchMode::Id`]. Use [`FieldMatchMode::Name`] when the + /// incoming Arrow schema does not carry Iceberg field-id metadata. + pub fn with_match_mode(mut self, match_mode: FieldMatchMode) -> Self { + self.match_mode = match_mode; + self + } + + /// Set the [`EncryptionManager`] used to encrypt written files. + pub fn with_encryption_manager(mut self, encryption_manager: Arc) -> Self { + self.encryption_manager = Some(encryption_manager); + self + } } -impl FileWriterBuilder for ParquetWriterBuilder { - type R = ParquetWriter; +fn parquet_compression(codec: CompressionCodec) -> Result { + let compression = match codec { + CompressionCodec::None => Compression::UNCOMPRESSED, + CompressionCodec::Snappy => Compression::SNAPPY, + CompressionCodec::Lzo => Compression::LZO, + CompressionCodec::Lz4 => Compression::LZ4, + CompressionCodec::Lz4Raw => Compression::LZ4_RAW, + CompressionCodec::Zstd(level) => { + let level = + ZstdLevel::try_new(level as i32).map_err(|e| invalid_level_error("zstd", e))?; + Compression::ZSTD(level) + } + CompressionCodec::Gzip(level) => { + let level = + GzipLevel::try_new(level as u32).map_err(|e| invalid_level_error("gzip", e))?; + Compression::GZIP(level) + } + CompressionCodec::Brotli(level) => { + let level = + BrotliLevel::try_new(level as u32).map_err(|e| invalid_level_error("brotli", e))?; + Compression::BROTLI(level) + } + }; + Ok(compression) +} + +fn invalid_level_error(codec: &str, source: impl Into) -> Error { + invalid_data!("Invalid {codec} compression level").with_source(source) +} - async fn build(self) -> Result { - let out_file = self - .file_io - .new_output(self.location_generator.generate_location( - self.partition_key.as_ref(), - &self.file_name_generator.generate_file_name(), - ))?; +impl FileWriterBuilder for ParquetWriterBuilder { + type R = ParquetWriter; + async fn build(&self, output_file: OutputFile) -> Result { + let key_metadata = self + .encryption_manager + .as_ref() + .map(|em| em.generate_key_metadata()); + let writer_properties = + resolve_writer_properties(self.props.clone(), key_metadata.as_ref())?; Ok(ParquetWriter { schema: self.schema.clone(), inner_writer: None, - writer_properties: self.props, + writer_properties, current_row_num: 0, - out_file, + output_file, nan_value_count_visitor: NanValueCountVisitor::new_with_match_mode(self.match_mode), + key_metadata, }) } } @@ -233,11 +288,8 @@ impl SchemaVisitor for IndexByParquetPathName { let full_name = self.field_names.iter().map(String::as_str).join("."); let field_id = self.field_id; if let Some(existing_field_id) = self.name_to_id.get(full_name.as_str()) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}" - ), + return Err(invalid_data!( + "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}" )); } else { self.name_to_id.insert(full_name, field_id); @@ -245,31 +297,115 @@ impl SchemaVisitor for IndexByParquetPathName { Ok(()) } + + fn variant(&mut self, _v: &VariantType) -> Result { + Err(Error::new( + ErrorKind::FeatureUnsupported, + "Writing variant columns to Parquet is not supported yet", + )) + } } /// `ParquetWriter`` is used to write arrow data into parquet file on storage. pub struct ParquetWriter { schema: SchemaRef, - out_file: OutputFile, - inner_writer: Option>>>, + output_file: OutputFile, + inner_writer: Option>, writer_properties: WriterProperties, current_row_num: usize, nan_value_count_visitor: NanValueCountVisitor, + key_metadata: Option, } /// Used to aggregate min and max value of each column. struct MinMaxColAggregator { lower_bounds: HashMap, upper_bounds: HashMap, + // None means at least one non-null row group has no bounding box: file bounds are unknown. + geo_bounds: HashMap)>, schema: SchemaRef, } +struct GeospatialBounds { + x: WraparoundInterval, + y: Interval, + z: Interval, + m: Interval, +} + +impl From<&BoundingBox> for GeospatialBounds { + fn from(bbox: &BoundingBox) -> Self { + Self { + x: WraparoundInterval::new(bbox.get_xmin(), bbox.get_xmax()), + y: Interval::new(bbox.get_ymin(), bbox.get_ymax()), + z: bbox + .get_zmin() + .zip(bbox.get_zmax()) + .map(|(lo, hi)| Interval::new(lo, hi)) + .unwrap_or_else(Interval::empty), + m: bbox + .get_mmin() + .zip(bbox.get_mmax()) + .map(|(lo, hi)| Interval::new(lo, hi)) + .unwrap_or_else(Interval::empty), + } + } +} + +impl GeospatialBounds { + fn merge(&mut self, bbox: &BoundingBox) { + let other = Self::from(bbox); + self.x = self.x.merge_interval(&other.x); + self.y.update_interval(&other.y); + self.z.update_interval(&other.z); + self.m.update_interval(&other.m); + } + + fn into_datums(self, ty: PrimitiveType) -> (Datum, Datum) { + let (xmin, xmax) = if matches!(ty, PrimitiveType::Geography(_)) { + // A full interval union uses infinities; Geography requires canonical longitudes. + (self.x.lo().max(-180.0), self.x.hi().min(180.0)) + } else if self.x.is_wraparound() { + // Parquet permits wrapping geometry boxes; Iceberg only permits wrapping geography. + (f64::NEG_INFINITY, f64::INFINITY) + } else { + (self.x.lo(), self.x.hi()) + }; + let dimensions = if !self.m.is_empty() { + 4 + } else if !self.z.is_empty() { + 3 + } else { + 2 + }; + let (zmin, zmax) = if self.z.is_empty() { + (f64::NAN, f64::NAN) + } else { + (self.z.lo(), self.z.hi()) + }; + // Iceberg bound serialization is XY[Z[M]] doubles, not WKB (spec Appendix D). + let encode = |point: [f64; 4]| { + PrimitiveLiteral::Binary( + point[..dimensions] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect(), + ) + }; + ( + Datum::new(ty.clone(), encode([xmin, self.y.lo(), zmin, self.m.lo()])), + Datum::new(ty, encode([xmax, self.y.hi(), zmax, self.m.hi()])), + ) + } +} + impl MinMaxColAggregator { /// Creates new and empty `MinMaxColAggregator` fn new(schema: SchemaRef) -> Self { Self { lower_bounds: HashMap::new(), upper_bounds: HashMap::new(), + geo_bounds: HashMap::new(), schema, } } @@ -296,8 +432,8 @@ impl MinMaxColAggregator { .or_insert(datum); } - /// Update statistics - fn update(&mut self, field_id: i32, value: Statistics) -> Result<()> { + /// Update bounds from a column chunk. + fn update(&mut self, field_id: i32, column: &ColumnChunkMetaData) -> Result<()> { let Some(ty) = self .schema .field_by_id(field_id) @@ -307,43 +443,86 @@ impl MinMaxColAggregator { // Ignore the field if it is not in schema. return Ok(()); }; - let Type::Primitive(ty) = ty.clone() else { + let Type::Primitive(ty) = ty else { return Err(Error::new( ErrorKind::Unexpected, - format!( - "Composed type {} is not supported for min max aggregation.", - ty - ), + format!("Composed type {ty} is not supported for min max aggregation."), )); }; + match ty { + PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => { + // Null-only row groups do not contribute coordinates. + if column.num_values() == 0 + || column + .statistics() + .and_then(Statistics::null_count_opt) + .is_some_and(|nulls| nulls == column.num_values() as u64) + { + return Ok(()); + } + // Parquet bounds already account for the logical type's edge interpolation. + let bbox = column + .geo_statistics() + .and_then(|stats| stats.bounding_box()); + self.geo_bounds + .entry(field_id) + .and_modify(|(_, state)| { + if let (Some(bounds), Some(bbox)) = (state.as_mut(), bbox) { + bounds.merge(bbox); + } else { + *state = None; + } + }) + .or_insert_with(|| (ty.clone(), bbox.map(GeospatialBounds::from))); + return Ok(()); + } + _ => {} + } + let Some(value) = column.statistics() else { + return Ok(()); + }; + let ty = ty.clone(); + if value.min_is_exact() { - let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, &value)? else { + let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, value)? else { return Err(Error::new( ErrorKind::Unexpected, - format!("Statistics {} is not match with field type {}.", value, ty), + format!("Statistics {value} is not match with field type {ty}."), )); }; - self.update_state_min(field_id, min_datum); + // Iceberg bounds must not be NaN. + if !min_datum.is_nan() { + self.update_state_min(field_id, min_datum); + } } if value.max_is_exact() { - let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, &value)? else { + let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, value)? else { return Err(Error::new( ErrorKind::Unexpected, - format!("Statistics {} is not match with field type {}.", value, ty), + format!("Statistics {value} is not match with field type {ty}."), )); }; - self.update_state_max(field_id, max_datum); + if !max_datum.is_nan() { + self.update_state_max(field_id, max_datum); + } } Ok(()) } /// Returns lower and upper bounds - fn produce(self) -> (HashMap, HashMap) { + fn produce(mut self) -> (HashMap, HashMap) { + for (field_id, (ty, bounds)) in self.geo_bounds { + if let Some(bounds) = bounds { + let (lower, upper) = bounds.into_datums(ty); + self.lower_bounds.insert(field_id, lower); + self.upper_bounds.insert(field_id, upper); + } + } (self.lower_bounds, self.upper_bounds) } } @@ -366,12 +545,10 @@ impl ParquetWriter { let reader = input_file.reader().await?; let mut parquet_reader = ArrowFileReader::new(file_metadata, reader); - let parquet_metadata = parquet_reader.get_metadata(None).await.map_err(|err| { - Error::new( - ErrorKind::DataInvalid, - format!("Error reading Parquet metadata: {}", err), - ) - })?; + let parquet_metadata = parquet_reader + .get_metadata(None) + .await + .map_err(|err| invalid_data!("Error reading Parquet metadata: {err}"))?; let mut builder = ParquetWriter::parquet_to_data_file_builder( table_metadata.current_schema().clone(), parquet_metadata, @@ -379,6 +556,7 @@ impl ParquetWriter { file_path, // TODO: Implement nan_value_counts here HashMap::new(), + None, )?; builder.partition_spec_id(table_metadata.default_partition_spec_id()); let data_file = builder.build().unwrap(); @@ -388,29 +566,6 @@ impl ParquetWriter { Ok(data_files) } - fn thrift_to_parquet_metadata(&self, file_metadata: FileMetaData) -> Result { - let mut buffer = Vec::new(); - { - let mut protocol = TCompactOutputProtocol::new(&mut buffer); - file_metadata - .write_to_out_protocol(&mut protocol) - .map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to write parquet metadata") - .with_source(err) - })?; - - protocol.flush().map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to flush protocol").with_source(err) - })?; - } - - let parquet_metadata = ParquetMetaDataReader::decode_metadata(&buffer).map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to decode parquet metadata").with_source(err) - })?; - - Ok(parquet_metadata) - } - /// `ParquetMetadata` to data file builder pub(crate) fn parquet_to_data_file_builder( schema: SchemaRef, @@ -418,6 +573,7 @@ impl ParquetWriter { written_size: usize, file_path: String, nan_value_counts: HashMap, + key_metadata: Option, ) -> Result { let index_by_parquet_path = { let mut visitor = IndexByParquetPathName::new(); @@ -444,12 +600,13 @@ impl ParquetWriter { *per_col_val_num.entry(field_id).or_insert(0) += column_chunk_metadata.num_values() as u64; - if let Some(statistics) = column_chunk_metadata.statistics() { - if let Some(null_count) = statistics.null_count_opt() { - *per_col_null_val_num.entry(field_id).or_insert(0) += null_count; - } + min_max_agg.update(field_id, column_chunk_metadata)?; - min_max_agg.update(field_id, statistics.clone())?; + if let Some(null_count) = column_chunk_metadata + .statistics() + .and_then(Statistics::null_count_opt) + { + *per_col_null_val_num.entry(field_id).or_insert(0) += null_count; } } } @@ -461,6 +618,11 @@ impl ParquetWriter { ) }; + let key_metadata = match key_metadata { + Some(m) => Some(m.encode()?.into_vec()), + None => None, + }; + let mut builder = DataFileBuilder::default(); builder .content(DataContentType::Data) @@ -477,13 +639,14 @@ impl ParquetWriter { // - We can ignore implementing distinct_counts due to this: https://lists.apache.org/thread/j52tsojv0x4bopxyzsp7m7bqt23n5fnd .lower_bounds(lower_bounds) .upper_bounds(upper_bounds) - .split_offsets( + .key_metadata(key_metadata) + .split_offsets(Some( metadata .row_groups() .iter() .filter_map(|group| group.file_offset()) .collect(), - ); + )); Ok(builder) } @@ -502,22 +665,19 @@ impl ParquetWriter { upper_bounds.get(&field.source_id), ) { if !field.transform.preserves_order() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "cannot infer partition value for non linear partition field (needs to preserve order): {} with transform {}", - field.name, field.transform - ), + return Err(invalid_data!( + "cannot infer partition value for non linear partition field (needs to preserve order): {} with transform {}", + field.name, + field.transform )); } if lower != upper { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "multiple partition values for field {}: lower: {:?}, upper: {:?}", - field.name, lower, upper - ), + return Err(invalid_data!( + "multiple partition values for field {}: lower: {:?}, upper: {:?}", + field.name, + lower, + upper )); } @@ -537,6 +697,40 @@ impl ParquetWriter { } } +fn resolve_writer_properties( + writer_properties: WriterProperties, + key_metadata: Option<&StandardKeyMetadata>, +) -> Result { + let Some(key_metadata) = key_metadata else { + return Ok(writer_properties); + }; + + if writer_properties.file_encryption_properties().is_some() { + return Err(Error::new( + ErrorKind::Unexpected, + "Parquet writer properties already have file encryption properties set", + )); + } + + let mut builder = + FileEncryptionProperties::builder(key_metadata.encryption_key().as_bytes().to_vec()); + if let Some(aad) = key_metadata.aad_prefix() { + builder = builder.with_aad_prefix(aad.to_vec()); + } + let file_encryption_properties = builder.build().map_err(|e| { + Error::new( + ErrorKind::Unexpected, + "Failed to build parquet file encryption properties", + ) + .with_source(e) + })?; + + Ok(writer_properties + .into_builder() + .with_file_encryption_properties(file_encryption_properties) + .build()) +} + impl FileWriter for ParquetWriter { async fn write(&mut self, batch: &arrow_array::RecordBatch) -> Result<()> { // Skip empty batch @@ -555,7 +749,7 @@ impl FileWriter for ParquetWriter { writer } else { let arrow_schema: ArrowSchemaRef = Arc::new(self.schema.as_ref().try_into()?); - let inner_writer = self.out_file.writer().await?; + let inner_writer = self.output_file.writer().await?; let async_writer = AsyncFileWriter::new(inner_writer); let writer = AsyncArrowWriter::try_new( async_writer, @@ -594,7 +788,7 @@ impl FileWriter for ParquetWriter { let written_size = writer.bytes_written(); if self.current_row_num == 0 { - self.out_file.delete().await.map_err(|err| { + self.output_file.delete().await.map_err(|err| { Error::new( ErrorKind::Unexpected, "Failed to delete empty parquet file.", @@ -603,21 +797,15 @@ impl FileWriter for ParquetWriter { })?; Ok(vec![]) } else { - let parquet_metadata = - Arc::new(self.thrift_to_parquet_metadata(metadata).map_err(|err| { - Error::new( - ErrorKind::Unexpected, - "Failed to convert metadata from thrift to parquet.", - ) - .with_source(err) - })?); + let parquet_metadata = Arc::new(metadata); Ok(vec![Self::parquet_to_data_file_builder( self.schema, parquet_metadata, written_size, - self.out_file.location().to_string(), + self.output_file.location().to_string(), self.nan_value_count_visitor.nan_value_counts, + self.key_metadata, )?]) } } @@ -625,7 +813,7 @@ impl FileWriter for ParquetWriter { impl CurrentFileStatus for ParquetWriter { fn current_file_path(&self) -> String { - self.out_file.location().to_string() + self.output_file.location().to_string() } fn current_row_num(&self) -> usize { @@ -649,16 +837,16 @@ impl CurrentFileStatus for ParquetWriter { /// # NOTES /// /// We keep this wrapper been used inside only. -struct AsyncFileWriter(W); +struct AsyncFileWriter(Box); -impl AsyncFileWriter { +impl AsyncFileWriter { /// Create a new `AsyncFileWriter` with the given writer. - pub fn new(writer: W) -> Self { + pub fn new(writer: Box) -> Self { Self(writer) } } -impl ArrowAsyncFileWriter for AsyncFileWriter { +impl ArrowAsyncFileWriter for AsyncFileWriter { fn write(&mut self, bs: Bytes) -> BoxFuture<'_, parquet::errors::Result<()>> { Box::pin(async { self.0 @@ -670,9 +858,11 @@ impl ArrowAsyncFileWriter for AsyncFileWriter { fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> { Box::pin(async { + // TODO(encryption): retain the stored file size in data-file key metadata. self.0 .close() .await + .map(|_| ()) .map_err(|err| parquet::errors::ParquetError::External(Box::new(err))) }) } @@ -692,18 +882,29 @@ mod tests { }; use arrow_schema::{DataType, Field, Fields, SchemaRef as ArrowSchemaRef}; use arrow_select::concat::concat_batches; + use futures::TryStreamExt; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; + use parquet::basic::{ + BrotliLevel, Compression, EdgeInterpolationAlgorithm, GzipLevel, LogicalType, ZstdLevel, + }; use parquet::file::statistics::ValueStatistics; - use rust_decimal::Decimal; + use parquet::schema::types::ColumnPath; + use parquet_geospatial::testing::wkb_point_xy; use tempfile::TempDir; use uuid::Uuid; use super::*; - use crate::arrow::schema_to_arrow_schema; - use crate::io::FileIOBuilder; + use crate::Runtime; + use crate::arrow::{ArrowReaderBuilder, schema_to_arrow_schema}; + use crate::io::FileIO; + use crate::scan::{FileScanTask, FileScanTaskStream}; + use crate::spec::decimal_utils::{decimal_mantissa, decimal_new, decimal_scale}; + use crate::spec::geospatial::EdgeInterpolationAlgorithm as IcebergEdgeInterpolationAlgorithm; use crate::spec::{PrimitiveLiteral, Struct, *}; + use crate::test_utils::make_encryption_manager; use crate::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, + DefaultFileNameGenerator, DefaultLocationGenerator, FileNameGenerator, LocationGenerator, }; use crate::writer::tests::check_parquet_data_file; @@ -859,10 +1060,25 @@ mod tests { assert_eq!(visitor.name_to_id, expect); } + #[test] + fn test_index_by_parquet_path_variant_is_unsupported() { + // Writing variant columns to Parquet is not supported yet; indexing a schema that + // contains one must error rather than silently miss-map columns. + let schema = Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "v", Type::Variant(VariantType)).into(), + ]) + .build() + .unwrap(); + let mut visitor = IndexByParquetPathName::new(); + let err = visit_schema(&schema, &mut visitor).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}"); + } + #[tokio::test] async fn test_parquet_writer() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -871,11 +1087,13 @@ mod tests { // prepare data let schema = { - let fields = vec![ - arrow_schema::Field::new("col", arrow_schema::DataType::Int64, true).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "0".to_string())]), - ), - ]; + let fields = + vec![ + Field::new("col", DataType::Int64, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "0".to_string(), + )])), + ]; Arc::new(arrow_schema::Schema::new(fields)) }; let col = Arc::new(Int64Array::from_iter_values(0..1024)) as ArrayRef; @@ -883,18 +1101,18 @@ mod tests { let to_write = RecordBatch::try_new(schema.clone(), vec![col]).unwrap(); let to_write_null = RecordBatch::try_new(schema.clone(), vec![null_col]).unwrap(); + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; + // write data let mut pw = ParquetWriterBuilder::new( WriterProperties::builder() - .set_max_row_group_size(128) + .set_max_row_group_row_count(Some(128)) .build(), Arc::new(to_write.schema().as_ref().try_into().unwrap()), - None, - file_io.clone(), - location_gen, - file_name_gen, ) - .build() + .build(output_file) .await?; pw.write(&to_write).await?; pw.write(&to_write_null).await?; @@ -905,7 +1123,7 @@ mod tests { .next() .unwrap() // Put dummy field for build successfully. - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() @@ -931,10 +1149,100 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_parquet_writer_encrypted_roundtrip() -> Result<()> { + let temp_dir = TempDir::new().unwrap(); + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + ])); + let iceberg_schema: SchemaRef = Arc::new(arrow_schema.as_ref().try_into().unwrap()); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![ + 10, 20, 30, + ])) as ArrayRef]) + .unwrap(); + + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; + let raw_properties = HashMap::from([( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "test-key".to_string(), + )]); + let table_properties = TableProperties::new(&raw_properties); + let mut parquet_writer = + ParquetWriterBuilder::from_table_properties(&table_properties, iceberg_schema.clone())? + .with_encryption_manager(make_encryption_manager("test-key")) + .build(output_file) + .await?; + parquet_writer.write(&batch).await?; + let data_file = parquet_writer + .close() + .await? + .into_iter() + .next() + .unwrap() + .content(DataContentType::Data) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap(); + + let raw = file_io + .new_input(data_file.file_path.clone())? + .read() + .await?; + assert!( + ParquetRecordBatchReaderBuilder::try_new(raw).is_err(), + "an encrypted parquet file must not be readable without decryption" + ); + + let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build(); + let task = FileScanTask::builder() + .with_file_size_in_bytes(data_file.file_size_in_bytes()) + .with_start(0) + .with_length(0) + .with_data_file_path(data_file.file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(iceberg_schema) + .with_project_field_ids(vec![1]) + .with_case_sensitive(false) + .with_key_metadata(data_file.key_metadata().map(Box::from)) + .build() + .unwrap(); + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let batches: Vec = reader + .read(tasks) + .unwrap() + .stream() + .try_collect() + .await + .unwrap(); + + assert_eq!(batches.len(), 1); + let ids = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ids.values(), &[10, 20, 30]); + + Ok(()) + } + #[tokio::test] async fn test_parquet_writer_with_complex_schema() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -963,11 +1271,11 @@ mod tests { (0..1024).map(|n| n.to_string()), )) as ArrayRef; let col3 = Arc::new({ - let list_parts = arrow_array::ListArray::from_iter_primitive::( + let list_parts = ListArray::from_iter_primitive::( (0..1024).map(|n| Some(vec![Some(n)])), ) .into_parts(); - arrow_array::ListArray::new( + ListArray::new( { if let DataType::List(field) = arrow_schema.field(3).data_type() { field.clone() @@ -1006,7 +1314,7 @@ mod tests { None, )); let col5 = Arc::new({ - let mut map_array_builder = arrow_array::builder::MapBuilder::new( + let mut map_array_builder = MapBuilder::new( None, arrow_array::builder::StringBuilder::new(), arrow_array::builder::ListBuilder::new(arrow_array::builder::PrimitiveBuilder::< @@ -1065,7 +1373,7 @@ mod tests { nulls, ) }; - arrow_array::MapArray::new( + MapArray::new( { if let DataType::Map(map_field, _) = arrow_schema.field(5).data_type() { map_field.clone() @@ -1083,18 +1391,15 @@ mod tests { col0, col1, col2, col3, col4, col5, ]) .unwrap(); + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; // write data - let mut pw = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - Arc::new(schema), - None, - file_io.clone(), - location_gen, - file_name_gen, - ) - .build() - .await?; + let mut pw = + ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(schema)) + .build(output_file) + .await?; pw.write(&to_write).await?; let res = pw.close().await?; assert_eq!(res.len(), 1); @@ -1103,7 +1408,7 @@ mod tests { .next() .unwrap() // Put dummy field for build successfully. - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() @@ -1160,8 +1465,8 @@ mod tests { #[tokio::test] async fn test_all_type_for_write() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); - let loccation_gen = DefaultLocationGenerator::with_data_location( + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); let file_name_gen = @@ -1179,13 +1484,13 @@ mod tests { ])) as ArrayRef; let col1 = Arc::new(Int32Array::from(vec![Some(1), Some(2), None, Some(4)])) as ArrayRef; let col2 = Arc::new(Int64Array::from(vec![Some(1), Some(2), None, Some(4)])) as ArrayRef; - let col3 = Arc::new(arrow_array::Float32Array::from(vec![ + let col3 = Arc::new(Float32Array::from(vec![ Some(0.5), Some(2.0), None, Some(3.5), ])) as ArrayRef; - let col4 = Arc::new(arrow_array::Float64Array::from(vec![ + let col4 = Arc::new(Float64Array::from(vec![ Some(0.5), Some(2.0), None, @@ -1197,7 +1502,7 @@ mod tests { None, Some("d"), ])) as ArrayRef; - let col6 = Arc::new(arrow_array::LargeBinaryArray::from_opt_vec(vec![ + let col6 = Arc::new(arrow_array::BinaryArray::from_opt_vec(vec![ Some(b"one"), None, Some(b""), @@ -1236,7 +1541,7 @@ mod tests { .with_timezone_utc(), ) as ArrayRef; let col13 = Arc::new( - arrow_array::Decimal128Array::from(vec![Some(1), Some(2), None, Some(100)]) + Decimal128Array::from(vec![Some(1), Some(2), None, Some(100)]) .with_precision_and_scale(10, 5) .unwrap(), ) as ArrayRef; @@ -1267,7 +1572,7 @@ mod tests { .unwrap(), ) as ArrayRef; let col16 = Arc::new( - arrow_array::Decimal128Array::from(vec![Some(1), Some(2), None, Some(100)]) + Decimal128Array::from(vec![Some(1), Some(2), None, Some(100)]) .with_precision_and_scale(38, 5) .unwrap(), ) as ArrayRef; @@ -1276,18 +1581,15 @@ mod tests { col14, col15, col16, ]) .unwrap(); + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; // write data - let mut pw = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - Arc::new(schema), - None, - file_io.clone(), - loccation_gen, - file_name_gen, - ) - .build() - .await?; + let mut pw = + ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(schema)) + .build(output_file) + .await?; pw.write(&to_write).await?; let res = pw.close().await?; assert_eq!(res.len(), 1); @@ -1296,7 +1598,7 @@ mod tests { .next() .unwrap() // Put dummy field for build successfully. - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() @@ -1317,7 +1619,7 @@ mod tests { (0, Datum::bool(false)), (1, Datum::int(1)), (2, Datum::long(1)), - (3, Datum::float(0.5)), + (3, Datum::float(0.5_f32)), (4, Datum::double(0.5)), (5, Datum::string("a")), (6, Datum::binary(vec![])), @@ -1357,7 +1659,7 @@ mod tests { (0, Datum::bool(true)), (1, Datum::int(4)), (2, Datum::long(4)), - (3, Datum::float(3.5)), + (3, Datum::float(3.5_f32)), (4, Datum::double(3.5)), (5, Datum::string("d")), (6, Datum::binary(vec![122, 122, 122, 122])), @@ -1404,8 +1706,8 @@ mod tests { #[tokio::test] async fn test_decimal_bound() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); - let loccation_gen = DefaultLocationGenerator::with_data_location( + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); let file_name_gen = @@ -1429,16 +1731,12 @@ mod tests { .unwrap(), ); let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap()); - let mut pw = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - schema.clone(), - None, - file_io.clone(), - loccation_gen.clone(), - file_name_gen.clone(), - ) - .build() - .await?; + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; + let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()) + .build(output_file) + .await?; let col0 = Arc::new( Decimal128Array::from(vec![Some(22000000000), Some(11000000000)]) .with_data_type(DataType::Decimal128(28, 10)), @@ -1451,19 +1749,19 @@ mod tests { .into_iter() .next() .unwrap() - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() .unwrap(); assert_eq!( data_file.upper_bounds().get(&0), - Some(Datum::decimal_with_precision(Decimal::new(22000000000_i64, 10), 28).unwrap()) + Some(Datum::decimal_with_precision(decimal_new(22000000000_i64, 10), 28).unwrap()) .as_ref() ); assert_eq!( data_file.lower_bounds().get(&0), - Some(Datum::decimal_with_precision(Decimal::new(11000000000_i64, 10), 28).unwrap()) + Some(Datum::decimal_with_precision(decimal_new(11000000000_i64, 10), 28).unwrap()) .as_ref() ); @@ -1485,16 +1783,12 @@ mod tests { .unwrap(), ); let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap()); - let mut pw = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - schema.clone(), - None, - file_io.clone(), - loccation_gen.clone(), - file_name_gen.clone(), - ) - .build() - .await?; + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; + let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()) + .build(output_file) + .await?; let col0 = Arc::new( Decimal128Array::from(vec![Some(-22000000000), Some(-11000000000)]) .with_data_type(DataType::Decimal128(28, 10)), @@ -1507,26 +1801,29 @@ mod tests { .into_iter() .next() .unwrap() - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() .unwrap(); assert_eq!( data_file.upper_bounds().get(&0), - Some(Datum::decimal_with_precision(Decimal::new(-11000000000_i64, 10), 28).unwrap()) + Some(Datum::decimal_with_precision(decimal_new(-11000000000_i64, 10), 28).unwrap()) .as_ref() ); assert_eq!( data_file.lower_bounds().get(&0), - Some(Datum::decimal_with_precision(Decimal::new(-22000000000_i64, 10), 28).unwrap()) + Some(Datum::decimal_with_precision(decimal_new(-22000000000_i64, 10), 28).unwrap()) .as_ref() ); - // test max and min of rust_decimal - let decimal_max = Decimal::MAX; - let decimal_min = Decimal::MIN; - assert_eq!(decimal_max.scale(), decimal_min.scale()); + // test 38-digit precision decimal values (Iceberg spec max) + // Note: fastnum D128::MAX/MIN have impractical exponents, so we use meaningful values + use crate::spec::decimal_utils::decimal_from_str_exact; + let decimal_max = decimal_from_str_exact("99999999999999999999999999999999999999").unwrap(); + let decimal_min = + decimal_from_str_exact("-99999999999999999999999999999999999999").unwrap(); + assert_eq!(decimal_scale(&decimal_max), decimal_scale(&decimal_min)); let schema = Arc::new( Schema::builder() .with_fields(vec![ @@ -1535,7 +1832,7 @@ mod tests { "decimal", Type::Primitive(PrimitiveType::Decimal { precision: 38, - scale: decimal_max.scale(), + scale: decimal_scale(&decimal_max), }), ) .into(), @@ -1544,20 +1841,16 @@ mod tests { .unwrap(), ); let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap()); - let mut pw = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - schema, - None, - file_io.clone(), - loccation_gen, - file_name_gen, - ) - .build() - .await?; + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; + let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema) + .build(output_file) + .await?; let col0 = Arc::new( Decimal128Array::from(vec![ - Some(decimal_max.mantissa()), - Some(decimal_min.mantissa()), + Some(decimal_mantissa(&decimal_max)), + Some(decimal_mantissa(&decimal_min)), ]) .with_data_type(DataType::Decimal128(38, 0)), ) as ArrayRef; @@ -1569,7 +1862,7 @@ mod tests { .into_iter() .next() .unwrap() - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() @@ -1658,7 +1951,7 @@ mod tests { #[tokio::test] async fn test_empty_write() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -1667,46 +1960,42 @@ mod tests { // Test that file will create if data to write let schema = { - let fields = vec![ - arrow_schema::Field::new("col", arrow_schema::DataType::Int64, true).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "0".to_string())]), - ), - ]; + let fields = + vec![ + Field::new("col", DataType::Int64, true).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "0".to_string(), + )])), + ]; Arc::new(arrow_schema::Schema::new(fields)) }; let col = Arc::new(Int64Array::from_iter_values(0..1024)) as ArrayRef; let to_write = RecordBatch::try_new(schema.clone(), vec![col]).unwrap(); + let file_path = location_gen.generate_location(None, &file_name_gen.generate_file_name()); + let output_file = file_io.new_output(&file_path)?; let mut pw = ParquetWriterBuilder::new( WriterProperties::builder().build(), Arc::new(to_write.schema().as_ref().try_into().unwrap()), - None, - file_io.clone(), - location_gen.clone(), - file_name_gen, ) - .build() + .build(output_file) .await?; pw.write(&to_write).await?; - let file_path = pw.out_file.location().to_string(); pw.close().await.unwrap(); - assert!(file_io.exists(file_path).await.unwrap()); + assert!(file_io.exists(&file_path).await.unwrap()); // Test that file will not create if no data to write let file_name_gen = DefaultFileNameGenerator::new("test_empty".to_string(), None, DataFileFormat::Parquet); + let file_path = location_gen.generate_location(None, &file_name_gen.generate_file_name()); + let output_file = file_io.new_output(&file_path)?; let pw = ParquetWriterBuilder::new( WriterProperties::builder().build(), Arc::new(to_write.schema().as_ref().try_into().unwrap()), - None, - file_io.clone(), - location_gen, - file_name_gen, ) - .build() + .build(output_file) .await?; - let file_path = pw.out_file.location().to_string(); pw.close().await.unwrap(); - assert!(!file_io.exists(file_path).await.unwrap()); + assert!(!file_io.exists(&file_path).await.unwrap()); Ok(()) } @@ -1714,7 +2003,7 @@ mod tests { #[tokio::test] async fn test_nan_val_cnts_primitive_type() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -1724,12 +2013,14 @@ mod tests { // prepare data let arrow_schema = { let fields = vec![ - Field::new("col", arrow_schema::DataType::Float32, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "0".to_string())]), - ), - Field::new("col2", arrow_schema::DataType::Float64, false).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]), - ), + Field::new("col", DataType::Float32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "0".to_string(), + )])), + Field::new("col2", DataType::Float64, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), ]; Arc::new(arrow_schema::Schema::new(fields)) }; @@ -1746,17 +2037,16 @@ mod tests { let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![float_32_col, float_64_col]).unwrap(); + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; // write data let mut pw = ParquetWriterBuilder::new( WriterProperties::builder().build(), Arc::new(to_write.schema().as_ref().try_into().unwrap()), - None, - file_io.clone(), - location_gen, - file_name_gen, ) - .build() + .build(output_file) .await?; pw.write(&to_write).await?; @@ -1767,7 +2057,7 @@ mod tests { .next() .unwrap() // Put dummy field for build successfully. - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() @@ -1778,11 +2068,11 @@ mod tests { assert_eq!(*data_file.value_counts(), HashMap::from([(0, 4), (1, 4)])); assert_eq!( *data_file.lower_bounds(), - HashMap::from([(0, Datum::float(1.0)), (1, Datum::double(1.0)),]) + HashMap::from([(0, Datum::float(1.0_f32)), (1, Datum::double(1.0)),]) ); assert_eq!( *data_file.upper_bounds(), - HashMap::from([(0, Datum::float(2.0)), (1, Datum::double(2.0)),]) + HashMap::from([(0, Datum::float(2.0_f32)), (1, Datum::double(2.0)),]) ); assert_eq!( *data_file.null_value_counts(), @@ -1803,7 +2093,7 @@ mod tests { #[tokio::test] async fn test_nan_val_cnts_struct_type() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -1827,7 +2117,7 @@ mod tests { let schema_struct_nested_fields = Fields::from(vec![ Field::new( "col6", - arrow_schema::DataType::Struct(schema_struct_nested_float_fields.clone()), + DataType::Struct(schema_struct_nested_float_fields.clone()), false, ) .with_metadata(HashMap::from([( @@ -1841,7 +2131,7 @@ mod tests { let fields = vec![ Field::new( "col3", - arrow_schema::DataType::Struct(schema_struct_float_fields.clone()), + DataType::Struct(schema_struct_float_fields.clone()), false, ) .with_metadata(HashMap::from([( @@ -1850,7 +2140,7 @@ mod tests { )])), Field::new( "col5", - arrow_schema::DataType::Struct(schema_struct_nested_fields.clone()), + DataType::Struct(schema_struct_nested_fields.clone()), false, ) .with_metadata(HashMap::from([( @@ -1887,17 +2177,16 @@ mod tests { struct_nested_float_field_col, ]) .unwrap(); + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; // write data let mut pw = ParquetWriterBuilder::new( WriterProperties::builder().build(), Arc::new(to_write.schema().as_ref().try_into().unwrap()), - None, - file_io.clone(), - location_gen, - file_name_gen, ) - .build() + .build(output_file) .await?; pw.write(&to_write).await?; @@ -1908,7 +2197,7 @@ mod tests { .next() .unwrap() // Put dummy field for build successfully. - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() @@ -1919,11 +2208,11 @@ mod tests { assert_eq!(*data_file.value_counts(), HashMap::from([(4, 4), (7, 4)])); assert_eq!( *data_file.lower_bounds(), - HashMap::from([(4, Datum::float(1.0)), (7, Datum::float(1.0)),]) + HashMap::from([(4, Datum::float(1.0_f32)), (7, Datum::float(1.0_f32)),]) ); assert_eq!( *data_file.upper_bounds(), - HashMap::from([(4, Datum::float(2.0)), (7, Datum::float(2.0)),]) + HashMap::from([(4, Datum::float(2.0_f32)), (7, Datum::float(2.0_f32)),]) ); assert_eq!( *data_file.null_value_counts(), @@ -1944,7 +2233,7 @@ mod tests { #[tokio::test] async fn test_nan_val_cnts_list_type() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -2047,6 +2336,9 @@ mod tests { // large_list_float_field_col, ]) .expect("Could not form record batch"); + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; // write data let mut pw = ParquetWriterBuilder::new( @@ -2058,12 +2350,8 @@ mod tests { .try_into() .expect("Could not convert iceberg schema"), ), - None, - file_io.clone(), - location_gen, - file_name_gen, ) - .build() + .build(output_file) .await?; pw.write(&to_write).await?; @@ -2073,7 +2361,7 @@ mod tests { .into_iter() .next() .unwrap() - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() @@ -2084,11 +2372,11 @@ mod tests { assert_eq!(*data_file.value_counts(), HashMap::from([(1, 4), (4, 4)])); assert_eq!( *data_file.lower_bounds(), - HashMap::from([(1, Datum::float(1.0)), (4, Datum::float(1.0))]) + HashMap::from([(1, Datum::float(1.0_f32)), (4, Datum::float(1.0_f32))]) ); assert_eq!( *data_file.upper_bounds(), - HashMap::from([(1, Datum::float(2.0)), (4, Datum::float(2.0))]) + HashMap::from([(1, Datum::float(2.0_f32)), (4, Datum::float(2.0_f32))]) ); assert_eq!( *data_file.null_value_counts(), @@ -2147,7 +2435,7 @@ mod tests { #[tokio::test] async fn test_nan_val_cnts_map_type() -> Result<()> { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -2229,6 +2517,9 @@ mod tests { struct_list_float_field_col, ]) .expect("Could not form record batch"); + let output_file = file_io.new_output( + location_gen.generate_location(None, &file_name_gen.generate_file_name()), + )?; // write data let mut pw = ParquetWriterBuilder::new( @@ -2240,12 +2531,8 @@ mod tests { .try_into() .expect("Could not convert iceberg schema"), ), - None, - file_io.clone(), - location_gen, - file_name_gen, ) - .build() + .build(output_file) .await?; pw.write(&to_write).await?; @@ -2255,7 +2542,7 @@ mod tests { .into_iter() .next() .unwrap() - .content(crate::spec::DataContentType::Data) + .content(DataContentType::Data) .partition(Struct::empty()) .partition_spec_id(0) .build() @@ -2271,18 +2558,18 @@ mod tests { *data_file.lower_bounds(), HashMap::from([ (1, Datum::int(1)), - (2, Datum::float(1.0)), + (2, Datum::float(1.0_f32)), (6, Datum::int(1)), - (7, Datum::float(1.0)) + (7, Datum::float(1.0_f32)) ]) ); assert_eq!( *data_file.upper_bounds(), HashMap::from([ (1, Datum::int(4)), - (2, Datum::float(2.0)), + (2, Datum::float(2.0_f32)), (6, Datum::int(4)), - (7, Datum::float(2.0)) + (7, Datum::float(2.0_f32)) ]) ); assert_eq!( @@ -2304,12 +2591,15 @@ mod tests { #[tokio::test] async fn test_write_empty_parquet_file() { let temp_dir = TempDir::new().unwrap(); - let file_io = FileIOBuilder::new_fs_io().build().unwrap(); + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); let file_name_gen = DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + let output_file = file_io + .new_output(location_gen.generate_location(None, &file_name_gen.generate_file_name())) + .unwrap(); // write data let pw = ParquetWriterBuilder::new( @@ -2325,12 +2615,8 @@ mod tests { .build() .expect("Failed to create schema"), ), - None, - file_io.clone(), - location_gen, - file_name_gen, ) - .build() + .build(output_file) .await .unwrap(); @@ -2341,38 +2627,777 @@ mod tests { assert_eq!(std::fs::read_dir(temp_dir.path()).unwrap().count(), 0); } - #[test] - fn test_min_max_aggregator() { + #[tokio::test] + async fn test_parquet_writer_uuid_logical_type() { + let temp_dir = TempDir::new().unwrap(); + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + let output_file = file_io + .new_output(location_gen.generate_location(None, &file_name_gen.generate_file_name())) + .unwrap(); + let schema = Arc::new( Schema::builder() .with_schema_id(1) .with_fields(vec![ - NestedField::required(0, "col", Type::Primitive(PrimitiveType::Int)) - .with_id(0) - .into(), + NestedField::required(0, "uuid", Type::Primitive(PrimitiveType::Uuid)).into(), ]) .build() - .expect("Failed to create schema"), + .unwrap(), ); - let mut min_max_agg = MinMaxColAggregator::new(schema); - let create_statistics = - |min, max| Statistics::Int32(ValueStatistics::new(min, max, None, None, false)); - min_max_agg - .update(0, create_statistics(None, Some(42))) - .unwrap(); - min_max_agg - .update(0, create_statistics(Some(0), Some(i32::MAX))) - .unwrap(); - min_max_agg - .update(0, create_statistics(Some(i32::MIN), None)) + let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap()); + let col = Arc::new( + arrow_array::FixedSizeBinaryArray::try_from_iter( + [Uuid::from_u128(0), Uuid::from_u128(1)] + .into_iter() + .map(|uuid| uuid.into_bytes()), + ) + .unwrap(), + ) as ArrayRef; + let to_write = RecordBatch::try_new(arrow_schema, vec![col]).unwrap(); + + let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema) + .build(output_file) + .await .unwrap(); - min_max_agg - .update(0, create_statistics(None, None)) + pw.write(&to_write).await.unwrap(); + let data_file = pw + .close() + .await + .unwrap() + .into_iter() + .next() + .unwrap() + .content(DataContentType::Data) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() .unwrap(); - let (lower_bounds, upper_bounds) = min_max_agg.produce(); - - assert_eq!(lower_bounds, HashMap::from([(0, Datum::int(i32::MIN))])); - assert_eq!(upper_bounds, HashMap::from([(0, Datum::int(i32::MAX))])); + let raw = file_io + .new_input(data_file.file_path.clone()) + .unwrap() + .read() + .await + .unwrap(); + let reader = ParquetRecordBatchReaderBuilder::try_new(raw).unwrap(); + let column = reader.metadata().file_metadata().schema_descr().column(0); + assert_eq!( + ( + column.physical_type(), + column.type_length(), + column.logical_type_ref() + ), + ( + parquet::basic::Type::FIXED_LEN_BYTE_ARRAY, + 16, + Some(&LogicalType::Uuid) + ) + ); + } + + struct GeospatialTestFile { + temp_dir: TempDir, + file_io: FileIO, + schema: SchemaRef, + batch: RecordBatch, + data_file: DataFile, + } + + async fn write_geospatial_test_file() -> Result { + let temp_dir = TempDir::new().unwrap(); + let file_io = FileIO::new_with_fs(); + let geom_type = PrimitiveType::Geometry(GeometryType::new(Some("srid:0".to_string()))?); + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::required(1, "geom", Type::Primitive(geom_type)).into(), + NestedField::optional( + 2, + "geog", + Type::Primitive(PrimitiveType::Geography(GeographyType::new( + None, + IcebergEdgeInterpolationAlgorithm::Karney, + )?)), + ) + .into(), + ]) + .build()?, + ); + let geometry = [ + parquet_geospatial::testing::wkb_point_xyzm(1.0, 20.0, f64::NAN, 40.0), + parquet_geospatial::testing::wkb_point_xyzm(-3.0, 4.0, f64::NAN, 70.0), + ]; + let geography = wkb_point_xy(3.0, 4.0); + let batch = RecordBatch::try_new(Arc::new(schema_to_arrow_schema(&schema)?), vec![ + Arc::new(arrow_array::LargeBinaryArray::from_iter_values(&geometry)), + Arc::new(arrow_array::LargeBinaryArray::from(vec![ + Some(geography.as_slice()), + None, + ])), + ])?; + let output = file_io.new_output(temp_dir.path().join("geo.parquet").to_str().unwrap())?; + let props = WriterProperties::builder() + .set_max_row_group_row_count(Some(1)) + .build(); + let mut writer = ParquetWriterBuilder::new(props, schema.clone()) + .build(output) + .await?; + writer.write(&batch).await?; + let data_file = writer + .close() + .await? + .into_iter() + .next() + .unwrap() + .content(DataContentType::Data) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap(); + + Ok(GeospatialTestFile { + temp_dir, + file_io, + schema, + batch, + data_file, + }) + } + + #[tokio::test] + async fn test_geospatial_parquet_roundtrip() -> Result<()> { + let file = write_geospatial_test_file().await?; + let input = file.file_io.new_input(file.data_file.file_path())?; + let mut parquet_reader = + ArrowFileReader::new(input.metadata().await?, input.reader().await?); + let metadata = parquet_reader.get_metadata(None).await?; + let schema_descr = metadata.file_metadata().schema_descr(); + assert_eq!(metadata.num_row_groups(), 2); + let logical_types: Vec<_> = schema_descr + .columns() + .iter() + .map(|column| column.logical_type_ref()) + .collect(); + let expected = [ + LogicalType::geometry(Some("srid:0".to_string())), + LogicalType::geography(None, Some(EdgeInterpolationAlgorithm::KARNEY)), + ]; + assert_eq!(logical_types, expected.iter().map(Some).collect::>()); + + let arrow_schema = parquet::arrow::parquet_to_arrow_schema( + schema_descr, + metadata.file_metadata().key_value_metadata(), + )?; + let unset_metadata: serde_json::Value = + serde_json::from_str(arrow_schema.field(0).extension_type_metadata().unwrap())?; + assert!(unset_metadata.get("crs").is_none()); + + let task = FileScanTask::builder() + .with_file_size_in_bytes(file.data_file.file_size_in_bytes()) + .with_start(0) + .with_length(0) + .with_data_file_path(file.data_file.file_path.clone()) + .with_data_file_format(DataFileFormat::Parquet) + .with_schema(file.schema) + .with_project_field_ids(vec![1, 2]) + .with_case_sensitive(false) + .build()?; + let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream; + let reader = ArrowReaderBuilder::new(file.file_io, Runtime::current()).build(); + let batches: Vec = reader.read(tasks)?.stream().try_collect().await?; + let read_back = concat_batches(batches[0].schema_ref(), &batches)?; + assert_eq!(read_back.columns(), file.batch.columns()); + Ok(()) + } + + fn coordinate_bytes(coordinates: &[f64]) -> Vec { + coordinates + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect() + } + + #[tokio::test] + async fn test_geometry_bounds_manifest_roundtrip() -> Result<()> { + let file = write_geospatial_test_file().await?; + let path = file.temp_dir.path().join("manifest.avro"); + let output = file.file_io.new_output(path.to_str().unwrap())?; + let mut writer = ManifestWriterBuilder::new( + output, + Some(1), + file.schema.clone(), + PartitionSpec::builder(file.schema).build()?, + ) + .build_v3_data(); + writer.add_file(file.data_file, 1)?; + writer.write_manifest_file().await?; + let manifest = Manifest::parse_avro(&std::fs::read(path).unwrap())?; + let data_file = manifest.entries()[0].data_file(); + let ty = PrimitiveType::Geometry(GeometryType::new(Some("srid:0".to_string()))?); + let expected_lower = HashMap::from([( + 1, + Datum::new( + ty.clone(), + PrimitiveLiteral::Binary(coordinate_bytes(&[-3.0, 4.0, f64::NAN, 40.0])), + ), + )]); + let expected_upper = HashMap::from([( + 1, + Datum::new( + ty, + PrimitiveLiteral::Binary(coordinate_bytes(&[1.0, 20.0, f64::NAN, 70.0])), + ), + )]); + assert_eq!( + (data_file.lower_bounds(), data_file.upper_bounds()), + (&expected_lower, &expected_upper), + ); + Ok(()) + } + + #[tokio::test] + async fn test_geometry_bounds_do_not_drive_scalar_metrics() -> Result<()> { + use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator; + use crate::expr::visitors::strict_metrics_evaluator::StrictMetricsEvaluator; + use crate::expr::{Bind, Reference}; + + let file = write_geospatial_test_file().await?; + let values = file + .batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let literal = Datum::try_from_bytes( + values.value(0), + file.schema + .field_by_id(1) + .unwrap() + .field_type + .as_primitive_type() + .unwrap() + .clone(), + )?; + let inequality = Reference::new("geom") + .less_than(literal.clone()) + .bind(file.schema.clone(), true)?; + let exclusion = Reference::new("geom") + .not_equal_to(literal) + .bind(file.schema, true)?; + assert_eq!( + ( + InclusiveMetricsEvaluator::eval(&inequality, &file.data_file, true)?, + StrictMetricsEvaluator::eval(&exclusion, &file.data_file)?, + ), + (true, false), + ); + Ok(()) + } + + fn geometry_bound_bytes(bbox: BoundingBox) -> (Vec, Vec) { + let ty = PrimitiveType::Geometry(GeometryType::default()); + let (lower, upper) = GeospatialBounds::from(&bbox).into_datums(ty); + ( + lower.to_bytes().unwrap().into_vec(), + upper.to_bytes().unwrap().into_vec(), + ) + } + + #[test] + fn test_geometry_xy_bounds() { + assert_eq!( + geometry_bound_bytes(BoundingBox::new(-3.0, 1.0, 4.0, 20.0)), + ( + coordinate_bytes(&[-3.0, 4.0]), + coordinate_bytes(&[1.0, 20.0]) + ), + ); + } + + #[test] + fn test_geometry_xyz_bounds() { + assert_eq!( + geometry_bound_bytes(BoundingBox::new(-3.0, 1.0, 4.0, 20.0).with_zrange(2.0, 9.0)), + ( + coordinate_bytes(&[-3.0, 4.0, 2.0]), + coordinate_bytes(&[1.0, 20.0, 9.0]) + ), + ); + } + + #[test] + fn test_wrapping_geometry_bounds_cover_all_x() { + assert_eq!( + geometry_bound_bytes(BoundingBox::new(170.0, -170.0, -10.0, 10.0)), + ( + coordinate_bytes(&[f64::NEG_INFINITY, -10.0]), + coordinate_bytes(&[f64::INFINITY, 10.0]), + ), + ); + } + + fn geospatial_stats_fixture(ty: PrimitiveType) -> (SchemaRef, ColumnChunkMetaData) { + let schema = Arc::new( + Schema::builder() + .with_fields(vec![ + NestedField::optional(1, "geo", Type::Primitive(ty)).into(), + ]) + .build() + .unwrap(), + ); + let parquet_schema = parquet::arrow::ArrowSchemaConverter::new() + .convert(&schema_to_arrow_schema(&schema).unwrap()) + .unwrap(); + let column = ColumnChunkMetaData::builder(parquet_schema.column(0)) + .set_num_values(1) + .build() + .unwrap(); + (schema, column) + } + + fn with_geo_box(column: &ColumnChunkMetaData, bbox: BoundingBox) -> ColumnChunkMetaData { + use parquet::geospatial::statistics::GeospatialStatistics; + column + .clone() + .into_builder() + .set_geo_statistics(Box::new(GeospatialStatistics::new(Some(bbox), None))) + .build() + .unwrap() + } + + #[test] + fn test_geometry_bounds_merge_across_null_row_group() -> Result<()> { + let (schema, column) = + geospatial_stats_fixture(PrimitiveType::Geometry(GeometryType::default())); + let first = with_geo_box( + &column, + BoundingBox::new(-3.0, 1.0, 4.0, 20.0).with_zrange(4.0, 9.0), + ); + let second = with_geo_box( + &column, + BoundingBox::new(0.0, 5.0, -20.0, 5.0).with_mrange(1.0, 3.0), + ); + let nulls = column + .into_builder() + .set_statistics(Statistics::ByteArray(ValueStatistics::new( + None, + None, + None, + Some(1), + false, + ))) + .build()?; + let mut bounds = MinMaxColAggregator::new(schema); + for column in [&first, &nulls, &second] { + bounds.update(1, column)?; + } + let (lower, upper) = bounds.produce(); + assert_eq!( + ( + lower[&1].to_bytes()?.into_vec(), + upper[&1].to_bytes()?.into_vec() + ), + ( + coordinate_bytes(&[-3.0, -20.0, 4.0, 1.0]), + coordinate_bytes(&[5.0, 20.0, 9.0, 3.0]) + ), + ); + Ok(()) + } + + #[test] + fn test_geometry_bounds_require_all_non_null_row_groups() -> Result<()> { + let (schema, unknown) = + geospatial_stats_fixture(PrimitiveType::Geometry(GeometryType::default())); + let known = with_geo_box(&unknown, BoundingBox::new(1.0, 2.0, 3.0, 4.0)); + let mut bounds = MinMaxColAggregator::new(schema); + for column in [&known, &unknown, &known] { + bounds.update(1, column)?; + } + assert_eq!(bounds.produce(), (HashMap::new(), HashMap::new())); + Ok(()) + } + + #[test] + fn test_geography_bounds_merge_from_parquet_statistics() -> Result<()> { + let (schema, column) = + geospatial_stats_fixture(PrimitiveType::Geography(GeographyType::default())); + let first = with_geo_box(&column, BoundingBox::new(170.0, -170.0, -20.0, 20.0)); + let second = with_geo_box(&column, BoundingBox::new(175.0, 178.0, -10.0, 30.0)); + let mut bounds = MinMaxColAggregator::new(schema); + bounds.update(1, &first)?; + bounds.update(1, &second)?; + let (lower, upper) = bounds.produce(); + assert_eq!( + ( + lower[&1].to_bytes()?.into_vec(), + upper[&1].to_bytes()?.into_vec() + ), + ( + coordinate_bytes(&[170.0, -20.0]), + coordinate_bytes(&[-170.0, 30.0]) + ), + ); + Ok(()) + } + + #[test] + fn test_geography_bounds_full_longitude_range() -> Result<()> { + let (schema, column) = + geospatial_stats_fixture(PrimitiveType::Geography(GeographyType::default())); + let first = with_geo_box(&column, BoundingBox::new(170.0, -170.0, -20.0, 20.0)); + let second = with_geo_box(&column, BoundingBox::new(-175.0, 175.0, -10.0, 30.0)); + let mut bounds = MinMaxColAggregator::new(schema); + bounds.update(1, &first)?; + bounds.update(1, &second)?; + let (lower, upper) = bounds.produce(); + assert_eq!( + ( + lower[&1].to_bytes()?.into_vec(), + upper[&1].to_bytes()?.into_vec() + ), + ( + coordinate_bytes(&[-180.0, -20.0]), + coordinate_bytes(&[180.0, 30.0]) + ), + ); + Ok(()) + } + + #[test] + fn test_min_max_aggregator() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(0, "col", Type::Primitive(PrimitiveType::Int)) + .with_id(0) + .into(), + ]) + .build() + .expect("Failed to create schema"), + ); + let parquet_schema = parquet::arrow::ArrowSchemaConverter::new() + .convert(&schema_to_arrow_schema(&schema).unwrap()) + .unwrap(); + let mut min_max_agg = MinMaxColAggregator::new(schema); + let create_statistics = |min, max| { + ColumnChunkMetaData::builder(parquet_schema.column(0)) + .set_statistics(Statistics::Int32(ValueStatistics::new( + min, max, None, None, false, + ))) + .build() + .unwrap() + }; + min_max_agg + .update(0, &create_statistics(None, Some(42))) + .unwrap(); + min_max_agg + .update(0, &create_statistics(Some(0), Some(i32::MAX))) + .unwrap(); + min_max_agg + .update(0, &create_statistics(Some(i32::MIN), None)) + .unwrap(); + min_max_agg + .update(0, &create_statistics(None, None)) + .unwrap(); + + let (lower_bounds, upper_bounds) = min_max_agg.produce(); + + assert_eq!(lower_bounds, HashMap::from([(0, Datum::int(i32::MIN))])); + assert_eq!(upper_bounds, HashMap::from([(0, Datum::int(i32::MAX))])); + } + + #[test] + fn test_min_max_aggregator_skips_nan_bounds() { + let schema = Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(0, "f", Type::Primitive(PrimitiveType::Float)).into(), + NestedField::required(1, "d", Type::Primitive(PrimitiveType::Double)).into(), + ]) + .build() + .expect("Failed to create schema"), + ); + let parquet_schema = parquet::arrow::ArrowSchemaConverter::new() + .convert(&schema_to_arrow_schema(&schema).unwrap()) + .unwrap(); + let mut min_max_agg = MinMaxColAggregator::new(schema); + let float_stats = |min, max| { + ColumnChunkMetaData::builder(parquet_schema.column(0)) + .set_statistics(Statistics::Float(ValueStatistics::new( + min, max, None, None, false, + ))) + .build() + .unwrap() + }; + let double_stats = |min, max| { + ColumnChunkMetaData::builder(parquet_schema.column(1)) + .set_statistics(Statistics::Double(ValueStatistics::new( + min, max, None, None, false, + ))) + .build() + .unwrap() + }; + + min_max_agg + .update(0, &float_stats(Some(-f32::NAN), Some(f32::NAN))) + .unwrap(); + min_max_agg + .update(0, &float_stats(Some(1.5), Some(2.5))) + .unwrap(); + min_max_agg + .update(1, &double_stats(Some(f64::NAN), Some(f64::NAN))) + .unwrap(); + + let (lower_bounds, upper_bounds) = min_max_agg.produce(); + + assert_eq!(lower_bounds, HashMap::from([(0, Datum::float(1.5_f32))])); + assert_eq!(upper_bounds, HashMap::from([(0, Datum::float(2.5_f32))])); + } + + fn cdc_test_schema() -> SchemaRef { + Arc::new( + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "payload", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(), + ) + } + + #[test] + fn test_from_table_properties_no_cdc_by_default() { + let raw_properties = HashMap::new(); + let tp = TableProperties::new(&raw_properties); + let builder = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()).unwrap(); + assert!(builder.props.content_defined_chunking().is_none()); + } + + #[tokio::test] + async fn test_from_table_properties_without_encryption_writes_plaintext() { + let raw_properties = HashMap::new(); + let tp = TableProperties::new(&raw_properties); + let tmp = TempDir::new().unwrap(); + let output = FileIO::new_with_fs() + .new_output(format!("{}/plain.parquet", tmp.path().to_str().unwrap())) + .unwrap(); + let writer = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) + .unwrap() + .build(output) + .await + .unwrap(); + + assert!(writer.key_metadata.is_none()); + assert!( + writer + .writer_properties + .file_encryption_properties() + .is_none() + ); + } + + #[tokio::test] + async fn test_from_table_properties_propagate_to_writer() { + // `build()` must carry the translated `WriterProperties` through to the + // `ParquetWriter` unchanged — otherwise the `write.parquet.*` settings + // derived in `from_table_properties` would never reach parquet-rs. + // + // Asserting on the writer's `WriterProperties` (rather than re-reading a + // written file) keeps this a direct propagation check: every future + // `write.parquet.*` option just adds an assertion on its corresponding + // `WriterProperties` getter here. + let raw_properties = HashMap::from([ + ( + TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(), + "true".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(), + "4096".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(), + "8192".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(), + "2".to_string(), + ), + ]); + let tp = TableProperties::new(&raw_properties); + + let tmp = TempDir::new().unwrap(); + let output = FileIO::new_with_fs() + .new_output(format!("{}/cdc.parquet", tmp.path().to_str().unwrap())) + .unwrap(); + let writer = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) + .unwrap() + .build(output) + .await + .unwrap(); + + let cdc = writer + .writer_properties + .content_defined_chunking() + .copied() + .expect("CDC should be enabled on the built writer"); + assert_eq!(cdc.min_chunk_size, 4096); + assert_eq!(cdc.max_chunk_size, 8192); + assert_eq!(cdc.norm_level, 2); + } + + #[test] + fn test_from_table_properties_sizing_defaults() { + // With no properties set, the writer must use Iceberg's defaults (which + // differ from parquet-rs's own defaults), not parquet-rs's. + let raw_properties = HashMap::new(); + let tp = TableProperties::new(&raw_properties); + let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) + .unwrap() + .props; + + assert_eq!( + props.max_row_group_bytes(), + Some(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT) + ); + assert_eq!( + props.data_page_size_limit(), + TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT + ); + assert_eq!( + props.data_page_row_count_limit(), + TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT + ); + assert_eq!( + props.dictionary_page_size_limit(), + TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT + ); + // Default codec is zstd at the Java-aligned default level (3). + assert_eq!( + props.compression(&ColumnPath::from("id")), + Compression::ZSTD(ZstdLevel::try_new(3).unwrap()) + ); + } + + #[test] + fn test_from_table_properties_sizing_and_compression_overrides() { + let raw_properties = HashMap::from([ + ( + TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(), + "1048576".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES.to_string(), + "65536".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT.to_string(), + "5000".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES.to_string(), + "131072".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), + "gzip".to_string(), + ), + ( + TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(), + "9".to_string(), + ), + ]); + let tp = TableProperties::new(&raw_properties); + let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()) + .unwrap() + .props; + + assert_eq!(props.max_row_group_bytes(), Some(1048576)); + assert_eq!(props.data_page_size_limit(), 65536); + assert_eq!(props.data_page_row_count_limit(), 5000); + assert_eq!(props.dictionary_page_size_limit(), 131072); + assert_eq!( + props.compression(&ColumnPath::from("id")), + Compression::GZIP(GzipLevel::try_new(9).unwrap()) + ); + } + + #[test] + fn test_from_table_properties_invalid_codec_errors() { + let entries = HashMap::from([( + TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), + "bogus".to_string(), + )]); + let err = ParquetWriterBuilder::from_table_properties( + &TableProperties::new(&entries), + cdc_test_schema(), + ) + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("bogus")); + } + + #[test] + fn test_parquet_compression_mapping() { + // Codecs without a level. + assert_eq!( + parquet_compression(CompressionCodec::None).unwrap(), + Compression::UNCOMPRESSED + ); + assert_eq!( + parquet_compression(CompressionCodec::Snappy).unwrap(), + Compression::SNAPPY + ); + assert_eq!( + parquet_compression(CompressionCodec::Lz4).unwrap(), + Compression::LZ4 + ); + assert_eq!( + parquet_compression(CompressionCodec::Lz4Raw).unwrap(), + Compression::LZ4_RAW + ); + assert_eq!( + parquet_compression(CompressionCodec::Lzo).unwrap(), + Compression::LZO + ); + + // Level-carrying codecs at their defaults. + assert_eq!( + parquet_compression(CompressionCodec::zstd_default()).unwrap(), + Compression::ZSTD(ZstdLevel::try_new(3).unwrap()) + ); + assert_eq!( + parquet_compression(CompressionCodec::gzip_default()).unwrap(), + Compression::GZIP(GzipLevel::try_new(6).unwrap()) + ); + assert_eq!( + parquet_compression(CompressionCodec::brotli_default()).unwrap(), + Compression::BROTLI(BrotliLevel::try_new(1).unwrap()) + ); + + // Explicit levels are honored. + assert_eq!( + parquet_compression(CompressionCodec::Zstd(10)).unwrap(), + Compression::ZSTD(ZstdLevel::try_new(10).unwrap()) + ); + } + + #[test] + fn test_parquet_compression_invalid_level() { + // zstd valid range is 1..=22; 99 is out of range. + let err = parquet_compression(CompressionCodec::Zstd(99)).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.to_string().contains("zstd")); } } diff --git a/crates/iceberg/src/writer/file_writer/rolling_writer.rs b/crates/iceberg/src/writer/file_writer/rolling_writer.rs index 181205c371..9a45cd0fb2 100644 --- a/crates/iceberg/src/writer/file_writer/rolling_writer.rs +++ b/crates/iceberg/src/writer/file_writer/rolling_writer.rs @@ -15,50 +15,104 @@ // specific language governing permissions and limitations // under the License. +use std::fmt::{Debug, Formatter}; + use arrow_array::RecordBatch; -use crate::spec::DataFileBuilder; +use crate::io::{FileIO, OutputFile}; +use crate::spec::{DataFileBuilder, PartitionKey, TableProperties}; use crate::writer::CurrentFileStatus; +use crate::writer::file_writer::location_generator::{FileNameGenerator, LocationGenerator}; use crate::writer::file_writer::{FileWriter, FileWriterBuilder}; use crate::{Error, ErrorKind, Result}; -/// Builder for creating a `RollingFileWriter` that rolls over to a new file -/// when the data size exceeds a target threshold. -#[derive(Clone)] -pub struct RollingFileWriterBuilder { +/// Builder for [`RollingFileWriter`]. +#[derive(Clone, Debug)] +pub struct RollingFileWriterBuilder< + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +> { inner_builder: B, target_file_size: usize, + file_io: FileIO, + location_generator: L, + file_name_generator: F, } -impl RollingFileWriterBuilder { - /// Creates a new `RollingFileWriterBuilder` with the specified inner builder and target size. +impl RollingFileWriterBuilder +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + /// Creates a new `RollingFileWriterBuilder` with the specified target file size. /// - /// # Arguments + /// # Parameters /// /// * `inner_builder` - The builder for the underlying file writer - /// * `target_file_size` - The target size in bytes before rolling over to a new file + /// * `target_file_size` - The target file size in bytes that triggers rollover + /// * `file_io` - The file IO interface for creating output files + /// * `location_generator` - Generator for file locations + /// * `file_name_generator` - Generator for file names + /// + /// # Returns /// - /// NOTE: The `target_file_size` does not exactly reflect the final size on physical storage. - /// This is because the input size is based on the Arrow in-memory format and cannot precisely control rollover behavior. - /// The actual file size on disk is expected to be slightly larger than `target_file_size`. - pub fn new(inner_builder: B, target_file_size: usize) -> Self { + /// A new `RollingFileWriterBuilder` instance + pub fn new( + inner_builder: B, + target_file_size: usize, + file_io: FileIO, + location_generator: L, + file_name_generator: F, + ) -> Self { Self { inner_builder, target_file_size, + file_io, + location_generator, + file_name_generator, } } -} -impl FileWriterBuilder for RollingFileWriterBuilder { - type R = RollingFileWriter; + /// Creates a new `RollingFileWriterBuilder` with the default target file size. + /// + /// # Parameters + /// + /// * `inner_builder` - The builder for the underlying file writer + /// * `file_io` - The file IO interface for creating output files + /// * `location_generator` - Generator for file locations + /// * `file_name_generator` - Generator for file names + /// + /// # Returns + /// + /// A new `RollingFileWriterBuilder` instance with default target file size + pub fn new_with_default_file_size( + inner_builder: B, + file_io: FileIO, + location_generator: L, + file_name_generator: F, + ) -> Self { + Self { + inner_builder, + target_file_size: TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, + file_io, + location_generator, + file_name_generator, + } + } - async fn build(self) -> Result { - Ok(RollingFileWriter { + /// Build a new [`RollingFileWriter`]. + pub fn build(&self) -> RollingFileWriter { + RollingFileWriter { inner: None, - inner_builder: self.inner_builder, + inner_builder: self.inner_builder.clone(), target_file_size: self.target_file_size, data_file_builders: vec![], - }) + file_io: self.file_io.clone(), + location_generator: self.location_generator.clone(), + file_name_generator: self.file_name_generator.clone(), + } } } @@ -68,14 +122,36 @@ impl FileWriterBuilder for RollingFileWriterBuilder { /// This writer wraps another file writer that tracks the amount of data written. /// When the data size exceeds the target size, it closes the current file and /// starts writing to a new one. -pub struct RollingFileWriter { +pub struct RollingFileWriter { inner: Option, inner_builder: B, target_file_size: usize, data_file_builders: Vec, + file_io: FileIO, + location_generator: L, + file_name_generator: F, } -impl RollingFileWriter { +impl Debug for RollingFileWriter +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RollingFileWriter") + .field("target_file_size", &self.target_file_size) + .field("file_io", &self.file_io) + .finish() + } +} + +impl RollingFileWriter +where + B: FileWriterBuilder, + L: LocationGenerator, + F: FileNameGenerator, +{ /// Determines if the writer should roll over to a new file. /// /// # Returns @@ -84,23 +160,55 @@ impl RollingFileWriter { fn should_roll(&self) -> bool { self.current_written_size() > self.target_file_size } -} -impl FileWriter for RollingFileWriter { - async fn write(&mut self, input: &RecordBatch) -> Result<()> { + fn new_output_file(&self, partition_key: &Option) -> Result { + self.file_io + .new_output(self.location_generator.generate_location( + partition_key.as_ref(), + &self.file_name_generator.generate_file_name(), + )) + } + + /// Writes a record batch to the current file, rolling over to a new file if necessary. + /// + /// # Parameters + /// + /// * `partition_key` - Optional partition key for the data + /// * `input` - The record batch to write + /// + /// # Returns + /// + /// A `Result` indicating success or failure + /// + /// # Errors + /// + /// Returns an error if the writer is not initialized or if writing fails + pub async fn write( + &mut self, + partition_key: &Option, + input: &RecordBatch, + ) -> Result<()> { if self.inner.is_none() { // initialize inner writer - self.inner = Some(self.inner_builder.clone().build().await?); + self.inner = Some( + self.inner_builder + .build(self.new_output_file(partition_key)?) + .await?, + ); } - if self.should_roll() { - if let Some(inner) = self.inner.take() { - // close the current writer, roll to a new file - self.data_file_builders.extend(inner.close().await?); - - // start a new writer - self.inner = Some(self.inner_builder.clone().build().await?); - } + if self.should_roll() + && let Some(inner) = self.inner.take() + { + // close the current writer, roll to a new file + self.data_file_builders.extend(inner.close().await?); + + // start a new writer + self.inner = Some( + self.inner_builder + .build(self.new_output_file(partition_key)?) + .await?, + ); } // write the input @@ -114,7 +222,13 @@ impl FileWriter for RollingFileWriter { } } - async fn close(mut self) -> Result> { + /// Closes the writer and returns all data file builders. + /// + /// # Returns + /// + /// A `Result` containing a vector of `DataFileBuilder` instances representing + /// all files that were written, including any that were created due to rollover + pub async fn close(mut self) -> Result> { // close the current writer and merge the output if let Some(current_writer) = self.inner { self.data_file_builders @@ -125,7 +239,9 @@ impl FileWriter for RollingFileWriter { } } -impl CurrentFileStatus for RollingFileWriter { +impl CurrentFileStatus + for RollingFileWriter +{ fn current_file_path(&self) -> String { self.inner.as_ref().unwrap().current_file_path() } @@ -141,7 +257,7 @@ impl CurrentFileStatus for RollingFileWriter { #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::collections::{HashMap, HashSet}; use std::sync::Arc; use arrow_array::{ArrayRef, Int32Array, StringArray}; @@ -152,8 +268,11 @@ mod tests { use tempfile::TempDir; use super::*; - use crate::io::FileIOBuilder; + use crate::arrow::test_utils::read_encrypted_parquet; + use crate::encryption::StandardKeyMetadata; + use crate::io::FileIO; use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Schema, Type}; + use crate::test_utils::make_encryption_manager; use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; use crate::writer::file_writer::ParquetWriterBuilder; use crate::writer::file_writer::location_generator::{ @@ -188,7 +307,7 @@ mod tests { #[tokio::test] async fn test_rolling_writer_basic() -> Result<()> { let temp_dir = TempDir::new()?; - let file_io = FileIOBuilder::new_fs_io().build()?; + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -199,25 +318,22 @@ mod tests { let schema = make_test_schema()?; // Create writer builders - let parquet_writer_builder = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - Arc::new(schema), - None, - file_io.clone(), - location_gen, - file_name_gen, - ); + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(schema)); // Set a large target size so no rolling occurs - let rolling_writer_builder = RollingFileWriterBuilder::new( + let rolling_file_writer_builder = RollingFileWriterBuilder::new( parquet_writer_builder, - 1024 * 1024, // 1MB, large enough to not trigger rolling + 1024 * 1024, + file_io.clone(), + location_gen, + file_name_gen, ); - let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder, None, 0); + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_file_writer_builder); // Create writer - let mut writer = data_file_writer_builder.build().await?; + let mut writer = data_file_writer_builder.build(None).await?; // Create test data let arrow_schema = make_test_arrow_schema(); @@ -249,7 +365,7 @@ mod tests { #[tokio::test] async fn test_rolling_writer_with_rolling() -> Result<()> { let temp_dir = TempDir::new()?; - let file_io = FileIOBuilder::new_fs_io().build()?; + let file_io = FileIO::new_with_fs(); let location_gen = DefaultLocationGenerator::with_data_location( temp_dir.path().to_str().unwrap().to_string(), ); @@ -260,22 +376,22 @@ mod tests { let schema = make_test_schema()?; // Create writer builders - let parquet_writer_builder = ParquetWriterBuilder::new( - WriterProperties::builder().build(), - Arc::new(schema), - None, - file_io.clone(), + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(schema)); + + // Set a very small target size to trigger rolling + let rolling_writer_builder = RollingFileWriterBuilder::new( + parquet_writer_builder, + 1024, + file_io, location_gen, file_name_gen, ); - // Set a very small target size to trigger rolling - let rolling_writer_builder = RollingFileWriterBuilder::new(parquet_writer_builder, 1024); - - let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder, None, 0); + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); // Create writer - let mut writer = data_file_writer_builder.build().await?; + let mut writer = data_file_writer_builder.build(None).await?; // Create test data let arrow_schema = make_test_arrow_schema(); @@ -286,7 +402,7 @@ mod tests { "Kelly", "Larry", "Mallory", "Shawn", ]; - let mut rng = rand::thread_rng(); + let mut rng = rand::rng(); let batch_num = 10; let batch_rows = 100; let expected_rows = batch_num * batch_rows; @@ -321,9 +437,115 @@ mod tests { let total_records: u64 = data_files.iter().map(|file| file.record_count).sum(); assert_eq!( total_records, expected_rows as u64, - "Expected {} total records across all files", - expected_rows + "Expected {expected_rows} total records across all files" + ); + + Ok(()) + } + + /// Every rolled file is independently encrypted + #[tokio::test] + async fn test_rolling_writer_encrypted() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + let schema = make_test_schema()?; + + let raw_properties = HashMap::from([( + TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(), + "test-key".to_string(), + )]); + let table_properties = TableProperties::new(&raw_properties); + let parquet_writer_builder = + ParquetWriterBuilder::from_table_properties(&table_properties, Arc::new(schema))? + .with_encryption_manager(make_encryption_manager("test-key")); + + // Set a very small target size to trigger rolling + let rolling_writer_builder = RollingFileWriterBuilder::new( + parquet_writer_builder, + 1024, + file_io.clone(), + location_gen, + file_name_gen, + ); + + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); + + // Create writer + let mut writer = data_file_writer_builder.build(None).await?; + + // Create test data + let arrow_schema = make_test_arrow_schema(); + let arrow_schema_ref = Arc::new(arrow_schema.clone()); + + let names = vec![ + "Alice", "Bob", "Charlie", "Dave", "Eve", "Frank", "Grace", "Heidi", "Ivan", "Judy", + "Kelly", "Larry", "Mallory", "Shawn", + ]; + + let mut rng = rand::rng(); + let batch_num = 10; + let batch_rows = 100; + let expected_rows = batch_num * batch_rows; + + for i in 0..batch_num { + let int_values: Vec = (0..batch_rows).map(|row| i * batch_rows + row).collect(); + let str_values: Vec<&str> = (0..batch_rows) + .map(|_| *names.iter().choose(&mut rng).unwrap()) + .collect(); + + let int_array = Arc::new(Int32Array::from(int_values)) as ArrayRef; + let str_array = Arc::new(StringArray::from(str_values)) as ArrayRef; + + let batch = + RecordBatch::try_new(Arc::clone(&arrow_schema_ref), vec![int_array, str_array]) + .expect("Failed to create RecordBatch"); + + writer.write(batch).await?; + } + + let data_files = writer.close().await?; + + assert!( + data_files.len() > 4, + "Expected at least 4 data files to be created, but got {}", + data_files.len() + ); + + // Verify total record count across all files + let total_records: u64 = data_files.iter().map(|file| file.record_count).sum(); + assert_eq!( + total_records, expected_rows as u64, + "Expected {expected_rows} total records across all files" + ); + + let mut distinct_keys = HashSet::new(); + for data_file in &data_files { + let key_metadata = StandardKeyMetadata::decode( + data_file + .key_metadata() + .expect("each rolled file must carry key metadata"), + )?; + assert!( + distinct_keys.insert(key_metadata.encryption_key().as_bytes().to_vec()), + "each rolled file must use a distinct DEK" + ); + } + + let data_file = &data_files[0]; + let key_metadata = StandardKeyMetadata::decode(data_file.key_metadata().unwrap())?; + let batches = read_encrypted_parquet( + &data_file.file_path, + key_metadata.encryption_key().as_bytes(), + key_metadata.aad_prefix(), ); + let read_rows: u64 = batches.iter().map(|b| b.num_rows() as u64).sum(); + assert_eq!(read_rows, data_file.record_count); Ok(()) } diff --git a/crates/iceberg/src/writer/mod.rs b/crates/iceberg/src/writer/mod.rs index 8f17d50e27..ca401097ab 100644 --- a/crates/iceberg/src/writer/mod.rs +++ b/crates/iceberg/src/writer/mod.rs @@ -60,6 +60,9 @@ //! async fn main() -> Result<()> { //! // Connect to a catalog. //! use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; +//! use iceberg::writer::file_writer::rolling_writer::{ +//! RollingFileWriter, RollingFileWriterBuilder, +//! }; //! let catalog = MemoryCatalogBuilder::default() //! .load( //! "memory", @@ -75,7 +78,7 @@ //! let table = catalog //! .load_table(&TableIdent::from_strs(["hello", "world"])?) //! .await?; -//! let location_generator = DefaultLocationGenerator::new(table.metadata().clone()).unwrap(); +//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -86,15 +89,20 @@ //! let parquet_writer_builder = ParquetWriterBuilder::new( //! WriterProperties::default(), //! table.metadata().current_schema().clone(), -//! None, +//! ); +//! +//! // Create a rolling file writer using parquet file writer builder. +//! let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( +//! parquet_writer_builder, //! table.file_io().clone(), //! location_generator.clone(), //! file_name_generator.clone(), //! ); +//! //! // Create a data file writer using parquet file writer builder. -//! let data_file_writer_builder = DataFileWriterBuilder::new(parquet_writer_builder, None, 0); +//! let data_file_writer_builder = DataFileWriterBuilder::new(rolling_file_writer_builder); //! // Build the data file writer -//! let mut data_file_writer = data_file_writer_builder.build().await.unwrap(); +//! let mut data_file_writer = data_file_writer_builder.build(None).await?; //! //! // Write the data using data_file_writer... //! @@ -113,7 +121,7 @@ //! use arrow_array::RecordBatch; //! use iceberg::io::FileIOBuilder; //! use iceberg::memory::MemoryCatalogBuilder; -//! use iceberg::spec::DataFile; +//! use iceberg::spec::{DataFile, PartitionKey}; //! use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; //! use iceberg::writer::file_writer::ParquetWriterBuilder; //! use iceberg::writer::file_writer::location_generator::{ @@ -140,9 +148,9 @@ //! impl IcebergWriterBuilder for LatencyRecordWriterBuilder { //! type R = LatencyRecordWriter; //! -//! async fn build(self) -> Result { +//! async fn build(&self, partition_key: Option) -> Result { //! Ok(LatencyRecordWriter { -//! inner_writer: self.inner_writer_builder.build().await?, +//! inner_writer: self.inner_writer_builder.build(partition_key).await?, //! }) //! } //! } @@ -174,6 +182,9 @@ //! // Connect to a catalog. //! use iceberg::memory::MEMORY_CATALOG_WAREHOUSE; //! use iceberg::spec::{Literal, PartitionKey, Struct}; +//! use iceberg::writer::file_writer::rolling_writer::{ +//! RollingFileWriter, RollingFileWriterBuilder, +//! }; //! //! let catalog = MemoryCatalogBuilder::default() //! .load( @@ -196,7 +207,7 @@ //! table.metadata().current_schema().clone(), //! Struct::from_iter(vec![Some(Literal::string("Seattle"))]), //! ); -//! let location_generator = DefaultLocationGenerator::new(table.metadata().clone()).unwrap(); +//! let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); //! let file_name_generator = DefaultFileNameGenerator::new( //! "test".to_string(), //! None, @@ -207,17 +218,167 @@ //! let parquet_writer_builder = ParquetWriterBuilder::new( //! WriterProperties::default(), //! table.metadata().current_schema().clone(), -//! Some(partition_key), +//! ); +//! +//! // Create a rolling file writer +//! let rolling_file_writer_builder = RollingFileWriterBuilder::new( +//! parquet_writer_builder, +//! 512 * 1024 * 1024, //! table.file_io().clone(), //! location_generator.clone(), //! file_name_generator.clone(), //! ); -//! // Create a data file writer builder using parquet file writer builder. -//! let data_file_writer_builder = DataFileWriterBuilder::new(parquet_writer_builder, None, 0); +//! +//! // Create a data file writer builder using rolling file writer. +//! let data_file_writer_builder = DataFileWriterBuilder::new(rolling_file_writer_builder); //! // Create latency record writer using data file writer builder. //! let latency_record_builder = LatencyRecordWriterBuilder::new(data_file_writer_builder); //! // Build the final writer -//! let mut latency_record_data_file_writer = latency_record_builder.build().await.unwrap(); +//! let mut latency_record_data_file_writer = latency_record_builder +//! .build(Some(partition_key)) +//! .await +//! .unwrap(); +//! +//! Ok(()) +//! } +//! ``` +//! +//! # Adding Partitioning to Data File Writers +//! +//! You can wrap a `DataFileWriter` with partitioning writers to handle partitioned tables. +//! Iceberg provides two partitioning strategies: +//! +//! ## FanoutWriter - For Unsorted Data +//! +//! Wraps the data file writer to handle unsorted data by maintaining multiple active writers. +//! Use this when your data is not pre-sorted by partition key. Writes to different partitions +//! can happen in any order, even interleaved. +//! +//! ```rust, no_run +//! # // Same setup as the simple example above... +//! # use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; +//! # use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; +//! # use iceberg::{Catalog, CatalogBuilder, Result, TableIdent}; +//! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; +//! # use iceberg::writer::file_writer::ParquetWriterBuilder; +//! # use iceberg::writer::file_writer::location_generator::{ +//! # DefaultFileNameGenerator, DefaultLocationGenerator, +//! # }; +//! # use parquet::file::properties::WriterProperties; +//! # use std::collections::HashMap; +//! # #[tokio::main] +//! # async fn main() -> Result<()> { +//! # let catalog = MemoryCatalogBuilder::default() +//! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) +//! # .await?; +//! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; +//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); +//! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); +//! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( +//! # parquet_writer_builder, table.file_io().clone(), location_generator, file_name_generator); +//! # let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); +//! +//! // Wrap the data file writer with FanoutWriter for partitioning +//! use iceberg::writer::partitioning::fanout_writer::FanoutWriter; +//! use iceberg::writer::partitioning::PartitioningWriter; +//! use iceberg::spec::{Literal, PartitionKey, Struct}; +//! +//! let mut fanout_writer = FanoutWriter::new(data_file_writer_builder); +//! +//! // Create partition keys for different regions +//! let schema = table.metadata().current_schema().clone(); +//! let partition_spec = table.metadata().default_partition_spec().as_ref().clone(); +//! +//! let partition_key_us = PartitionKey::new( +//! partition_spec.clone(), +//! schema.clone(), +//! Struct::from_iter([Some(Literal::string("US"))]), +//! ); +//! +//! let partition_key_eu = PartitionKey::new( +//! partition_spec.clone(), +//! schema.clone(), +//! Struct::from_iter([Some(Literal::string("EU"))]), +//! ); +//! +//! // Write to different partitions in any order - can interleave partition writes +//! // fanout_writer.write(partition_key_us.clone(), batch_us1).await?; +//! // fanout_writer.write(partition_key_eu.clone(), batch_eu1).await?; +//! // fanout_writer.write(partition_key_us.clone(), batch_us2).await?; // Back to US - OK! +//! // fanout_writer.write(partition_key_eu.clone(), batch_eu2).await?; // Back to EU - OK! +//! +//! let data_files = fanout_writer.close().await?; +//! # Ok(()) +//! # } +//! ``` +//! +//! ## ClusteredWriter - For Sorted Data +//! +//! Wraps the data file writer for pre-sorted data. More memory efficient as it maintains +//! only one active writer at a time, but requires input sorted by partition key. +//! +//! ```rust, no_run +//! # // Same setup as the simple example above... +//! # use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; +//! # use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; +//! # use iceberg::{Catalog, CatalogBuilder, Result, TableIdent}; +//! # use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; +//! # use iceberg::writer::file_writer::ParquetWriterBuilder; +//! # use iceberg::writer::file_writer::location_generator::{ +//! # DefaultFileNameGenerator, DefaultLocationGenerator, +//! # }; +//! # use parquet::file::properties::WriterProperties; +//! # use std::collections::HashMap; +//! # #[tokio::main] +//! # async fn main() -> Result<()> { +//! # let catalog = MemoryCatalogBuilder::default() +//! # .load("memory", HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), "file:///path/to/warehouse".to_string())])) +//! # .await?; +//! # let table = catalog.load_table(&TableIdent::from_strs(["hello", "world"])?).await?; +//! # let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); +//! # let file_name_generator = DefaultFileNameGenerator::new("test".to_string(), None, iceberg::spec::DataFileFormat::Parquet); +//! # let parquet_writer_builder = ParquetWriterBuilder::new(WriterProperties::default(), table.metadata().current_schema().clone()); +//! # let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( +//! # parquet_writer_builder, table.file_io().clone(), location_generator, file_name_generator); +//! # let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); +//! +//! // Wrap the data file writer with ClusteredWriter for sorted partitioning +//! use iceberg::writer::partitioning::clustered_writer::ClusteredWriter; +//! use iceberg::writer::partitioning::PartitioningWriter; +//! use iceberg::spec::{Literal, PartitionKey, Struct}; +//! +//! let mut clustered_writer = ClusteredWriter::new(data_file_writer_builder); +//! +//! // Create partition keys (must write in sorted order) +//! let schema = table.metadata().current_schema().clone(); +//! let partition_spec = table.metadata().default_partition_spec().as_ref().clone(); +//! +//! let partition_key_asia = PartitionKey::new( +//! partition_spec.clone(), +//! schema.clone(), +//! Struct::from_iter([Some(Literal::string("ASIA"))]), +//! ); +//! +//! let partition_key_eu = PartitionKey::new( +//! partition_spec.clone(), +//! schema.clone(), +//! Struct::from_iter([Some(Literal::string("EU"))]), +//! ); +//! +//! let partition_key_us = PartitionKey::new( +//! partition_spec.clone(), +//! schema.clone(), +//! Struct::from_iter([Some(Literal::string("US"))]), +//! ); +//! +//! // Write to partitions in sorted order (ASIA -> EU -> US) +//! // clustered_writer.write(partition_key_asia, batch_asia).await?; +//! // clustered_writer.write(partition_key_eu, batch_eu).await?; +//! // clustered_writer.write(partition_key_us, batch_us).await?; +//! // Writing back to ASIA would fail since data must be sorted! +//! +//! let data_files = clustered_writer.close().await?; //! //! Ok(()) //! } @@ -225,24 +386,23 @@ pub mod base_writer; pub mod file_writer; +pub mod partitioning; use arrow_array::RecordBatch; use crate::Result; -use crate::spec::DataFile; +use crate::spec::{DataFile, PartitionKey}; type DefaultInput = RecordBatch; type DefaultOutput = Vec; /// The builder for iceberg writer. #[async_trait::async_trait] -pub trait IcebergWriterBuilder: - Send + Clone + 'static -{ +pub trait IcebergWriterBuilder: Send + Sync + 'static { /// The associated writer type. type R: IcebergWriter; - /// Build the iceberg writer. - async fn build(self) -> Result; + /// Build the iceberg writer with an optional partition key. + async fn build(&self, partition_key: Option) -> Result; } /// The iceberg writer used to write data to iceberg table. diff --git a/crates/iceberg/src/writer/partitioning/clustered_writer.rs b/crates/iceberg/src/writer/partitioning/clustered_writer.rs new file mode 100644 index 0000000000..21b52bd813 --- /dev/null +++ b/crates/iceberg/src/writer/partitioning/clustered_writer.rs @@ -0,0 +1,593 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! This module provides the `ClusteredWriter` implementation. + +use std::collections::HashSet; +use std::marker::PhantomData; + +use async_trait::async_trait; + +use crate::spec::{PartitionKey, Struct}; +use crate::writer::partitioning::PartitioningWriter; +use crate::writer::{DefaultInput, DefaultOutput, IcebergWriter, IcebergWriterBuilder}; +use crate::{Error, ErrorKind, Result}; + +/// A writer that writes data to a single partition at a time. +/// +/// This writer expects input data to be sorted by partition key. It maintains only one +/// active writer at a time, making it memory efficient for sorted data. +/// +/// # Type Parameters +/// +/// * `B` - The inner writer builder type +/// * `I` - Input type (defaults to `RecordBatch`) +/// * `O` - Output collection type (defaults to `Vec`) +pub struct ClusteredWriter +where + B: IcebergWriterBuilder, + O: IntoIterator + FromIterator<::Item>, + ::Item: Clone, +{ + inner_builder: B, + current_writer: Option, + current_partition: Option, + closed_partitions: HashSet, + output: Vec<::Item>, + _phantom: PhantomData, +} + +impl ClusteredWriter +where + B: IcebergWriterBuilder, + I: Send + 'static, + O: IntoIterator + FromIterator<::Item>, + ::Item: Send + Clone, +{ + /// Create a new `ClusteredWriter`. + pub fn new(inner_builder: B) -> Self { + Self { + inner_builder, + current_writer: None, + current_partition: None, + closed_partitions: HashSet::new(), + output: Vec::new(), + _phantom: PhantomData, + } + } + + /// Closes the current writer if it exists, flushes the written data to output, and record closed partition. + async fn close_current_writer(&mut self) -> Result<()> { + if let Some(mut writer) = self.current_writer.take() { + self.output.extend(writer.close().await?); + + // Add the current partition to the set of closed partitions + if let Some(current_partition) = self.current_partition.take() { + self.closed_partitions.insert(current_partition); + } + } + + Ok(()) + } +} + +#[async_trait] +impl PartitioningWriter for ClusteredWriter +where + B: IcebergWriterBuilder, + I: Send + 'static, + O: IntoIterator + FromIterator<::Item> + Send + 'static, + ::Item: Send + Clone, +{ + async fn write(&mut self, partition_key: PartitionKey, input: I) -> Result<()> { + let partition_value = partition_key.data(); + + // Check if this partition has been closed already + if self.closed_partitions.contains(partition_value) { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "The input is not sorted! Cannot write to partition that was previously closed: {partition_key:?}" + ), + )); + } + + // Check if we need to switch to a new partition + let need_new_writer = match &self.current_partition { + Some(current) => current != partition_value, + None => true, + }; + + if need_new_writer { + self.close_current_writer().await?; + + // Create a new writer for the new partition + self.current_writer = Some( + self.inner_builder + .build(Some(partition_key.clone())) + .await?, + ); + self.current_partition = Some(partition_value.clone()); + } + + // do write + self.current_writer + .as_mut() + .expect("Writer should be initialized") + .write(input) + .await + } + + async fn close(mut self) -> Result { + self.close_current_writer().await?; + + // Collect all output items into the output collection type + Ok(O::from_iter(self.output)) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{Float64Array, Int32Array, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use parquet::file::properties::WriterProperties; + use tempfile::TempDir; + + use super::*; + use crate::arrow::schema_to_arrow_schema; + use crate::io::FileIO; + use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Type}; + use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + + #[tokio::test] + async fn test_clustered_writer_single_partition() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + // Create schema with partition field + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "region", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build()?, + ); + + // Create partition spec and key + let partition_spec = crate::spec::PartitionSpec::builder(schema.clone()).build()?; + let partition_value = Struct::from_iter([Some(crate::spec::Literal::string("US"))]); + let partition_key = + PartitionKey::new(partition_spec, schema.clone(), partition_value.clone()); + + // Create writer builder + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + + // Create rolling file writer builder + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + + // Create data file writer builder + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); + + // Create clustered writer + let mut writer = ClusteredWriter::new(data_file_writer_builder); + + // Create test data with proper field ID metadata + let arrow_schema = Schema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 1.to_string(), + )])), + Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 2.to_string(), + )])), + Field::new("region", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 3.to_string(), + )])), + ]); + + let batch1 = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["Alice", "Bob"])), + Arc::new(StringArray::from(vec!["US", "US"])), + ])?; + + let batch2 = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![3, 4])), + Arc::new(StringArray::from(vec!["Charlie", "Dave"])), + Arc::new(StringArray::from(vec!["US", "US"])), + ])?; + + // Write data to the same partition (this should work) + writer.write(partition_key.clone(), batch1).await?; + writer.write(partition_key.clone(), batch2).await?; + + // Close writer and get data files + let data_files = writer.close().await?; + + // Verify at least one file was created + assert!( + !data_files.is_empty(), + "Expected at least one data file to be created" + ); + + // Verify that all data files have the correct partition value + for data_file in &data_files { + assert_eq!(data_file.partition, partition_value); + } + + Ok(()) + } + + #[tokio::test] + async fn test_clustered_writer_sorted_partitions() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + // Create schema with partition field + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "region", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build()?, + ); + + // Create partition spec + let partition_spec = crate::spec::PartitionSpec::builder(schema.clone()).build()?; + + // Create partition keys for different regions (in sorted order) + let partition_value_asia = Struct::from_iter([Some(crate::spec::Literal::string("ASIA"))]); + let partition_key_asia = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_asia.clone(), + ); + + let partition_value_eu = Struct::from_iter([Some(crate::spec::Literal::string("EU"))]); + let partition_key_eu = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_eu.clone(), + ); + + let partition_value_us = Struct::from_iter([Some(crate::spec::Literal::string("US"))]); + let partition_key_us = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_us.clone(), + ); + + // Create writer builder + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + + // Create rolling file writer builder + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + + // Create data file writer builder + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); + + // Create clustered writer + let mut writer = ClusteredWriter::new(data_file_writer_builder); + + // Create test data with proper field ID metadata + let arrow_schema = Schema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 1.to_string(), + )])), + Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 2.to_string(), + )])), + Field::new("region", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 3.to_string(), + )])), + ]); + + // Create batches for different partitions (in sorted order) + let batch_asia = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["Alice", "Bob"])), + Arc::new(StringArray::from(vec!["ASIA", "ASIA"])), + ])?; + + let batch_eu = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![3, 4])), + Arc::new(StringArray::from(vec!["Charlie", "Dave"])), + Arc::new(StringArray::from(vec!["EU", "EU"])), + ])?; + + let batch_us = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![5, 6])), + Arc::new(StringArray::from(vec!["Eve", "Frank"])), + Arc::new(StringArray::from(vec!["US", "US"])), + ])?; + + // Write data in sorted partition order (this should work) + writer.write(partition_key_asia.clone(), batch_asia).await?; + writer.write(partition_key_eu.clone(), batch_eu).await?; + writer.write(partition_key_us.clone(), batch_us).await?; + + // Close writer and get data files + let data_files = writer.close().await?; + + // Verify files were created for all partitions + assert!( + data_files.len() >= 3, + "Expected at least 3 data files (one per partition), got {}", + data_files.len() + ); + + // Verify that we have files for each partition + let mut partitions_found = HashSet::new(); + for data_file in &data_files { + partitions_found.insert(data_file.partition.clone()); + } + + assert!( + partitions_found.contains(&partition_value_asia), + "Missing ASIA partition" + ); + assert!( + partitions_found.contains(&partition_value_eu), + "Missing EU partition" + ); + assert!( + partitions_found.contains(&partition_value_us), + "Missing US partition" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_clustered_writer_unsorted_partitions_error() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + // Create schema with partition field + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "region", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build()?, + ); + + // Create partition spec + let partition_spec = crate::spec::PartitionSpec::builder(schema.clone()).build()?; + + // Create partition keys for different regions + let partition_value_us = Struct::from_iter([Some(crate::spec::Literal::string("US"))]); + let partition_key_us = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_us.clone(), + ); + + let partition_value_eu = Struct::from_iter([Some(crate::spec::Literal::string("EU"))]); + let partition_key_eu = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_eu.clone(), + ); + + // Create writer builder + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + + // Create rolling file writer builder + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + + // Create data file writer builder + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); + + // Create clustered writer + let mut writer = ClusteredWriter::new(data_file_writer_builder); + + // Create test data with proper field ID metadata + let arrow_schema = Schema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 1.to_string(), + )])), + Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 2.to_string(), + )])), + Field::new("region", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 3.to_string(), + )])), + ]); + + // Create batches for different partitions + let batch_us = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["Alice", "Bob"])), + Arc::new(StringArray::from(vec!["US", "US"])), + ])?; + + let batch_eu = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![3, 4])), + Arc::new(StringArray::from(vec!["Charlie", "Dave"])), + Arc::new(StringArray::from(vec!["EU", "EU"])), + ])?; + + let batch_us2 = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![5])), + Arc::new(StringArray::from(vec!["Eve"])), + Arc::new(StringArray::from(vec!["US"])), + ])?; + + // Write data to US partition first + writer.write(partition_key_us.clone(), batch_us).await?; + + // Write data to EU partition (this closes US partition) + writer.write(partition_key_eu.clone(), batch_eu).await?; + + // Try to write to US partition again - this should fail because data is not sorted + let result = writer.write(partition_key_us.clone(), batch_us2).await; + + assert!(result.is_err(), "Expected error when writing unsorted data"); + + let error = result.unwrap_err(); + assert!( + error.to_string().contains("The input is not sorted"), + "Expected 'input is not sorted' error, got: {error}" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_clustered_writer_signed_zero_partitions() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "d", Type::Primitive(PrimitiveType::Double)).into(), + ]) + .build()?, + ); + let partition_spec = crate::spec::PartitionSpec::builder(schema.clone()) + .add_partition_field("d", "d", crate::spec::Transform::Identity)? + .build()?; + + let partition_value = |d: f64| Struct::from_iter([Some(crate::spec::Literal::double(d))]); + let partition_key = + |d: f64| PartitionKey::new(partition_spec.clone(), schema.clone(), partition_value(d)); + + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema)?); + let batch = |id: i32, d: f64| { + RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![id])), + Arc::new(Float64Array::from(vec![d])), + ]) + }; + + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + + // -0.0 and 0.0 are different partition values, as in iceberg-java, so 0.0 right + // after -0.0 starts a new data file. + let mut writer = + ClusteredWriter::new(DataFileWriterBuilder::new(rolling_writer_builder.clone())); + writer.write(partition_key(-0.0), batch(1, -0.0)?).await?; + writer.write(partition_key(0.0), batch(2, 0.0)?).await?; + let partitions_written: Vec = writer + .close() + .await? + .into_iter() + .map(|data_file| data_file.partition) + .collect(); + assert_eq!(partitions_written, vec![ + partition_value(-0.0), + partition_value(0.0) + ]); + + // 0.0 is not the closed -0.0 partition, so this input is still sorted. + let mut writer = ClusteredWriter::new(DataFileWriterBuilder::new(rolling_writer_builder)); + writer.write(partition_key(-0.0), batch(1, -0.0)?).await?; + writer.write(partition_key(1.0), batch(3, 1.0)?).await?; + writer.write(partition_key(0.0), batch(2, 0.0)?).await?; + let partitions_written: Vec = writer + .close() + .await? + .into_iter() + .map(|data_file| data_file.partition) + .collect(); + assert_eq!(partitions_written, vec![ + partition_value(-0.0), + partition_value(1.0), + partition_value(0.0) + ]); + + Ok(()) + } +} diff --git a/crates/iceberg/src/writer/partitioning/fanout_writer.rs b/crates/iceberg/src/writer/partitioning/fanout_writer.rs new file mode 100644 index 0000000000..2cdd865375 --- /dev/null +++ b/crates/iceberg/src/writer/partitioning/fanout_writer.rs @@ -0,0 +1,467 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! This module provides the `FanoutWriter` implementation. + +use std::collections::HashMap; +use std::marker::PhantomData; + +use async_trait::async_trait; + +use crate::spec::{PartitionKey, Struct}; +use crate::writer::partitioning::PartitioningWriter; +use crate::writer::{DefaultInput, DefaultOutput, IcebergWriter, IcebergWriterBuilder}; +use crate::{Error, ErrorKind, Result}; + +/// A writer that can write data to multiple partitions simultaneously. +/// +/// Unlike `ClusteredWriter` which expects sorted input and maintains only one active writer, +/// `FanoutWriter` can handle unsorted data by maintaining multiple active writers in a map. +/// This allows writing to any partition at any time, but uses more memory as all writers +/// remain active until the writer is closed. +/// +/// # Type Parameters +/// +/// * `B` - The inner writer builder type +/// * `I` - Input type (defaults to `RecordBatch`) +/// * `O` - Output collection type (defaults to `Vec`) +pub struct FanoutWriter +where + B: IcebergWriterBuilder, + O: IntoIterator + FromIterator<::Item>, + ::Item: Clone, +{ + inner_builder: B, + partition_writers: HashMap, + output: Vec<::Item>, + _phantom: PhantomData, +} + +impl FanoutWriter +where + B: IcebergWriterBuilder, + I: Send + 'static, + O: IntoIterator + FromIterator<::Item>, + ::Item: Send + Clone, +{ + /// Create a new `FanoutWriter`. + pub fn new(inner_builder: B) -> Self { + Self { + inner_builder, + partition_writers: HashMap::new(), + output: Vec::new(), + _phantom: PhantomData, + } + } + + /// Get or create a writer for the specified partition. + async fn get_or_create_writer(&mut self, partition_key: &PartitionKey) -> Result<&mut B::R> { + if !self.partition_writers.contains_key(partition_key.data()) { + let writer = self + .inner_builder + .build(Some(partition_key.clone())) + .await?; + self.partition_writers + .insert(partition_key.data().clone(), writer); + } + + self.partition_writers + .get_mut(partition_key.data()) + .ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "Failed to get partition writer after creation", + ) + }) + } +} + +#[async_trait] +impl PartitioningWriter for FanoutWriter +where + B: IcebergWriterBuilder, + I: Send + 'static, + O: IntoIterator + FromIterator<::Item> + Send + 'static, + ::Item: Send + Clone, +{ + async fn write(&mut self, partition_key: PartitionKey, input: I) -> Result<()> { + let writer = self.get_or_create_writer(&partition_key).await?; + writer.write(input).await + } + + async fn close(mut self) -> Result { + // Close all partition writers + for (_, mut writer) in self.partition_writers { + self.output.extend(writer.close().await?); + } + + // Collect all output items into the output collection type + Ok(O::from_iter(self.output)) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{Float64Array, Int32Array, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use parquet::file::properties::WriterProperties; + use tempfile::TempDir; + + use super::*; + use crate::arrow::schema_to_arrow_schema; + use crate::io::FileIO; + use crate::spec::{ + DataFileFormat, Literal, NestedField, PartitionKey, PartitionSpec, PrimitiveType, Struct, + Transform, Type, + }; + use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + + #[tokio::test] + async fn test_fanout_writer_single_partition() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + // Create schema with partition field + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "region", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build()?, + ); + + // Create partition spec - using the same pattern as data_file_writer tests + let partition_spec = PartitionSpec::builder(schema.clone()).build()?; + let partition_value = Struct::from_iter([Some(Literal::string("US"))]); + let partition_key = + PartitionKey::new(partition_spec, schema.clone(), partition_value.clone()); + + // Create writer builder + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + + // Create rolling file writer builder + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + + // Create data file writer builder + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); + + // Create fanout writer + let mut writer = FanoutWriter::new(data_file_writer_builder); + + // Create test data with proper field ID metadata + let arrow_schema = Schema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 1.to_string(), + )])), + Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 2.to_string(), + )])), + Field::new("region", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 3.to_string(), + )])), + ]); + + let batch1 = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["Alice", "Bob"])), + Arc::new(StringArray::from(vec!["US", "US"])), + ])?; + + let batch2 = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![3, 4])), + Arc::new(StringArray::from(vec!["Charlie", "Dave"])), + Arc::new(StringArray::from(vec!["US", "US"])), + ])?; + + // Write data to the same partition + writer.write(partition_key.clone(), batch1).await?; + writer.write(partition_key.clone(), batch2).await?; + + // Close writer and get data files + let data_files = writer.close().await?; + + // Verify at least one file was created + assert!( + !data_files.is_empty(), + "Expected at least one data file to be created" + ); + + // Verify that all data files have the correct partition value + for data_file in &data_files { + assert_eq!(data_file.partition, partition_value); + } + + Ok(()) + } + + #[tokio::test] + async fn test_fanout_writer_multiple_partitions() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + // Create schema with partition field + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::required(3, "region", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build()?, + ); + + // Create partition spec + let partition_spec = PartitionSpec::builder(schema.clone()).build()?; + + // Create partition keys for different regions + let partition_value_us = Struct::from_iter([Some(Literal::string("US"))]); + let partition_key_us = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_us.clone(), + ); + + let partition_value_eu = Struct::from_iter([Some(Literal::string("EU"))]); + let partition_key_eu = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_eu.clone(), + ); + + let partition_value_asia = Struct::from_iter([Some(Literal::string("ASIA"))]); + let partition_key_asia = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_asia.clone(), + ); + + // Create writer builder + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + + // Create rolling file writer builder + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + + // Create data file writer builder + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); + + // Create fanout writer + let mut writer = FanoutWriter::new(data_file_writer_builder); + + // Create test data with proper field ID metadata + let arrow_schema = Schema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 1.to_string(), + )])), + Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 2.to_string(), + )])), + Field::new("region", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + 3.to_string(), + )])), + ]); + + // Create batches for different partitions + let batch_us1 = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["Alice", "Bob"])), + Arc::new(StringArray::from(vec!["US", "US"])), + ])?; + + let batch_eu1 = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![3, 4])), + Arc::new(StringArray::from(vec!["Charlie", "Dave"])), + Arc::new(StringArray::from(vec!["EU", "EU"])), + ])?; + + let batch_us2 = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![5])), + Arc::new(StringArray::from(vec!["Eve"])), + Arc::new(StringArray::from(vec!["US"])), + ])?; + + let batch_asia1 = RecordBatch::try_new(Arc::new(arrow_schema.clone()), vec![ + Arc::new(Int32Array::from(vec![6, 7])), + Arc::new(StringArray::from(vec!["Frank", "Grace"])), + Arc::new(StringArray::from(vec!["ASIA", "ASIA"])), + ])?; + + // Write data in mixed partition order to demonstrate fanout capability + // This is the key difference from ClusteredWriter - we can write to any partition at any time + writer.write(partition_key_us.clone(), batch_us1).await?; + writer.write(partition_key_eu.clone(), batch_eu1).await?; + writer.write(partition_key_us.clone(), batch_us2).await?; // Back to US partition + writer + .write(partition_key_asia.clone(), batch_asia1) + .await?; + + // Close writer and get data files + let data_files = writer.close().await?; + + // Verify files were created for all partitions + assert!( + data_files.len() >= 3, + "Expected at least 3 data files (one per partition), got {}", + data_files.len() + ); + + // Verify that we have files for each partition + let mut partitions_found = std::collections::HashSet::new(); + for data_file in &data_files { + partitions_found.insert(data_file.partition.clone()); + } + + assert!( + partitions_found.contains(&partition_value_us), + "Missing US partition" + ); + assert!( + partitions_found.contains(&partition_value_eu), + "Missing EU partition" + ); + assert!( + partitions_found.contains(&partition_value_asia), + "Missing ASIA partition" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_fanout_writer_signed_zero_partitions() -> Result<()> { + let temp_dir = TempDir::new()?; + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "d", Type::Primitive(PrimitiveType::Double)).into(), + ]) + .build()?, + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .add_partition_field("d", "d", Transform::Identity)? + .build()?; + + let partition_value_neg_zero = Struct::from_iter([Some(Literal::double(-0.0))]); + let partition_key_neg_zero = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_neg_zero.clone(), + ); + + let partition_value_pos_zero = Struct::from_iter([Some(Literal::double(0.0))]); + let partition_key_pos_zero = PartitionKey::new( + partition_spec.clone(), + schema.clone(), + partition_value_pos_zero.clone(), + ); + + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io.clone(), + location_gen, + file_name_gen, + ); + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); + + let mut writer = FanoutWriter::new(data_file_writer_builder); + + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema)?); + let batch_neg_zero = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(Float64Array::from(vec![-0.0])), + ])?; + let batch_pos_zero = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![2])), + Arc::new(Float64Array::from(vec![0.0])), + ])?; + + writer.write(partition_key_neg_zero, batch_neg_zero).await?; + writer.write(partition_key_pos_zero, batch_pos_zero).await?; + + let data_files = writer.close().await?; + + // -0.0 and 0.0 are different partition values, as in iceberg-java, so each row + // goes to its own data file. + assert_eq!(data_files.len(), 2); + for data_file in &data_files { + assert_eq!(data_file.record_count, 1); + } + let partitions_found: std::collections::HashSet = data_files + .iter() + .map(|data_file| data_file.partition.clone()) + .collect(); + assert_eq!( + partitions_found, + std::collections::HashSet::from([partition_value_neg_zero, partition_value_pos_zero]) + ); + + Ok(()) + } +} diff --git a/crates/iceberg/src/writer/partitioning/mod.rs b/crates/iceberg/src/writer/partitioning/mod.rs new file mode 100644 index 0000000000..c8106041ac --- /dev/null +++ b/crates/iceberg/src/writer/partitioning/mod.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Partitioning writers for handling partitioned Iceberg tables. +//! +//! This module provides two strategies for writing to partitioned tables: +//! - [`FanoutWriter`](fanout_writer::FanoutWriter): Handles unsorted data by maintaining multiple active writers +//! - [`ClusteredWriter`](clustered_writer::ClusteredWriter): Optimized for pre-sorted data with single active writer + +pub mod clustered_writer; +pub mod fanout_writer; +pub mod unpartitioned_writer; + +use crate::Result; +use crate::spec::PartitionKey; +use crate::writer::{DefaultInput, DefaultOutput}; + +/// A writer that can write data to partitioned tables. +/// +/// This trait provides methods for writing data with partition keys and +/// closing the writer to retrieve the output. +#[async_trait::async_trait] +pub trait PartitioningWriter: Send + 'static { + /// Write data with a partition key. + /// + /// # Parameters + /// + /// * `partition_key` - Partition key to determine which partition to write to + /// * `input` - The input data to write + /// + /// # Returns + /// + /// `Ok(())` on success, or an error if the write operation fails. + async fn write(&mut self, partition_key: PartitionKey, input: I) -> Result<()>; + + /// Close the writer and return the output. + /// + /// # Returns + /// + /// The accumulated output from all write operations. + async fn close(self) -> Result; +} diff --git a/crates/iceberg/src/writer/partitioning/unpartitioned_writer.rs b/crates/iceberg/src/writer/partitioning/unpartitioned_writer.rs new file mode 100644 index 0000000000..de127e0c9d --- /dev/null +++ b/crates/iceberg/src/writer/partitioning/unpartitioned_writer.rs @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! This module provides the `UnpartitionedWriter` implementation. + +use std::marker::PhantomData; + +use crate::Result; +use crate::writer::{DefaultInput, DefaultOutput, IcebergWriter, IcebergWriterBuilder}; + +/// A simple wrapper around `IcebergWriterBuilder` for unpartitioned tables. +/// +/// This writer lazily creates the underlying writer on the first write operation +/// and writes all data to a single file (or set of files if rolling). +/// +/// # Type Parameters +/// +/// * `B` - The inner writer builder type +/// * `I` - Input type (defaults to `RecordBatch`) +/// * `O` - Output collection type (defaults to `Vec`) +pub struct UnpartitionedWriter +where + B: IcebergWriterBuilder, + O: IntoIterator + FromIterator<::Item>, + ::Item: Clone, +{ + inner_builder: B, + writer: Option, + output: Vec<::Item>, + _phantom: PhantomData, +} + +impl UnpartitionedWriter +where + B: IcebergWriterBuilder, + I: Send + 'static, + O: IntoIterator + FromIterator<::Item>, + ::Item: Send + Clone, +{ + /// Create a new `UnpartitionedWriter`. + pub fn new(inner_builder: B) -> Self { + Self { + inner_builder, + writer: None, + output: Vec::new(), + _phantom: PhantomData, + } + } + + /// Write data to the writer. + /// + /// The underlying writer is lazily created on the first write operation. + /// + /// # Parameters + /// + /// * `input` - The input data to write + /// + /// # Returns + /// + /// `Ok(())` on success, or an error if the write operation fails. + pub async fn write(&mut self, input: I) -> Result<()> { + // Lazily create writer on first write + if self.writer.is_none() { + self.writer = Some(self.inner_builder.build(None).await?); + } + + // Write directly to inner writer + self.writer + .as_mut() + .expect("Writer should be initialized") + .write(input) + .await + } + + /// Close the writer and return all written data files. + /// + /// This method consumes the writer to prevent further use. + /// + /// # Returns + /// + /// The accumulated output from all write operations, or an empty collection + /// if no data was written. + pub async fn close(mut self) -> Result { + if let Some(mut writer) = self.writer.take() { + self.output.extend(writer.close().await?); + } + Ok(O::from_iter(self.output)) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use arrow_array::{Int32Array, RecordBatch, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + use parquet::file::properties::WriterProperties; + use tempfile::TempDir; + + use super::*; + use crate::Result; + use crate::io::FileIO; + use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Struct, Type}; + use crate::writer::base_writer::data_file_writer::DataFileWriterBuilder; + use crate::writer::file_writer::ParquetWriterBuilder; + use crate::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, + }; + use crate::writer::file_writer::rolling_writer::RollingFileWriterBuilder; + + #[tokio::test] + async fn test_unpartitioned_writer() -> Result<()> { + let temp_dir = TempDir::new()?; + + // Build Iceberg schema + let schema = Arc::new( + crate::spec::Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build()?, + ); + + // Build Arrow schema + let arrow_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "1".to_string(), + )])), + Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( + PARQUET_FIELD_ID_META_KEY.to_string(), + "2".to_string(), + )])), + ])); + + // Build writer + let file_io = FileIO::new_with_fs(); + let location_gen = DefaultLocationGenerator::with_data_location( + temp_dir.path().to_str().unwrap().to_string(), + ); + let file_name_gen = + DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet); + let parquet_writer_builder = + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()); + let rolling_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, + file_io, + location_gen, + file_name_gen, + ); + let writer_builder = DataFileWriterBuilder::new(rolling_writer_builder); + + let mut writer = UnpartitionedWriter::new(writer_builder); + + // Write two batches + let batch1 = RecordBatch::try_new(arrow_schema.clone(), vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["Alice", "Bob"])), + ])?; + let batch2 = RecordBatch::try_new(arrow_schema, vec![ + Arc::new(Int32Array::from(vec![3, 4])), + Arc::new(StringArray::from(vec!["Charlie", "Dave"])), + ])?; + + writer.write(batch1).await?; + writer.write(batch2).await?; + + let data_files = writer.close().await?; + + // Verify files have empty partition and correct format + assert!(!data_files.is_empty()); + for file in &data_files { + assert_eq!(file.partition, Struct::empty()); + assert_eq!(file.file_format, DataFileFormat::Parquet); + assert_eq!(file.record_count, 4); + } + + Ok(()) + } +} diff --git a/crates/iceberg/testdata/example_table_metadata_v2_deep_history.json b/crates/iceberg/testdata/example_table_metadata_v2_deep_history.json new file mode 100644 index 0000000000..a354958697 --- /dev/null +++ b/crates/iceberg/testdata/example_table_metadata_v2_deep_history.json @@ -0,0 +1,104 @@ +{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "current-schema-id": 1, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ] + }, + { + "type": "struct", + "schema-id": 1, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"}, + {"id": 2, "name": "y", "required": true, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": true, "type": "long"} + ] + } + ], + "default-spec-id": 0, + "partition-specs": [ + { + "spec-id": 0, + "fields": [ + {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000} + ] + } + ], + "last-partition-id": 1000, + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, + {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"} + ] + } + ], + "properties": {}, + "current-snapshot-id": 3059729675574597004, + "snapshots": [ + { + "snapshot-id": 3051729675574597004, + "timestamp-ms": 1515100955770, + "sequence-number": 0, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-3051729675574597004.avro" + }, + { + "snapshot-id": 3055729675574597004, + "parent-snapshot-id": 3051729675574597004, + "timestamp-ms": 1555100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-3055729675574597004.avro", + "schema-id": 1 + }, + { + "snapshot-id": 3056729675574597004, + "parent-snapshot-id": 3055729675574597004, + "timestamp-ms": 1575100955770, + "sequence-number": 2, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-3056729675574597004.avro", + "schema-id": 1 + }, + { + "snapshot-id": 3057729675574597004, + "parent-snapshot-id": 3056729675574597004, + "timestamp-ms": 1595100955770, + "sequence-number": 3, + "summary": {"operation": "overwrite"}, + "manifest-list": "s3://bucket/metadata/snap-3057729675574597004.avro", + "schema-id": 1 + }, + { + "snapshot-id": 3059729675574597004, + "parent-snapshot-id": 3057729675574597004, + "timestamp-ms": 1602638573590, + "sequence-number": 4, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-3059729675574597004.avro", + "schema-id": 1 + } + ], + "snapshot-log": [ + {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, + {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, + {"snapshot-id": 3056729675574597004, "timestamp-ms": 1575100955770}, + {"snapshot-id": 3057729675574597004, "timestamp-ms": 1595100955770}, + {"snapshot-id": 3059729675574597004, "timestamp-ms": 1602638573590} + ], + "metadata-log": [], + "refs": {"main": {"snapshot-id": 3059729675574597004, "type": "branch"}} +} diff --git a/crates/iceberg/testdata/example_table_metadata_v2_partition_evolution.json b/crates/iceberg/testdata/example_table_metadata_v2_partition_evolution.json new file mode 100644 index 0000000000..70743c394f --- /dev/null +++ b/crates/iceberg/testdata/example_table_metadata_v2_partition_evolution.json @@ -0,0 +1,128 @@ +{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "{{ table_location }}", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "current-schema-id": 1, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ]}, + { + "type": "struct", + "schema-id": 1, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"}, + {"id": 2, "name": "y", "required": true, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": true, "type": "long"}, + {"id": 4, "name": "a", "required": true, "type": "string"}, + {"id": 5, "name": "dbl", "required": true, "type": "double"}, + {"id": 6, "name": "i32", "required": true, "type": "int"}, + {"id": 7, "name": "i64", "required": true, "type": "long"}, + {"id": 8, "name": "bool", "required": true, "type": "boolean"} + ] + } + ], + "default-spec-id": 2, + "partition-specs": [ + { + "spec-id": 0, + "fields": [ + {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000} + ] + }, + { + "spec-id": 1, + "fields": [ + { + "name": "x", + "transform": "identity", + "source-id": 1, + "field-id": 1000 + }, + { + "name": "a_trunc_3", + "transform": "truncate[3]", + "source-id": 4, + "field-id": 1001 + } + ] + }, + { + "spec-id": 2, + "fields": [ + { + "name": "x", + "transform": "identity", + "source-id": 1, + "field-id": 1000 + }, + { + "name": "a_trunc_3", + "transform": "truncate[3]", + "source-id": 4, + "field-id": 1001 + }, + { + "name": "z_bucket_32", + "transform": "bucket[32]", + "source-id": 3, + "field-id": 1002 + } + ] + } + ], + "last-partition-id": 1002, + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, + {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"} + ] + } + ], + "properties": {"read.split.target.size": "134217728"}, + "current-snapshot-id": 3059729675574597004, + "snapshots": [ + { + "snapshot-id": 3051729675574597004, + "timestamp-ms": 1515100955770, + "sequence-number": 0, + "summary": {"operation": "append"}, + "manifest-list": "{{ manifest_list_1_location }}" + }, + { + "snapshot-id": 3055729675574597004, + "parent-snapshot-id": 3051729675574597004, + "timestamp-ms": 1555100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "{{ manifest_list_2_location }}", + "schema-id": 1 + }, + { + "snapshot-id": 3059729675574597004, + "parent-snapshot-id": 3055729675574597004, + "timestamp-ms": 1555100955771, + "sequence-number": 2, + "summary": {"operation": "append"}, + "manifest-list": "{{ manifest_list_3_location }}", + "schema-id": 1 + } + ], + "snapshot-log": [ + {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, + {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, + {"snapshot-id": 3059729675574597004, "timestamp-ms": 1555100955770} + ], + "metadata-log": [{"metadata-file": "{{ table_metadata_1_location }}", "timestamp-ms": 1515100}], + "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}} +} \ No newline at end of file diff --git a/crates/iceberg/testdata/manifests/README.md b/crates/iceberg/testdata/manifests/README.md new file mode 100644 index 0000000000..6b68b71ed0 --- /dev/null +++ b/crates/iceberg/testdata/manifests/README.md @@ -0,0 +1,99 @@ + + + +# Manifest test data + +`repeated-decimal-type-definitions.avro` is a V2 data manifest written by +iceberg-rust at commit `ecba0b5`, which used `apache-avro` 0.21. Its table schema +has two optional `decimal(10, 2)` columns, `d1` (id 1) and `d2` (id 2), and its +partition spec has an identity partition on each. It holds one entry for +`s3://b/t/a.parquet` with partition values `123.45` and `-6.78`. + +The Avro schema in its header defines the named fixed type `decimal_10_2` twice. +The Avro specification allows only one definition of each name, and +`apache-avro` 0.22 rejects the header, but iceberg-rust wrote manifests like +this one before it moved to 0.22. Tests use it to check that such manifests +still read. + +To regenerate it, check out a commit that uses `apache-avro` 0.21, write the +manifest described above with `ManifestWriterBuilder::build_v2_data`, and copy +the file here. + +`pyiceberg-v2-data.avro` is a V2 data manifest written by PyIceberg 0.12.0 +(apache/iceberg-python commit `0d584073`) with deflate compression. Its table +schema has optional columns `id` (long), `category` (string), and `score` +(double), and its partition spec has an identity partition on `category`. It +holds two entries with column metrics. Its Avro schema is PyIceberg's, so tests +use it to check reading a manifest whose writer schema iceberg-rust didn't +generate. + +To regenerate it, run this script with PyIceberg installed, passing the output +path: + +```python +import sys + +from pyiceberg.io.pyarrow import PyArrowFileIO +from pyiceberg.manifest import DataFile, DataFileContent, FileFormat, ManifestEntry, ManifestEntryStatus, write_manifest +from pyiceberg.partitioning import PartitionField, PartitionSpec +from pyiceberg.schema import Schema +from pyiceberg.transforms import IdentityTransform +from pyiceberg.typedef import Record +from pyiceberg.types import DoubleType, LongType, NestedField, StringType + +SCHEMA = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "category", StringType(), required=False), + NestedField(3, "score", DoubleType(), required=False), +) +SPEC = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="category"), spec_id=0) + + +def entry(i: int) -> ManifestEntry: + data_file = DataFile.from_args( + content=DataFileContent.DATA, + file_path=f"s3://bucket/table/data/category=c{i}/{i:05d}.parquet", + file_format=FileFormat.PARQUET, + partition=Record(f"c{i}"), + record_count=100 + i, + file_size_in_bytes=1000 + i, + column_sizes={1: 10 + i, 2: 20 + i, 3: 30 + i}, + value_counts={1: 100 + i, 2: 100 + i, 3: 100 + i}, + null_value_counts={1: 0, 2: i, 3: 1}, + nan_value_counts={3: i}, + lower_bounds={1: (i).to_bytes(8, "little"), 2: f"c{i}".encode()}, + upper_bounds={1: (i + 50).to_bytes(8, "little"), 2: f"c{i}".encode()}, + split_offsets=[4], + sort_order_id=0, + ) + return ManifestEntry.from_args( + status=ManifestEntryStatus.ADDED, snapshot_id=7, sequence_number=1, file_sequence_number=1, data_file=data_file + ) + + +def main(path: str) -> None: + with write_manifest(2, SPEC, SCHEMA, PyArrowFileIO().new_output(path), snapshot_id=7, avro_compression="deflate") as writer: + for i in range(2): + writer.add_entry(entry(i)) + + +if __name__ == "__main__": + main(sys.argv[1]) +``` diff --git a/crates/iceberg/testdata/manifests/pyiceberg-v2-data.avro b/crates/iceberg/testdata/manifests/pyiceberg-v2-data.avro new file mode 100644 index 0000000000..c8c6866449 Binary files /dev/null and b/crates/iceberg/testdata/manifests/pyiceberg-v2-data.avro differ diff --git a/crates/iceberg/testdata/manifests/repeated-decimal-type-definitions.avro b/crates/iceberg/testdata/manifests/repeated-decimal-type-definitions.avro new file mode 100644 index 0000000000..598268c030 Binary files /dev/null and b/crates/iceberg/testdata/manifests/repeated-decimal-type-definitions.avro differ diff --git a/crates/iceberg/testdata/manifests_lists/README.md b/crates/iceberg/testdata/manifests_lists/README.md new file mode 100644 index 0000000000..73364777f5 --- /dev/null +++ b/crates/iceberg/testdata/manifests_lists/README.md @@ -0,0 +1,41 @@ + + +# Encrypted manifest list test data + +`manifest-list-v3-encrypted.avro` is an AES-GCM (AGS1 stream format) encrypted +manifest list containing zero entries. It was generated using an `EncryptionManager` +seeded with the master key bytes below. + +The corresponding table metadata fixture at +`../table_metadata/TableMetadataV3ValidEncryption.json` contains: +- A snapshot whose `key-id` references the wrapped DEK entry +- `encryption-keys` with a KEK (wrapped by the master key) and a wrapped DEK entry + (wrapped by the KEK) + +To decrypt this file in tests, seed a `MemoryKeyManagementClient` with: +``` +key-id: "master-1" +bytes: [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f] +``` + +If you need to regenerate this file (e.g. after changing the encryption format), +use `add_master_key_bytes` with the bytes above, encrypt an empty manifest list, +and update both this file and the encryption-keys in the JSON fixture. diff --git a/crates/iceberg/testdata/manifests_lists/manifest-list-v3-encrypted.avro b/crates/iceberg/testdata/manifests_lists/manifest-list-v3-encrypted.avro new file mode 100644 index 0000000000..4963f0effb Binary files /dev/null and b/crates/iceberg/testdata/manifests_lists/manifest-list-v3-encrypted.avro differ diff --git a/crates/iceberg/testdata/table_metadata/TableMetadataV3ValidEncryption.json b/crates/iceberg/testdata/table_metadata/TableMetadataV3ValidEncryption.json new file mode 100644 index 0000000000..1c096417fb --- /dev/null +++ b/crates/iceberg/testdata/table_metadata/TableMetadataV3ValidEncryption.json @@ -0,0 +1,84 @@ +{ + "format-version": 3, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "memory:///table", + "last-sequence-number": 0, + "last-updated-ms": 1602638573590, + "last-column-id": 1, + "current-schema-id": 0, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "fields": [ + { + "id": 1, + "name": "x", + "required": true, + "type": "long" + } + ] + } + ], + "default-spec-id": 0, + "partition-specs": [ + { + "spec-id": 0, + "fields": [] + } + ], + "last-partition-id": 1000, + "default-sort-order-id": 0, + "sort-orders": [ + { + "order-id": 0, + "fields": [] + } + ], + "properties": { + "encryption.key-id": "master-1" + }, + "current-snapshot-id": 1, + "snapshots": [ + { + "snapshot-id": 1, + "sequence-number": 0, + "timestamp-ms": 0, + "manifest-list": "memory:///table/metadata/manifest-list-enc.avro", + "summary": { + "operation": "append" + }, + "schema-id": 0, + "key-id": "1fb95f04-7f98-4b1f-a08f-e810cfb907ff" + } + ], + "encryption-keys": [ + { + "key-id": "5b29941b-b3b4-4d57-8758-a9d1bf340b39", + "encrypted-key-metadata": "WVMIPwafypGJRgXOICFKcL+m8K4B40gEIJFsgC7rlitqN9PUpDeT19OZPs8=", + "encrypted-by-id": "master-1", + "properties": { + "KEY_TIMESTAMP": "1781172301703" + } + }, + { + "key-id": "1fb95f04-7f98-4b1f-a08f-e810cfb907ff", + "encrypted-key-metadata": "XcrqLxxyf9a7FAFXM63u3MEH8CSWqh6g1H8fJD2/6zx0EjcPzPaP/5AtRpO2s6p2isV8s7a4m77zj+GIqNmIpbA2Xw==", + "encrypted-by-id": "5b29941b-b3b4-4d57-8758-a9d1bf340b39" + } + ], + "snapshot-log": [ + { + "snapshot-id": 1, + "timestamp-ms": 0 + } + ], + "metadata-log": [], + "refs": { + "main": { + "snapshot-id": 1, + "type": "branch" + } + }, + "next-row-id": 0 +} diff --git a/crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json b/crates/iceberg/testdata/table_metadata/TableMetadataV3ValidMinimal.json similarity index 53% rename from crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json rename to crates/iceberg/testdata/table_metadata/TableMetadataV3ValidMinimal.json index 0dc89de58c..bf85114c02 100644 --- a/crates/integrations/datafusion/tests/test_data/TableMetadataV2Valid.json +++ b/crates/iceberg/testdata/table_metadata/TableMetadataV3ValidMinimal.json @@ -1,11 +1,12 @@ { - "format-version": 2, + "format-version": 3, "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "location": "s3://bucket/test/location", "last-sequence-number": 34, "last-updated-ms": 1602638573590, "last-column-id": 3, - "current-schema-id": 1, + "current-schema-id": 0, + "next-row-id": 0, "schemas": [ { "type": "struct", @@ -15,23 +16,9 @@ "id": 1, "name": "x", "required": true, - "type": "long" - } - ] - }, - { - "type": "struct", - "schema-id": 1, - "identifier-field-ids": [ - 1, - 2 - ], - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" + "type": "long", + "initial-default": 1, + "write-default": 1 }, { "id": 2, @@ -83,40 +70,5 @@ } ] } - ], - "properties": {}, - "current-snapshot-id": 3055729675574597004, - "snapshots": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770, - "sequence-number": 0, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/1.avro" - }, - { - "snapshot-id": 3055729675574597004, - "parent-snapshot-id": 3051729675574597004, - "timestamp-ms": 1555100955770, - "sequence-number": 1, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/2.avro", - "schema-id": 1 - } - ], - "snapshot-log": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770 - }, - { - "snapshot-id": 3055729675574597004, - "timestamp-ms": 1555100955770 - } - ], - "metadata-log": [] + ] } \ No newline at end of file diff --git a/crates/iceberg/tests/file_io_s3_test.rs b/crates/iceberg/tests/file_io_s3_test.rs deleted file mode 100644 index b7c484de8f..0000000000 --- a/crates/iceberg/tests/file_io_s3_test.rs +++ /dev/null @@ -1,259 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for FileIO S3. -#[cfg(all(test, feature = "storage-s3"))] -mod tests { - use std::net::{IpAddr, SocketAddr}; - use std::sync::{Arc, RwLock}; - - use async_trait::async_trait; - use ctor::{ctor, dtor}; - use iceberg::io::{ - CustomAwsCredentialLoader, FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, - S3_SECRET_ACCESS_KEY, - }; - use iceberg_test_utils::docker::DockerCompose; - use iceberg_test_utils::{normalize_test_name, set_up}; - use reqsign::{AwsCredential, AwsCredentialLoad}; - use reqwest::Client; - - const MINIO_PORT: u16 = 9000; - static DOCKER_COMPOSE_ENV: RwLock> = RwLock::new(None); - - #[ctor] - fn before_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - let docker_compose = DockerCompose::new( - normalize_test_name(module_path!()), - format!("{}/testdata/file_io_s3", env!("CARGO_MANIFEST_DIR")), - ); - docker_compose.up(); - guard.replace(docker_compose); - } - - #[dtor] - fn after_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - guard.take(); - } - - async fn get_file_io() -> FileIO { - set_up(); - - let container_ip = get_container_ip("minio"); - let minio_socket_addr = SocketAddr::new(container_ip, MINIO_PORT); - - FileIOBuilder::new("s3") - .with_props(vec![ - (S3_ENDPOINT, format!("http://{}", minio_socket_addr)), - (S3_ACCESS_KEY_ID, "admin".to_string()), - (S3_SECRET_ACCESS_KEY, "password".to_string()), - (S3_REGION, "us-east-1".to_string()), - ]) - .build() - .unwrap() - } - - fn get_container_ip(service_name: &str) -> IpAddr { - let guard = DOCKER_COMPOSE_ENV.read().unwrap(); - let docker_compose = guard.as_ref().unwrap(); - docker_compose.get_container_ip(service_name) - } - - #[tokio::test] - async fn test_file_io_s3_exists() { - let file_io = get_file_io().await; - assert!(!file_io.exists("s3://bucket2/any").await.unwrap()); - assert!(file_io.exists("s3://bucket1/").await.unwrap()); - } - - #[tokio::test] - async fn test_file_io_s3_output() { - let file_io = get_file_io().await; - assert!(!file_io.exists("s3://bucket1/test_output").await.unwrap()); - let output_file = file_io.new_output("s3://bucket1/test_output").unwrap(); - { - output_file.write("123".into()).await.unwrap(); - } - assert!(file_io.exists("s3://bucket1/test_output").await.unwrap()); - } - - #[tokio::test] - async fn test_file_io_s3_input() { - let file_io = get_file_io().await; - let output_file = file_io.new_output("s3://bucket1/test_input").unwrap(); - { - output_file.write("test_input".into()).await.unwrap(); - } - - let input_file = file_io.new_input("s3://bucket1/test_input").unwrap(); - - { - let buffer = input_file.read().await.unwrap(); - assert_eq!(buffer, "test_input".as_bytes()); - } - } - - // Mock credential loader for testing - struct MockCredentialLoader { - credential: Option, - } - - impl MockCredentialLoader { - fn new(credential: Option) -> Self { - Self { credential } - } - - fn new_minio() -> Self { - Self::new(Some(AwsCredential { - access_key_id: "admin".to_string(), - secret_access_key: "password".to_string(), - session_token: None, - expires_in: None, - })) - } - } - - #[async_trait] - impl AwsCredentialLoad for MockCredentialLoader { - async fn load_credential(&self, _client: Client) -> anyhow::Result> { - Ok(self.credential.clone()) - } - } - - #[test] - fn test_file_io_builder_extension_system() { - // Test adding and retrieving extensions - let test_string = "test_extension_value".to_string(); - let builder = FileIOBuilder::new_fs_io().with_extension(test_string.clone()); - - // Test retrieving the extension - let extension: Option> = builder.extension(); - assert!(extension.is_some()); - assert_eq!(*extension.unwrap(), test_string); - - // Test that non-existent extension returns None - let non_existent: Option> = builder.extension(); - assert!(non_existent.is_none()); - } - - #[test] - fn test_file_io_builder_multiple_extensions() { - // Test adding multiple different types of extensions - let test_string = "test_value".to_string(); - let test_number = 42i32; - - let builder = FileIOBuilder::new_fs_io() - .with_extension(test_string.clone()) - .with_extension(test_number); - - // Retrieve both extensions - let string_ext: Option> = builder.extension(); - let number_ext: Option> = builder.extension(); - - assert!(string_ext.is_some()); - assert!(number_ext.is_some()); - assert_eq!(*string_ext.unwrap(), test_string); - assert_eq!(*number_ext.unwrap(), test_number); - } - - #[test] - fn test_custom_aws_credential_loader_instantiation() { - // Test creating CustomAwsCredentialLoader with mock loader - let mock_loader = MockCredentialLoader::new_minio(); - let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); - - // Test that the loader can be used in FileIOBuilder - let builder = FileIOBuilder::new("s3") - .with_extension(custom_loader.clone()) - .with_props(vec![ - (S3_ENDPOINT, "http://localhost:9000".to_string()), - ("bucket", "test-bucket".to_string()), - (S3_REGION, "us-east-1".to_string()), - ]); - - // Verify the extension was stored - let retrieved_loader: Option> = builder.extension(); - assert!(retrieved_loader.is_some()); - } - - #[tokio::test] - async fn test_s3_with_custom_credential_loader_integration() { - let _file_io = get_file_io().await; - - // Create a mock credential loader - let mock_loader = MockCredentialLoader::new_minio(); - let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); - - // Get container info for endpoint - let container_ip = get_container_ip("minio"); - let minio_socket_addr = SocketAddr::new(container_ip, MINIO_PORT); - - // Build FileIO with custom credential loader - let file_io_with_custom_creds = FileIOBuilder::new("s3") - .with_extension(custom_loader) - .with_props(vec![ - (S3_ENDPOINT, format!("http://{}", minio_socket_addr)), - (S3_REGION, "us-east-1".to_string()), - ]) - .build() - .unwrap(); - - // Test that the FileIO was built successfully with the custom loader - match file_io_with_custom_creds.exists("s3://bucket1/any").await { - Ok(_) => {} - Err(e) => panic!("Failed to check existence of bucket: {e}"), - } - } - - #[tokio::test] - async fn test_s3_with_custom_credential_loader_integration_failure() { - let _file_io = get_file_io().await; - - // Create a mock credential loader with no credentials - let mock_loader = MockCredentialLoader::new(None); - let custom_loader = CustomAwsCredentialLoader::new(Arc::new(mock_loader)); - - // Get container info for endpoint - let container_ip = get_container_ip("minio"); - let minio_socket_addr = SocketAddr::new(container_ip, MINIO_PORT); - - // Build FileIO with custom credential loader - let file_io_with_custom_creds = FileIOBuilder::new("s3") - .with_extension(custom_loader) - .with_props(vec![ - (S3_ENDPOINT, format!("http://{}", minio_socket_addr)), - (S3_REGION, "us-east-1".to_string()), - ]) - .build() - .unwrap(); - - // Test that the FileIO was built successfully with the custom loader - match file_io_with_custom_creds.exists("s3://bucket1/any").await { - Ok(_) => panic!( - "Expected error, but got Ok - the credential loader should fail to provide valid credentials" - ), - Err(e) => { - assert!( - e.to_string() - .contains("no valid credential found and anonymous access is not allowed") - ); - } - } - } -} diff --git a/crates/integration_tests/Cargo.toml b/crates/integration_tests/Cargo.toml index 07eea5f375..cd9a9c4627 100644 --- a/crates/integration_tests/Cargo.toml +++ b/crates/integration_tests/Cargo.toml @@ -20,6 +20,7 @@ edition = { workspace = true } homepage = { workspace = true } license = { workspace = true } name = "iceberg-integration-tests" +publish = false repository = { workspace = true } rust-version = { workspace = true } version = { workspace = true } @@ -27,14 +28,15 @@ version = { workspace = true } [dependencies] arrow-array = { workspace = true } arrow-schema = { workspace = true } -ctor = { workspace = true } -datafusion = { workspace = true } futures = { workspace = true } iceberg = { workspace = true } iceberg-catalog-rest = { workspace = true } -iceberg-datafusion = { workspace = true } +iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] } iceberg_test_utils = { path = "../test_utils", features = ["tests"] } +ordered-float = "2.10.1" parquet = { workspace = true } tokio = { workspace = true } uuid = { workspace = true } -ordered-float = "2.10.1" + +[lints] +workspace = true diff --git a/crates/integration_tests/DEPENDENCIES.rust.tsv b/crates/integration_tests/DEPENDENCIES.rust.tsv deleted file mode 100644 index 891d23966e..0000000000 --- a/crates/integration_tests/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,406 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@55.2.0 X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-csv@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-json@55.2.0 X -arrow-ord@55.2.0 X -arrow-row@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-compression@0.4.19 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -blake2@0.10.6 X X -blake3@1.8.2 X X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -bzip2@0.5.2 X X -bzip2-sys@0.1.13+1.0.8 X X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -chrono-tz@0.10.4 X X -comfy-table@7.2.0 X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.3.1 X X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -csv@1.3.1 X X -csv-core@0.1.12 X X -ctor@0.2.9 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -dashmap@6.1.0 X -datafusion@48.0.1 X -datafusion-catalog@48.0.1 X -datafusion-catalog-listing@48.0.1 X -datafusion-common@48.0.1 X -datafusion-common-runtime@48.0.1 X -datafusion-datasource@48.0.1 X -datafusion-datasource-csv@48.0.1 X -datafusion-datasource-json@48.0.1 X -datafusion-datasource-parquet@48.0.1 X -datafusion-doc@48.0.1 X -datafusion-execution@48.0.1 X -datafusion-expr@48.0.1 X -datafusion-expr-common@48.0.1 X -datafusion-functions@48.0.1 X -datafusion-functions-aggregate@48.0.1 X -datafusion-functions-aggregate-common@48.0.1 X -datafusion-functions-nested@48.0.1 X -datafusion-functions-table@48.0.1 X -datafusion-functions-window@48.0.1 X -datafusion-functions-window-common@48.0.1 X -datafusion-macros@48.0.1 X -datafusion-optimizer@48.0.1 X -datafusion-physical-expr@48.0.1 X -datafusion-physical-expr-common@48.0.1 X -datafusion-physical-optimizer@48.0.1 X -datafusion-physical-plan@48.0.1 X -datafusion-session@48.0.1 X -datafusion-sql@48.0.1 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -errno@0.3.13 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -fixedbitset@0.5.7 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -h2@0.4.12 X -half@2.6.0 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -humantime@2.2.0 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-catalog-rest@0.7.0 X -iceberg-datafusion@0.7.0 X -iceberg-integration-tests@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -linux-raw-sys@0.9.4 X X X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -lzma-sys@0.1.20 X X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -object_store@0.12.3 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -petgraph@0.8.2 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -psm@0.1.26 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustix@1.0.8 X X X -rustls@0.23.31 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -same-file@1.0.6 X X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simdutf8@0.1.5 X X -siphasher@1.0.1 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.0 X X -sqlparser@0.55.0 X -sqlparser_derive@0.3.0 X -stable_deref_trait@1.2.0 X X -stacker@0.1.21 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -tempfile@3.22.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.2.1 X X -untrusted@0.9.0 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -web-time@1.1.0 X X -webpki-roots@1.0.2 X -winapi-util@0.1.11 X X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.61.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -xz2@0.1.7 X X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integration_tests/src/lib.rs b/crates/integration_tests/src/lib.rs index 422cd55863..c137b46f86 100644 --- a/crates/integration_tests/src/lib.rs +++ b/crates/integration_tests/src/lib.rs @@ -16,49 +16,46 @@ // under the License. use std::collections::HashMap; +use std::sync::OnceLock; -use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY}; +use iceberg::io::{ + S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, +}; use iceberg_catalog_rest::REST_CATALOG_PROP_URI; -use iceberg_test_utils::docker::DockerCompose; -use iceberg_test_utils::{normalize_test_name, set_up}; +use iceberg_test_utils::{get_object_store_endpoint, get_rest_catalog_endpoint, set_up}; -const REST_CATALOG_PORT: u16 = 8181; - -pub struct TestFixture { - pub _docker_compose: DockerCompose, +/// Global test fixture that uses environment-based configuration. +/// This assumes Docker containers are started externally (e.g., via `make docker-up`). +pub struct GlobalTestFixture { pub catalog_config: HashMap, } -pub fn set_test_fixture(func: &str) -> TestFixture { - set_up(); - let docker_compose = DockerCompose::new( - normalize_test_name(format!("{}_{func}", module_path!())), - format!("{}/testdata", env!("CARGO_MANIFEST_DIR")), - ); +static GLOBAL_FIXTURE: OnceLock = OnceLock::new(); - // Stop any containers from previous runs and start new ones - docker_compose.down(); - docker_compose.up(); +impl GlobalTestFixture { + /// Creates a new GlobalTestFixture from environment variables. + /// Uses default localhost endpoints if environment variables are not set. + pub fn from_env() -> Self { + set_up(); - let rest_catalog_ip = docker_compose.get_container_ip("rest"); - let minio_ip = docker_compose.get_container_ip("minio"); + let rest_endpoint = get_rest_catalog_endpoint(); + let object_store_endpoint = get_object_store_endpoint(); - let catalog_config = HashMap::from([ - ( - REST_CATALOG_PROP_URI.to_string(), - format!("http://{}:{}", rest_catalog_ip, REST_CATALOG_PORT), - ), - ( - S3_ENDPOINT.to_string(), - format!("http://{}:{}", minio_ip, 9000), - ), - (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), - (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), - (S3_REGION.to_string(), "us-east-1".to_string()), - ]); + let catalog_config = HashMap::from([ + (REST_CATALOG_PROP_URI.to_string(), rest_endpoint), + (S3_ENDPOINT.to_string(), object_store_endpoint), + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + ]); - TestFixture { - _docker_compose: docker_compose, - catalog_config, + GlobalTestFixture { catalog_config } } } + +/// Returns a reference to the global test fixture. +/// This fixture assumes Docker containers are started externally. +pub fn get_test_fixture() -> &'static GlobalTestFixture { + GLOBAL_FIXTURE.get_or_init(GlobalTestFixture::from_env) +} diff --git a/crates/integration_tests/testdata/docker-compose.yaml b/crates/integration_tests/testdata/docker-compose.yaml deleted file mode 100644 index cf0240d1a5..0000000000 --- a/crates/integration_tests/testdata/docker-compose.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -networks: - rest_bridge: - -services: - rest: - image: apache/iceberg-rest-fixture:1.10.0 - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog - - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory - - CATALOG_WAREHOUSE=s3://icebergdata/demo - - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://minio:9000 - depends_on: - - minio - networks: - rest_bridge: - ports: - - 8181:8181 - expose: - - 8181 - - minio: - image: minio/minio:RELEASE.2025-05-24T17-08-30Z - environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=minio - - MINIO_DEFAULT_BUCKETS=icebergdata - hostname: icebergdata.minio - networks: - rest_bridge: - ports: - - 9001:9001 - expose: - - 9001 - - 9000 - command: ["server", "/data", "--console-address", ":9001"] - - mc: - depends_on: - - minio - image: minio/mc:RELEASE.2025-05-21T01-59-54Z - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - entrypoint: > - /bin/sh -c " until (/usr/bin/mc alias set minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; /usr/bin/mc rm -r --force minio/icebergdata; /usr/bin/mc mb minio/icebergdata; /usr/bin/mc policy set public minio/icebergdata; tail -f /dev/null " - networks: - rest_bridge: - - spark-iceberg: - build: spark/ - networks: - rest_bridge: - depends_on: - - rest - - minio - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 diff --git a/crates/integration_tests/testdata/spark/Dockerfile b/crates/integration_tests/testdata/spark/Dockerfile deleted file mode 100644 index 339051bfc1..0000000000 --- a/crates/integration_tests/testdata/spark/Dockerfile +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM python:3.9-bullseye - -RUN apt-get -qq update && \ - apt-get -qq install -y --no-install-recommends sudo curl openjdk-11-jdk && \ - apt-get -qq clean && \ - rm -rf /var/lib/apt/lists/* - -ENV SPARK_HOME=${SPARK_HOME:-"/opt/spark"} -ENV HADOOP_HOME=${HADOOP_HOME:-"/opt/hadoop"} -ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$PYTHONPATH - -RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events -WORKDIR ${SPARK_HOME} - -ENV SPARK_VERSION=3.5.6 -ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12 -ENV ICEBERG_VERSION=1.10.0 - -RUN curl --retry 5 -s -C - https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \ - && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \ - && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz - -# Download iceberg spark runtime -RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar -Lo iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \ - && mv iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar /opt/spark/jars - -# Download AWS bundle -RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar - -COPY spark-defaults.conf /opt/spark/conf -ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}" - -RUN chmod u+x /opt/spark/sbin/* && \ - chmod u+x /opt/spark/bin/* - -WORKDIR '/home/' - -COPY entrypoint.sh . -COPY provision.py . - -HEALTHCHECK --retries=120 --interval=1s \ - CMD ls /tmp/ready || exit 1 - -ENTRYPOINT ["./entrypoint.sh"] diff --git a/crates/integration_tests/tests/shared_tests/mod.rs b/crates/integration_tests/tests/common/mod.rs similarity index 86% rename from crates/integration_tests/tests/shared_tests/mod.rs rename to crates/integration_tests/tests/common/mod.rs index 065b14d5da..b7197a3a46 100644 --- a/crates/integration_tests/tests/shared_tests/mod.rs +++ b/crates/integration_tests/tests/common/mod.rs @@ -16,24 +16,20 @@ // under the License. use std::collections::HashMap; +use std::sync::Arc; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent}; use iceberg_catalog_rest::RestCatalogBuilder; - -use crate::get_shared_containers; - -mod append_data_file_test; -mod append_partition_data_file_test; -mod conflict_commit_test; -mod datafusion; -mod read_evolved_schema; -mod read_positional_deletes; -mod scan_all_type; +use iceberg_integration_tests::get_test_fixture; +use iceberg_storage_opendal::OpenDalStorageFactory; pub async fn random_ns() -> Namespace { - let fixture = get_shared_containers(); + let fixture = get_test_fixture(); let rest_catalog = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) .load("rest", fixture.catalog_config.clone()) .await .unwrap(); @@ -54,7 +50,7 @@ pub async fn random_ns() -> Namespace { ns } -fn test_schema() -> Schema { +pub fn test_schema() -> Schema { Schema::builder() .with_schema_id(1) .with_identifier_field_ids(vec![2]) diff --git a/crates/integration_tests/tests/shared_tests/conflict_commit_test.rs b/crates/integration_tests/tests/conflict_commit_test.rs similarity index 84% rename from crates/integration_tests/tests/shared_tests/conflict_commit_test.rs rename to crates/integration_tests/tests/conflict_commit_test.rs index 4b85612ff5..73da9736ef 100644 --- a/crates/integration_tests/tests/shared_tests/conflict_commit_test.rs +++ b/crates/integration_tests/tests/conflict_commit_test.rs @@ -17,9 +17,12 @@ //! Integration tests for rest catalog. +mod common; + use std::sync::Arc; use arrow_array::{ArrayRef, BooleanArray, Int32Array, RecordBatch, StringArray}; +use common::{random_ns, test_schema}; use futures::TryStreamExt; use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; @@ -27,18 +30,21 @@ use iceberg::writer::file_writer::ParquetWriterBuilder; use iceberg::writer::file_writer::location_generator::{ DefaultFileNameGenerator, DefaultLocationGenerator, }; +use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; use iceberg::{Catalog, CatalogBuilder, TableCreation}; use iceberg_catalog_rest::RestCatalogBuilder; +use iceberg_integration_tests::get_test_fixture; +use iceberg_storage_opendal::OpenDalStorageFactory; use parquet::file::properties::WriterProperties; -use crate::get_shared_containers; -use crate::shared_tests::{random_ns, test_schema}; - #[tokio::test] async fn test_append_data_file_conflict() { - let fixture = get_shared_containers(); + let fixture = get_test_fixture(); let rest_catalog = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) .load("rest", fixture.catalog_config.clone()) .await .unwrap(); @@ -64,7 +70,7 @@ async fn test_append_data_file_conflict() { .try_into() .unwrap(), ); - let location_generator = DefaultLocationGenerator::new(table.metadata().clone()).unwrap(); + let location_generator = DefaultLocationGenerator::new(table.metadata()).unwrap(); let file_name_generator = DefaultFileNameGenerator::new( "test".to_string(), None, @@ -73,13 +79,15 @@ async fn test_append_data_file_conflict() { let parquet_writer_builder = ParquetWriterBuilder::new( WriterProperties::default(), table.metadata().current_schema().clone(), - None, + ); + let rolling_file_writer_builder = RollingFileWriterBuilder::new_with_default_file_size( + parquet_writer_builder, table.file_io().clone(), location_generator.clone(), file_name_generator.clone(), ); - let data_file_writer_builder = DataFileWriterBuilder::new(parquet_writer_builder, None, 0); - let mut data_file_writer = data_file_writer_builder.build().await.unwrap(); + let data_file_writer_builder = DataFileWriterBuilder::new(rolling_file_writer_builder); + let mut data_file_writer = data_file_writer_builder.build(None).await.unwrap(); let col1 = StringArray::from(vec![Some("foo"), Some("bar"), None, Some("baz")]); let col2 = Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4)]); let col3 = BooleanArray::from(vec![Some(true), Some(false), None, Some(false)]); @@ -111,7 +119,7 @@ async fn test_append_data_file_conflict() { .select_all() .build() .unwrap() - .to_arrow() + .to_arrow(Default::default()) .await .unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); diff --git a/crates/integration_tests/tests/shared_tests/read_evolved_schema.rs b/crates/integration_tests/tests/read_evolved_schema.rs similarity index 90% rename from crates/integration_tests/tests/shared_tests/read_evolved_schema.rs rename to crates/integration_tests/tests/read_evolved_schema.rs index fe8a488fa3..844ad81136 100644 --- a/crates/integration_tests/tests/shared_tests/read_evolved_schema.rs +++ b/crates/integration_tests/tests/read_evolved_schema.rs @@ -17,20 +17,25 @@ //! Integration tests for rest catalog. +use std::sync::Arc; + use arrow_array::{Decimal128Array, Float64Array, Int64Array, StringArray}; use futures::TryStreamExt; use iceberg::expr::Reference; use iceberg::spec::Datum; use iceberg::{Catalog, CatalogBuilder, TableIdent}; use iceberg_catalog_rest::RestCatalogBuilder; +use iceberg_integration_tests::get_test_fixture; +use iceberg_storage_opendal::OpenDalStorageFactory; use ordered_float::OrderedFloat; -use crate::get_shared_containers; - #[tokio::test] async fn test_evolved_schema() { - let fixture = get_shared_containers(); + let fixture = get_test_fixture(); let rest_catalog = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) .load("rest", fixture.catalog_config.clone()) .await .unwrap(); @@ -43,7 +48,7 @@ async fn test_evolved_schema() { let predicate = Reference::new("language").not_equal_to(Datum::string("Rust")); let scan = table.scan().with_filter(predicate).build(); - let batch_stream = scan.unwrap().to_arrow().await.unwrap(); + let batch_stream = scan.unwrap().to_arrow(Default::default()).await.unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); @@ -79,7 +84,7 @@ async fn test_evolved_schema() { // Reference::new("foo").not_equal_to(Datum::int(22)); let scan = table.scan().build(); - let batch_stream = scan.unwrap().to_arrow().await.unwrap(); + let batch_stream = scan.unwrap().to_arrow(Default::default()).await.unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); let mut actual = vec![ @@ -109,7 +114,7 @@ async fn test_evolved_schema() { .await .unwrap(); let scan = table.scan().build(); - let batch_stream = scan.unwrap().to_arrow().await.unwrap(); + let batch_stream = scan.unwrap().to_arrow(Default::default()).await.unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); let mut actual_foo = vec![ diff --git a/crates/integration_tests/tests/shared_tests/read_positional_deletes.rs b/crates/integration_tests/tests/read_positional_deletes.rs similarity index 81% rename from crates/integration_tests/tests/shared_tests/read_positional_deletes.rs rename to crates/integration_tests/tests/read_positional_deletes.rs index 7641810073..64d07f86fc 100644 --- a/crates/integration_tests/tests/shared_tests/read_positional_deletes.rs +++ b/crates/integration_tests/tests/read_positional_deletes.rs @@ -17,16 +17,21 @@ //! Integration tests for rest catalog. +use std::sync::Arc; + use futures::TryStreamExt; use iceberg::{Catalog, CatalogBuilder, TableIdent}; use iceberg_catalog_rest::RestCatalogBuilder; - -use crate::get_shared_containers; +use iceberg_integration_tests::get_test_fixture; +use iceberg_storage_opendal::OpenDalStorageFactory; #[tokio::test] async fn test_read_table_with_positional_deletes() { - let fixture = get_shared_containers(); + let fixture = get_test_fixture(); let rest_catalog = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) .load("rest", fixture.catalog_config.clone()) .await .unwrap(); @@ -40,7 +45,7 @@ async fn test_read_table_with_positional_deletes() { .unwrap(); let scan = table.scan().build().unwrap(); - println!("{:?}", scan); + println!("{scan:?}"); let plan: Vec<_> = scan .plan_files() @@ -49,14 +54,14 @@ async fn test_read_table_with_positional_deletes() { .try_collect() .await .unwrap(); - println!("{:?}", plan); + println!("{plan:?}"); // Scan plan phase should include delete files in file plan // when with_delete_file_processing_enabled == true - assert_eq!(plan[0].deletes.len(), 1); + assert_eq!(plan[0].deletes().len(), 1); // we should see two rows deleted, returning 10 rows instead of 12 - let batch_stream = scan.to_arrow().await.unwrap(); + let batch_stream = scan.to_arrow(Default::default()).await.unwrap(); let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); let num_rows: usize = batches.iter().map(|v| v.num_rows()).sum(); assert_eq!(num_rows, 10); diff --git a/crates/integration_tests/tests/shared_tests/append_data_file_test.rs b/crates/integration_tests/tests/shared_tests/append_data_file_test.rs deleted file mode 100644 index 9a3d62137e..0000000000 --- a/crates/integration_tests/tests/shared_tests/append_data_file_test.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for rest catalog. - -use std::sync::Arc; - -use arrow_array::{ArrayRef, BooleanArray, Int32Array, RecordBatch, StringArray}; -use futures::TryStreamExt; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; -use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; -use iceberg::writer::file_writer::ParquetWriterBuilder; -use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, -}; -use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, CatalogBuilder, TableCreation}; -use iceberg_catalog_rest::RestCatalogBuilder; -use parquet::arrow::arrow_reader::ArrowReaderOptions; -use parquet::file::properties::WriterProperties; - -use crate::get_shared_containers; -use crate::shared_tests::{random_ns, test_schema}; - -#[tokio::test] -async fn test_append_data_file() { - let fixture = get_shared_containers(); - let rest_catalog = RestCatalogBuilder::default() - .load("rest", fixture.catalog_config.clone()) - .await - .unwrap(); - let ns = random_ns().await; - let schema = test_schema(); - - let table_creation = TableCreation::builder() - .name("t1".to_string()) - .schema(schema.clone()) - .build(); - - let table = rest_catalog - .create_table(ns.name(), table_creation) - .await - .unwrap(); - - // Create the writer and write the data - let schema: Arc = Arc::new( - table - .metadata() - .current_schema() - .as_ref() - .try_into() - .unwrap(), - ); - let location_generator = DefaultLocationGenerator::new(table.metadata().clone()).unwrap(); - let file_name_generator = DefaultFileNameGenerator::new( - "test".to_string(), - None, - iceberg::spec::DataFileFormat::Parquet, - ); - let parquet_writer_builder = ParquetWriterBuilder::new( - WriterProperties::default(), - table.metadata().current_schema().clone(), - None, - table.file_io().clone(), - location_generator.clone(), - file_name_generator.clone(), - ); - let data_file_writer_builder = DataFileWriterBuilder::new(parquet_writer_builder, None, 0); - let mut data_file_writer = data_file_writer_builder.build().await.unwrap(); - let col1 = StringArray::from(vec![Some("foo"), Some("bar"), None, Some("baz")]); - let col2 = Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4)]); - let col3 = BooleanArray::from(vec![Some(true), Some(false), None, Some(false)]); - let batch = RecordBatch::try_new(schema.clone(), vec![ - Arc::new(col1) as ArrayRef, - Arc::new(col2) as ArrayRef, - Arc::new(col3) as ArrayRef, - ]) - .unwrap(); - data_file_writer.write(batch.clone()).await.unwrap(); - let data_file = data_file_writer.close().await.unwrap(); - - // check parquet file schema - let content = table - .file_io() - .new_input(data_file[0].file_path()) - .unwrap() - .read() - .await - .unwrap(); - let parquet_reader = parquet::arrow::arrow_reader::ArrowReaderMetadata::load( - &content, - ArrowReaderOptions::default(), - ) - .unwrap(); - let field_ids: Vec = parquet_reader - .parquet_schema() - .columns() - .iter() - .map(|col| col.self_type().get_basic_info().id()) - .collect(); - assert_eq!(field_ids, vec![1, 2, 3]); - - // commit result - let tx = Transaction::new(&table); - let append_action = tx.fast_append().add_data_files(data_file.clone()); - let tx = append_action.apply(tx).unwrap(); - let table = tx.commit(&rest_catalog).await.unwrap(); - - // check result - let batch_stream = table - .scan() - .select_all() - .build() - .unwrap() - .to_arrow() - .await - .unwrap(); - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches.len(), 1); - assert_eq!(batches[0], batch); -} diff --git a/crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs b/crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs deleted file mode 100644 index c737357ee9..0000000000 --- a/crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs +++ /dev/null @@ -1,239 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration test for partition data file - -use std::sync::Arc; - -use arrow_array::{ArrayRef, BooleanArray, Int32Array, RecordBatch, StringArray}; -use futures::TryStreamExt; -use iceberg::spec::{ - Literal, PartitionKey, PrimitiveLiteral, Struct, Transform, UnboundPartitionSpec, -}; -use iceberg::table::Table; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; -use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; -use iceberg::writer::file_writer::ParquetWriterBuilder; -use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, -}; -use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, CatalogBuilder, TableCreation}; -use iceberg_catalog_rest::RestCatalogBuilder; -use parquet::file::properties::WriterProperties; - -use crate::get_shared_containers; -use crate::shared_tests::{random_ns, test_schema}; - -#[tokio::test] -async fn test_append_partition_data_file() { - let fixture = get_shared_containers(); - let rest_catalog = RestCatalogBuilder::default() - .load("rest", fixture.catalog_config.clone()) - .await - .unwrap(); - let ns = random_ns().await; - let schema = test_schema(); - - let unbound_partition_spec = UnboundPartitionSpec::builder() - .add_partition_field(2, "id", Transform::Identity) - .expect("could not add partition field") - .build(); - - let partition_spec = unbound_partition_spec - .bind(schema.clone()) - .expect("could not bind to schema"); - - let table_creation = TableCreation::builder() - .name("t1".to_string()) - .schema(schema.clone()) - .partition_spec(partition_spec.clone()) - .build(); - - let table = rest_catalog - .create_table(ns.name(), table_creation) - .await - .unwrap(); - - // Create the writer and write the data - let schema: Arc = Arc::new( - table - .metadata() - .current_schema() - .as_ref() - .try_into() - .unwrap(), - ); - - let first_partition_id_value = 100; - let partition_key = PartitionKey::new( - partition_spec.clone(), - table.metadata().current_schema().clone(), - Struct::from_iter(vec![Some(Literal::int(first_partition_id_value))]), - ); - - let location_generator = DefaultLocationGenerator::new(table.metadata().clone()).unwrap(); - let file_name_generator = DefaultFileNameGenerator::new( - "test".to_string(), - None, - iceberg::spec::DataFileFormat::Parquet, - ); - - let parquet_writer_builder = ParquetWriterBuilder::new( - WriterProperties::default(), - table.metadata().current_schema().clone(), - Some(partition_key), - table.file_io().clone(), - location_generator.clone(), - file_name_generator.clone(), - ); - - let mut data_file_writer_valid = DataFileWriterBuilder::new( - parquet_writer_builder.clone(), - Some(Struct::from_iter([Some(Literal::Primitive( - PrimitiveLiteral::Int(first_partition_id_value), - ))])), - 0, - ) - .build() - .await - .unwrap(); - - let col1 = StringArray::from(vec![Some("foo1"), Some("foo2")]); - let col2 = Int32Array::from(vec![ - Some(first_partition_id_value), - Some(first_partition_id_value), - ]); - let col3 = BooleanArray::from(vec![Some(true), Some(false)]); - let batch = RecordBatch::try_new(schema.clone(), vec![ - Arc::new(col1) as ArrayRef, - Arc::new(col2) as ArrayRef, - Arc::new(col3) as ArrayRef, - ]) - .unwrap(); - - data_file_writer_valid.write(batch.clone()).await.unwrap(); - let data_file_valid = data_file_writer_valid.close().await.unwrap(); - - // commit result - let tx = Transaction::new(&table); - let append_action = tx.fast_append().add_data_files(data_file_valid.clone()); - let tx = append_action.apply(tx).unwrap(); - let table = tx.commit(&rest_catalog).await.unwrap(); - - // check result - let batch_stream = table - .scan() - .select_all() - .build() - .unwrap() - .to_arrow() - .await - .unwrap(); - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches.len(), 1); - assert_eq!(batches[0], batch); - - test_schema_incompatible_partition_type( - parquet_writer_builder.clone(), - batch.clone(), - table.clone(), - &rest_catalog, - ) - .await; - - test_schema_incompatible_partition_fields( - parquet_writer_builder, - batch, - table, - &rest_catalog, - first_partition_id_value, - ) - .await; -} - -async fn test_schema_incompatible_partition_type( - parquet_writer_builder: ParquetWriterBuilder< - DefaultLocationGenerator, - DefaultFileNameGenerator, - >, - batch: RecordBatch, - table: Table, - catalog: &dyn Catalog, -) { - // test writing different "type" of partition than mentioned in schema - let mut data_file_writer_invalid = DataFileWriterBuilder::new( - parquet_writer_builder.clone(), - Some(Struct::from_iter([Some(Literal::Primitive( - PrimitiveLiteral::Boolean(true), - ))])), - 0, - ) - .build() - .await - .unwrap(); - - data_file_writer_invalid.write(batch.clone()).await.unwrap(); - let data_file_invalid = data_file_writer_invalid.close().await.unwrap(); - - let tx = Transaction::new(&table); - let append_action = tx.fast_append().add_data_files(data_file_invalid.clone()); - let tx = append_action.apply(tx).unwrap(); - - if tx.commit(catalog).await.is_ok() { - panic!("diverging partition info should have returned error"); - } -} - -async fn test_schema_incompatible_partition_fields( - parquet_writer_builder: ParquetWriterBuilder< - DefaultLocationGenerator, - DefaultFileNameGenerator, - >, - batch: RecordBatch, - table: Table, - catalog: &dyn Catalog, - first_partition_id_value: i32, -) { - // test writing different number of partition fields than mentioned in schema - - let mut data_file_writer_invalid = DataFileWriterBuilder::new( - parquet_writer_builder, - Some(Struct::from_iter([ - Some(Literal::Primitive(PrimitiveLiteral::Int( - first_partition_id_value, - ))), - Some(Literal::Primitive(PrimitiveLiteral::Int( - first_partition_id_value, - ))), - ])), - 0, - ) - .build() - .await - .unwrap(); - - data_file_writer_invalid.write(batch.clone()).await.unwrap(); - let data_file_invalid = data_file_writer_invalid.close().await.unwrap(); - - let tx = Transaction::new(&table); - let append_action = tx.fast_append().add_data_files(data_file_invalid.clone()); - let tx = append_action.apply(tx).unwrap(); - if tx.commit(catalog).await.is_ok() { - panic!("passing different number of partition fields should have returned error"); - } -} diff --git a/crates/integration_tests/tests/shared_tests/datafusion.rs b/crates/integration_tests/tests/shared_tests/datafusion.rs deleted file mode 100644 index 81bbb5f54c..0000000000 --- a/crates/integration_tests/tests/shared_tests/datafusion.rs +++ /dev/null @@ -1,142 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::sync::Arc; - -use arrow_schema::TimeUnit; -use datafusion::arrow::datatypes::{DataType, Field, Schema}; -use datafusion::assert_batches_eq; -use datafusion::catalog::TableProvider; -use datafusion::error::DataFusionError; -use datafusion::prelude::SessionContext; -use iceberg::{Catalog, CatalogBuilder, TableIdent}; -use iceberg_catalog_rest::RestCatalogBuilder; -use iceberg_datafusion::IcebergTableProvider; -use parquet::arrow::PARQUET_FIELD_ID_META_KEY; - -use crate::get_shared_containers; - -#[tokio::test] -async fn test_basic_queries() -> Result<(), DataFusionError> { - let fixture = get_shared_containers(); - let rest_catalog = RestCatalogBuilder::default() - .load("rest", fixture.catalog_config.clone()) - .await - .unwrap(); - - let table = rest_catalog - .load_table(&TableIdent::from_strs(["default", "types_test"]).unwrap()) - .await - .unwrap(); - - let ctx = SessionContext::new(); - - let table_provider = Arc::new( - IcebergTableProvider::try_new_from_table(table) - .await - .unwrap(), - ); - - let schema = table_provider.schema(); - - assert_eq!( - schema.as_ref(), - &Schema::new(vec![ - Field::new("cboolean", DataType::Boolean, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("ctinyint", DataType::Int32, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("csmallint", DataType::Int32, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - Field::new("cint", DataType::Int32, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "4".to_string(), - )])), - Field::new("cbigint", DataType::Int64, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "5".to_string(), - )])), - Field::new("cfloat", DataType::Float32, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "6".to_string(), - )])), - Field::new("cdouble", DataType::Float64, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "7".to_string(), - )])), - Field::new("cdecimal", DataType::Decimal128(8, 2), true).with_metadata(HashMap::from( - [(PARQUET_FIELD_ID_META_KEY.to_string(), "8".to_string(),)] - )), - Field::new("cdate", DataType::Date32, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "9".to_string(), - )])), - Field::new( - "ctimestamp_ntz", - DataType::Timestamp(TimeUnit::Microsecond, None), - true - ) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "10".to_string(), - )])), - Field::new( - "ctimestamp", - DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("+00:00"))), - true - ) - .with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "11".to_string(), - )])), - Field::new("cstring", DataType::Utf8, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "12".to_string(), - )])), - Field::new("cbinary", DataType::LargeBinary, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "13".to_string(), - )])), - ]) - ); - - ctx.register_table("types_table", table_provider)?; - - let batches = ctx - .sql("SELECT * FROM types_table ORDER BY cbigint LIMIT 3") - .await? - .collect() - .await?; - let expected = [ - "+----------+----------+-----------+------+---------+--------+---------+----------+------------+---------------------+----------------------+---------+----------+", - "| cboolean | ctinyint | csmallint | cint | cbigint | cfloat | cdouble | cdecimal | cdate | ctimestamp_ntz | ctimestamp | cstring | cbinary |", - "+----------+----------+-----------+------+---------+--------+---------+----------+------------+---------------------+----------------------+---------+----------+", - "| false | -128 | 0 | 0 | 0 | 0.0 | 0.0 | 0.00 | 1970-01-01 | 1970-01-01T00:00:00 | 1970-01-01T00:00:00Z | 0 | 00000000 |", - "| true | -127 | 1 | 1 | 1 | 1.0 | 1.0 | 0.01 | 1970-01-02 | 1970-01-01T00:00:01 | 1970-01-01T00:00:01Z | 1 | 00000001 |", - "| false | -126 | 2 | 2 | 2 | 2.0 | 2.0 | 0.02 | 1970-01-03 | 1970-01-01T00:00:02 | 1970-01-01T00:00:02Z | 2 | 00000002 |", - "+----------+----------+-----------+------+---------+--------+---------+----------+------------+---------------------+----------------------+---------+----------+", - ]; - assert_batches_eq!(expected, &batches); - Ok(()) -} diff --git a/crates/integration_tests/tests/shared_tests/scan_all_type.rs b/crates/integration_tests/tests/shared_tests/scan_all_type.rs deleted file mode 100644 index b8e7c5941e..0000000000 --- a/crates/integration_tests/tests/shared_tests/scan_all_type.rs +++ /dev/null @@ -1,363 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for rest catalog. - -use std::collections::HashMap; -use std::sync::Arc; - -use arrow_array::builder::{Int32Builder, ListBuilder, MapBuilder, StringBuilder}; -use arrow_array::{ - Array, ArrayRef, BooleanArray, Date32Array, Decimal128Array, FixedSizeBinaryArray, - Float32Array, Float64Array, Int32Array, Int64Array, LargeBinaryArray, MapArray, RecordBatch, - StringArray, StructArray, Time64MicrosecondArray, TimestampMicrosecondArray, -}; -use arrow_schema::{DataType, Field, Fields}; -use futures::TryStreamExt; -use iceberg::arrow::{DEFAULT_MAP_FIELD_NAME, UTC_TIME_ZONE}; -use iceberg::spec::{ - LIST_FIELD_NAME, ListType, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedField, - PrimitiveType, Schema, StructType, Type, -}; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; -use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; -use iceberg::writer::file_writer::ParquetWriterBuilder; -use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, -}; -use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, CatalogBuilder, TableCreation}; -use iceberg_catalog_rest::RestCatalogBuilder; -use parquet::arrow::PARQUET_FIELD_ID_META_KEY; -use parquet::file::properties::WriterProperties; -use uuid::Uuid; - -use crate::get_shared_containers; -use crate::shared_tests::random_ns; - -#[tokio::test] -async fn test_scan_all_type() { - let fixture = get_shared_containers(); - let rest_catalog = RestCatalogBuilder::default() - .load("rest", fixture.catalog_config.clone()) - .await - .unwrap(); - let ns = random_ns().await; - - let schema = Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![2]) - .with_fields(vec![ - // test all type - NestedField::required(1, "int", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "long", Type::Primitive(PrimitiveType::Long)).into(), - NestedField::required(3, "float", Type::Primitive(PrimitiveType::Float)).into(), - NestedField::required(4, "double", Type::Primitive(PrimitiveType::Double)).into(), - NestedField::required( - 5, - "decimal", - Type::Primitive(PrimitiveType::Decimal { - precision: 20, - scale: 5, - }), - ) - .into(), - NestedField::required(6, "string", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(7, "boolean", Type::Primitive(PrimitiveType::Boolean)).into(), - NestedField::required(8, "binary", Type::Primitive(PrimitiveType::Binary)).into(), - NestedField::required(9, "date", Type::Primitive(PrimitiveType::Date)).into(), - NestedField::required(10, "time", Type::Primitive(PrimitiveType::Time)).into(), - NestedField::required(11, "timestamp", Type::Primitive(PrimitiveType::Timestamp)) - .into(), - NestedField::required(12, "fixed", Type::Primitive(PrimitiveType::Fixed(10))).into(), - NestedField::required(13, "uuid", Type::Primitive(PrimitiveType::Uuid)).into(), - NestedField::required( - 14, - "timestamptz", - Type::Primitive(PrimitiveType::Timestamptz), - ) - .into(), - NestedField::required( - 15, - "struct", - Type::Struct(StructType::new(vec![ - NestedField::required(18, "int", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(19, "string", Type::Primitive(PrimitiveType::String)) - .into(), - ])), - ) - .into(), - NestedField::required( - 16, - "list", - Type::List(ListType::new( - NestedField::list_element(20, Type::Primitive(PrimitiveType::Int), true).into(), - )), - ) - .into(), - NestedField::required( - 17, - "map", - Type::Map(MapType::new( - NestedField::map_key_element(21, Type::Primitive(PrimitiveType::Int)).into(), - NestedField::map_value_element( - 22, - Type::Primitive(PrimitiveType::String), - true, - ) - .into(), - )), - ) - .into(), - ]) - .build() - .unwrap(); - - let table_creation = TableCreation::builder() - .name("t1".to_string()) - .schema(schema.clone()) - .build(); - - let table = rest_catalog - .create_table(ns.name(), table_creation) - .await - .unwrap(); - - // Create the writer and write the data - let schema: Arc = Arc::new( - table - .metadata() - .current_schema() - .as_ref() - .try_into() - .unwrap(), - ); - let location_generator = DefaultLocationGenerator::new(table.metadata().clone()).unwrap(); - let file_name_generator = DefaultFileNameGenerator::new( - "test".to_string(), - None, - iceberg::spec::DataFileFormat::Parquet, - ); - let parquet_writer_builder = ParquetWriterBuilder::new( - WriterProperties::default(), - table.metadata().current_schema().clone(), - None, - table.file_io().clone(), - location_generator.clone(), - file_name_generator.clone(), - ); - let data_file_writer_builder = DataFileWriterBuilder::new(parquet_writer_builder, None, 0); - let mut data_file_writer = data_file_writer_builder.build().await.unwrap(); - - // Prepare data - let col1 = Int32Array::from(vec![1, 2, 3, 4, 5]); - let col2 = Int64Array::from(vec![1, 2, 3, 4, 5]); - let col3 = Float32Array::from(vec![1.1, 2.2, 3.3, 4.4, 5.5]); - let col4 = Float64Array::from(vec![1.1, 2.2, 3.3, 4.4, 5.5]); - let col5 = Decimal128Array::from(vec![ - Some(1.into()), - Some(2.into()), - Some(3.into()), - Some(4.into()), - Some(5.into()), - ]) - .with_data_type(DataType::Decimal128(20, 5)); - let col6 = StringArray::from(vec!["a", "b", "c", "d", "e"]); - let col7 = BooleanArray::from(vec![true, false, true, false, true]); - let col8 = LargeBinaryArray::from_opt_vec(vec![ - Some(b"a"), - Some(b"b"), - Some(b"c"), - Some(b"d"), - Some(b"e"), - ]); - let col9 = Date32Array::from(vec![1, 2, 3, 4, 5]); - let col10 = Time64MicrosecondArray::from(vec![1, 2, 3, 4, 5]); - let col11 = TimestampMicrosecondArray::from(vec![1, 2, 3, 4, 5]); - let col12 = FixedSizeBinaryArray::try_from_iter( - vec![ - vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - ] - .into_iter(), - ) - .unwrap(); - assert_eq!(col12.data_type(), &DataType::FixedSizeBinary(10)); - let col13 = FixedSizeBinaryArray::try_from_iter( - vec![ - Uuid::new_v4().as_bytes().to_vec(), - Uuid::new_v4().as_bytes().to_vec(), - Uuid::new_v4().as_bytes().to_vec(), - Uuid::new_v4().as_bytes().to_vec(), - Uuid::new_v4().as_bytes().to_vec(), - ] - .into_iter(), - ) - .unwrap(); - assert_eq!(col13.data_type(), &DataType::FixedSizeBinary(16)); - let col14 = TimestampMicrosecondArray::from(vec![1, 2, 3, 4, 5]).with_timezone(UTC_TIME_ZONE); - let col15 = StructArray::from(vec![ - ( - Arc::new( - Field::new("int", DataType::Int32, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - 18.to_string(), - )])), - ), - Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])) as ArrayRef, - ), - ( - Arc::new( - Field::new("string", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - 19.to_string(), - )])), - ), - Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])) as ArrayRef, - ), - ]); - let col16 = { - let mut builder = ListBuilder::new(Int32Builder::new()).with_field(Arc::new( - Field::new(LIST_FIELD_NAME, DataType::Int32, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - 20.to_string(), - )])), - )); - builder.append_value([Some(1), Some(2), Some(3), Some(4), Some(5)]); - builder.append_value([Some(1), Some(2), Some(3), Some(4), Some(5)]); - builder.append_value([Some(1), Some(2), Some(3), Some(4), Some(5)]); - builder.append_value([Some(1), Some(2), Some(3), Some(4), Some(5)]); - builder.append_value([Some(1), Some(2), Some(3), Some(4), Some(5)]); - builder.finish() - }; - let col17 = { - let string_builder = StringBuilder::new(); - let int_builder = Int32Builder::with_capacity(4); - let mut builder = MapBuilder::new(None, int_builder, string_builder); - builder.keys().append_value(1); - builder.values().append_value("a"); - builder.append(true).unwrap(); - builder.keys().append_value(2); - builder.values().append_value("b"); - builder.append(true).unwrap(); - builder.keys().append_value(3); - builder.values().append_value("c"); - builder.append(true).unwrap(); - builder.keys().append_value(4); - builder.values().append_value("d"); - builder.append(true).unwrap(); - builder.keys().append_value(5); - builder.values().append_value("e"); - builder.append(true).unwrap(); - let array = builder.finish(); - let (_field, offsets, entries, nulls, ordered) = array.into_parts(); - let new_struct_fields = Fields::from(vec![ - Field::new(MAP_KEY_FIELD_NAME, DataType::Int32, false).with_metadata(HashMap::from([ - (PARQUET_FIELD_ID_META_KEY.to_string(), 21.to_string()), - ])), - Field::new(MAP_VALUE_FIELD_NAME, DataType::Utf8, false).with_metadata(HashMap::from([ - (PARQUET_FIELD_ID_META_KEY.to_string(), 22.to_string()), - ])), - ]); - let entries = { - let (_, arrays, nulls) = entries.into_parts(); - StructArray::new(new_struct_fields.clone(), arrays, nulls) - }; - let field = Arc::new(Field::new( - DEFAULT_MAP_FIELD_NAME, - DataType::Struct(new_struct_fields), - false, - )); - MapArray::new(field, offsets, entries, nulls, ordered) - }; - - let batch = RecordBatch::try_new(schema.clone(), vec![ - Arc::new(col1) as ArrayRef, - Arc::new(col2) as ArrayRef, - Arc::new(col3) as ArrayRef, - Arc::new(col4) as ArrayRef, - Arc::new(col5) as ArrayRef, - Arc::new(col6) as ArrayRef, - Arc::new(col7) as ArrayRef, - Arc::new(col8) as ArrayRef, - Arc::new(col9) as ArrayRef, - Arc::new(col10) as ArrayRef, - Arc::new(col11) as ArrayRef, - Arc::new(col12) as ArrayRef, - Arc::new(col13) as ArrayRef, - Arc::new(col14) as ArrayRef, - Arc::new(col15) as ArrayRef, - Arc::new(col16) as ArrayRef, - Arc::new(col17) as ArrayRef, - ]) - .unwrap(); - data_file_writer.write(batch.clone()).await.unwrap(); - let data_file = data_file_writer.close().await.unwrap(); - - // commit result - let tx = Transaction::new(&table); - let append_action = tx.fast_append().add_data_files(data_file.clone()); - let tx = append_action.apply(tx).unwrap(); - let table = tx.commit(&rest_catalog).await.unwrap(); - - // check result - let batch_stream = table - .scan() - .select(vec![ - "int", - "long", - "float", - "double", - "decimal", - "string", - "boolean", - "binary", - "date", - "time", - "timestamp", - "fixed", - "uuid", - "timestamptz", - "struct", - "list", - "map", - ]) - .build() - .unwrap() - .to_arrow() - .await - .unwrap(); - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches.len(), 1); - assert_eq!(batches[0], batch); - - // check result - let batch_stream = table - .scan() - .select_all() - .build() - .unwrap() - .to_arrow() - .await - .unwrap(); - let batches: Vec<_> = batch_stream.try_collect().await.unwrap(); - assert_eq!(batches.len(), 1); - assert_eq!(batches[0], batch); -} diff --git a/crates/integrations/cache-moka/Cargo.toml b/crates/integrations/cache-moka/Cargo.toml index 87e3a74a69..5b78955e63 100644 --- a/crates/integrations/cache-moka/Cargo.toml +++ b/crates/integrations/cache-moka/Cargo.toml @@ -18,13 +18,21 @@ [package] name = "iceberg-cache-moka" +categories = ["caching"] +description = "Apache Iceberg Moka cache implementation" +keywords = ["iceberg", "cache", "moka"] + edition = { workspace = true } homepage = { workspace = true } license = { workspace = true } +publish = true repository = { workspace = true } rust-version = { workspace = true } version = { workspace = true } [dependencies] iceberg = { workspace = true } -moka = { version = "0.12.10", features = ["sync"] } +moka = { workspace = true, features = ["sync"] } + +[lints] +workspace = true diff --git a/crates/integrations/cache-moka/DEPENDENCIES.rust.tsv b/crates/integrations/cache-moka/DEPENDENCIES.rust.tsv deleted file mode 100644 index 59b898d3fe..0000000000 --- a/crates/integrations/cache-moka/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,324 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayvec@0.7.6 X X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-ord@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -gloo-timers@0.3.0 X X -half@2.6.0 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-cache-moka@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustls@0.23.31 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simdutf8@0.1.5 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.0 X X -stable_deref_trait@1.2.0 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -untrusted@0.9.0 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -webpki-roots@1.0.2 X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integrations/cache-moka/LICENSE b/crates/integrations/cache-moka/LICENSE new file mode 120000 index 0000000000..5853aaea53 --- /dev/null +++ b/crates/integrations/cache-moka/LICENSE @@ -0,0 +1 @@ +../../../LICENSE \ No newline at end of file diff --git a/crates/integrations/cache-moka/NOTICE b/crates/integrations/cache-moka/NOTICE new file mode 120000 index 0000000000..295f6bdb3a --- /dev/null +++ b/crates/integrations/cache-moka/NOTICE @@ -0,0 +1 @@ +../../../NOTICE \ No newline at end of file diff --git a/crates/integrations/cache-moka/public-api.txt b/crates/integrations/cache-moka/public-api.txt new file mode 100644 index 0000000000..8ac55e9305 --- /dev/null +++ b/crates/integrations/cache-moka/public-api.txt @@ -0,0 +1,11 @@ +pub mod iceberg_cache_moka +pub struct iceberg_cache_moka::MokaObjectCacheProvider +impl iceberg_cache_moka::MokaObjectCacheProvider +pub fn iceberg_cache_moka::MokaObjectCacheProvider::new() -> Self +pub fn iceberg_cache_moka::MokaObjectCacheProvider::with_manifest_cache(self, cache: moka::sync::cache::Cache>) -> Self +pub fn iceberg_cache_moka::MokaObjectCacheProvider::with_manifest_list_cache(self, cache: moka::sync::cache::Cache>) -> Self +impl core::default::Default for iceberg_cache_moka::MokaObjectCacheProvider +pub fn iceberg_cache_moka::MokaObjectCacheProvider::default() -> Self +impl iceberg::cache::ObjectCacheProvide for iceberg_cache_moka::MokaObjectCacheProvider +pub fn iceberg_cache_moka::MokaObjectCacheProvider::manifest_cache(&self) -> &dyn iceberg::cache::ObjectCache> +pub fn iceberg_cache_moka::MokaObjectCacheProvider::manifest_list_cache(&self) -> &dyn iceberg::cache::ObjectCache> diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv deleted file mode 100644 index 8fb3e04f80..0000000000 --- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,401 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anyhow@1.0.99 X X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@55.2.0 X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-csv@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-json@55.2.0 X -arrow-ord@55.2.0 X -arrow-row@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-compression@0.4.19 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -blake2@0.10.6 X X -blake3@1.8.2 X X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -bzip2@0.5.2 X X -bzip2-sys@0.1.13+1.0.8 X X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -chrono-tz@0.10.4 X X -comfy-table@7.2.0 X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.3.1 X X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -csv@1.3.1 X X -csv-core@0.1.12 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -dashmap@6.1.0 X -datafusion@48.0.1 X -datafusion-catalog@48.0.1 X -datafusion-catalog-listing@48.0.1 X -datafusion-common@48.0.1 X -datafusion-common-runtime@48.0.1 X -datafusion-datasource@48.0.1 X -datafusion-datasource-csv@48.0.1 X -datafusion-datasource-json@48.0.1 X -datafusion-datasource-parquet@48.0.1 X -datafusion-doc@48.0.1 X -datafusion-execution@48.0.1 X -datafusion-expr@48.0.1 X -datafusion-expr-common@48.0.1 X -datafusion-functions@48.0.1 X -datafusion-functions-aggregate@48.0.1 X -datafusion-functions-aggregate-common@48.0.1 X -datafusion-functions-nested@48.0.1 X -datafusion-functions-table@48.0.1 X -datafusion-functions-window@48.0.1 X -datafusion-functions-window-common@48.0.1 X -datafusion-macros@48.0.1 X -datafusion-optimizer@48.0.1 X -datafusion-physical-expr@48.0.1 X -datafusion-physical-expr-common@48.0.1 X -datafusion-physical-optimizer@48.0.1 X -datafusion-physical-plan@48.0.1 X -datafusion-session@48.0.1 X -datafusion-sql@48.0.1 X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -equivalent@1.0.2 X X -errno@0.3.13 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -fixedbitset@0.5.7 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -form_urlencoded@1.2.2 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -half@2.6.0 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@1.3.1 X X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -humantime@2.2.0 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-datafusion@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libm@0.2.15 X -libz-rs-sys@0.5.2 X -linux-raw-sys@0.9.4 X X X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -lzma-sys@0.1.20 X X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -object_store@0.12.3 X X -once_cell@1.21.3 X X -opendal@0.54.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -petgraph@0.8.2 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -psm@0.1.26 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustix@1.0.8 X X X -rustls@0.23.31 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -ryu@1.0.20 X X -same-file@1.0.6 X X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -simdutf8@0.1.5 X X -siphasher@1.0.1 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.0 X X -sqlparser@0.55.0 X -sqlparser_derive@0.3.0 X -stable_deref_trait@1.2.0 X X -stacker@0.1.21 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -tempfile@3.22.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.2.1 X X -untrusted@0.9.0 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -web-time@1.1.0 X X -webpki-roots@1.0.2 X -winapi-util@0.1.11 X X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.61.0 X X -windows-targets@0.52.6 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -xz2@0.1.7 X X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integrations/datafusion/README.md b/crates/integrations/datafusion/README.md deleted file mode 100644 index 134a8eff4b..0000000000 --- a/crates/integrations/datafusion/README.md +++ /dev/null @@ -1,22 +0,0 @@ - - -# Apache Iceberg DataFusion Integration - -This crate contains the integration of Apache DataFusion and Apache Iceberg. diff --git a/crates/integrations/datafusion/src/catalog.rs b/crates/integrations/datafusion/src/catalog.rs deleted file mode 100644 index c3cbcc88b4..0000000000 --- a/crates/integrations/datafusion/src/catalog.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::any::Any; -use std::collections::HashMap; -use std::sync::Arc; - -use datafusion::catalog::{CatalogProvider, SchemaProvider}; -use futures::future::try_join_all; -use iceberg::{Catalog, NamespaceIdent, Result}; - -use crate::schema::IcebergSchemaProvider; - -/// Provides an interface to manage and access multiple schemas -/// within an Iceberg [`Catalog`]. -/// -/// Acts as a centralized catalog provider that aggregates -/// multiple [`SchemaProvider`], each associated with distinct namespaces. -#[derive(Debug)] -pub struct IcebergCatalogProvider { - /// A `HashMap` where keys are namespace names - /// and values are dynamic references to objects implementing the - /// [`SchemaProvider`] trait. - schemas: HashMap>, -} - -impl IcebergCatalogProvider { - /// Asynchronously tries to construct a new [`IcebergCatalogProvider`] - /// using the given client to fetch and initialize schema providers for - /// each namespace in the Iceberg [`Catalog`]. - /// - /// This method retrieves the list of namespace names - /// attempts to create a schema provider for each namespace, and - /// collects these providers into a `HashMap`. - pub async fn try_new(client: Arc) -> Result { - // TODO: - // Schemas and providers should be cached and evicted based on time - // As of right now; schemas might become stale. - let schema_names: Vec<_> = client - .list_namespaces(None) - .await? - .iter() - .flat_map(|ns| ns.as_ref().clone()) - .collect(); - - let providers = try_join_all( - schema_names - .iter() - .map(|name| { - IcebergSchemaProvider::try_new( - client.clone(), - NamespaceIdent::new(name.clone()), - ) - }) - .collect::>(), - ) - .await?; - - let schemas: HashMap> = schema_names - .into_iter() - .zip(providers.into_iter()) - .map(|(name, provider)| { - let provider = Arc::new(provider) as Arc; - (name, provider) - }) - .collect(); - - Ok(IcebergCatalogProvider { schemas }) - } -} - -impl CatalogProvider for IcebergCatalogProvider { - fn as_any(&self) -> &dyn Any { - self - } - - fn schema_names(&self) -> Vec { - self.schemas.keys().cloned().collect() - } - - fn schema(&self, name: &str) -> Option> { - self.schemas.get(name).cloned() - } -} diff --git a/crates/integrations/datafusion/src/error.rs b/crates/integrations/datafusion/src/error.rs deleted file mode 100644 index 273d92fa62..0000000000 --- a/crates/integrations/datafusion/src/error.rs +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use anyhow::anyhow; -use iceberg::{Error, ErrorKind}; - -/// Converts a datafusion error into an iceberg error. -pub fn from_datafusion_error(error: datafusion::error::DataFusionError) -> Error { - Error::new( - ErrorKind::Unexpected, - "Operation failed for hitting datafusion error".to_string(), - ) - .with_source(anyhow!("datafusion error: {:?}", error)) -} -/// Converts an iceberg error into a datafusion error. -pub fn to_datafusion_error(error: Error) -> datafusion::error::DataFusionError { - datafusion::error::DataFusionError::External(error.into()) -} diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs deleted file mode 100644 index 0ee0504faa..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ /dev/null @@ -1,513 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::any::Any; -use std::fmt::{Debug, Formatter}; -use std::sync::Arc; - -use datafusion::arrow::array::{Array, ArrayRef, RecordBatch, StringArray, UInt64Array}; -use datafusion::arrow::datatypes::{ - DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, -}; -use datafusion::common::{DataFusionError, Result as DFResult}; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use futures::StreamExt; -use iceberg::Catalog; -use iceberg::spec::{DataFile, deserialize_data_file_from_json}; -use iceberg::table::Table; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; - -use crate::physical_plan::DATA_FILES_COL_NAME; -use crate::to_datafusion_error; - -/// IcebergCommitExec is responsible for collecting the files written and use -/// [`Transaction::fast_append`] to commit the data files written. -#[derive(Debug)] -pub(crate) struct IcebergCommitExec { - table: Table, - catalog: Arc, - input: Arc, - schema: ArrowSchemaRef, - count_schema: ArrowSchemaRef, - plan_properties: PlanProperties, -} - -impl IcebergCommitExec { - pub fn new( - table: Table, - catalog: Arc, - input: Arc, - schema: ArrowSchemaRef, - ) -> Self { - let plan_properties = Self::compute_properties(schema.clone()); - - Self { - table, - catalog, - input, - schema, - count_schema: Self::make_count_schema(), - plan_properties, - } - } - - // Compute the plan properties for this execution plan - fn compute_properties(schema: ArrowSchemaRef) -> PlanProperties { - PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - ) - } - - // Create a record batch with just the count of rows written - fn make_count_batch(count: u64) -> DFResult { - let count_array = Arc::new(UInt64Array::from(vec![count])) as ArrayRef; - - RecordBatch::try_from_iter_with_nullable(vec![("count", count_array, false)]).map_err(|e| { - DataFusionError::ArrowError(e, Some("Failed to make count batch!".to_string())) - }) - } - - fn make_count_schema() -> ArrowSchemaRef { - // Define a schema. - Arc::new(ArrowSchema::new(vec![Field::new( - "count", - DataType::UInt64, - false, - )])) - } -} - -impl DisplayAs for IcebergCommitExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default => { - write!(f, "IcebergCommitExec: table={}", self.table.identifier()) - } - DisplayFormatType::Verbose => { - write!( - f, - "IcebergCommitExec: table={}, schema={:?}", - self.table.identifier(), - self.schema - ) - } - DisplayFormatType::TreeRender => { - write!(f, "IcebergCommitExec: table={}", self.table.identifier()) - } - } - } -} - -impl ExecutionPlan for IcebergCommitExec { - fn name(&self) -> &str { - "IcebergCommitExec" - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn properties(&self) -> &PlanProperties { - &self.plan_properties - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.input] - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> DFResult> { - if children.len() != 1 { - return Err(DataFusionError::Internal(format!( - "IcebergCommitExec expects exactly one child, but provided {}", - children.len() - ))); - } - - Ok(Arc::new(IcebergCommitExec::new( - self.table.clone(), - self.catalog.clone(), - children[0].clone(), - self.schema.clone(), - ))) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> DFResult { - // IcebergCommitExec only has one partition (partition 0) - if partition != 0 { - return Err(DataFusionError::Internal(format!( - "IcebergCommitExec only has one partition, but got partition {}", - partition - ))); - } - - let table = self.table.clone(); - let input_plan = self.input.clone(); - let count_schema = Arc::clone(&self.count_schema); - - // todo revisit this - let spec_id = self.table.metadata().default_partition_spec_id(); - let partition_type = self.table.metadata().default_partition_type().clone(); - let current_schema = self.table.metadata().current_schema().clone(); - - let catalog = Arc::clone(&self.catalog); - - // Process the input streams from all partitions and commit the data files - let stream = futures::stream::once(async move { - let mut data_files: Vec = Vec::new(); - let mut total_record_count: u64 = 0; - - // Execute and collect results from the input coalesced plan - let mut batch_stream = input_plan.execute(0, context)?; - - while let Some(batch_result) = batch_stream.next().await { - let batch = batch_result?; - - let files_array = batch - .column_by_name(DATA_FILES_COL_NAME) - .ok_or_else(|| { - DataFusionError::Internal( - "Expected 'data_files' column in input batch".to_string(), - ) - })? - .as_any() - .downcast_ref::() - .ok_or_else(|| { - DataFusionError::Internal( - "Expected 'data_files' column to be StringArray".to_string(), - ) - })?; - - // Deserialize all data files from the StringArray - let batch_files: Vec = files_array - .into_iter() - .flatten() - .map(|f| -> DFResult { - // Parse JSON to DataFileSerde and convert to DataFile - deserialize_data_file_from_json( - f, - spec_id, - &partition_type, - ¤t_schema, - ) - .map_err(to_datafusion_error) - }) - .collect::>()?; - - // add record_counts from the current batch to total record count - total_record_count += batch_files.iter().map(|f| f.record_count()).sum::(); - - // Add all deserialized files to our collection - data_files.extend(batch_files); - } - - // If no data files were collected, return an empty result - if data_files.is_empty() { - return Ok(RecordBatch::new_empty(count_schema)); - } - - // Create a transaction and commit the data files - let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(data_files); - - // Apply the action and commit the transaction - let _updated_table = action - .apply(tx) - .map_err(to_datafusion_error)? - .commit(catalog.as_ref()) - .await - .map_err(to_datafusion_error)?; - - Self::make_count_batch(total_record_count) - }) - .boxed(); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&self.count_schema), - stream, - ))) - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::fmt; - use std::sync::Arc; - - use datafusion::arrow::array::{ArrayRef, RecordBatch, StringArray}; - use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; - use datafusion::execution::context::TaskContext; - use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; - use datafusion::physical_plan::common::collect; - use datafusion::physical_plan::execution_plan::Boundedness; - use datafusion::physical_plan::stream::RecordBatchStreamAdapter; - use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; - use futures::StreamExt; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, NestedField, PrimitiveType, Schema, - Struct, Type, - }; - use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; - - use super::*; - use crate::physical_plan::DATA_FILES_COL_NAME; - - // A mock execution plan that returns record batches with serialized data files - #[derive(Debug)] - struct MockWriteExec { - schema: Arc, - data_files_json: Vec, - plan_properties: PlanProperties, - } - - impl MockWriteExec { - fn new(data_files_json: Vec) -> Self { - let schema = Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])); - - let plan_properties = PlanProperties::new( - EquivalenceProperties::new(schema.clone()), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - ); - - Self { - schema, - data_files_json, - plan_properties, - } - } - } - - impl ExecutionPlan for MockWriteExec { - fn name(&self) -> &str { - "MockWriteExec" - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn schema(&self) -> Arc { - self.schema.clone() - } - - fn properties(&self) -> &PlanProperties { - &self.plan_properties - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> datafusion::common::Result> { - Ok(self) - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> datafusion::common::Result { - // Create a record batch with the serialized data files - let array = Arc::new(StringArray::from(self.data_files_json.clone())) as ArrayRef; - let batch = RecordBatch::try_new(self.schema.clone(), vec![array])?; - - // Create a stream that returns this batch - let stream = futures::stream::once(async move { Ok(batch) }).boxed(); - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - stream, - ))) - } - } - - // Implement DisplayAs for MockDataFilesExec - impl DisplayAs for MockWriteExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> fmt::Result { - match t { - DisplayFormatType::Default - | DisplayFormatType::Verbose - | DisplayFormatType::TreeRender => { - write!(f, "MockDataFilesExec: files={}", self.data_files_json.len()) - } - } - } - } - - #[tokio::test] - async fn test_iceberg_commit_exec() -> Result<(), Box> { - // Create a memory catalog with in-memory file IO - let catalog = Arc::new( - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([( - MEMORY_CATALOG_WAREHOUSE.to_string(), - "memory://root".to_string(), - )]), - ) - .await - .unwrap(), - ); - - // Create a namespace - let namespace = NamespaceIdent::new("test_namespace".to_string()); - catalog.create_namespace(&namespace, HashMap::new()).await?; - - // Create a schema for the table - let schema = Schema::builder() - .with_schema_id(1) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - // Create a table - let table_creation = TableCreation::builder() - .name("test_table".to_string()) - .schema(schema) - .location("memory://root/test_table".to_string()) - .properties(HashMap::new()) - .build(); - - let table = catalog.create_table(&namespace, table_creation).await?; - - // Create data files - let data_file1 = DataFileBuilder::default() - .content(DataContentType::Data) - .file_path("path/to/file1.parquet".to_string()) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(1024) - .record_count(100) - .partition_spec_id(table.metadata().default_partition_spec_id()) - .partition(Struct::empty()) - .build()?; - - let data_file2 = DataFileBuilder::default() - .content(DataContentType::Data) - .file_path("path/to/file2.parquet".to_string()) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(2048) - .record_count(200) - .partition_spec_id(table.metadata().default_partition_spec_id()) - .partition(Struct::empty()) - .build()?; - - // Serialize data files to JSON - let partition_type = table.metadata().default_partition_type().clone(); - let data_file1_json = iceberg::spec::serialize_data_file_to_json( - data_file1.clone(), - &partition_type, - table.metadata().format_version(), - )?; - - let data_file2_json = iceberg::spec::serialize_data_file_to_json( - data_file2.clone(), - &partition_type, - table.metadata().format_version(), - )?; - - // Create a mock execution plan that returns the serialized data files - let input_exec = Arc::new(MockWriteExec::new(vec![data_file1_json, data_file2_json])); - - // Create the IcebergCommitExec - let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])); - - let commit_exec = - IcebergCommitExec::new(table.clone(), catalog.clone(), input_exec, arrow_schema); - - // Execute the commit exec - let task_ctx = Arc::new(TaskContext::default()); - let stream = commit_exec.execute(0, task_ctx)?; - let batches = collect(stream).await?; - - // Verify the results - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert_eq!(batch.num_columns(), 1); - assert_eq!(batch.num_rows(), 1); - - // The output should be a record batch with a single column "count" and a single row - // with the total record count (100 + 200 = 300) - let count_array = batch.column(0); - assert_eq!(count_array.len(), 1); - assert_eq!(count_array.data_type(), &DataType::UInt64); - - // Verify that the count is correct - let count = count_array.as_any().downcast_ref::().unwrap(); - assert_eq!(count.value(0), 300); - - // Verify that the table has been updated with the new files - let updated_table = catalog - .load_table(&TableIdent::from_strs(["test_namespace", "test_table"]).unwrap()) - .await?; - let current_snapshot = updated_table.metadata().current_snapshot().unwrap(); - - // Load the manifest list to verify the data files were added - let manifest_list = current_snapshot - .load_manifest_list(updated_table.file_io(), updated_table.metadata()) - .await?; - - // There should be at least one manifest - assert!(!manifest_list.entries().is_empty()); - - // Load the first manifest and verify it contains our data files - let manifest = manifest_list.entries()[0] - .load_manifest(updated_table.file_io()) - .await?; - - // Verify that the manifest contains our data files - let manifest_files: Vec = manifest - .entries() - .iter() - .map(|entry| entry.data_file().file_path().to_string()) - .collect(); - - assert!(manifest_files.contains(&"path/to/file1.parquet".to_string())); - assert!(manifest_files.contains(&"path/to/file2.parquet".to_string())); - - Ok(()) - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs b/crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs deleted file mode 100644 index 2974dd2642..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/expr_to_predicate.rs +++ /dev/null @@ -1,432 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::vec; - -use datafusion::arrow::datatypes::DataType; -use datafusion::logical_expr::{Expr, Operator}; -use datafusion::scalar::ScalarValue; -use iceberg::expr::{BinaryExpression, Predicate, PredicateOperator, Reference, UnaryExpression}; -use iceberg::spec::Datum; - -// A datafusion expression could be an Iceberg predicate, column, or literal. -enum TransformedResult { - Predicate(Predicate), - Column(Reference), - Literal(Datum), - NotTransformed, -} - -enum OpTransformedResult { - Operator(PredicateOperator), - And, - Or, - NotTransformed, -} - -/// Converts DataFusion filters ([`Expr`]) to an iceberg [`Predicate`]. -/// If none of the filters could be converted, return `None` which adds no predicates to the scan operation. -/// If the conversion was successful, return the converted predicates combined with an AND operator. -pub fn convert_filters_to_predicate(filters: &[Expr]) -> Option { - filters - .iter() - .filter_map(convert_filter_to_predicate) - .reduce(Predicate::and) -} - -fn convert_filter_to_predicate(expr: &Expr) -> Option { - match to_iceberg_predicate(expr) { - TransformedResult::Predicate(predicate) => Some(predicate), - TransformedResult::Column(_) | TransformedResult::Literal(_) => { - unreachable!("Not a valid expression: {:?}", expr) - } - _ => None, - } -} - -fn to_iceberg_predicate(expr: &Expr) -> TransformedResult { - match expr { - Expr::BinaryExpr(binary) => { - let left = to_iceberg_predicate(&binary.left); - let right = to_iceberg_predicate(&binary.right); - let op = to_iceberg_operation(binary.op); - match op { - OpTransformedResult::Operator(op) => to_iceberg_binary_predicate(left, right, op), - OpTransformedResult::And => to_iceberg_and_predicate(left, right), - OpTransformedResult::Or => to_iceberg_or_predicate(left, right), - OpTransformedResult::NotTransformed => TransformedResult::NotTransformed, - } - } - Expr::Not(exp) => { - let expr = to_iceberg_predicate(exp); - match expr { - TransformedResult::Predicate(p) => TransformedResult::Predicate(!p), - _ => TransformedResult::NotTransformed, - } - } - Expr::Column(column) => TransformedResult::Column(Reference::new(column.name())), - Expr::Literal(literal, _) => match scalar_value_to_datum(literal) { - Some(data) => TransformedResult::Literal(data), - None => TransformedResult::NotTransformed, - }, - Expr::InList(inlist) => { - let mut datums = vec![]; - for expr in &inlist.list { - let p = to_iceberg_predicate(expr); - match p { - TransformedResult::Literal(l) => datums.push(l), - _ => return TransformedResult::NotTransformed, - } - } - - let expr = to_iceberg_predicate(&inlist.expr); - match expr { - TransformedResult::Column(r) => match inlist.negated { - false => TransformedResult::Predicate(r.is_in(datums)), - true => TransformedResult::Predicate(r.is_not_in(datums)), - }, - _ => TransformedResult::NotTransformed, - } - } - Expr::IsNull(expr) => { - let p = to_iceberg_predicate(expr); - match p { - TransformedResult::Column(r) => TransformedResult::Predicate(Predicate::Unary( - UnaryExpression::new(PredicateOperator::IsNull, r), - )), - _ => TransformedResult::NotTransformed, - } - } - Expr::IsNotNull(expr) => { - let p = to_iceberg_predicate(expr); - match p { - TransformedResult::Column(r) => TransformedResult::Predicate(Predicate::Unary( - UnaryExpression::new(PredicateOperator::NotNull, r), - )), - _ => TransformedResult::NotTransformed, - } - } - Expr::Cast(c) => { - if c.data_type == DataType::Date32 || c.data_type == DataType::Date64 { - // Casts to date truncate the expression, we cannot simply extract it as it - // can create erroneous predicates. - return TransformedResult::NotTransformed; - } - to_iceberg_predicate(&c.expr) - } - _ => TransformedResult::NotTransformed, - } -} - -fn to_iceberg_operation(op: Operator) -> OpTransformedResult { - match op { - Operator::Eq => OpTransformedResult::Operator(PredicateOperator::Eq), - Operator::NotEq => OpTransformedResult::Operator(PredicateOperator::NotEq), - Operator::Lt => OpTransformedResult::Operator(PredicateOperator::LessThan), - Operator::LtEq => OpTransformedResult::Operator(PredicateOperator::LessThanOrEq), - Operator::Gt => OpTransformedResult::Operator(PredicateOperator::GreaterThan), - Operator::GtEq => OpTransformedResult::Operator(PredicateOperator::GreaterThanOrEq), - // AND OR - Operator::And => OpTransformedResult::And, - Operator::Or => OpTransformedResult::Or, - // Others not supported - _ => OpTransformedResult::NotTransformed, - } -} - -fn to_iceberg_and_predicate( - left: TransformedResult, - right: TransformedResult, -) -> TransformedResult { - match (left, right) { - (TransformedResult::Predicate(left), TransformedResult::Predicate(right)) => { - TransformedResult::Predicate(left.and(right)) - } - (TransformedResult::Predicate(left), _) => TransformedResult::Predicate(left), - (_, TransformedResult::Predicate(right)) => TransformedResult::Predicate(right), - _ => TransformedResult::NotTransformed, - } -} - -fn to_iceberg_or_predicate(left: TransformedResult, right: TransformedResult) -> TransformedResult { - match (left, right) { - (TransformedResult::Predicate(left), TransformedResult::Predicate(right)) => { - TransformedResult::Predicate(left.or(right)) - } - _ => TransformedResult::NotTransformed, - } -} - -fn to_iceberg_binary_predicate( - left: TransformedResult, - right: TransformedResult, - op: PredicateOperator, -) -> TransformedResult { - let (r, d, op) = match (left, right) { - (TransformedResult::NotTransformed, _) => return TransformedResult::NotTransformed, - (_, TransformedResult::NotTransformed) => return TransformedResult::NotTransformed, - (TransformedResult::Column(r), TransformedResult::Literal(d)) => (r, d, op), - (TransformedResult::Literal(d), TransformedResult::Column(r)) => { - (r, d, reverse_predicate_operator(op)) - } - _ => return TransformedResult::NotTransformed, - }; - TransformedResult::Predicate(Predicate::Binary(BinaryExpression::new(op, r, d))) -} - -fn reverse_predicate_operator(op: PredicateOperator) -> PredicateOperator { - match op { - PredicateOperator::Eq => PredicateOperator::Eq, - PredicateOperator::NotEq => PredicateOperator::NotEq, - PredicateOperator::GreaterThan => PredicateOperator::LessThan, - PredicateOperator::GreaterThanOrEq => PredicateOperator::LessThanOrEq, - PredicateOperator::LessThan => PredicateOperator::GreaterThan, - PredicateOperator::LessThanOrEq => PredicateOperator::GreaterThanOrEq, - _ => unreachable!("Reverse {}", op), - } -} - -const MILLIS_PER_DAY: i64 = 24 * 60 * 60 * 1000; -/// Convert a scalar value to an iceberg datum. -fn scalar_value_to_datum(value: &ScalarValue) -> Option { - match value { - ScalarValue::Int8(Some(v)) => Some(Datum::int(*v as i32)), - ScalarValue::Int16(Some(v)) => Some(Datum::int(*v as i32)), - ScalarValue::Int32(Some(v)) => Some(Datum::int(*v)), - ScalarValue::Int64(Some(v)) => Some(Datum::long(*v)), - ScalarValue::Float32(Some(v)) => Some(Datum::double(*v as f64)), - ScalarValue::Float64(Some(v)) => Some(Datum::double(*v)), - ScalarValue::Utf8(Some(v)) => Some(Datum::string(v.clone())), - ScalarValue::LargeUtf8(Some(v)) => Some(Datum::string(v.clone())), - ScalarValue::Date32(Some(v)) => Some(Datum::date(*v)), - ScalarValue::Date64(Some(v)) => Some(Datum::date((*v / MILLIS_PER_DAY) as i32)), - _ => None, - } -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - - use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; - use datafusion::common::DFSchema; - use datafusion::logical_expr::utils::split_conjunction; - use datafusion::prelude::{Expr, SessionContext}; - use iceberg::expr::{Predicate, Reference}; - use iceberg::spec::Datum; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; - - use super::convert_filters_to_predicate; - - fn create_test_schema() -> DFSchema { - let arrow_schema = Schema::new(vec![ - Field::new("foo", DataType::Int32, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("bar", DataType::Utf8, true).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("ts", DataType::Timestamp(TimeUnit::Second, None), true).with_metadata( - HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())]), - ), - ]); - DFSchema::try_from_qualified_schema("my_table", &arrow_schema).unwrap() - } - - fn convert_to_iceberg_predicate(sql: &str) -> Option { - let df_schema = create_test_schema(); - let expr = SessionContext::new() - .parse_sql_expr(sql, &df_schema) - .unwrap(); - let exprs: Vec = split_conjunction(&expr).into_iter().cloned().collect(); - convert_filters_to_predicate(&exprs[..]) - } - - #[test] - fn test_predicate_conversion_with_single_condition() { - let predicate = convert_to_iceberg_predicate("foo = 1").unwrap(); - assert_eq!(predicate, Reference::new("foo").equal_to(Datum::long(1))); - - let predicate = convert_to_iceberg_predicate("foo != 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").not_equal_to(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo > 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").greater_than(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo >= 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").greater_than_or_equal_to(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo < 1").unwrap(); - assert_eq!(predicate, Reference::new("foo").less_than(Datum::long(1))); - - let predicate = convert_to_iceberg_predicate("foo <= 1").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").less_than_or_equal_to(Datum::long(1)) - ); - - let predicate = convert_to_iceberg_predicate("foo is null").unwrap(); - assert_eq!(predicate, Reference::new("foo").is_null()); - - let predicate = convert_to_iceberg_predicate("foo is not null").unwrap(); - assert_eq!(predicate, Reference::new("foo").is_not_null()); - - let predicate = convert_to_iceberg_predicate("foo in (5, 6)").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").is_in([Datum::long(5), Datum::long(6)]) - ); - - let predicate = convert_to_iceberg_predicate("foo not in (5, 6)").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").is_not_in([Datum::long(5), Datum::long(6)]) - ); - - let predicate = convert_to_iceberg_predicate("not foo = 1").unwrap(); - assert_eq!(predicate, !Reference::new("foo").equal_to(Datum::long(1))); - } - - #[test] - fn test_predicate_conversion_with_single_unsupported_condition() { - let predicate = convert_to_iceberg_predicate("foo + 1 = 1"); - assert_eq!(predicate, None); - - let predicate = convert_to_iceberg_predicate("length(bar) = 1"); - assert_eq!(predicate, None); - - let predicate = convert_to_iceberg_predicate("foo in (1, 2, foo)"); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_single_condition_rev() { - let predicate = convert_to_iceberg_predicate("1 < foo").unwrap(); - assert_eq!( - predicate, - Reference::new("foo").greater_than(Datum::long(1)) - ); - } - - #[test] - fn test_predicate_conversion_with_and_condition() { - let sql = "foo > 1 and bar = 'test'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::and( - Reference::new("foo").greater_than(Datum::long(1)), - Reference::new("bar").equal_to(Datum::string("test")), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_and_condition_unsupported() { - let sql = "foo > 1 and length(bar) = 1"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Reference::new("foo").greater_than(Datum::long(1)); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_and_condition_both_unsupported() { - let sql = "foo in (1, 2, foo) and length(bar) = 1"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_or_condition_unsupported() { - let sql = "foo > 1 or length(bar) = 1"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } - - #[test] - fn test_predicate_conversion_with_or_condition_supported() { - let sql = "foo > 1 or bar = 'test'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Predicate::or( - Reference::new("foo").greater_than(Datum::long(1)), - Reference::new("bar").equal_to(Datum::string("test")), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_complex_binary_expr() { - let sql = "(foo > 1 and bar = 'test') or foo < 0 "; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - - let inner_predicate = Predicate::and( - Reference::new("foo").greater_than(Datum::long(1)), - Reference::new("bar").equal_to(Datum::string("test")), - ); - let expected_predicate = Predicate::or( - inner_predicate, - Reference::new("foo").less_than(Datum::long(0)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_one_and_expr_supported() { - let sql = "(foo > 1 and length(bar) = 1 ) or foo < 0 "; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - - let inner_predicate = Reference::new("foo").greater_than(Datum::long(1)); - let expected_predicate = Predicate::or( - inner_predicate, - Reference::new("foo").less_than(Datum::long(0)), - ); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_complex_binary_expr_unsupported() { - let sql = "(foo > 1 or length(bar) = 1 ) and foo < 0 "; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = Reference::new("foo").less_than(Datum::long(0)); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_cast() { - let sql = "ts >= timestamp '2023-01-05T00:00:00'"; - let predicate = convert_to_iceberg_predicate(sql).unwrap(); - let expected_predicate = - Reference::new("ts").greater_than_or_equal_to(Datum::string("2023-01-05T00:00:00")); - assert_eq!(predicate, expected_predicate); - } - - #[test] - fn test_predicate_conversion_with_date_cast() { - let sql = "ts >= date '2023-01-05T11:00:00'"; - let predicate = convert_to_iceberg_predicate(sql); - assert_eq!(predicate, None); - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs b/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs deleted file mode 100644 index 9a9d0aa0d9..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use datafusion::catalog::TableProvider; -use datafusion::physical_expr::EquivalenceProperties; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; -use futures::TryStreamExt; - -use crate::metadata_table::IcebergMetadataTableProvider; - -#[derive(Debug)] -pub struct IcebergMetadataScan { - provider: IcebergMetadataTableProvider, - properties: PlanProperties, -} - -impl IcebergMetadataScan { - pub fn new(provider: IcebergMetadataTableProvider) -> Self { - let properties = PlanProperties::new( - EquivalenceProperties::new(provider.schema()), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - ); - Self { - provider, - properties, - } - } -} - -impl DisplayAs for IcebergMetadataScan { - fn fmt_as( - &self, - _t: datafusion::physical_plan::DisplayFormatType, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - write!(f, "IcebergMetadataScan") - } -} - -impl ExecutionPlan for IcebergMetadataScan { - fn name(&self) -> &str { - "IcebergMetadataScan" - } - - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn properties(&self) -> &PlanProperties { - &self.properties - } - - fn children(&self) -> Vec<&std::sync::Arc> { - vec![] - } - - fn with_new_children( - self: std::sync::Arc, - _children: Vec>, - ) -> datafusion::error::Result> { - Ok(self) - } - - fn execute( - &self, - _partition: usize, - _context: std::sync::Arc, - ) -> datafusion::error::Result { - let fut = self.provider.clone().scan(); - let stream = futures::stream::once(fut).try_flatten(); - let schema = self.provider.schema(); - Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) - } -} diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs deleted file mode 100644 index d4751a19c8..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ /dev/null @@ -1,217 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::any::Any; -use std::pin::Pin; -use std::sync::Arc; -use std::vec; - -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::error::Result as DFResult; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::EquivalenceProperties; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, ExecutionPlan, Partitioning, PlanProperties}; -use datafusion::prelude::Expr; -use futures::{Stream, TryStreamExt}; -use iceberg::expr::Predicate; -use iceberg::table::Table; - -use super::expr_to_predicate::convert_filters_to_predicate; -use crate::to_datafusion_error; - -/// Manages the scanning process of an Iceberg [`Table`], encapsulating the -/// necessary details and computed properties required for execution planning. -#[derive(Debug)] -pub struct IcebergTableScan { - /// A table in the catalog. - table: Table, - /// Snapshot of the table to scan. - snapshot_id: Option, - /// Stores certain, often expensive to compute, - /// plan properties used in query optimization. - plan_properties: PlanProperties, - /// Projection column names, None means all columns - projection: Option>, - /// Filters to apply to the table scan - predicates: Option, -} - -impl IcebergTableScan { - /// Creates a new [`IcebergTableScan`] object. - pub(crate) fn new( - table: Table, - snapshot_id: Option, - schema: ArrowSchemaRef, - projection: Option<&Vec>, - filters: &[Expr], - ) -> Self { - let output_schema = match projection { - None => schema.clone(), - Some(projection) => Arc::new(schema.project(projection).unwrap()), - }; - let plan_properties = Self::compute_properties(output_schema.clone()); - let projection = get_column_names(schema.clone(), projection); - let predicates = convert_filters_to_predicate(filters); - - Self { - table, - snapshot_id, - plan_properties, - projection, - predicates, - } - } - - pub fn table(&self) -> &Table { - &self.table - } - - pub fn snapshot_id(&self) -> Option { - self.snapshot_id - } - - pub fn projection(&self) -> Option<&[String]> { - self.projection.as_deref() - } - - pub fn predicates(&self) -> Option<&Predicate> { - self.predicates.as_ref() - } - - /// Computes [`PlanProperties`] used in query optimization. - fn compute_properties(schema: ArrowSchemaRef) -> PlanProperties { - // TODO: - // This is more or less a placeholder, to be replaced - // once we support output-partitioning - PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - ) - } -} - -impl ExecutionPlan for IcebergTableScan { - fn name(&self) -> &str { - "IcebergTableScan" - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn children(&self) -> Vec<&Arc<(dyn ExecutionPlan + 'static)>> { - vec![] - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> DFResult> { - Ok(self) - } - - fn properties(&self) -> &PlanProperties { - &self.plan_properties - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> DFResult { - let fut = get_batch_stream( - self.table.clone(), - self.snapshot_id, - self.projection.clone(), - self.predicates.clone(), - ); - let stream = futures::stream::once(fut).try_flatten(); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - stream, - ))) - } -} - -impl DisplayAs for IcebergTableScan { - fn fmt_as( - &self, - _t: datafusion::physical_plan::DisplayFormatType, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - write!( - f, - "IcebergTableScan projection:[{}] predicate:[{}]", - self.projection - .clone() - .map_or(String::new(), |v| v.join(",")), - self.predicates - .clone() - .map_or(String::from(""), |p| format!("{}", p)) - ) - } -} - -/// Asynchronously retrieves a stream of [`RecordBatch`] instances -/// from a given table. -/// -/// This function initializes a [`TableScan`], builds it, -/// and then converts it into a stream of Arrow [`RecordBatch`]es. -async fn get_batch_stream( - table: Table, - snapshot_id: Option, - column_names: Option>, - predicates: Option, -) -> DFResult> + Send>>> { - let scan_builder = match snapshot_id { - Some(snapshot_id) => table.scan().snapshot_id(snapshot_id), - None => table.scan(), - }; - - let mut scan_builder = match column_names { - Some(column_names) => scan_builder.select(column_names), - None => scan_builder.select_all(), - }; - if let Some(pred) = predicates { - scan_builder = scan_builder.with_filter(pred); - } - let table_scan = scan_builder.build().map_err(to_datafusion_error)?; - - let stream = table_scan - .to_arrow() - .await - .map_err(to_datafusion_error)? - .map_err(to_datafusion_error); - Ok(Box::pin(stream)) -} - -fn get_column_names( - schema: ArrowSchemaRef, - projection: Option<&Vec>, -) -> Option> { - projection.map(|v| { - v.iter() - .map(|p| schema.field(*p).name().clone()) - .collect::>() - }) -} diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs deleted file mode 100644 index 625405c95b..0000000000 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ /dev/null @@ -1,620 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::any::Any; -use std::fmt::{Debug, Formatter}; -use std::str::FromStr; -use std::sync::Arc; - -use datafusion::arrow::array::{ArrayRef, RecordBatch, StringArray}; -use datafusion::arrow::datatypes::{ - DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, -}; -use datafusion::common::Result as DFResult; -use datafusion::error::DataFusionError; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, - execute_input_stream, -}; -use futures::StreamExt; -use iceberg::arrow::{FieldMatchMode, schema_to_arrow_schema}; -use iceberg::spec::{ - DataFileFormat, PROPERTY_DEFAULT_FILE_FORMAT, PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT, - PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, - serialize_data_file_to_json, -}; -use iceberg::table::Table; -use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; -use iceberg::writer::file_writer::ParquetWriterBuilder; -use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, -}; -use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; -use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Error, ErrorKind}; -use parquet::file::properties::WriterProperties; -use uuid::Uuid; - -use crate::physical_plan::DATA_FILES_COL_NAME; -use crate::to_datafusion_error; - -/// An execution plan node that writes data to an Iceberg table. -/// -/// This execution plan takes input data from a child execution plan and writes it to an Iceberg table. -/// It handles the creation of data files in the appropriate format and returns information about the written files as its output. -/// -/// The output of this execution plan is a record batch containing a single column with serialized -/// data file information that can be used for committing the write operation to the table. -#[derive(Debug)] -pub(crate) struct IcebergWriteExec { - table: Table, - input: Arc, - result_schema: ArrowSchemaRef, - plan_properties: PlanProperties, -} - -impl IcebergWriteExec { - pub fn new(table: Table, input: Arc, schema: ArrowSchemaRef) -> Self { - let plan_properties = Self::compute_properties(&input, schema); - - Self { - table, - input, - result_schema: Self::make_result_schema(), - plan_properties, - } - } - - fn compute_properties( - input: &Arc, - schema: ArrowSchemaRef, - ) -> PlanProperties { - PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(input.output_partitioning().partition_count()), - EmissionType::Final, - Boundedness::Bounded, - ) - } - - // Create a record batch with serialized data files - fn make_result_batch(data_files: Vec) -> DFResult { - let files_array = Arc::new(StringArray::from(data_files)) as ArrayRef; - - RecordBatch::try_new(Self::make_result_schema(), vec![files_array]).map_err(|e| { - DataFusionError::ArrowError(e, Some("Failed to make result batch".to_string())) - }) - } - - fn make_result_schema() -> ArrowSchemaRef { - // Define a schema. - Arc::new(ArrowSchema::new(vec![Field::new( - DATA_FILES_COL_NAME, - DataType::Utf8, - false, - )])) - } -} - -impl DisplayAs for IcebergWriteExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default => { - write!(f, "IcebergWriteExec: table={}", self.table.identifier()) - } - DisplayFormatType::Verbose => { - write!( - f, - "IcebergWriteExec: table={}, result_schema={:?}", - self.table.identifier(), - self.result_schema - ) - } - DisplayFormatType::TreeRender => { - write!(f, "IcebergWriteExec: table={}", self.table.identifier()) - } - } - } -} - -impl ExecutionPlan for IcebergWriteExec { - fn name(&self) -> &str { - "IcebergWriteExec" - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn properties(&self) -> &PlanProperties { - &self.plan_properties - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.input] - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> DFResult> { - if children.len() != 1 { - return Err(DataFusionError::Internal(format!( - "IcebergWriteExec expects exactly one child, but provided {}", - children.len() - ))); - } - - Ok(Arc::new(Self::new( - self.table.clone(), - Arc::clone(&children[0]), - self.schema(), - ))) - } - - /// Executes the write operation for the given partition. - /// - /// This function: - /// 1. Sets up a data file writer based on the table's configuration - /// 2. Processes input data from the child execution plan - /// 3. Writes the data to files using the configured writer - /// 4. Returns a stream containing information about the written data files - /// - /// The output of this function is a stream of record batches with the following structure: - /// - /// ```text - /// +------------------+ - /// | data_files | - /// +------------------+ - /// | "{"file_path":.. | <- JSON string representing a data file - /// +------------------+ - /// ``` - /// - /// Each row in the output contains a JSON string representing a data file that was written. - /// - /// This output can be used by a subsequent operation to commit the added files to the table. - fn execute( - &self, - partition: usize, - context: Arc, - ) -> DFResult { - if !self - .table - .metadata() - .default_partition_spec() - .is_unpartitioned() - { - // TODO add support for partitioned tables - return Err(DataFusionError::NotImplemented( - "IcebergWriteExec does not support partitioned tables yet".to_string(), - )); - } - - let spec_id = self.table.metadata().default_partition_spec_id(); - let partition_type = self.table.metadata().default_partition_type().clone(); - let format_version = self.table.metadata().format_version(); - - // Check data file format - let file_format = DataFileFormat::from_str( - self.table - .metadata() - .properties() - .get(PROPERTY_DEFAULT_FILE_FORMAT) - .unwrap_or(&PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string()), - ) - .map_err(to_datafusion_error)?; - if file_format != DataFileFormat::Parquet { - return Err(to_datafusion_error(Error::new( - ErrorKind::FeatureUnsupported, - format!( - "File format {} is not supported for insert_into yet!", - file_format - ), - ))); - } - - // Create data file writer builder - let parquet_file_writer_builder = ParquetWriterBuilder::new_with_match_mode( - WriterProperties::default(), - self.table.metadata().current_schema().clone(), - None, - FieldMatchMode::Name, - self.table.file_io().clone(), - DefaultLocationGenerator::new(self.table.metadata().clone()) - .map_err(to_datafusion_error)?, - // todo filename prefix/suffix should be configurable - DefaultFileNameGenerator::new(Uuid::now_v7().to_string(), None, file_format), - ); - let target_file_size = match self - .table - .metadata() - .properties() - .get(PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES) - { - Some(value_str) => value_str - .parse::() - .map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - "Invalid value for write.target-file-size-bytes", - ) - .with_source(e) - }) - .map_err(to_datafusion_error)?, - None => PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, - }; - let rolling_writer_builder = - RollingFileWriterBuilder::new(parquet_file_writer_builder, target_file_size); - let data_file_writer_builder = - DataFileWriterBuilder::new(rolling_writer_builder, None, spec_id); - - // Get input data - let data = execute_input_stream( - Arc::clone(&self.input), - Arc::new( - schema_to_arrow_schema(self.table.metadata().current_schema()) - .map_err(to_datafusion_error)?, - ), - partition, - Arc::clone(&context), - )?; - - // Create write stream - let stream = futures::stream::once(async move { - let mut writer = data_file_writer_builder - .build() - .await - .map_err(to_datafusion_error)?; - let mut input_stream = data; - - while let Some(batch) = input_stream.next().await { - writer.write(batch?).await.map_err(to_datafusion_error)?; - } - - let data_files = writer.close().await.map_err(to_datafusion_error)?; - - // Convert builders to data files and then to JSON strings - let data_files_strs: Vec = data_files - .into_iter() - .map(|data_file| { - serialize_data_file_to_json(data_file, &partition_type, format_version) - .map_err(to_datafusion_error) - }) - .collect::>>()?; - - Self::make_result_batch(data_files_strs) - }) - .boxed(); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&self.result_schema), - stream, - ))) - } -} - -#[cfg(test)] -mod tests { - use std::any::Any; - use std::collections::HashMap; - use std::fmt::{Debug, Formatter}; - use std::sync::Arc; - - use datafusion::arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray}; - use datafusion::arrow::datatypes::{ - DataType, Field, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef, - }; - use datafusion::common::Result as DFResult; - use datafusion::execution::{SendableRecordBatchStream, TaskContext}; - use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; - use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; - use datafusion::physical_plan::stream::RecordBatchStreamAdapter; - use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; - use futures::{StreamExt, stream}; - use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; - use iceberg::spec::{ - DataFileFormat, NestedField, PrimitiveType, Schema, Type, deserialize_data_file_from_json, - }; - use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, Result, TableCreation}; - use parquet::arrow::PARQUET_FIELD_ID_META_KEY; - use tempfile::TempDir; - - use super::*; - - /// A simple execution plan that returns a predefined set of record batches - struct MockExecutionPlan { - schema: ArrowSchemaRef, - batches: Vec, - properties: PlanProperties, - } - - impl MockExecutionPlan { - fn new(schema: ArrowSchemaRef, batches: Vec) -> Self { - let properties = PlanProperties::new( - EquivalenceProperties::new(schema.clone()), - Partitioning::UnknownPartitioning(1), - EmissionType::Final, - Boundedness::Bounded, - ); - - Self { - schema, - batches, - properties, - } - } - } - - impl Debug for MockExecutionPlan { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "MockExecutionPlan") - } - } - - impl DisplayAs for MockExecutionPlan { - fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default - | DisplayFormatType::Verbose - | DisplayFormatType::TreeRender => { - write!(f, "MockExecutionPlan") - } - } - } - } - - impl ExecutionPlan for MockExecutionPlan { - fn name(&self) -> &str { - "MockExecutionPlan" - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn properties(&self) -> &PlanProperties { - &self.properties - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn with_new_children( - self: Arc, - _children: Vec>, - ) -> DFResult> { - Ok(self) - } - - fn execute( - &self, - _partition: usize, - _context: Arc, - ) -> DFResult { - let batches = self.batches.clone(); - let stream = stream::iter(batches.into_iter().map(Ok)); - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema.clone(), - stream.boxed(), - ))) - } - } - - /// Helper function to create a temporary directory and return its path - fn temp_path() -> String { - let temp_dir = TempDir::new().unwrap(); - temp_dir.path().to_str().unwrap().to_string() - } - - /// Helper function to create a memory catalog - async fn get_iceberg_catalog() -> MemoryCatalog { - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), temp_path())]), - ) - .await - .unwrap() - } - - /// Helper function to create a test table schema - fn get_test_schema() -> Result { - Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build() - } - - /// Helper function to create a table creation - fn get_table_creation( - location: impl ToString, - name: impl ToString, - schema: Schema, - ) -> TableCreation { - TableCreation::builder() - .location(location.to_string()) - .name(name.to_string()) - .properties(HashMap::new()) - .schema(schema) - .build() - } - - #[tokio::test] - async fn test_iceberg_write_exec() -> Result<()> { - // 1. Set up test environment - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_namespace".to_string()); - - // Create namespace - iceberg_catalog - .create_namespace(&namespace, HashMap::new()) - .await?; - - // Create schema - let schema = get_test_schema()?; - - // Create table - let table_name = "test_table"; - let table_location = temp_path(); - let creation = get_table_creation(table_location, table_name, schema); - let table = iceberg_catalog.create_table(&namespace, creation).await?; - - // 2. Create test data - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("name", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - ])); - - let id_array = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef; - let name_array = Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie"])) as ArrayRef; - - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_array, name_array]) - .map_err(|e| { - Error::new( - ErrorKind::Unexpected, - format!("Failed to create record batch: {}", e), - ) - })?; - - // 3. Create mock input execution plan - let input_plan = Arc::new(MockExecutionPlan::new(arrow_schema.clone(), vec![ - batch.clone(), - ])); - - // 4. Create IcebergWriteExec - let write_exec = IcebergWriteExec::new(table.clone(), input_plan, arrow_schema); - - // 5. Execute the plan - let task_ctx = Arc::new(TaskContext::default()); - let stream = write_exec.execute(0, task_ctx).map_err(|e| { - Error::new( - ErrorKind::Unexpected, - format!("Failed to execute plan: {}", e), - ) - })?; - - // Collect the results - let mut results = vec![]; - let mut stream = stream; - while let Some(batch) = stream.next().await { - results.push(batch.map_err(|e| { - Error::new(ErrorKind::Unexpected, format!("Failed to get batch: {}", e)) - })?); - } - - // 6. Verify the results - assert_eq!(results.len(), 1, "Expected one result batch"); - let result_batch = &results[0]; - - // Check schema - assert_eq!( - result_batch.schema().as_ref(), - &ArrowSchema::new(vec![Field::new(DATA_FILES_COL_NAME, DataType::Utf8, false)]) - ); - - // Check data - assert_eq!(result_batch.num_rows(), 1, "Expected one data file"); - - // Get the data file JSON - let data_file_json = result_batch - .column(0) - .as_any() - .downcast_ref::() - .expect("Expected StringArray") - .value(0); - - // Deserialize the data file JSON - let partition_type = table.metadata().default_partition_type(); - let spec_id = table.metadata().default_partition_spec_id(); - let schema = table.metadata().current_schema(); - - let data_file = - deserialize_data_file_from_json(data_file_json, spec_id, partition_type, schema) - .expect("Failed to deserialize data file JSON"); - - // Verify data file properties - assert_eq!( - data_file.record_count(), - 3, - "Expected 3 records in the data file" - ); - assert!( - data_file.file_size_in_bytes() > 0, - "File size should be greater than 0" - ); - assert_eq!( - data_file.file_format(), - DataFileFormat::Parquet, - "Expected Parquet file format" - ); - - // Verify column statistics - assert!( - data_file.column_sizes().get(&1).unwrap() > &0, - "Column 1 size should be greater than 0" - ); - assert!( - data_file.column_sizes().get(&2).unwrap() > &0, - "Column 2 size should be greater than 0" - ); - - assert_eq!( - *data_file.value_counts().get(&1).unwrap(), - 3, - "Expected 3 values for column 1" - ); - assert_eq!( - *data_file.value_counts().get(&2).unwrap(), - 3, - "Expected 3 values for column 2" - ); - - // Verify lower and upper bounds - assert!( - data_file.lower_bounds().contains_key(&1) || data_file.lower_bounds().contains_key(&2), - "Expected lower bounds to contain at least one column" - ); - assert!( - data_file.upper_bounds().contains_key(&1) || data_file.upper_bounds().contains_key(&2), - "Expected upper bounds to contain at least one column" - ); - - // Check that the file path exists - let file_path = data_file.file_path(); - assert!(!file_path.is_empty(), "File path should not be empty"); - - // 7. Verify the file exists - let file_io = table.file_io(); - assert!(file_io.exists(file_path).await?, "Data file should exist"); - - Ok(()) - } -} diff --git a/crates/integrations/datafusion/src/schema.rs b/crates/integrations/datafusion/src/schema.rs deleted file mode 100644 index 3920ee73ca..0000000000 --- a/crates/integrations/datafusion/src/schema.rs +++ /dev/null @@ -1,129 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::any::Any; -use std::collections::HashMap; -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::catalog::SchemaProvider; -use datafusion::datasource::TableProvider; -use datafusion::error::{DataFusionError, Result as DFResult}; -use futures::future::try_join_all; -use iceberg::inspect::MetadataTableType; -use iceberg::{Catalog, NamespaceIdent, Result}; - -use crate::table::IcebergTableProvider; - -/// Represents a [`SchemaProvider`] for the Iceberg [`Catalog`], managing -/// access to table providers within a specific namespace. -#[derive(Debug)] -pub(crate) struct IcebergSchemaProvider { - /// A `HashMap` where keys are table names - /// and values are dynamic references to objects implementing the - /// [`TableProvider`] trait. - tables: HashMap>, -} - -impl IcebergSchemaProvider { - /// Asynchronously tries to construct a new [`IcebergSchemaProvider`] - /// using the given client to fetch and initialize table providers for - /// the provided namespace in the Iceberg [`Catalog`]. - /// - /// This method retrieves a list of table names - /// attempts to create a table provider for each table name, and - /// collects these providers into a `HashMap`. - pub(crate) async fn try_new( - client: Arc, - namespace: NamespaceIdent, - ) -> Result { - // TODO: - // Tables and providers should be cached based on table_name - // if we have a cache miss; we update our internal cache & check again - // As of right now; tables might become stale. - let table_names: Vec<_> = client - .list_tables(&namespace) - .await? - .iter() - .map(|tbl| tbl.name().to_string()) - .collect(); - - let providers = try_join_all( - table_names - .iter() - .map(|name| IcebergTableProvider::try_new(client.clone(), namespace.clone(), name)) - .collect::>(), - ) - .await?; - - let tables: HashMap> = table_names - .into_iter() - .zip(providers.into_iter()) - .map(|(name, provider)| (name, Arc::new(provider))) - .collect(); - - Ok(IcebergSchemaProvider { tables }) - } -} - -#[async_trait] -impl SchemaProvider for IcebergSchemaProvider { - fn as_any(&self) -> &dyn Any { - self - } - - fn table_names(&self) -> Vec { - self.tables - .keys() - .flat_map(|table_name| { - [table_name.clone()] - .into_iter() - .chain(MetadataTableType::all_types().map(|metadata_table_name| { - format!("{}${}", table_name.clone(), metadata_table_name.as_str()) - })) - }) - .collect() - } - - fn table_exist(&self, name: &str) -> bool { - if let Some((table_name, metadata_table_name)) = name.split_once('$') { - self.tables.contains_key(table_name) - && MetadataTableType::try_from(metadata_table_name).is_ok() - } else { - self.tables.contains_key(name) - } - } - - async fn table(&self, name: &str) -> DFResult>> { - if let Some((table_name, metadata_table_name)) = name.split_once('$') { - let metadata_table_type = - MetadataTableType::try_from(metadata_table_name).map_err(DataFusionError::Plan)?; - if let Some(table) = self.tables.get(table_name) { - let metadata_table = table.metadata_table(metadata_table_type); - return Ok(Some(Arc::new(metadata_table))); - } else { - return Ok(None); - } - } - - Ok(self - .tables - .get(name) - .cloned() - .map(|t| t as Arc)) - } -} diff --git a/crates/integrations/datafusion/src/table/metadata_table.rs b/crates/integrations/datafusion/src/table/metadata_table.rs deleted file mode 100644 index 38148b4084..0000000000 --- a/crates/integrations/datafusion/src/table/metadata_table.rs +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::any::Any; -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::catalog::Session; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::Expr; -use datafusion::physical_plan::ExecutionPlan; -use futures::TryStreamExt; -use futures::stream::BoxStream; -use iceberg::arrow::schema_to_arrow_schema; -use iceberg::inspect::MetadataTableType; -use iceberg::table::Table; - -use crate::physical_plan::metadata_scan::IcebergMetadataScan; -use crate::to_datafusion_error; - -/// Represents a [`TableProvider`] for the Iceberg [`Catalog`], -/// managing access to a [`MetadataTable`]. -#[derive(Debug, Clone)] -pub struct IcebergMetadataTableProvider { - pub(crate) table: Table, - pub(crate) r#type: MetadataTableType, -} - -#[async_trait] -impl TableProvider for IcebergMetadataTableProvider { - fn as_any(&self) -> &dyn Any { - self - } - - fn schema(&self) -> ArrowSchemaRef { - let metadata_table = self.table.inspect(); - let schema = match self.r#type { - MetadataTableType::Snapshots => metadata_table.snapshots().schema(), - MetadataTableType::Manifests => metadata_table.manifests().schema(), - }; - schema_to_arrow_schema(&schema).unwrap().into() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - _projection: Option<&Vec>, - _filters: &[Expr], - _limit: Option, - ) -> DFResult> { - Ok(Arc::new(IcebergMetadataScan::new(self.clone()))) - } -} - -impl IcebergMetadataTableProvider { - pub async fn scan(self) -> DFResult>> { - let metadata_table = self.table.inspect(); - let stream = match self.r#type { - MetadataTableType::Snapshots => metadata_table.snapshots().scan().await, - MetadataTableType::Manifests => metadata_table.manifests().scan().await, - } - .map_err(to_datafusion_error)?; - let stream = stream.map_err(to_datafusion_error); - Ok(Box::pin(stream)) - } -} diff --git a/crates/integrations/datafusion/src/table/mod.rs b/crates/integrations/datafusion/src/table/mod.rs deleted file mode 100644 index a8c49837c3..0000000000 --- a/crates/integrations/datafusion/src/table/mod.rs +++ /dev/null @@ -1,306 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod metadata_table; -pub mod table_provider_factory; - -use std::any::Any; -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::arrow::datatypes::SchemaRef as ArrowSchemaRef; -use datafusion::catalog::Session; -use datafusion::common::DataFusionError; -use datafusion::datasource::{TableProvider, TableType}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::dml::InsertOp; -use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; -use datafusion::physical_plan::ExecutionPlan; -use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; -use iceberg::arrow::schema_to_arrow_schema; -use iceberg::inspect::MetadataTableType; -use iceberg::table::Table; -use iceberg::{Catalog, Error, ErrorKind, NamespaceIdent, Result, TableIdent}; -use metadata_table::IcebergMetadataTableProvider; - -use crate::physical_plan::commit::IcebergCommitExec; -use crate::physical_plan::scan::IcebergTableScan; -use crate::physical_plan::write::IcebergWriteExec; - -/// Represents a [`TableProvider`] for the Iceberg [`Catalog`], -/// managing access to a [`Table`]. -#[derive(Debug, Clone)] -pub struct IcebergTableProvider { - /// A table in the catalog. - table: Table, - /// Table snapshot id that will be queried via this provider. - snapshot_id: Option, - /// A reference-counted arrow `Schema`. - schema: ArrowSchemaRef, - /// The catalog that the table belongs to. - catalog: Option>, -} - -impl IcebergTableProvider { - pub(crate) fn new(table: Table, schema: ArrowSchemaRef) -> Self { - IcebergTableProvider { - table, - snapshot_id: None, - schema, - catalog: None, - } - } - /// Asynchronously tries to construct a new [`IcebergTableProvider`] - /// using the given client and table name to fetch an actual [`Table`] - /// in the provided namespace. - pub(crate) async fn try_new( - client: Arc, - namespace: NamespaceIdent, - name: impl Into, - ) -> Result { - let ident = TableIdent::new(namespace, name.into()); - let table = client.load_table(&ident).await?; - - let schema = Arc::new(schema_to_arrow_schema(table.metadata().current_schema())?); - - Ok(IcebergTableProvider { - table, - snapshot_id: None, - schema, - catalog: Some(client), - }) - } - - /// Asynchronously tries to construct a new [`IcebergTableProvider`] - /// using the given table. Can be used to create a table provider from an existing table regardless of the catalog implementation. - pub async fn try_new_from_table(table: Table) -> Result { - let schema = Arc::new(schema_to_arrow_schema(table.metadata().current_schema())?); - Ok(IcebergTableProvider { - table, - snapshot_id: None, - schema, - catalog: None, - }) - } - - /// Asynchronously tries to construct a new [`IcebergTableProvider`] - /// using a specific snapshot of the given table. Can be used to create a table provider from an existing table regardless of the catalog implementation. - pub async fn try_new_from_table_snapshot(table: Table, snapshot_id: i64) -> Result { - let snapshot = table - .metadata() - .snapshot_by_id(snapshot_id) - .ok_or_else(|| { - Error::new( - ErrorKind::Unexpected, - format!( - "snapshot id {snapshot_id} not found in table {}", - table.identifier().name() - ), - ) - })?; - let schema = snapshot.schema(table.metadata())?; - let schema = Arc::new(schema_to_arrow_schema(&schema)?); - Ok(IcebergTableProvider { - table, - snapshot_id: Some(snapshot_id), - schema, - catalog: None, - }) - } - - pub(crate) fn metadata_table(&self, r#type: MetadataTableType) -> IcebergMetadataTableProvider { - IcebergMetadataTableProvider { - table: self.table.clone(), - r#type, - } - } -} - -#[async_trait] -impl TableProvider for IcebergTableProvider { - fn as_any(&self) -> &dyn Any { - self - } - - fn schema(&self) -> ArrowSchemaRef { - self.schema.clone() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], - _limit: Option, - ) -> DFResult> { - Ok(Arc::new(IcebergTableScan::new( - self.table.clone(), - self.snapshot_id, - self.schema.clone(), - projection, - filters, - ))) - } - - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> DFResult> { - // Push down all filters, as a single source of truth, the scanner will drop the filters which couldn't be push down - Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) - } - - async fn insert_into( - &self, - _state: &dyn Session, - input: Arc, - _insert_op: InsertOp, - ) -> DFResult> { - if !self - .table - .metadata() - .default_partition_spec() - .is_unpartitioned() - { - // TODO add insert into support for partitioned tables - return Err(DataFusionError::NotImplemented( - "IcebergTableProvider::insert_into does not support partitioned tables yet" - .to_string(), - )); - } - - let Some(catalog) = self.catalog.clone() else { - return Err(DataFusionError::Execution( - "Catalog cannot be none for insert_into".to_string(), - )); - }; - - let write_plan = Arc::new(IcebergWriteExec::new( - self.table.clone(), - input, - self.schema.clone(), - )); - - // Merge the outputs of write_plan into one so we can commit all files together - let coalesce_partitions = Arc::new(CoalescePartitionsExec::new(write_plan)); - - Ok(Arc::new(IcebergCommitExec::new( - self.table.clone(), - catalog, - coalesce_partitions, - self.schema.clone(), - ))) - } -} - -#[cfg(test)] -mod tests { - use datafusion::common::Column; - use datafusion::prelude::SessionContext; - use iceberg::TableIdent; - use iceberg::io::FileIO; - use iceberg::table::{StaticTable, Table}; - - use super::*; - - async fn get_test_table_from_metadata_file() -> Table { - let metadata_file_name = "TableMetadataV2Valid.json"; - let metadata_file_path = format!( - "{}/tests/test_data/{}", - env!("CARGO_MANIFEST_DIR"), - metadata_file_name - ); - let file_io = FileIO::from_path(&metadata_file_path) - .unwrap() - .build() - .unwrap(); - let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap(); - let static_table = - StaticTable::from_metadata_file(&metadata_file_path, static_identifier, file_io) - .await - .unwrap(); - static_table.into_table() - } - - #[tokio::test] - async fn test_try_new_from_table() { - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - let df = ctx.sql("SELECT * FROM mytable").await.unwrap(); - let df_schema = df.schema(); - let df_columns = df_schema.fields(); - assert_eq!(df_columns.len(), 3); - let x_column = df_columns.first().unwrap(); - let column_data = format!( - "{:?}:{:?}", - x_column.name(), - x_column.data_type().to_string() - ); - assert_eq!(column_data, "\"x\":\"Int64\""); - let has_column = df_schema.has_column(&Column::from_name("z")); - assert!(has_column); - } - - #[tokio::test] - async fn test_try_new_from_table_snapshot() { - let table = get_test_table_from_metadata_file().await; - let snapshot_id = table.metadata().snapshots().next().unwrap().snapshot_id(); - let table_provider = - IcebergTableProvider::try_new_from_table_snapshot(table.clone(), snapshot_id) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - let df = ctx.sql("SELECT * FROM mytable").await.unwrap(); - let df_schema = df.schema(); - let df_columns = df_schema.fields(); - assert_eq!(df_columns.len(), 3); - let x_column = df_columns.first().unwrap(); - let column_data = format!( - "{:?}:{:?}", - x_column.name(), - x_column.data_type().to_string() - ); - assert_eq!(column_data, "\"x\":\"Int64\""); - let has_column = df_schema.has_column(&Column::from_name("z")); - assert!(has_column); - } - - #[tokio::test] - async fn test_physical_input_schema_consistent_with_logical_input_schema() { - let table = get_test_table_from_metadata_file().await; - let table_provider = IcebergTableProvider::try_new_from_table(table.clone()) - .await - .unwrap(); - let ctx = SessionContext::new(); - ctx.register_table("mytable", Arc::new(table_provider)) - .unwrap(); - let df = ctx.sql("SELECT count(*) FROM mytable").await.unwrap(); - let physical_plan = df.create_physical_plan().await; - assert!(physical_plan.is_ok()) - } -} diff --git a/crates/integrations/datafusion/src/table/table_provider_factory.rs b/crates/integrations/datafusion/src/table/table_provider_factory.rs deleted file mode 100644 index a6d3146e54..0000000000 --- a/crates/integrations/datafusion/src/table/table_provider_factory.rs +++ /dev/null @@ -1,304 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::borrow::Cow; -use std::collections::HashMap; -use std::sync::Arc; - -use async_trait::async_trait; -use datafusion::catalog::{Session, TableProvider, TableProviderFactory}; -use datafusion::error::Result as DFResult; -use datafusion::logical_expr::CreateExternalTable; -use datafusion::sql::TableReference; -use iceberg::arrow::schema_to_arrow_schema; -use iceberg::io::FileIO; -use iceberg::table::StaticTable; -use iceberg::{Error, ErrorKind, Result, TableIdent}; - -use super::IcebergTableProvider; -use crate::to_datafusion_error; - -/// A factory that implements DataFusion's `TableProviderFactory` to create `IcebergTableProvider` instances. -/// -/// # Example -/// -/// The following example demonstrates how to create an Iceberg external table using SQL in -/// a DataFusion session with `IcebergTableProviderFactory`: -/// -/// ``` -/// use std::sync::Arc; -/// -/// use datafusion::execution::session_state::SessionStateBuilder; -/// use datafusion::prelude::*; -/// use datafusion::sql::TableReference; -/// use iceberg_datafusion::IcebergTableProviderFactory; -/// -/// #[tokio::main] -/// async fn main() { -/// // Create a new session context -/// let mut state = SessionStateBuilder::new().with_default_features().build(); -/// -/// // Register the IcebergTableProviderFactory in the session -/// state.table_factories_mut().insert( -/// "ICEBERG".to_string(), -/// Arc::new(IcebergTableProviderFactory::new()), -/// ); -/// -/// let ctx = SessionContext::new_with_state(state); -/// -/// // Define the table reference and the location of the Iceberg metadata file -/// let table_ref = TableReference::bare("my_iceberg_table"); -/// // /path/to/iceberg/metadata -/// let metadata_file_path = format!( -/// "{}/testdata/table_metadata/{}", -/// env!("CARGO_MANIFEST_DIR"), -/// "TableMetadataV2.json" -/// ); -/// -/// // SQL command to create the Iceberg external table -/// let sql = format!( -/// "CREATE EXTERNAL TABLE {} STORED AS ICEBERG LOCATION '{}'", -/// table_ref, metadata_file_path -/// ); -/// -/// // Execute the SQL to create the external table -/// ctx.sql(&sql).await.expect("Failed to create table"); -/// -/// // Verify the table was created by retrieving the table provider -/// let table_provider = ctx -/// .table_provider(table_ref) -/// .await -/// .expect("Table not found"); -/// -/// println!("Iceberg external table created successfully."); -/// } -/// ``` -/// -/// # Note -/// This factory is designed to work with the DataFusion query engine, -/// specifically for handling Iceberg tables in external table commands. -/// Currently, this implementation supports only reading Iceberg tables, with -/// the creation of new tables not yet available. -/// -/// # Errors -/// An error will be returned if any unsupported feature, such as partition columns, -/// order expressions, constraints, or column defaults, is detected in the table creation command. -#[derive(Debug, Default)] -pub struct IcebergTableProviderFactory {} - -impl IcebergTableProviderFactory { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -impl TableProviderFactory for IcebergTableProviderFactory { - async fn create( - &self, - _state: &dyn Session, - cmd: &CreateExternalTable, - ) -> DFResult> { - check_cmd(cmd).map_err(to_datafusion_error)?; - - let table_name = &cmd.name; - let metadata_file_path = &cmd.location; - let options = &cmd.options; - - let table_name_with_ns = complement_namespace_if_necessary(table_name); - - let table = create_static_table(table_name_with_ns, metadata_file_path, options) - .await - .map_err(to_datafusion_error)? - .into_table(); - - let schema = schema_to_arrow_schema(table.metadata().current_schema()) - .map_err(to_datafusion_error)?; - - Ok(Arc::new(IcebergTableProvider::new(table, Arc::new(schema)))) - } -} - -fn check_cmd(cmd: &CreateExternalTable) -> Result<()> { - let CreateExternalTable { - schema, - table_partition_cols, - order_exprs, - constraints, - column_defaults, - .. - } = cmd; - - // Check if any of the fields violate the constraints in a single condition - let is_invalid = !schema.fields().is_empty() - || !table_partition_cols.is_empty() - || !order_exprs.is_empty() - || !constraints.is_empty() - || !column_defaults.is_empty(); - - if is_invalid { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "Currently we only support reading existing icebergs tables in external table command. To create new table, please use catalog provider.", - )); - } - - Ok(()) -} - -/// Complements the namespace of a table name if necessary. -/// -/// # Note -/// If the table name is a bare name, it will be complemented with the 'default' namespace. -/// Otherwise, it will be returned as is. Because Iceberg tables are always namespaced, but DataFusion -/// external table commands maybe not include the namespace, this function ensures that the namespace is always present. -/// -/// # See also -/// - [`iceberg::NamespaceIdent`] -/// - [`datafusion::sql::planner::SqlToRel::external_table_to_plan`] -fn complement_namespace_if_necessary(table_name: &TableReference) -> Cow<'_, TableReference> { - match table_name { - TableReference::Bare { table } => { - Cow::Owned(TableReference::partial("default", table.as_ref())) - } - other => Cow::Borrowed(other), - } -} - -async fn create_static_table( - table_name: Cow<'_, TableReference>, - metadata_file_path: &str, - props: &HashMap, -) -> Result { - let table_ident = TableIdent::from_strs(table_name.to_vec())?; - let file_io = FileIO::from_path(metadata_file_path)? - .with_props(props) - .build()?; - StaticTable::from_metadata_file(metadata_file_path, table_ident, file_io).await -} - -#[cfg(test)] -mod tests { - - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::catalog::TableProviderFactory; - use datafusion::common::{Constraints, DFSchema}; - use datafusion::execution::session_state::SessionStateBuilder; - use datafusion::logical_expr::CreateExternalTable; - use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; - use datafusion::prelude::SessionContext; - use datafusion::sql::TableReference; - - use super::*; - - fn table_metadata_v2_schema() -> Schema { - Schema::new(vec![ - Field::new("x", DataType::Int64, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "1".to_string(), - )])), - Field::new("y", DataType::Int64, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])), - Field::new("z", DataType::Int64, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "3".to_string(), - )])), - ]) - } - - fn table_metadata_location() -> String { - format!( - "{}/testdata/table_metadata/{}", - env!("CARGO_MANIFEST_DIR"), - "TableMetadataV2.json" - ) - } - - fn create_external_table_cmd() -> CreateExternalTable { - let metadata_file_path = table_metadata_location(); - - CreateExternalTable { - name: TableReference::partial("static_ns", "static_table"), - location: metadata_file_path, - schema: Arc::new(DFSchema::empty()), - file_type: "iceberg".to_string(), - options: Default::default(), - table_partition_cols: Default::default(), - order_exprs: Default::default(), - constraints: Constraints::empty(), - column_defaults: Default::default(), - if_not_exists: Default::default(), - temporary: false, - definition: Default::default(), - unbounded: Default::default(), - } - } - - #[tokio::test] - async fn test_schema_of_created_table() { - let factory = IcebergTableProviderFactory::new(); - - let state = SessionStateBuilder::new().build(); - let cmd = create_external_table_cmd(); - - let table_provider = factory - .create(&state, &cmd) - .await - .expect("create table failed"); - - let expected_schema = table_metadata_v2_schema(); - let actual_schema = table_provider.schema(); - - assert_eq!(actual_schema.as_ref(), &expected_schema); - } - - #[tokio::test] - async fn test_schema_of_created_external_table_sql() { - let mut state = SessionStateBuilder::new().with_default_features().build(); - state.table_factories_mut().insert( - "ICEBERG".to_string(), - Arc::new(IcebergTableProviderFactory::new()), - ); - let ctx = SessionContext::new_with_state(state); - - // All external tables in DataFusion use bare names. - // See https://github.com/apache/datafusion/blob/main/datafusion/sql/src/statement.rs#L1038-#L1039 - let table_ref = TableReference::bare("static_table"); - - // Create the external table - let sql = format!( - "CREATE EXTERNAL TABLE {} STORED AS ICEBERG LOCATION '{}'", - table_ref, - table_metadata_location() - ); - let _df = ctx.sql(&sql).await.expect("create table failed"); - - // Get the created external table - let table_provider = ctx - .table_provider(table_ref) - .await - .expect("table not found"); - - // Check the schema of the created table - let expected_schema = table_metadata_v2_schema(); - let actual_schema = table_provider.schema(); - - assert_eq!(actual_schema.as_ref(), &expected_schema); - } -} diff --git a/crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json b/crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json deleted file mode 100644 index a7b47217f5..0000000000 --- a/crates/integrations/datafusion/testdata/table_metadata/TableMetadataV2.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "format-version": 2, - "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", - "location": "s3://bucket/test/location", - "last-sequence-number": 34, - "last-updated-ms": 1602638573590, - "last-column-id": 3, - "current-schema-id": 1, - "schemas": [ - { - "type": "struct", - "schema-id": 0, - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - } - ] - }, - { - "type": "struct", - "schema-id": 1, - "identifier-field-ids": [ - 1, - 2 - ], - "fields": [ - { - "id": 1, - "name": "x", - "required": true, - "type": "long" - }, - { - "id": 2, - "name": "y", - "required": true, - "type": "long" - }, - { - "id": 3, - "name": "z", - "required": true, - "type": "long" - } - ] - } - ], - "default-spec-id": 0, - "partition-specs": [ - { - "spec-id": 0, - "fields": [ - { - "name": "x", - "transform": "identity", - "source-id": 1, - "field-id": 1000 - } - ] - } - ], - "last-partition-id": 1000, - "default-sort-order-id": 3, - "sort-orders": [ - { - "order-id": 3, - "fields": [ - { - "transform": "identity", - "source-id": 2, - "direction": "asc", - "null-order": "nulls-first" - }, - { - "transform": "bucket[4]", - "source-id": 3, - "direction": "desc", - "null-order": "nulls-last" - } - ] - } - ], - "properties": {}, - "current-snapshot-id": 3055729675574597004, - "snapshots": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770, - "sequence-number": 0, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/1.avro" - }, - { - "snapshot-id": 3055729675574597004, - "parent-snapshot-id": 3051729675574597004, - "timestamp-ms": 1555100955770, - "sequence-number": 1, - "summary": { - "operation": "append" - }, - "manifest-list": "s3://a/b/2.avro", - "schema-id": 1 - } - ], - "snapshot-log": [ - { - "snapshot-id": 3051729675574597004, - "timestamp-ms": 1515100955770 - }, - { - "snapshot-id": 3055729675574597004, - "timestamp-ms": 1555100955770 - } - ], - "metadata-log": [] -} \ No newline at end of file diff --git a/crates/integrations/datafusion/tests/integration_datafusion_test.rs b/crates/integrations/datafusion/tests/integration_datafusion_test.rs deleted file mode 100644 index cb4987a973..0000000000 --- a/crates/integrations/datafusion/tests/integration_datafusion_test.rs +++ /dev/null @@ -1,812 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for Iceberg Datafusion with Hive Metastore. - -use std::collections::HashMap; -use std::sync::Arc; -use std::vec; - -use datafusion::arrow::array::{Array, StringArray, UInt64Array}; -use datafusion::arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; -use datafusion::execution::context::SessionContext; -use datafusion::parquet::arrow::PARQUET_FIELD_ID_META_KEY; -use expect_test::expect; -use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; -use iceberg::spec::{NestedField, PrimitiveType, Schema, StructType, Type}; -use iceberg::test_utils::check_record_batches; -use iceberg::{Catalog, CatalogBuilder, MemoryCatalog, NamespaceIdent, Result, TableCreation}; -use iceberg_datafusion::IcebergCatalogProvider; -use tempfile::TempDir; - -fn temp_path() -> String { - let temp_dir = TempDir::new().unwrap(); - temp_dir.path().to_str().unwrap().to_string() -} - -async fn get_iceberg_catalog() -> MemoryCatalog { - MemoryCatalogBuilder::default() - .load( - "memory", - HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), temp_path())]), - ) - .await - .unwrap() -} - -fn get_struct_type() -> StructType { - StructType::new(vec![ - NestedField::required(4, "s_foo1", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(5, "s_foo2", Type::Primitive(PrimitiveType::String)).into(), - ]) -} - -async fn set_test_namespace(catalog: &MemoryCatalog, namespace: &NamespaceIdent) -> Result<()> { - let properties = HashMap::new(); - - catalog.create_namespace(namespace, properties).await?; - - Ok(()) -} - -fn get_table_creation( - location: impl ToString, - name: impl ToString, - schema: Option, -) -> Result { - let schema = match schema { - None => Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo1", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "foo2", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?, - Some(schema) => schema, - }; - - let creation = TableCreation::builder() - .location(location.to_string()) - .name(name.to_string()) - .properties(HashMap::new()) - .schema(schema) - .build(); - - Ok(creation) -} - -#[tokio::test] -async fn test_provider_plan_stream_schema() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_provider_get_table_schema".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let creation = get_table_creation(temp_path(), "my_table", None)?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_provider_get_table_schema").unwrap(); - - let table = schema.table("my_table").await.unwrap().unwrap(); - let table_schema = table.schema(); - - let expected = [("foo1", &DataType::Int32), ("foo2", &DataType::Utf8)]; - - for (field, exp) in table_schema.fields().iter().zip(expected.iter()) { - assert_eq!(field.name(), exp.0); - assert_eq!(field.data_type(), exp.1); - assert!(!field.is_nullable()) - } - - let df = ctx - .sql("select foo2 from catalog.test_provider_get_table_schema.my_table") - .await - .unwrap(); - - let task_ctx = Arc::new(df.task_ctx()); - let plan = df.create_physical_plan().await.unwrap(); - let stream = plan.execute(1, task_ctx).unwrap(); - - // Ensure both the plan and the stream conform to the same schema - assert_eq!(plan.schema(), stream.schema()); - assert_eq!( - stream.schema().as_ref(), - &ArrowSchema::new(vec![ - Field::new("foo2", DataType::Utf8, false).with_metadata(HashMap::from([( - PARQUET_FIELD_ID_META_KEY.to_string(), - "2".to_string(), - )])) - ]), - ); - - Ok(()) -} - -#[tokio::test] -async fn test_provider_list_table_names() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_provider_list_table_names".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let creation = get_table_creation(temp_path(), "my_table", None)?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_provider_list_table_names").unwrap(); - - let result = schema.table_names(); - - expect![[r#" - [ - "my_table", - "my_table$snapshots", - "my_table$manifests", - ] - "#]] - .assert_debug_eq(&result); - - Ok(()) -} - -#[tokio::test] -async fn test_provider_list_schema_names() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_provider_list_schema_names".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - let provider = ctx.catalog("catalog").unwrap(); - - let expected = ["test_provider_list_schema_names"]; - let result = provider.schema_names(); - - assert!( - expected - .iter() - .all(|item| result.contains(&item.to_string())) - ); - Ok(()) -} - -#[tokio::test] -async fn test_table_projection() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("ns".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo1", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "foo2", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "foo3", Type::Struct(get_struct_type())).into(), - ]) - .build()?; - let creation = get_table_creation(temp_path(), "t1", Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - let table_df = ctx.table("catalog.ns.t1").await.unwrap(); - - let records = table_df - .clone() - .explain(false, false) - .unwrap() - .collect() - .await - .unwrap(); - assert_eq!(1, records.len()); - let record = &records[0]; - // the first column is plan_type, the second column plan string. - let s = record - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(2, s.len()); - // the first row is logical_plan, the second row is physical_plan - assert!(s.value(1).contains("projection:[foo1,foo2,foo3]")); - - // datafusion doesn't support query foo3.s_foo1, use foo3 instead - let records = table_df - .select_columns(&["foo1", "foo3"]) - .unwrap() - .explain(false, false) - .unwrap() - .collect() - .await - .unwrap(); - assert_eq!(1, records.len()); - let record = &records[0]; - let s = record - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(2, s.len()); - assert!( - s.value(1) - .contains("IcebergTableScan projection:[foo1,foo3]") - ); - - Ok(()) -} - -#[tokio::test] -async fn test_table_predict_pushdown() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("ns".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - let creation = get_table_creation(temp_path(), "t1", Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - let records = ctx - .sql("select * from catalog.ns.t1 where (foo > 1 and length(bar) = 1 ) or bar is null") - .await - .unwrap() - .explain(false, false) - .unwrap() - .collect() - .await - .unwrap(); - assert_eq!(1, records.len()); - let record = &records[0]; - // the first column is plan_type, the second column plan string. - let s = record - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(2, s.len()); - // the first row is logical_plan, the second row is physical_plan - let expected = "predicate:[(foo > 1) OR (bar IS NULL)]"; - assert!(s.value(1).trim().contains(expected)); - Ok(()) -} - -#[tokio::test] -async fn test_metadata_table() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("ns".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - let creation = get_table_creation(temp_path(), "t1", Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - let snapshots = ctx - .sql("select * from catalog.ns.t1$snapshots") - .await - .unwrap() - .collect() - .await - .unwrap(); - check_record_batches( - snapshots, - expect![[r#" - Field { name: "committed_at", data_type: Timestamp(Microsecond, Some("+00:00")), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "1"} }, - Field { name: "snapshot_id", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "2"} }, - Field { name: "parent_id", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "3"} }, - Field { name: "operation", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "4"} }, - Field { name: "manifest_list", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "5"} }, - Field { name: "summary", data_type: Map(Field { name: "key_value", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "7"} }, Field { name: "value", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "8"} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "6"} }"#]], - expect![[r#" - committed_at: PrimitiveArray - [ - ], - snapshot_id: PrimitiveArray - [ - ], - parent_id: PrimitiveArray - [ - ], - operation: StringArray - [ - ], - manifest_list: StringArray - [ - ], - summary: MapArray - [ - ]"#]], - &[], - None, - ); - - let manifests = ctx - .sql("select * from catalog.ns.t1$manifests") - .await - .unwrap() - .collect() - .await - .unwrap(); - check_record_batches( - manifests, - expect![[r#" - Field { name: "content", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "14"} }, - Field { name: "path", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "1"} }, - Field { name: "length", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "2"} }, - Field { name: "partition_spec_id", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "3"} }, - Field { name: "added_snapshot_id", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "4"} }, - Field { name: "added_data_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "5"} }, - Field { name: "existing_data_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "6"} }, - Field { name: "deleted_data_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "7"} }, - Field { name: "added_delete_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "15"} }, - Field { name: "existing_delete_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "16"} }, - Field { name: "deleted_delete_files_count", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "17"} }, - Field { name: "partition_summaries", data_type: List(Field { name: "item", data_type: Struct([Field { name: "contains_null", data_type: Boolean, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "10"} }, Field { name: "contains_nan", data_type: Boolean, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "11"} }, Field { name: "lower_bound", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "12"} }, Field { name: "upper_bound", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "13"} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "9"} }), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "8"} }"#]], - expect![[r#" - content: PrimitiveArray - [ - ], - path: StringArray - [ - ], - length: PrimitiveArray - [ - ], - partition_spec_id: PrimitiveArray - [ - ], - added_snapshot_id: PrimitiveArray - [ - ], - added_data_files_count: PrimitiveArray - [ - ], - existing_data_files_count: PrimitiveArray - [ - ], - deleted_data_files_count: PrimitiveArray - [ - ], - added_delete_files_count: PrimitiveArray - [ - ], - existing_delete_files_count: PrimitiveArray - [ - ], - deleted_delete_files_count: PrimitiveArray - [ - ], - partition_summaries: ListArray - [ - ]"#]], - &[], - None, - ); - - Ok(()) -} - -#[tokio::test] -async fn test_insert_into() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_insert_into".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - - let creation = get_table_creation(temp_path(), "my_table", None)?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - // Verify table schema - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_insert_into").unwrap(); - let table = schema.table("my_table").await.unwrap().unwrap(); - let table_schema = table.schema(); - - let expected = [("foo1", &DataType::Int32), ("foo2", &DataType::Utf8)]; - for (field, exp) in table_schema.fields().iter().zip(expected.iter()) { - assert_eq!(field.name(), exp.0); - assert_eq!(field.data_type(), exp.1); - assert!(!field.is_nullable()) - } - - // Insert data into the table - let df = ctx - .sql("INSERT INTO catalog.test_insert_into.my_table VALUES (1, 'alan'), (2, 'turing')") - .await - .unwrap(); - - // Verify the insert operation result - let batches = df.collect().await.unwrap(); - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert!( - batch.num_rows() == 1 && batch.num_columns() == 1, - "Results should only have one row and one column that has the number of rows inserted" - ); - // Verify the number of rows inserted - let rows_inserted = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(rows_inserted.value(0), 2); - - // Refresh context to avoid getting stale table - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - ctx.register_catalog("catalog", catalog); - - // Query the table to verify the inserted data - let df = ctx - .sql("SELECT * FROM catalog.test_insert_into.my_table") - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Use check_record_batches to verify the data - check_record_batches( - batches, - expect![[r#" - Field { name: "foo1", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "1"} }, - Field { name: "foo2", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "2"} }"#]], - expect![[r#" - foo1: PrimitiveArray - [ - 1, - 2, - ], - foo2: StringArray - [ - "alan", - "turing", - ]"#]], - &[], - Some("foo1"), - ); - - Ok(()) -} - -fn get_nested_struct_type() -> StructType { - // Create a nested struct type with: - // - address: STRUCT - // - contact: STRUCT - StructType::new(vec![ - NestedField::optional( - 10, - "address", - Type::Struct(StructType::new(vec![ - NestedField::required(11, "street", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(12, "city", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(13, "zip", Type::Primitive(PrimitiveType::Int)).into(), - ])), - ) - .into(), - NestedField::optional( - 20, - "contact", - Type::Struct(StructType::new(vec![ - NestedField::optional(21, "email", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(22, "phone", Type::Primitive(PrimitiveType::String)).into(), - ])), - ) - .into(), - ]) -} - -#[tokio::test] -async fn test_insert_into_nested() -> Result<()> { - let iceberg_catalog = get_iceberg_catalog().await; - let namespace = NamespaceIdent::new("test_insert_nested".to_string()); - set_test_namespace(&iceberg_catalog, &namespace).await?; - let table_name = "nested_table"; - - // Create a schema with nested fields - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), - NestedField::optional(3, "profile", Type::Struct(get_nested_struct_type())).into(), - ]) - .build()?; - - // Create the table with the nested schema - let creation = get_table_creation(temp_path(), table_name, Some(schema))?; - iceberg_catalog.create_table(&namespace, creation).await?; - - let client = Arc::new(iceberg_catalog); - let catalog = Arc::new(IcebergCatalogProvider::try_new(client.clone()).await?); - - let ctx = SessionContext::new(); - ctx.register_catalog("catalog", catalog); - - // Verify table schema - let provider = ctx.catalog("catalog").unwrap(); - let schema = provider.schema("test_insert_nested").unwrap(); - let table = schema.table("nested_table").await.unwrap().unwrap(); - let table_schema = table.schema(); - - // Verify the schema has the expected structure - assert_eq!(table_schema.fields().len(), 3); - assert_eq!(table_schema.field(0).name(), "id"); - assert_eq!(table_schema.field(1).name(), "name"); - assert_eq!(table_schema.field(2).name(), "profile"); - assert!(matches!( - table_schema.field(2).data_type(), - DataType::Struct(_) - )); - - // In DataFusion, we need to use named_struct to create struct values - // Insert data with nested structs - let insert_sql = r#" - INSERT INTO catalog.test_insert_nested.nested_table - SELECT - 1 as id, - 'Alice' as name, - named_struct( - 'address', named_struct( - 'street', '123 Main St', - 'city', 'San Francisco', - 'zip', 94105 - ), - 'contact', named_struct( - 'email', 'alice@example.com', - 'phone', '555-1234' - ) - ) as profile - UNION ALL - SELECT - 2 as id, - 'Bob' as name, - named_struct( - 'address', named_struct( - 'street', '456 Market St', - 'city', 'San Jose', - 'zip', 95113 - ), - 'contact', named_struct( - 'email', 'bob@example.com', - 'phone', NULL - ) - ) as profile - "#; - - // Execute the insert - let df = ctx.sql(insert_sql).await.unwrap(); - let batches = df.collect().await.unwrap(); - - // Verify the insert operation result - assert_eq!(batches.len(), 1); - let batch = &batches[0]; - assert!(batch.num_rows() == 1 && batch.num_columns() == 1); - - // Verify the number of rows inserted - let rows_inserted = batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(rows_inserted.value(0), 2); - - // Refresh context to avoid getting stale table - let catalog = Arc::new(IcebergCatalogProvider::try_new(client).await?); - ctx.register_catalog("catalog", catalog); - - // Query the table to verify the inserted data - let df = ctx - .sql("SELECT * FROM catalog.test_insert_nested.nested_table ORDER BY id") - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Use check_record_batches to verify the data - check_record_batches( - batches, - expect![[r#" - Field { name: "id", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "1"} }, - Field { name: "name", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "2"} }, - Field { name: "profile", data_type: Struct([Field { name: "address", data_type: Struct([Field { name: "street", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "6"} }, Field { name: "city", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "7"} }, Field { name: "zip", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "8"} }]), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "4"} }, Field { name: "contact", data_type: Struct([Field { name: "email", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "9"} }, Field { name: "phone", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "10"} }]), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "5"} }]), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "3"} }"#]], - expect![[r#" - id: PrimitiveArray - [ - 1, - 2, - ], - name: StringArray - [ - "Alice", - "Bob", - ], - profile: StructArray - -- validity: - [ - valid, - valid, - ] - [ - -- child 0: "address" (Struct([Field { name: "street", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "6"} }, Field { name: "city", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "7"} }, Field { name: "zip", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "8"} }])) - StructArray - -- validity: - [ - valid, - valid, - ] - [ - -- child 0: "street" (Utf8) - StringArray - [ - "123 Main St", - "456 Market St", - ] - -- child 1: "city" (Utf8) - StringArray - [ - "San Francisco", - "San Jose", - ] - -- child 2: "zip" (Int32) - PrimitiveArray - [ - 94105, - 95113, - ] - ] - -- child 1: "contact" (Struct([Field { name: "email", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "9"} }, Field { name: "phone", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "10"} }])) - StructArray - -- validity: - [ - valid, - valid, - ] - [ - -- child 0: "email" (Utf8) - StringArray - [ - "alice@example.com", - "bob@example.com", - ] - -- child 1: "phone" (Utf8) - StringArray - [ - "555-1234", - null, - ] - ] - ]"#]], - &[], - Some("id"), - ); - - // Query with explicit field access to verify nested data - let df = ctx - .sql( - r#" - SELECT - id, - name, - profile.address.street, - profile.address.city, - profile.address.zip, - profile.contact.email, - profile.contact.phone - FROM catalog.test_insert_nested.nested_table - ORDER BY id - "#, - ) - .await - .unwrap(); - - let batches = df.collect().await.unwrap(); - - // Use check_record_batches to verify the flattened data - check_record_batches( - batches, - expect![[r#" - Field { name: "id", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "1"} }, - Field { name: "name", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "2"} }, - Field { name: "catalog.test_insert_nested.nested_table.profile[address][street]", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "6"} }, - Field { name: "catalog.test_insert_nested.nested_table.profile[address][city]", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "7"} }, - Field { name: "catalog.test_insert_nested.nested_table.profile[address][zip]", data_type: Int32, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "8"} }, - Field { name: "catalog.test_insert_nested.nested_table.profile[contact][email]", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "9"} }, - Field { name: "catalog.test_insert_nested.nested_table.profile[contact][phone]", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {"PARQUET:field_id": "10"} }"#]], - expect![[r#" - id: PrimitiveArray - [ - 1, - 2, - ], - name: StringArray - [ - "Alice", - "Bob", - ], - catalog.test_insert_nested.nested_table.profile[address][street]: StringArray - [ - "123 Main St", - "456 Market St", - ], - catalog.test_insert_nested.nested_table.profile[address][city]: StringArray - [ - "San Francisco", - "San Jose", - ], - catalog.test_insert_nested.nested_table.profile[address][zip]: PrimitiveArray - [ - 94105, - 95113, - ], - catalog.test_insert_nested.nested_table.profile[contact][email]: StringArray - [ - "alice@example.com", - "bob@example.com", - ], - catalog.test_insert_nested.nested_table.profile[contact][phone]: StringArray - [ - "555-1234", - null, - ]"#]], - &[], - Some("id"), - ); - - Ok(()) -} diff --git a/crates/integrations/playground/Cargo.toml b/crates/integrations/playground/Cargo.toml deleted file mode 100644 index 8ba983d058..0000000000 --- a/crates/integrations/playground/Cargo.toml +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -description = "Apache iceberg client" -edition.workspace = true -homepage.workspace = true -license.workspace = true -name = "iceberg-playground" -readme = "README.md" -repository.workspace = true -rust-version.workspace = true -version.workspace = true - -[dependencies] -anyhow = { workspace = true } -clap = { workspace = true } -datafusion = { workspace = true } -datafusion-cli = { workspace = true } -dirs = { workspace = true } -fs-err = { workspace = true } -iceberg = { workspace = true } -iceberg-catalog-rest = { workspace = true } -iceberg-datafusion = { workspace = true } -mimalloc = { workspace = true } -stacker = { workspace = true } -tokio = { workspace = true } -toml = { workspace = true } -tracing = { workspace = true } -tracing-subscriber = { workspace = true } - -[package.metadata.cargo-machete] -# These dependencies are added to ensure minimal dependency version -ignored = ["stacker", "mimalloc"] diff --git a/crates/integrations/playground/DEPENDENCIES.rust.tsv b/crates/integrations/playground/DEPENDENCIES.rust.tsv deleted file mode 100644 index 84dd20ed3e..0000000000 --- a/crates/integrations/playground/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,509 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 CDLA-Permissive-2.0 ISC LGPL-2.1-or-later MIT MIT-0 MPL-2.0 Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -adler32@1.2.0 X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anstream@0.6.20 X X -anstyle@1.0.11 X X -anstyle-parse@0.2.7 X X -anstyle-query@1.1.4 X X -anstyle-wincon@3.0.10 X X -anyhow@1.0.99 X X -apache-avro@0.17.0 X -apache-avro@0.20.0 X -array-init@2.1.0 X X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@55.2.0 X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-csv@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-json@55.2.0 X -arrow-ord@55.2.0 X -arrow-row@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -as-any@0.3.2 X X -async-compression@0.4.19 X X -async-lock@3.4.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.0 X X -aws-config@1.8.6 X -aws-credential-types@1.2.6 X -aws-runtime@1.5.10 X -aws-sdk-sso@1.83.0 X -aws-sdk-ssooidc@1.84.0 X -aws-sdk-sts@1.85.0 X -aws-sigv4@1.3.4 X -aws-smithy-async@1.2.5 X -aws-smithy-http@0.62.3 X -aws-smithy-http-client@1.1.1 X -aws-smithy-json@0.61.5 X -aws-smithy-observability@0.1.3 X -aws-smithy-query@0.60.7 X -aws-smithy-runtime@1.9.1 X -aws-smithy-runtime-api@1.9.0 X -aws-smithy-types@1.3.2 X -aws-smithy-xml@0.60.10 X -aws-types@1.3.8 X -backon@1.5.2 X -backtrace@0.3.75 X X -base64@0.22.1 X X -base64-simd@0.8.0 X -bigdecimal@0.4.8 X X -bimap@0.6.3 X X -bitflags@2.9.4 X X -blake2@0.10.6 X X -blake3@1.8.2 X X X -block-buffer@0.10.4 X X -bon@3.7.2 X X -bon-macros@3.7.2 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -bytemuck@1.23.2 X X X -byteorder@1.5.0 X X -bytes@1.10.1 X -bytes-utils@0.1.4 X X -bzip2@0.4.4 X X -bzip2@0.5.2 X X -bzip2-sys@0.1.13+1.0.8 X X -cc@1.2.36 X X -cfg-if@1.0.3 X X -cfg_aliases@0.2.1 X -chrono@0.4.42 X X -chrono-tz@0.10.4 X X -clap@4.5.47 X X -clap_builder@4.5.47 X X -clap_derive@4.5.47 X X -clap_lex@0.7.5 X X -clipboard-win@5.4.1 X -colorchoice@1.0.4 X X -comfy-table@7.2.0 X -concurrent-queue@2.5.0 X X -const-oid@0.9.6 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.3.1 X X X -core-foundation@0.10.1 X X -core-foundation-sys@0.8.7 X X -core2@0.4.0 X X -cpufeatures@0.2.17 X X -crc32c@0.6.8 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.15 X X -crossbeam-epoch@0.9.18 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -csv@1.3.1 X X -csv-core@0.1.12 X X -darling@0.20.11 X -darling@0.21.3 X -darling_core@0.20.11 X -darling_core@0.21.3 X -darling_macro@0.20.11 X -darling_macro@0.21.3 X -dary_heap@0.3.7 X X -dashmap@6.1.0 X -datafusion@48.0.1 X -datafusion-catalog@48.0.1 X -datafusion-catalog-listing@48.0.1 X -datafusion-cli@48.0.1 X -datafusion-common@48.0.1 X -datafusion-common-runtime@48.0.1 X -datafusion-datasource@48.0.1 X -datafusion-datasource-avro@48.0.1 X -datafusion-datasource-csv@48.0.1 X -datafusion-datasource-json@48.0.1 X -datafusion-datasource-parquet@48.0.1 X -datafusion-doc@48.0.1 X -datafusion-execution@48.0.1 X -datafusion-expr@48.0.1 X -datafusion-expr-common@48.0.1 X -datafusion-functions@48.0.1 X -datafusion-functions-aggregate@48.0.1 X -datafusion-functions-aggregate-common@48.0.1 X -datafusion-functions-nested@48.0.1 X -datafusion-functions-table@48.0.1 X -datafusion-functions-window@48.0.1 X -datafusion-functions-window-common@48.0.1 X -datafusion-macros@48.0.1 X -datafusion-optimizer@48.0.1 X -datafusion-physical-expr@48.0.1 X -datafusion-physical-expr-common@48.0.1 X -datafusion-physical-optimizer@48.0.1 X -datafusion-physical-plan@48.0.1 X -datafusion-session@48.0.1 X -datafusion-sql@48.0.1 X -deranged@0.5.3 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -digest@0.10.7 X X -dirs@6.0.0 X X -dirs-sys@0.5.0 X X -displaydoc@0.2.5 X X -dissimilar@1.0.10 X -either@1.15.0 X X -endian-type@0.1.2 X -env_filter@0.1.3 X X -env_logger@0.11.8 X X -equivalent@1.0.2 X X -errno@0.3.13 X X -error-code@3.3.2 X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastrand@2.3.0 X X -fd-lock@4.0.4 X X -find-msvc-tools@0.1.1 X X -fixedbitset@0.5.7 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -form_urlencoded@1.2.2 X X -fs-err@3.1.1 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generator@0.8.7 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -glob@0.3.3 X X -gloo-timers@0.3.0 X X -h2@0.4.12 X -half@2.6.0 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hermit-abi@0.5.2 X X -hex@0.4.3 X X -hmac@0.12.1 X X -home@0.5.11 X X -http@0.2.12 X X -http@1.3.1 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -humantime@2.2.0 X X -hyper@1.7.0 X -hyper-rustls@0.27.7 X X X -hyper-util@0.1.16 X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.7.0 X -iceberg-catalog-rest@0.7.0 X -iceberg-datafusion@0.7.0 X -iceberg-playground@0.7.0 X -iceberg_test_utils@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -ipnet@2.11.0 X X -iri-string@0.7.8 X X -is_terminal_polyfill@1.70.1 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.15 X X -jiff@0.2.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lazy_static@1.5.0 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libflate@2.1.0 X -libflate_lz77@2.1.0 X -libm@0.2.15 X -libmimalloc-sys@0.1.44 X -libredox@0.1.9 X -libz-rs-sys@0.5.2 X -linux-raw-sys@0.9.4 X X X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -loom@0.7.2 X -lz4_flex@0.11.5 X -lzma-sys@0.1.20 X X -matchers@0.2.0 X -md-5@0.10.6 X X -memchr@2.7.5 X X -mimalloc@0.1.48 X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -moka@0.12.10 X X -murmur3@0.5.2 X X -nibble_vec@0.1.0 X -nix@0.30.1 X -nu-ansi-term@0.50.1 X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-conv@0.1.0 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -num_cpus@1.17.0 X X -object@0.36.7 X X -object_store@0.12.3 X X -once_cell@1.21.3 X X -once_cell_polyfill@1.70.1 X X -opendal@0.54.0 X -openssl-probe@0.1.6 X X -option-ext@0.2.0 X -ordered-float@2.10.1 X -ordered-float@4.6.0 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -percent-encoding@2.3.2 X X -petgraph@0.8.2 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -portable-atomic-util@0.2.4 X X -potential_utf@0.1.3 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -psm@0.1.26 X X -quad-rand@0.2.3 X -quick-xml@0.37.5 X -quick-xml@0.38.3 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -radix_trie@0.2.1 X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.17 X -redox_users@0.5.2 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -reqsign@0.16.5 X -reqwest@0.12.23 X X -ring@0.17.14 X X -rle-decode-fast@1.0.3 X X -roaring@0.11.2 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustix@1.0.8 X X X -rustls@0.23.31 X X X -rustls-native-certs@0.8.1 X X X -rustls-pemfile@2.2.0 X X X -rustls-pki-types@1.12.0 X X -rustls-webpki@0.103.4 X -rustversion@1.0.22 X X -rustyline@16.0.0 X -ryu@1.0.20 X X -same-file@1.0.6 X X -schannel@0.1.27 X -scoped-tls@1.0.1 X X -scopeguard@1.2.0 X X -security-framework@3.4.0 X X -security-framework-sys@2.15.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_json@1.0.143 X X -serde_repr@0.1.20 X X -serde_spanned@0.6.9 X X -serde_urlencoded@0.7.1 X X -serde_with@3.14.0 X X -serde_with_macros@3.14.0 X X -sha1@0.10.6 X X -sha2@0.10.9 X X -sharded-slab@0.1.7 X -shlex@1.3.0 X X -signal-hook-registry@1.4.6 X X -simdutf8@0.1.5 X X -siphasher@1.0.1 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -socket2@0.6.0 X X -sqlparser@0.55.0 X -sqlparser_derive@0.3.0 X -stable_deref_trait@1.2.0 X X -stacker@0.1.21 X X -static_assertions@1.1.0 X X -strsim@0.11.1 X -strum@0.26.3 X -strum@0.27.2 X -strum_macros@0.26.4 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.106 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -tempfile@3.22.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thread_local@1.1.9 X X -threadpool@1.8.1 X X -thrift@0.17.0 X -time@0.3.43 X X -time-core@0.1.6 X X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-rustls@0.26.2 X X -tokio-util@0.7.16 X -toml@0.8.23 X X -toml_datetime@0.6.11 X X -toml_edit@0.22.27 X X -toml_write@0.1.2 X X -tower@0.5.2 X -tower-http@0.6.6 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.19.1 X X -typed-builder@0.20.1 X X -typed-builder-macro@0.19.1 X X -typed-builder-macro@0.20.1 X X -typenum@1.18.0 X X -unicode-ident@1.0.18 X X X -unicode-segmentation@1.12.0 X X -unicode-width@0.2.1 X X -untrusted@0.9.0 X -url@2.5.7 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -utf8parse@0.2.2 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -vsimd@0.8.0 X -walkdir@2.5.0 X X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -wasm-streams@0.4.2 X X -web-sys@0.3.78 X X -web-time@1.1.0 X X -webpki-roots@1.0.2 X -winapi-util@0.1.11 X X -windows@0.61.3 X X -windows-collections@0.2.0 X X -windows-core@0.61.2 X X -windows-future@0.2.1 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-numerics@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.0 X X -windows-targets@0.52.6 X X -windows-targets@0.53.3 X X -windows-threading@0.1.0 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.0 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.0 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.0 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.0 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.0 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.0 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.0 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.0 X X -winnow@0.7.13 X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -xmlparser@0.13.6 X X -xz2@0.1.7 X X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zeroize@1.8.1 X X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/integrations/playground/README.md b/crates/integrations/playground/README.md deleted file mode 100644 index c471724598..0000000000 --- a/crates/integrations/playground/README.md +++ /dev/null @@ -1,25 +0,0 @@ - - - -# Introduction - -Iceberg Playground (`iceberg-playground`) is a small command line utility that runs SQL queries against tables, -which is backed by the DataFusion engine. - diff --git a/crates/integrations/playground/src/catalog.rs b/crates/integrations/playground/src/catalog.rs deleted file mode 100644 index f7fd14ca84..0000000000 --- a/crates/integrations/playground/src/catalog.rs +++ /dev/null @@ -1,131 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::any::Any; -use std::collections::HashMap; -use std::path::Path; -use std::sync::Arc; - -use anyhow::anyhow; -use datafusion::catalog::{CatalogProvider, CatalogProviderList}; -use fs_err::read_to_string; -use iceberg::CatalogBuilder; -use iceberg_catalog_rest::RestCatalogBuilder; -use iceberg_datafusion::IcebergCatalogProvider; -use toml::{Table as TomlTable, Value}; - -const CONFIG_NAME_CATALOGS: &str = "catalogs"; - -#[derive(Debug)] -pub struct IcebergCatalogList { - catalogs: HashMap>, -} - -impl IcebergCatalogList { - pub async fn parse(path: &Path) -> anyhow::Result { - let toml_table: TomlTable = toml::from_str(&read_to_string(path)?)?; - Self::parse_table(&toml_table).await - } - - pub async fn parse_table(configs: &TomlTable) -> anyhow::Result { - if let Value::Array(catalogs_config) = - configs.get(CONFIG_NAME_CATALOGS).ok_or_else(|| { - anyhow::Error::msg(format!("{CONFIG_NAME_CATALOGS} entry not found in config")) - })? - { - let mut catalogs = HashMap::with_capacity(catalogs_config.len()); - for config in catalogs_config { - if let Value::Table(table_config) = config { - let (name, catalog_provider) = - IcebergCatalogList::parse_one(table_config).await?; - catalogs.insert(name, catalog_provider); - } else { - return Err(anyhow!("{CONFIG_NAME_CATALOGS} entry must be a table")); - } - } - Ok(Self { catalogs }) - } else { - Err(anyhow!("{CONFIG_NAME_CATALOGS} must be an array of table!")) - } - } - - async fn parse_one( - config: &TomlTable, - ) -> anyhow::Result<(String, Arc)> { - let name = config - .get("name") - .ok_or_else(|| anyhow::anyhow!("name not found for catalog"))? - .as_str() - .ok_or_else(|| anyhow::anyhow!("name is not string"))?; - - let r#type = config - .get("type") - .ok_or_else(|| anyhow::anyhow!("type not found for catalog"))? - .as_str() - .ok_or_else(|| anyhow::anyhow!("type is not string"))?; - - if r#type != "rest" { - return Err(anyhow::anyhow!("Only rest catalog is supported for now!")); - } - - let catalog_config = config - .get("config") - .ok_or_else(|| anyhow::anyhow!("config not found for catalog {name}"))? - .as_table() - .ok_or_else(|| anyhow::anyhow!("config is not table for catalog {name}"))?; - - // parse all config into props - let mut props = HashMap::with_capacity(catalog_config.len()); - for (key, value) in catalog_config { - let value_str = value - .as_str() - .ok_or_else(|| anyhow::anyhow!("props {key} is not string"))?; - props.insert(key.to_string(), value_str.to_string()); - } - let catalog = RestCatalogBuilder::default().load(name, props).await?; - - Ok(( - name.to_string(), - Arc::new(IcebergCatalogProvider::try_new(Arc::new(catalog)).await?), - )) - } -} - -impl CatalogProviderList for IcebergCatalogList { - fn as_any(&self) -> &dyn Any { - self - } - - fn register_catalog( - &self, - _name: String, - _catalog: Arc, - ) -> Option> { - tracing::error!("Registering catalog is not supported yet"); - None - } - - fn catalog_names(&self) -> Vec { - self.catalogs.keys().cloned().collect() - } - - fn catalog(&self, name: &str) -> Option> { - self.catalogs - .get(name) - .map(|c| c.clone() as Arc) - } -} diff --git a/crates/integrations/playground/src/main.rs b/crates/integrations/playground/src/main.rs deleted file mode 100644 index 6f3de1bc17..0000000000 --- a/crates/integrations/playground/src/main.rs +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::path::PathBuf; -use std::process::ExitCode; -use std::str::FromStr; -use std::sync::Arc; - -use clap::Parser; -use datafusion::execution::runtime_env::RuntimeEnvBuilder; -use datafusion::prelude::{SessionConfig, SessionContext}; -use datafusion_cli::exec; -use datafusion_cli::print_format::PrintFormat; -use datafusion_cli::print_options::{MaxRows, PrintOptions}; -use iceberg_playground::{ICEBERG_PLAYGROUND_VERSION, IcebergCatalogList}; - -#[derive(Debug, Parser, PartialEq)] -#[clap(author, version, about, long_about= None)] -struct Args { - #[clap( - short = 'r', - long, - help = "Parse catalog config instead of using ~/.icebergrc" - )] - rc: Option, - - #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)] - format: PrintFormat, - - #[clap( - short, - long, - help = "Reduce printing other than the results and work quietly" - )] - quiet: bool, - - #[clap( - long, - help = "The max number of rows to display for 'Table' format\n[possible values: numbers(0/10/...), inf(no limit)]", - default_value = "40" - )] - maxrows: MaxRows, - - #[clap(long, help = "Enables console syntax highlighting")] - color: bool, -} - -#[tokio::main] -/// Calls [`main_inner`], then handles printing errors and returning the correct exit code -pub async fn main() -> ExitCode { - tracing_subscriber::fmt::init(); - - if let Err(e) = main_inner().await { - println!("Error: {e}"); - return ExitCode::FAILURE; - } - - ExitCode::SUCCESS -} - -async fn main_inner() -> anyhow::Result<()> { - let args = Args::parse(); - - if !args.quiet { - println!("ICEBERG PLAYGROUND v{}", ICEBERG_PLAYGROUND_VERSION); - } - - let session_config = SessionConfig::from_env()?.with_information_schema(true); - - let rt_builder = RuntimeEnvBuilder::new(); - - let runtime_env = rt_builder.build_arc()?; - - // enable dynamic file query - let ctx = SessionContext::new_with_config_rt(session_config, runtime_env).enable_url_table(); - ctx.refresh_catalogs().await?; - - let mut print_options = PrintOptions { - format: args.format, - quiet: args.quiet, - maxrows: args.maxrows, - color: args.color, - }; - - let rc = match args.rc { - Some(file) => PathBuf::from_str(&file)?, - None => dirs::home_dir() - .map(|h| h.join(".icebergrc")) - .ok_or_else(|| anyhow::anyhow!("cannot find home directory"))?, - }; - - let catalogs = Arc::new(IcebergCatalogList::parse(&rc).await?); - ctx.register_catalog_list(catalogs); - - Ok(exec::exec_from_repl(&ctx, &mut print_options).await?) -} diff --git a/crates/integrations/datafusion/Cargo.toml b/crates/property-macro/Cargo.toml similarity index 62% rename from crates/integrations/datafusion/Cargo.toml rename to crates/property-macro/Cargo.toml index 0ee1738b4f..d48872d490 100644 --- a/crates/integrations/datafusion/Cargo.toml +++ b/crates/property-macro/Cargo.toml @@ -18,27 +18,34 @@ [package] edition = { workspace = true } homepage = { workspace = true } -name = "iceberg-datafusion" +name = "iceberg-property-macro" +publish = true +readme = "README.md" rust-version = { workspace = true } version = { workspace = true } -categories = ["database"] -description = "Apache Iceberg DataFusion Integration" -keywords = ["iceberg", "integrations", "datafusion"] license = { workspace = true } repository = { workspace = true } +categories = ["database"] +description = "Apache Iceberg Rust property macros, internal to the iceberg crate" +keywords = ["iceberg"] + +[lib] +proc-macro = true + [dependencies] -anyhow = { workspace = true } -async-trait = { workspace = true } -datafusion = { workspace = true } -futures = { workspace = true } -iceberg = { workspace = true } -parquet = { workspace = true } -tokio = { workspace = true } -uuid = { workspace = true } +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true, features = ["full"] } [dev-dependencies] -expect-test = { workspace = true } -parquet = { workspace = true } -tempfile = { workspace = true } +# Path-only so cargo strips it when packaging; a versioned dev-dependency would +# create a publish cycle, since `iceberg` depends on this crate. +iceberg = { path = "../iceberg" } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +trybuild = { workspace = true } + +[lints] +workspace = true diff --git a/crates/property-macro/LICENSE b/crates/property-macro/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/crates/property-macro/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/property-macro/NOTICE b/crates/property-macro/NOTICE new file mode 120000 index 0000000000..fb376cfaa4 --- /dev/null +++ b/crates/property-macro/NOTICE @@ -0,0 +1 @@ +../../NOTICE \ No newline at end of file diff --git a/crates/property-macro/README.md b/crates/property-macro/README.md new file mode 100644 index 0000000000..1bcb124263 --- /dev/null +++ b/crates/property-macro/README.md @@ -0,0 +1,354 @@ + + +# Iceberg property macros + +This crate contains the property macros used internally by +[`iceberg`](https://crates.io/crates/iceberg). It is published only because +`iceberg` depends on it and has no API stability guarantees, so depend on +`iceberg` rather than on this crate. + +## `Properties` derive macro + +`#[derive(Properties)]` parses an owned typed struct from a flat +`HashMap` and can generate opt-in read-only getters. It +deliberately does not generate property-map serialization or implement +`Default`, `Serialize`, `Deserialize`, or any other trait. + +### Generated API + +For every annotated struct, `#[derive(Properties)]` generates this inherent +constructor: + +```text +impl MyProperties { + pub fn from_properties( + properties: &HashMap, + ) -> iceberg::Result; +} +``` + +`from_properties` borrows the source map, parses every modeled property, and +uses its annotated default when a property is absent. Unknown keys are ignored. +An invalid value returns an `iceberg::Error` with `ErrorKind::DataInvalid` and a +message containing its primary property key. + +Adding `getter` to a field generates a public immutable accessor with the field +name. Primitive `Copy` types, references, pointers, and compositions of those +types return `T`; other types return `&T`. Because a procedural macro cannot +resolve trait implementations, a user-defined `Copy` type returns `&T`. +Documentation attributes on the field are copied to the generated getter. The +macro generates no setters, backing fields, or conversion back to a property +map. + +`Option` key fields and `HashMap` prefix fields must use the +corresponding types from `std`. The macro recognizes those field shapes +syntactically and generates code using the standard-library variants. + +### Complete example + +This example covers exact keys and defaults, optional values, case-insensitive +booleans, prefixed maps, nested groups, custom single-value parsing, custom +multi-key parsing, lists of additional keys, read-only getters, ignored unknown +keys, and contextual errors. + +```rust +use std::collections::HashMap; + +use iceberg::{Error, ErrorKind}; +use iceberg_property_macro::Properties; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const FANOUT: &str = "write.datafusion.fanout.enabled"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const LOCATION: &str = "write.data.path"; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; +const DEPTH: &str = "dimensions.depth"; + +fn parse_location(value: &str) -> iceberg::Result { + let location = value.trim().trim_end_matches('/'); + if location.is_empty() { + Err(Error::new( + ErrorKind::DataInvalid, + "location must not be empty", + )) + } else { + Ok(location.to_string()) + } +} + +fn parse_dimensions( + properties: &HashMap, + width_key: &str, + additional_keys: &[&str], + default: (u64, u64, u64), +) -> iceberg::Result<(u64, u64, u64)> { + if additional_keys.len() != 2 { + return Err(Error::new( + ErrorKind::DataInvalid, + "dimensions require height and depth keys", + )); + } + let parse = |key: &str, default| { + properties + .get(key) + .map(|value| { + value.parse::().map_err(|error| { + Error::new(ErrorKind::DataInvalid, error.to_string()) + }) + }) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok(( + parse(width_key, default.0)?, + parse(additional_keys[0], default.1)?, + parse(additional_keys[1], default.2)?, + )) +} + +#[derive(Debug, Properties)] +struct CommitProperties { + /// Maximum number of times to retry a commit. + #[property(key = RETRIES, default = 4, getter)] + retries: usize, +} + +#[derive(Debug, Properties)] +struct TableLikeProperties { + /// Nested groups parse from the same flat property map. + #[property(nested, getter)] + commit: CommitProperties, + + /// Option distinguishes an absent property from a present value. + #[property(key = OWNER, default = None, getter)] + owner: Option, + + /// This DataFusion-specific boolean is parsed case-insensitively. + /// Its `true` default is engine-specific, rather than an Iceberg-wide default. + #[property(key = FANOUT, default = true, getter)] + fanout_enabled: bool, + + /// A prefix captures suffix/value pairs into a typed map. + #[property(prefix = COLUMN_FPP_PREFIX, getter)] + column_fpp: HashMap, + + /// An explicitly configured data path override. + /// `None` means callers should use `/data`. + /// A single-key parser validates and normalizes configured values. + #[property( + key = LOCATION, + default = None, + parse_with = parse_location, + getter + )] + location: Option, + + /// A full-map parser can model one field with multiple property keys. + #[property( + key = WIDTH, + additional_keys = [HEIGHT, DEPTH], + default = (640, 480, 320), + parse_properties_with = parse_dimensions, + getter + )] + dimensions: (u64, u64, u64), +} + +fn main() -> iceberg::Result<()> { + let defaults = TableLikeProperties::from_properties(&HashMap::new())?; + assert_eq!(defaults.commit().retries(), 4); + assert_eq!(defaults.owner(), &None); + assert!(defaults.fanout_enabled()); + assert!(defaults.column_fpp().is_empty()); + assert_eq!(defaults.location(), &None); + assert_eq!(defaults.dimensions(), (640, 480, 320)); + + let raw = HashMap::from([ + (RETRIES.to_string(), "8".to_string()), + (OWNER.to_string(), "iceberg".to_string()), + (FANOUT.to_string(), "FALSE".to_string()), + (format!("{COLUMN_FPP_PREFIX}id"), "0.01".to_string()), + (LOCATION.to_string(), " s3://bucket/table/ ".to_string()), + (WIDTH.to_string(), "1920".to_string()), + (HEIGHT.to_string(), "1080".to_string()), + (DEPTH.to_string(), "720".to_string()), + ("unmodeled".to_string(), "ignored".to_string()), + ]); + + let properties = TableLikeProperties::from_properties(&raw)?; + assert_eq!(properties.commit().retries(), 8); + assert_eq!(properties.owner().as_deref(), Some("iceberg")); + assert!(!properties.fanout_enabled()); + assert_eq!(properties.column_fpp()["id"], 0.01); + assert_eq!( + properties.location().as_deref(), + Some("s3://bucket/table") + ); + assert_eq!(properties.dimensions(), (1920, 1080, 720)); + + let error = TableLikeProperties::from_properties(&HashMap::from([( + LOCATION.to_string(), + "/".to_string(), + )])) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(format!("{error}").contains(LOCATION)); + + Ok(()) +} +``` + +### Using ordinary derives together + +`Properties` does not implicitly derive other traits, so `Default`, +`Serialize`, and `Deserialize` can be selected independently and behave like +ordinary Rust derives: + +```rust +use std::collections::HashMap; + +use iceberg_property_macro::Properties; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Default, Serialize, Deserialize, Properties)] +struct ReadProperties { + #[property( + key = "commit.retry.num-retries", + default = 4, + getter + )] + retries: u64, +} + +fn main() -> Result<(), Box> { + // The property annotation supplies the default used by from_properties. + let properties = ReadProperties::from_properties(&HashMap::new())?; + assert_eq!(properties.retries(), 4); + + // The ordinary Default derive uses the field's Rust default instead. + let defaults = ReadProperties::default(); + assert_eq!(defaults.retries(), 0); + + // Ordinary Serde derives use Rust field names, not property keys. + let json = serde_json::to_string(&properties)?; + assert_eq!(json, r#"{"retries":4}"#); + let decoded: ReadProperties = serde_json::from_str(r#"{"retries":7}"#)?; + assert_eq!(decoded.retries(), 7); + Ok(()) +} +``` + +### Property declarations + +All field settings must be grouped under `#[property(...)]`. This keeps `key`, +`default`, `prefix`, `nested`, parser hooks, additional keys, and getter +generation in one attribute and avoids collisions with ordinary Rust derives. + +Exact-key fields require a `default`. The `prefix` setting requires +`HashMap` and returns the entries matched by the prefix, or an empty +map when none match. `nested` embeds another `Properties` struct while reading +the same flat map. Neither prefix nor nested fields accept a `default`. +`parse_with` customizes parsing for one exact-key field; on an `Option` field, +the parser produces `T` and the macro wraps a present value in `Some`. +`parse_properties_with` receives the complete property map, and its required +`additional_keys` setting supplies the list of secondary keys. + +Boolean values are parsed case-insensitively. Other values require `FromStr` +unless a custom parser is supplied. String-literal and path defaults are +converted into their field type with `Into`. + +`from_properties` and both custom parser hooks use `iceberg::Result`. Generated +`FromStr` failures use `ErrorKind::DataInvalid`. The macro preserves errors from +custom parsers and adds the primary property key as error context. + +## `properties_view!` function-like macro + +`properties_view!` independently defines a lightweight borrowed view over a +flat property map. Its struct-shaped fields are declarations used to generate +getters rather than stored fields. Constructing the view does not parse +anything; each generated getter parses and returns only its declared property. +Missing properties use their annotated defaults, while invalid configured +values return an error from the corresponding getter. + +```rust +use std::collections::HashMap; + +use iceberg_property_macro::properties_view; + +properties_view! { + #[derive(Debug)] + pub struct WriteProperties { + // `getter` makes a generated getter public. Without it, the getter is + // private, regardless of the declaration's visibility modifier. + + /// Number of times to retry a commit. + #[property(key = "commit.retry.num-retries", default = 4, getter)] + commit_num_retries: usize, + + /// Optional configured base directory for metadata files. + #[property(key = "write.metadata.path", default = None)] + raw_write_metadata_path: Option, + } +} + +impl WriteProperties<'_> { + /// Returns the configured metadata path or the table's metadata directory. + pub fn write_metadata_path(&self, table_path: &str) -> iceberg::Result { + Ok(self.raw_write_metadata_path()?.unwrap_or_else(|| { + format!("{}/metadata", table_path.trim_end_matches('/')) + })) + } +} + +fn read(properties: &HashMap) -> iceberg::Result<()> { + let view = WriteProperties::new(properties); + let _retries = view.commit_num_retries()?; + let _metadata_path = view.write_metadata_path("s3://bucket/table")?; + Ok(()) +} +``` + +For this declaration, the macro generates +`impl<'properties> WriteProperties<'properties>` with a +`pub fn new(properties: &'properties HashMap) -> Self` +constructor. The constructor has the same visibility as the view declaration: +`pub` here, `pub(crate)` for a `pub(crate)` view, and private for a private view. +It only stores the map reference and does not parse any properties. + +The generated type has one lifetime parameter and contains only a reference to +the source `HashMap`; no declared field is stored in the view. Every declaration +generates a getter, but only the `getter` option makes that method public. Field +visibility modifiers are ignored, consistent with how `Properties` uses +`getter` to control generated accessors. Omitting `getter` makes it possible to +keep a raw generated method private and expose a public method that adds +application-specific behavior, as shown by `write_metadata_path` above. + +Non-`Copy` values are returned by value because the view does not retain parsed +results, so calling the same getter repeatedly reparses that field. + +The view supports exact keys, optional values, case-insensitive booleans, +prefixed maps, nested views, `parse_with`, and `parse_properties_with` using the +same property declaration syntax as `Properties`. A nested field's type must be +another view type, such as `CommitProperties<'_>`. The nested getter returns a +view tied directly to the source map's lifetime, so it remains usable after the +parent view is dropped. diff --git a/crates/property-macro/public-api.txt b/crates/property-macro/public-api.txt new file mode 100644 index 0000000000..23e6c068d0 --- /dev/null +++ b/crates/property-macro/public-api.txt @@ -0,0 +1,3 @@ +pub mod iceberg_property_macro +pub proc macro iceberg_property_macro::#[derive(Properties)] +pub proc macro iceberg_property_macro::properties_view!() diff --git a/crates/property-macro/src/lib.rs b/crates/property-macro/src/lib.rs new file mode 100644 index 0000000000..e2e3d871da --- /dev/null +++ b/crates/property-macro/src/lib.rs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![doc = include_str!("../README.md")] + +use proc_macro::TokenStream; +use syn::{DeriveInput, ItemStruct, parse_macro_input}; + +mod properties; +mod properties_view; + +/// Derives property-map parsing and opt-in read-only accessors for a struct. +#[proc_macro_derive(Properties, attributes(property))] +pub fn derive_properties(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + + match properties::expand_properties(input) { + Ok(tokens) => tokens.into(), + Err(error) => error.into_compile_error().into(), + } +} + +/// Defines a borrowed view that parses properties independently through generated getters. +#[proc_macro] +pub fn properties_view(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as ItemStruct); + + match properties_view::expand_properties_view(input) { + Ok(tokens) => tokens.into(), + Err(error) => error.into_compile_error().into(), + } +} diff --git a/crates/property-macro/src/properties.rs b/crates/property-macro/src/properties.rs new file mode 100644 index 0000000000..abedd9de56 --- /dev/null +++ b/crates/property-macro/src/properties.rs @@ -0,0 +1,674 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::{ + Attribute, Data, DeriveInput, Error, Expr, ExprLit, ExprPath, Field, Fields, GenericArgument, + Ident, Lit, Path, PathArguments, Token, Type, +}; + +pub(crate) struct PropertyField { + pub(crate) ident: Ident, + pub(crate) ty: Type, + key: Option, + additional_keys: Option>, + prefix: Option, + pub(crate) nested: bool, + default: Option, + parse_with: Option, + parse_properties_with: Option, + option_inner_type: Option, + map_value_type: Option, + pub(crate) public_getter: bool, + pub(crate) doc_attributes: Vec, +} + +enum PropertyOption { + Key(Expr), + AdditionalKeys(Vec), + Prefix(Expr), + Nested, + Default(Expr), + ParseWith(Path), + ParsePropertiesWith(Path), + Getter, +} + +#[derive(Default)] +pub(crate) struct PropertyOptions { + key: Option, + additional_keys: Option>, + prefix: Option, + nested: bool, + default: Option, + parse_with: Option, + parse_properties_with: Option, + public_getter: bool, +} + +impl Parse for PropertyOption { + fn parse(input: ParseStream<'_>) -> syn::Result { + let name = input.parse::()?; + let option_name = name.to_string(); + if option_name == "nested" { + return Ok(Self::Nested); + } + if option_name == "getter" { + return Ok(Self::Getter); + } + + input.parse::()?; + let expression = input.parse::()?; + match option_name.as_str() { + "key" => Ok(Self::Key(expression)), + "additional_keys" => { + expression_list(expression, "additional_keys").map(Self::AdditionalKeys) + } + "prefix" => Ok(Self::Prefix(expression)), + "default" => Ok(Self::Default(expression)), + "parse_with" => expression_path(expression, "parse_with").map(Self::ParseWith), + "parse_properties_with" => { + expression_path(expression, "parse_properties_with").map(Self::ParsePropertiesWith) + } + _ => Err(Error::new_spanned(name, "unknown property option")), + } + } +} + +pub(crate) fn expand_properties(input: DeriveInput) -> syn::Result { + let struct_name = input.ident; + let generics = input.generics; + let fields = match input.data { + Data::Struct(data) => match data.fields { + Fields::Named(fields) => fields.named, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs with named fields", + )); + } + }, + _ => { + return Err(Error::new_spanned( + struct_name, + "Properties can only be derived for structs", + )); + } + }; + + let fields = fields + .iter() + .map(|field| parse_property_field(field, property_options(field)?)) + .collect::>>()?; + let parses = fields + .iter() + .map(parse_field) + .collect::>>()?; + let accessors = fields.iter().map(field_getter); + let (impl_generics, type_generics, where_clause) = generics.split_for_impl(); + + Ok(quote! { + impl #impl_generics #struct_name #type_generics #where_clause { + #(#accessors)* + + /// Parses this typed property set from a flat string-to-string map. + pub fn from_properties( + properties: &::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + >, + ) -> ::iceberg::Result { + Ok(Self { + #(#parses,)* + }) + } + } + }) +} + +pub(crate) fn parse_property_field( + field: &Field, + property_options: PropertyOptions, +) -> syn::Result { + let ident = field + .ident + .clone() + .ok_or_else(|| Error::new_spanned(field, "Properties fields must be named"))?; + let PropertyOptions { + key, + additional_keys, + prefix, + nested, + default, + parse_with, + parse_properties_with, + public_getter, + } = property_options; + + if usize::from(key.is_some()) + usize::from(prefix.is_some()) + usize::from(nested) != 1 { + return Err(Error::new_spanned( + field, + "Properties fields must declare exactly one of key, prefix, or nested in #[property(...)]", + )); + } + + if nested && default.is_some() { + return Err(Error::new_spanned( + field, + "nested fields obtain defaults from their own property annotations and cannot declare default in #[property(...)]", + )); + } + if prefix.is_some() && default.is_some() { + return Err(Error::new_spanned( + field, + "prefix fields collect matching properties and cannot declare default in #[property(...)]", + )); + } + if key.is_some() && default.is_none() { + return Err(Error::new_spanned( + field, + "Properties key fields must declare default in #[property(...)]", + )); + } + + let map_value_type = hash_map_value_type(&field.ty); + if prefix.is_some() && map_value_type.is_none() { + return Err(Error::new_spanned( + &field.ty, + "property prefix fields must have type HashMap", + )); + } + + if additional_keys.is_some() && parse_properties_with.is_none() { + return Err(Error::new_spanned( + field, + "additional_keys requires parse_properties_with in #[property(...)]", + )); + } + if parse_properties_with.is_some() && additional_keys.is_none() { + return Err(Error::new_spanned( + field, + "parse_properties_with requires additional_keys in #[property(...)]", + )); + } + if (prefix.is_some() || nested) + && (additional_keys.is_some() || parse_with.is_some() || parse_properties_with.is_some()) + { + return Err(Error::new_spanned( + field, + "prefix and nested fields do not support custom parse functions", + )); + } + if parse_with.is_some() && parse_properties_with.is_some() { + return Err(Error::new_spanned( + field, + "fields cannot declare both parse_with and parse_properties_with", + )); + } + Ok(PropertyField { + ident, + ty: field.ty.clone(), + key, + additional_keys, + prefix, + nested, + default, + parse_with, + parse_properties_with, + option_inner_type: option_inner_type(&field.ty), + map_value_type, + public_getter, + doc_attributes: field + .attrs + .iter() + .filter(|attribute| attribute.path().is_ident("doc")) + .cloned() + .collect(), + }) +} + +pub(crate) fn property_options(field: &Field) -> syn::Result { + let Some(attribute) = find_attribute(&field.attrs, "property")? else { + return Err(Error::new_spanned( + field, + "Properties fields must declare #[property(...)]", + )); + }; + + let parsed = + attribute.parse_args_with(Punctuated::::parse_terminated)?; + if parsed.is_empty() { + return Err(Error::new_spanned( + attribute, + "property must declare at least one option", + )); + } + + let mut options = PropertyOptions::default(); + for option in parsed { + match option { + PropertyOption::Key(value) => { + set_property_option(&mut options.key, value, attribute, "key")? + } + PropertyOption::AdditionalKeys(value) => set_property_option( + &mut options.additional_keys, + value, + attribute, + "additional_keys", + )?, + PropertyOption::Prefix(value) => { + set_property_option(&mut options.prefix, value, attribute, "prefix")? + } + PropertyOption::Nested => { + if options.nested { + return Err(Error::new_spanned( + attribute, + "duplicate nested property option", + )); + } + options.nested = true; + } + PropertyOption::Default(value) => { + set_property_option(&mut options.default, value, attribute, "default")? + } + PropertyOption::ParseWith(value) => { + set_property_option(&mut options.parse_with, value, attribute, "parse_with")? + } + PropertyOption::ParsePropertiesWith(value) => set_property_option( + &mut options.parse_properties_with, + value, + attribute, + "parse_properties_with", + )?, + PropertyOption::Getter => { + if options.public_getter { + return Err(Error::new_spanned(attribute, "duplicate property accessor")); + } + options.public_getter = true; + } + } + } + + Ok(options) +} + +fn set_property_option( + target: &mut Option, + value: T, + attribute: &Attribute, + name: &str, +) -> syn::Result<()> { + if target.is_some() { + return Err(Error::new_spanned( + attribute, + format!("duplicate {name} property option"), + )); + } + *target = Some(value); + Ok(()) +} + +fn field_getter(field: &PropertyField) -> TokenStream2 { + if !field.public_getter { + return TokenStream2::new(); + } + let ident = &field.ident; + let ty = &field.ty; + let docs = &field.doc_attributes; + if is_copy_type(ty) { + quote! { + #(#docs)* + pub fn #ident(&self) -> #ty { + self.#ident + } + } + } else { + quote! { + #(#docs)* + pub fn #ident(&self) -> &#ty { + &self.#ident + } + } + } +} + +fn expression_path(expression: Expr, name: &str) -> syn::Result { + match expression { + Expr::Path(ExprPath { path, .. }) => Ok(path), + _ => Err(Error::new_spanned( + expression, + format!("{name} must be a path"), + )), + } +} + +fn expression_list(expression: Expr, name: &str) -> syn::Result> { + let Expr::Array(array) = expression else { + return Err(Error::new_spanned( + expression, + format!("{name} must be an array of keys"), + )); + }; + if array.elems.is_empty() { + return Err(Error::new_spanned( + array, + format!("{name} must contain at least one key"), + )); + } + Ok(array.elems.into_iter().collect()) +} + +fn find_attribute<'a>( + attributes: &'a [Attribute], + name: &str, +) -> syn::Result> { + let mut matching = attributes + .iter() + .filter(|attribute| attribute.path().is_ident(name)); + let first = matching.next(); + if let Some(duplicate) = matching.next() { + return Err(Error::new_spanned( + duplicate, + format!("duplicate #[{name}] attribute"), + )); + } + Ok(first) +} + +fn parse_field(field: &PropertyField) -> syn::Result { + let ident = &field.ident; + let parse = parse_field_value(field, ParseTarget::Owned)?; + Ok(quote!(#ident: #parse)) +} + +#[derive(Clone, Copy)] +pub(crate) enum ParseTarget { + Owned, + View, +} + +pub(crate) fn parse_field_value( + field: &PropertyField, + target: ParseTarget, +) -> syn::Result { + if field.nested { + let ty = &field.ty; + let constructor = match target { + ParseTarget::Owned => quote!(<#ty>::from_properties(properties)?), + ParseTarget::View => quote!(<#ty>::new(properties)), + }; + return Ok(constructor); + } + + let ty = &field.ty; + + if let Some(parse_properties_with) = &field.parse_properties_with { + let key = field.key.as_ref().ok_or_else(|| { + Error::new_spanned( + &field.ident, + "parse_properties_with fields must declare key", + ) + })?; + let additional_keys = field.additional_keys.as_ref().ok_or_else(|| { + Error::new_spanned( + &field.ident, + "parse_properties_with fields must declare additional_keys", + ) + })?; + let default = typed_default(field)?; + return Ok(quote! {{ + let parsed: ::iceberg::Result<#ty> = #parse_properties_with( + properties, + #key, + &[#(#additional_keys),*], + #default, + ); + parsed.map_err(|error| error.with_context("property", #key))? + }}); + } + + if let Some(prefix) = &field.prefix { + let value_type = field.map_value_type.as_ref().ok_or_else(|| { + Error::new_spanned( + &field.ty, + "property prefix fields must have type HashMap", + ) + })?; + let parse = if is_bool(value_type) { + quote!(value.to_ascii_lowercase().parse::<#value_type>()) + } else { + quote!(value.parse::<#value_type>()) + }; + return Ok(quote! { + properties + .iter() + .filter_map(|(key, value)| { + key.strip_prefix(#prefix).map(|suffix| { + #parse + .map(|parsed| (suffix.to_string(), parsed)) + .map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {key}: {error}"), + ) + }) + }) + }) + .collect::<::iceberg::Result<::std::collections::HashMap<_, _>>>()? + }); + } + + let key = field + .key + .as_ref() + .ok_or_else(|| Error::new_spanned(&field.ident, "property fields must declare key"))?; + let default = typed_default(field)?; + let parse = match (&field.parse_with, &field.option_inner_type) { + (Some(parse_with), Some(inner_type)) => quote! { + { + let parsed: ::iceberg::Result<#inner_type> = #parse_with(value); + Some(parsed.map_err(|error| error.with_context("property", #key))?) + } + }, + (Some(parse_with), None) => quote! { + { + let parsed: ::iceberg::Result<#ty> = #parse_with(value); + parsed.map_err(|error| error.with_context("property", #key))? + } + }, + (None, Some(inner_type)) if is_bool(inner_type) => quote! { + Some(value.to_ascii_lowercase().parse::<#inner_type>().map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {}: {error}", #key), + ) + })?) + }, + (None, Some(inner_type)) => quote! { + Some(value.parse::<#inner_type>().map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {}: {error}", #key), + ) + })?) + }, + (None, None) if is_bool(ty) => quote! { + value.to_ascii_lowercase().parse::<#ty>().map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {}: {error}", #key), + ) + })? + }, + (None, None) => quote! { + value.parse::<#ty>().map_err(|error| { + ::iceberg::Error::new( + ::iceberg::ErrorKind::DataInvalid, + format!("Invalid value for {}: {error}", #key), + ) + })? + }, + }; + + Ok(quote! { + match properties.get(#key) { + Some(value) => #parse, + None => #default, + } + }) +} + +fn typed_default(field: &PropertyField) -> syn::Result { + let ty = &field.ty; + let default = field.default.as_ref().ok_or_else(|| { + Error::new_spanned(&field.ident, "property key fields must declare default") + })?; + let default = default_value(default, ty); + Ok(quote!({ + let value: #ty = #default; + value + })) +} + +fn default_value(default: &Expr, ty: &Type) -> TokenStream2 { + if matches!( + default, + Expr::Lit(ExprLit { + lit: Lit::Str(_), + .. + }) | Expr::Path(_) + ) { + quote!(::std::convert::Into::<#ty>::into(#default)) + } else { + quote!(#default) + } +} + +fn option_inner_type(ty: &Type) -> Option { + let Type::Path(type_path) = ty else { + return None; + }; + + let segment = type_path.path.segments.last()?; + if segment.ident != "Option" { + return None; + } + + let PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let Some(GenericArgument::Type(inner_type)) = arguments.args.first() else { + return None; + }; + + Some(inner_type.clone()) +} + +fn hash_map_value_type(ty: &Type) -> Option { + let Type::Path(type_path) = ty else { + return None; + }; + + let segment = type_path.path.segments.last()?; + if segment.ident != "HashMap" { + return None; + } + + let PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return None; + }; + let mut arguments = arguments.args.iter(); + let Some(GenericArgument::Type(key_type)) = arguments.next() else { + return None; + }; + let Some(GenericArgument::Type(value_type)) = arguments.next() else { + return None; + }; + if !is_named_type(key_type, "String") { + return None; + } + + Some(value_type.clone()) +} + +fn is_bool(ty: &Type) -> bool { + is_named_type(ty, "bool") +} + +fn is_copy_type(ty: &Type) -> bool { + match ty { + Type::Array(array) => is_copy_type(&array.elem), + Type::BareFn(_) | Type::Never(_) | Type::Ptr(_) => true, + Type::Group(group) => is_copy_type(&group.elem), + Type::Paren(paren) => is_copy_type(&paren.elem), + Type::Reference(reference) => reference.mutability.is_none(), + Type::Tuple(tuple) => tuple.elems.iter().all(is_copy_type), + Type::Path(type_path) if type_path.qself.is_none() => { + let Some(segment) = type_path.path.segments.last() else { + return false; + }; + if matches!( + segment.ident.to_string().as_str(), + "bool" + | "char" + | "f32" + | "f64" + | "i8" + | "i16" + | "i32" + | "i64" + | "i128" + | "isize" + | "u8" + | "u16" + | "u32" + | "u64" + | "u128" + | "usize" + ) { + return true; + } + if segment.ident != "Option" && segment.ident != "Result" { + return false; + } + let PathArguments::AngleBracketed(arguments) = &segment.arguments else { + return false; + }; + arguments.args.iter().all(|argument| match argument { + GenericArgument::Lifetime(_) => true, + GenericArgument::Type(ty) => is_copy_type(ty), + _ => false, + }) + } + _ => false, + } +} + +fn is_named_type(ty: &Type, name: &str) -> bool { + let Type::Path(type_path) = ty else { + return false; + }; + + type_path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == name) +} diff --git a/crates/property-macro/src/properties_view.rs b/crates/property-macro/src/properties_view.rs new file mode 100644 index 0000000000..7536f57636 --- /dev/null +++ b/crates/property-macro/src/properties_view.rs @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::{Error, Fields, GenericArgument, ItemStruct, Lifetime, PathArguments, Type}; + +use crate::properties::{ + ParseTarget, PropertyField, parse_field_value, parse_property_field, property_options, +}; + +pub(crate) fn expand_properties_view(input: ItemStruct) -> syn::Result { + if !input.generics.params.is_empty() || input.generics.where_clause.is_some() { + return Err(Error::new_spanned( + input.generics, + "properties_view! does not support generic declarations", + )); + } + + let attributes = input.attrs; + let visibility = input.vis; + let struct_name = input.ident; + let fields = match input.fields { + Fields::Named(fields) => fields.named, + _ => { + return Err(Error::new_spanned( + struct_name, + "properties_view! requires a struct-shaped declaration with named fields", + )); + } + }; + let fields = fields + .iter() + .map(|field| parse_property_field(field, property_options(field)?)) + .collect::>>()?; + let properties_lifetime: Lifetime = syn::parse_quote!('properties); + let getters = fields + .iter() + .map(|field| view_field_getter(field, &properties_lifetime)) + .collect::>>()?; + + Ok(quote! { + #(#attributes)* + #visibility struct #struct_name<#properties_lifetime> { + properties: &#properties_lifetime ::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + >, + } + + impl<#properties_lifetime> #struct_name<#properties_lifetime> { + /// Creates a property view without parsing any values. + #visibility fn new( + properties: &#properties_lifetime ::std::collections::HashMap< + ::std::string::String, + ::std::string::String, + >, + ) -> Self { + Self { properties } + } + + #(#getters)* + } + }) +} + +fn view_field_getter( + field: &PropertyField, + properties_lifetime: &Lifetime, +) -> syn::Result { + let ident = &field.ident; + let ty = view_field_type(field, properties_lifetime)?; + let docs = &field.doc_attributes; + let visibility = if field.public_getter { + quote!(pub) + } else { + TokenStream2::new() + }; + let parse = parse_field_value(field, ParseTarget::View)?; + + Ok(quote! { + #(#docs)* + #visibility fn #ident(&self) -> ::iceberg::Result<#ty> { + let properties = self.properties; + let value = #parse; + Ok(value) + } + }) +} + +fn view_field_type(field: &PropertyField, properties_lifetime: &Lifetime) -> syn::Result { + let mut ty = field.ty.clone(); + if !field.nested { + return Ok(ty); + } + + let Type::Path(type_path) = &mut ty else { + return Err(Error::new_spanned( + &field.ty, + "nested property view types must have a lifetime argument", + )); + }; + let Some(segment) = type_path.path.segments.last_mut() else { + return Err(Error::new_spanned( + &field.ty, + "nested property view types must have a lifetime argument", + )); + }; + let PathArguments::AngleBracketed(arguments) = &mut segment.arguments else { + return Err(Error::new_spanned( + &field.ty, + "nested property view types must have a lifetime argument", + )); + }; + let Some(lifetime) = arguments + .args + .iter_mut() + .find_map(|argument| match argument { + GenericArgument::Lifetime(lifetime) => Some(lifetime), + _ => None, + }) + else { + return Err(Error::new_spanned( + &field.ty, + "nested property view types must have a lifetime argument", + )); + }; + *lifetime = properties_lifetime.clone(); + Ok(ty) +} + +#[cfg(test)] +mod tests { + use syn::{ImplItem, Item, Visibility, parse_quote, parse2}; + + use super::*; + + fn method_visibility(input: ItemStruct, method_name: &str) -> Visibility { + let expanded = expand_properties_view(input).unwrap(); + let file = parse2::(expanded).unwrap(); + + file.items + .into_iter() + .find_map(|item| match item { + Item::Impl(item_impl) => item_impl.items.into_iter().find_map(|item| match item { + ImplItem::Fn(method) if method.sig.ident == method_name => Some(method.vis), + _ => None, + }), + _ => None, + }) + .unwrap() + } + + #[test] + fn constructor_uses_the_struct_visibility() { + let visibility = method_visibility( + parse_quote! { + pub(crate) struct TestProperties { + #[property(key = "retries", default = 4)] + retries: u64, + } + }, + "new", + ); + + assert!(matches!(visibility, Visibility::Restricted(_))); + } + + #[test] + fn getter_option_overrides_explicit_visibility() { + let visibility = method_visibility( + parse_quote! { + struct TestProperties { + #[property(key = "retries", default = 4, getter)] + pub(crate) retries: u64, + } + }, + "retries", + ); + + assert!(matches!(visibility, Visibility::Public(_))); + } + + #[test] + fn field_visibility_does_not_make_the_getter_public() { + let visibility = method_visibility( + parse_quote! { + struct TestProperties { + #[property(key = "retries", default = 4)] + pub retries: u64, + } + }, + "retries", + ); + + assert!(matches!(visibility, Visibility::Inherited)); + } +} diff --git a/crates/integrations/datafusion/src/physical_plan/mod.rs b/crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.rs similarity index 78% rename from crates/integrations/datafusion/src/physical_plan/mod.rs rename to crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.rs index fcfd11a453..3036fd3b96 100644 --- a/crates/integrations/datafusion/src/physical_plan/mod.rs +++ b/crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.rs @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. -pub(crate) mod commit; -pub(crate) mod expr_to_predicate; -pub(crate) mod metadata_scan; -pub(crate) mod scan; -pub(crate) mod write; -pub(crate) const DATA_FILES_COL_NAME: &str = "data_files"; +use iceberg_property_macro::Properties; -pub use scan::IcebergTableScan; +#[derive(Properties)] +struct AdditionalKeysWithoutParser { + #[property(key = "value", additional_keys = ["other"], default = 1)] + value: u64, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.stderr b/crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.stderr new file mode 100644 index 0000000000..b0fa9be55c --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/additional_keys_without_parse_properties.stderr @@ -0,0 +1,6 @@ +error: additional_keys requires parse_properties_with in #[property(...)] + --> tests/compile-fail/derive-properties/additional_keys_without_parse_properties.rs:23:5 + | +23 | / #[property(key = "value", additional_keys = ["other"], default = 1)] +24 | | value: u64, + | |______________^ diff --git a/crates/integrations/playground/src/lib.rs b/crates/property-macro/tests/compile-fail/derive-properties/missing_property.rs similarity index 84% rename from crates/integrations/playground/src/lib.rs rename to crates/property-macro/tests/compile-fail/derive-properties/missing_property.rs index de62f4b143..07ed0f770b 100644 --- a/crates/integrations/playground/src/lib.rs +++ b/crates/property-macro/tests/compile-fail/derive-properties/missing_property.rs @@ -15,8 +15,12 @@ // specific language governing permissions and limitations // under the License. -#![doc = include_str!("../README.md")] -pub const ICEBERG_PLAYGROUND_VERSION: &str = env!("CARGO_PKG_VERSION"); -mod catalog; -pub use catalog::*; +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct MissingProperty { + value: u64, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/derive-properties/missing_property.stderr b/crates/property-macro/tests/compile-fail/derive-properties/missing_property.stderr new file mode 100644 index 0000000000..5be8dfe96c --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/missing_property.stderr @@ -0,0 +1,5 @@ +error: Properties fields must declare #[property(...)] + --> tests/compile-fail/derive-properties/missing_property.rs:23:5 + | +23 | value: u64, + | ^^^^^^^^^^ diff --git a/crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.rs b/crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.rs new file mode 100644 index 0000000000..97aefaa567 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct MultipleSources { + #[property(key = "value", prefix = "values.", default = HashMap::new())] + value: HashMap, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.stderr b/crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.stderr new file mode 100644 index 0000000000..37dc87f5e1 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/multiple_sources.stderr @@ -0,0 +1,6 @@ +error: Properties fields must declare exactly one of key, prefix, or nested in #[property(...)] + --> tests/compile-fail/derive-properties/multiple_sources.rs:25:5 + | +25 | / #[property(key = "value", prefix = "values.", default = HashMap::new())] +26 | | value: HashMap, + | |_______________________________^ diff --git a/crates/property-macro/tests/compile-fail/derive-properties/nested_default.rs b/crates/property-macro/tests/compile-fail/derive-properties/nested_default.rs new file mode 100644 index 0000000000..601b1df1e4 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/nested_default.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct Inner { + #[property(key = "value", default = 1)] + value: u64, +} + +#[derive(Properties)] +struct NestedDefault { + #[property(nested, default = unreachable!())] + inner: Inner, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/derive-properties/nested_default.stderr b/crates/property-macro/tests/compile-fail/derive-properties/nested_default.stderr new file mode 100644 index 0000000000..878985957a --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/nested_default.stderr @@ -0,0 +1,6 @@ +error: nested fields obtain defaults from their own property annotations and cannot declare default in #[property(...)] + --> tests/compile-fail/derive-properties/nested_default.rs:29:5 + | +29 | / #[property(nested, default = unreachable!())] +30 | | inner: Inner, + | |________________^ diff --git a/crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.rs b/crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.rs new file mode 100644 index 0000000000..96dd5b18d2 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.rs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +fn parse_all( + _properties: &HashMap, + _key: &str, + _additional_keys: &[&str], + default: u64, +) -> iceberg::Result { + Ok(default) +} + +#[derive(Properties)] +struct ParsePropertiesWithoutAdditionalKeys { + #[property(key = "value", default = 1, parse_properties_with = parse_all)] + value: u64, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.stderr b/crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.stderr new file mode 100644 index 0000000000..e4eae1fea6 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/parse_properties_without_additional_keys.stderr @@ -0,0 +1,6 @@ +error: parse_properties_with requires additional_keys in #[property(...)] + --> tests/compile-fail/derive-properties/parse_properties_without_additional_keys.rs:34:5 + | +34 | / #[property(key = "value", default = 1, parse_properties_with = parse_all)] +35 | | value: u64, + | |______________^ diff --git a/crates/property-macro/tests/compile-fail/derive-properties/prefix_default.rs b/crates/property-macro/tests/compile-fail/derive-properties/prefix_default.rs new file mode 100644 index 0000000000..d7c4ec3397 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/prefix_default.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +use std::collections::HashMap; + +use iceberg_property_macro::Properties; + +#[derive(Properties)] +struct PrefixDefault { + #[property(prefix = "values.", default = HashMap::new())] + values: HashMap, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/derive-properties/prefix_default.stderr b/crates/property-macro/tests/compile-fail/derive-properties/prefix_default.stderr new file mode 100644 index 0000000000..902e95782b --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/prefix_default.stderr @@ -0,0 +1,6 @@ +error: prefix fields collect matching properties and cannot declare default in #[property(...)] + --> tests/compile-fail/derive-properties/prefix_default.rs:25:5 + | +25 | / #[property(prefix = "values.", default = HashMap::new())] +26 | | values: HashMap, + | |________________________________^ diff --git a/crates/integrations/datafusion/src/lib.rs b/crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.rs similarity index 80% rename from crates/integrations/datafusion/src/lib.rs rename to crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.rs index 09d1cac4ce..1d904daece 100644 --- a/crates/integrations/datafusion/src/lib.rs +++ b/crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.rs @@ -15,14 +15,13 @@ // specific language governing permissions and limitations // under the License. -mod catalog; -pub use catalog::*; -mod error; -pub use error::*; +use iceberg_property_macro::Properties; -pub mod physical_plan; -mod schema; -pub mod table; -pub use table::table_provider_factory::IcebergTableProviderFactory; -pub use table::*; +#[derive(Properties)] +struct PrefixNonMap { + #[property(prefix = "values.")] + value: u64, +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.stderr b/crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.stderr new file mode 100644 index 0000000000..41454f966b --- /dev/null +++ b/crates/property-macro/tests/compile-fail/derive-properties/prefix_non_map.stderr @@ -0,0 +1,5 @@ +error: property prefix fields must have type HashMap + --> tests/compile-fail/derive-properties/prefix_non_map.rs:24:12 + | +24 | value: u64, + | ^^^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.rs b/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.rs new file mode 100644 index 0000000000..71aa916ffc --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct AdditionalKeysWithoutParser { + #[property(key = "value", additional_keys = ["other"], default = 1)] + value: u64, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.stderr new file mode 100644 index 0000000000..a5c74eee0a --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.stderr @@ -0,0 +1,6 @@ +error: additional_keys requires parse_properties_with in #[property(...)] + --> tests/compile-fail/properties-view/view_additional_keys_without_parse_properties.rs:22:9 + | +22 | / #[property(key = "value", additional_keys = ["other"], default = 1)] +23 | | value: u64, + | |__________________^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.rs b/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.rs new file mode 100644 index 0000000000..af83d0fa82 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct GenericView { + #[property(key = "value", default = None)] + value: Option, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.stderr new file mode 100644 index 0000000000..2dc8658cd7 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_generic_declaration.stderr @@ -0,0 +1,5 @@ +error: properties_view! does not support generic declarations + --> tests/compile-fail/properties-view/view_generic_declaration.rs:21:23 + | +21 | struct GenericView { + | ^^^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.rs b/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.rs new file mode 100644 index 0000000000..c0d7f84f5c --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.rs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct MissingProperty { + value: u64, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.stderr new file mode 100644 index 0000000000..8f0a522acd --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_missing_property.stderr @@ -0,0 +1,5 @@ +error: Properties fields must declare #[property(...)] + --> tests/compile-fail/properties-view/view_missing_property.rs:22:9 + | +22 | value: u64, + | ^^^^^^^^^^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.rs b/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.rs new file mode 100644 index 0000000000..db8d68bcce --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::properties_view; + +properties_view! { + struct MultipleSources { + #[property(key = "value", prefix = "values.", default = HashMap::new())] + value: HashMap, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.stderr new file mode 100644 index 0000000000..83cc348cbc --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_multiple_sources.stderr @@ -0,0 +1,14 @@ +error: Properties fields must declare exactly one of key, prefix, or nested in #[property(...)] + --> tests/compile-fail/properties-view/view_multiple_sources.rs:24:9 + | +24 | / #[property(key = "value", prefix = "values.", default = HashMap::new())] +25 | | value: HashMap, + | |___________________________________^ + +warning: unused import: `std::collections::HashMap` + --> tests/compile-fail/properties-view/view_multiple_sources.rs:18:5 + | +18 | use std::collections::HashMap; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.rs b/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.rs new file mode 100644 index 0000000000..06ef741f2c --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.rs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct Inner { + #[property(key = "value", default = 1)] + value: u64, + } +} + +properties_view! { + struct NestedDefault { + #[property(nested, default = unreachable!())] + inner: Inner<'_>, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.stderr new file mode 100644 index 0000000000..2c024f1e61 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_nested_default.stderr @@ -0,0 +1,6 @@ +error: nested fields obtain defaults from their own property annotations and cannot declare default in #[property(...)] + --> tests/compile-fail/properties-view/view_nested_default.rs:29:9 + | +29 | / #[property(nested, default = unreachable!())] +30 | | inner: Inner<'_>, + | |________________________^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.rs b/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.rs new file mode 100644 index 0000000000..cbba6e50e1 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.rs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::properties_view; + +fn parse_all( + _properties: &HashMap, + _key: &str, + _additional_keys: &[&str], + default: u64, +) -> iceberg::Result { + Ok(default) +} + +properties_view! { + struct ParsePropertiesWithoutAdditionalKeys { + #[property(key = "value", default = 1, parse_properties_with = parse_all)] + value: u64, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.stderr new file mode 100644 index 0000000000..89a2af6048 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.stderr @@ -0,0 +1,6 @@ +error: parse_properties_with requires additional_keys in #[property(...)] + --> tests/compile-fail/properties-view/view_parse_properties_without_additional_keys.rs:33:9 + | +33 | / #[property(key = "value", default = 1, parse_properties_with = parse_all)] +34 | | value: u64, + | |__________________^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.rs b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.rs new file mode 100644 index 0000000000..8b32a39588 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use iceberg_property_macro::properties_view; + +properties_view! { + struct PrefixDefault { + #[property(prefix = "values.", default = HashMap::new())] + values: HashMap, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.stderr new file mode 100644 index 0000000000..560ae271b2 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_default.stderr @@ -0,0 +1,14 @@ +error: prefix fields collect matching properties and cannot declare default in #[property(...)] + --> tests/compile-fail/properties-view/view_prefix_default.rs:24:9 + | +24 | / #[property(prefix = "values.", default = HashMap::new())] +25 | | values: HashMap, + | |____________________________________^ + +warning: unused import: `std::collections::HashMap` + --> tests/compile-fail/properties-view/view_prefix_default.rs:18:5 + | +18 | use std::collections::HashMap; + | ^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.rs b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.rs new file mode 100644 index 0000000000..05b6952f34 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct PrefixNonMap { + #[property(prefix = "values.")] + value: u64, + } +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.stderr new file mode 100644 index 0000000000..534e925890 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_prefix_non_map.stderr @@ -0,0 +1,5 @@ +error: property prefix fields must have type HashMap + --> tests/compile-fail/properties-view/view_prefix_non_map.rs:23:16 + | +23 | value: u64, + | ^^^ diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.rs b/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.rs new file mode 100644 index 0000000000..33dfd284de --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.rs @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iceberg_property_macro::properties_view; + +properties_view! { + struct TupleView(#[property(key = "value", default = 1)] u64); +} + +fn main() {} diff --git a/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.stderr b/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.stderr new file mode 100644 index 0000000000..415ff5f276 --- /dev/null +++ b/crates/property-macro/tests/compile-fail/properties-view/view_tuple_declaration.stderr @@ -0,0 +1,5 @@ +error: properties_view! requires a struct-shaped declaration with named fields + --> tests/compile-fail/properties-view/view_tuple_declaration.rs:21:12 + | +21 | struct TupleView(#[property(key = "value", default = 1)] u64); + | ^^^^^^^^^ diff --git a/crates/property-macro/tests/compile_fail.rs b/crates/property-macro/tests/compile_fail.rs new file mode 100644 index 0000000000..52749a1cbf --- /dev/null +++ b/crates/property-macro/tests/compile_fail.rs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[test] +fn reports_invalid_properties_derive_declarations() { + trybuild::TestCases::new().compile_fail("tests/compile-fail/derive-properties/*.rs"); +} + +#[test] +fn reports_invalid_property_view_declarations() { + trybuild::TestCases::new().compile_fail("tests/compile-fail/properties-view/*.rs"); +} diff --git a/crates/property-macro/tests/properties_derive.rs b/crates/property-macro/tests/properties_derive.rs new file mode 100644 index 0000000000..1b8edff146 --- /dev/null +++ b/crates/property-macro/tests/properties_derive.rs @@ -0,0 +1,313 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::str::FromStr; + +use iceberg::{Error, ErrorKind}; +use iceberg_property_macro::Properties; +use serde::{Deserialize, Serialize}; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const FORMAT: &str = "write.format.default"; +const FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; +const DEPTH: &str = "dimensions.depth"; + +fn parse_dimensions( + properties: &HashMap, + width_key: &str, + additional_keys: &[&str], + default: (u64, u64, u64), +) -> iceberg::Result<(u64, u64, u64)> { + if additional_keys.len() != 2 { + return Err(Error::new( + ErrorKind::DataInvalid, + "dimensions require height and depth keys", + )); + } + let parse = |property_key: &str, default| { + properties + .get(property_key) + .map(|value| { + value + .parse::() + .map_err(|error| Error::new(ErrorKind::DataInvalid, error.to_string())) + }) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok(( + parse(width_key, default.0)?, + parse(additional_keys[0], default.1)?, + parse(additional_keys[1], default.2)?, + )) +} + +#[derive(Debug, Properties)] +struct TestProperties { + #[property(key = RETRIES, default = 4, getter)] + retries: u64, + + #[property(key = OWNER, default = None, getter)] + owner: Option, + + #[property(key = FORMAT, default = "parquet", getter)] + format: String, + + #[property(key = FANOUT_ENABLED, default = true, getter)] + fanout_enabled: bool, + + #[property( + prefix = COLUMN_FPP_PREFIX, + getter + )] + column_fpp: HashMap, + + #[property( + key = WIDTH, + additional_keys = [HEIGHT, DEPTH], + default = (640, 480, 320), + parse_properties_with = parse_dimensions, + getter + )] + dimensions: (u64, u64, u64), +} + +#[test] +fn reads_defaults_through_generated_getters() { + let properties = TestProperties::from_properties(&HashMap::new()).unwrap(); + + assert_eq!(properties.retries(), 4); + assert_eq!(properties.owner(), &None); + assert_eq!(properties.format(), "parquet"); + assert!(properties.fanout_enabled()); + assert!(properties.column_fpp().is_empty()); + assert_eq!(properties.dimensions(), (640, 480, 320)); +} + +#[test] +fn reads_overrides_and_ignores_unknown_properties() { + let raw = HashMap::from([ + (RETRIES.to_string(), "8".to_string()), + (OWNER.to_string(), "iceberg".to_string()), + (FORMAT.to_string(), "orc".to_string()), + (FANOUT_ENABLED.to_string(), "FALSE".to_string()), + (format!("{COLUMN_FPP_PREFIX}id"), "0.01".to_string()), + (WIDTH.to_string(), "1920".to_string()), + (HEIGHT.to_string(), "1080".to_string()), + (DEPTH.to_string(), "720".to_string()), + ("unknown".to_string(), "ignored".to_string()), + ]); + let properties = TestProperties::from_properties(&raw).unwrap(); + + assert_eq!(properties.retries(), 8); + assert_eq!(properties.owner().as_deref(), Some("iceberg")); + assert_eq!(properties.format(), "orc"); + assert!(!properties.fanout_enabled()); + assert_eq!(properties.column_fpp()["id"], 0.01); + assert_eq!(properties.dimensions(), (1920, 1080, 720)); +} + +#[test] +fn reports_the_property_with_an_invalid_value() { + let numeric_error = TestProperties::from_properties(&HashMap::from([( + RETRIES.to_string(), + "many".to_string(), + )])) + .unwrap_err(); + assert_eq!(numeric_error.kind(), ErrorKind::DataInvalid); + assert!(numeric_error.message().contains(RETRIES)); + + let boolean_error = TestProperties::from_properties(&HashMap::from([( + FANOUT_ENABLED.to_string(), + "sometimes".to_string(), + )])) + .unwrap_err(); + assert_eq!(boolean_error.kind(), ErrorKind::DataInvalid); + assert!(boolean_error.message().contains(FANOUT_ENABLED)); + + let prefixed_key = format!("{COLUMN_FPP_PREFIX}id"); + let prefix_error = TestProperties::from_properties(&HashMap::from([( + prefixed_key.clone(), + "low".to_string(), + )])) + .unwrap_err(); + assert_eq!(prefix_error.kind(), ErrorKind::DataInvalid); + assert!(prefix_error.message().contains(&prefixed_key)); + + let dimensions_error = + TestProperties::from_properties(&HashMap::from([(WIDTH.to_string(), "wide".to_string())])) + .unwrap_err(); + assert_eq!(dimensions_error.kind(), ErrorKind::DataInvalid); + assert!(format!("{dimensions_error}").contains(WIDTH)); +} + +#[derive(Debug, Properties)] +struct CommitProperties { + /// Maximum number of times to retry a commit. + #[property(key = RETRIES, default = 4, getter)] + retries: u64, +} + +#[derive(Debug, Properties)] +struct NestedProperties { + #[property(nested, getter)] + commit: CommitProperties, +} + +#[test] +fn nested_properties_read_the_same_flat_map() { + let raw = HashMap::from([(RETRIES.to_string(), "9".to_string())]); + let properties = NestedProperties::from_properties(&raw).unwrap(); + + assert_eq!(properties.commit().retries(), 9); +} + +fn parse_non_empty(value: &str) -> iceberg::Result { + let value = value.trim(); + if value.is_empty() { + Err(Error::new( + ErrorKind::DataInvalid, + "value must not be empty", + )) + } else { + Ok(value.to_string()) + } +} + +#[derive(Debug, Properties)] +struct ValidatedProperties { + #[property( + key = "location", + default = "default", + parse_with = parse_non_empty, + getter + )] + location: String, +} + +#[test] +fn custom_single_value_parser_can_validate_and_normalize() { + let defaults = ValidatedProperties::from_properties(&HashMap::new()).unwrap(); + assert_eq!(defaults.location(), "default"); + + let parsed = ValidatedProperties::from_properties(&HashMap::from([( + "location".to_string(), + " path ".to_string(), + )])) + .unwrap(); + assert_eq!(parsed.location(), "path"); + + let error = ValidatedProperties::from_properties(&HashMap::from([( + "location".to_string(), + " ".to_string(), + )])) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert_eq!(error.message(), "value must not be empty"); + assert!(format!("{error}").contains("property: location")); +} + +#[derive(Debug, Properties)] +struct OptionalValidatedProperties { + #[property( + key = "optional-location", + default = None, + parse_with = parse_non_empty, + getter + )] + location: Option, +} + +#[test] +fn custom_single_value_parser_wraps_present_optional_values() { + let defaults = OptionalValidatedProperties::from_properties(&HashMap::new()).unwrap(); + assert_eq!(defaults.location(), &None); + + let parsed = OptionalValidatedProperties::from_properties(&HashMap::from([( + "optional-location".to_string(), + " path ".to_string(), + )])) + .unwrap(); + assert_eq!(parsed.location().as_deref(), Some("path")); + + let error = OptionalValidatedProperties::from_properties(&HashMap::from([( + "optional-location".to_string(), + " ".to_string(), + )])) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert_eq!(error.message(), "value must not be empty"); + assert!(format!("{error}").contains("property: optional-location")); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Meters(u64); + +impl FromStr for Meters { + type Err = ::Err; + + fn from_str(value: &str) -> Result { + value.parse().map(Self) + } +} + +#[derive(Debug, Properties)] +struct CopyGetterProperties { + #[property(key = "distance", default = Meters(1), getter)] + distance: Meters, + + #[property(key = "count", default = Some(2), getter)] + count: Option, +} + +#[test] +fn returns_only_structurally_known_copy_types_by_value() { + let properties = CopyGetterProperties::from_properties(&HashMap::new()).unwrap(); + + let _: &Meters = properties.distance(); + let _: Option = properties.count(); + assert_eq!(properties.distance(), &Meters(1)); + assert_eq!(properties.count(), Some(2)); +} + +#[derive(Debug, Default, Serialize, Deserialize, Properties)] +struct DerivedTraitProperties { + #[property(key = RETRIES, default = 4, getter)] + retries: u64, +} + +#[test] +fn coexists_with_default_serialize_and_deserialize_derives() { + let defaults = DerivedTraitProperties::default(); + assert_eq!(defaults.retries(), 0); + + let properties = DerivedTraitProperties::from_properties(&HashMap::new()).unwrap(); + assert_eq!(properties.retries(), 4); + assert_eq!( + serde_json::to_string(&properties).unwrap(), + r#"{"retries":4}"# + ); + + let decoded: DerivedTraitProperties = serde_json::from_str(r#"{"retries":7}"#).unwrap(); + assert_eq!(decoded.retries(), 7); +} diff --git a/crates/property-macro/tests/properties_view.rs b/crates/property-macro/tests/properties_view.rs new file mode 100644 index 0000000000..2a6cb05804 --- /dev/null +++ b/crates/property-macro/tests/properties_view.rs @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::mem::size_of; + +use iceberg::{Error, ErrorKind}; +use iceberg_property_macro::properties_view; + +const RETRIES: &str = "commit.retry.num-retries"; +const OWNER: &str = "owner"; +const FORMAT: &str = "write.format.default"; +const FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; +const COLUMN_FPP_PREFIX: &str = "write.parquet.bloom-filter-fpp.column."; +const WIDTH: &str = "dimensions.width"; +const HEIGHT: &str = "dimensions.height"; +const DEPTH: &str = "dimensions.depth"; + +fn parse_dimensions( + properties: &HashMap, + width_key: &str, + additional_keys: &[&str], + default: (u64, u64, u64), +) -> iceberg::Result<(u64, u64, u64)> { + if additional_keys.len() != 2 { + return Err(Error::new( + ErrorKind::DataInvalid, + "dimensions require height and depth keys", + )); + } + let parse = |property_key: &str, default| { + properties + .get(property_key) + .map(|value| { + value + .parse::() + .map_err(|error| Error::new(ErrorKind::DataInvalid, error.to_string())) + }) + .transpose() + .map(|value| value.unwrap_or(default)) + }; + + Ok(( + parse(width_key, default.0)?, + parse(additional_keys[0], default.1)?, + parse(additional_keys[1], default.2)?, + )) +} + +fn parse_non_empty(value: &str) -> iceberg::Result { + let value = value.trim(); + if value.is_empty() { + Err(Error::new( + ErrorKind::DataInvalid, + "value must not be empty", + )) + } else { + Ok(value.to_string()) + } +} + +properties_view! { + #[derive(Debug)] + struct TestPropertiesView { + /// Maximum number of times to retry a commit. + #[property(key = RETRIES, default = 4, getter)] + retries: u64, + + #[property(key = OWNER, default = None, getter)] + owner: Option, + + #[property(key = FORMAT, default = "parquet", getter)] + format: String, + + #[property(key = FANOUT_ENABLED, default = true, getter)] + fanout_enabled: bool, + + #[property(prefix = COLUMN_FPP_PREFIX, getter)] + column_fpp: HashMap, + + #[property( + key = "location", + default = "default", + parse_with = parse_non_empty, + getter + )] + location: String, + + #[property( + key = WIDTH, + additional_keys = [HEIGHT, DEPTH], + default = (640, 480, 320), + parse_properties_with = parse_dimensions, + getter + )] + dimensions: (u64, u64, u64), + } +} + +properties_view! { + #[derive(Debug)] + struct CommitPropertiesView { + #[property(key = RETRIES, default = 4, getter)] + retries: u64, + } +} + +properties_view! { + #[derive(Debug)] + struct NestedPropertiesView { + #[property(nested, getter)] + commit: CommitPropertiesView<'_>, + } +} + +#[test] +fn property_view_is_only_a_reference_to_the_source_map() { + assert_eq!( + size_of::>(), + size_of::<&HashMap>() + ); +} + +#[test] +fn property_view_parses_only_the_requested_field() { + let raw = HashMap::from([ + (RETRIES.to_string(), "many".to_string()), + (OWNER.to_string(), "iceberg".to_string()), + (FORMAT.to_string(), "orc".to_string()), + (FANOUT_ENABLED.to_string(), "FALSE".to_string()), + (WIDTH.to_string(), "1920".to_string()), + (HEIGHT.to_string(), "1080".to_string()), + (DEPTH.to_string(), "720".to_string()), + ]); + let properties = TestPropertiesView::new(&raw); + + let error = properties.retries().unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.message().contains(RETRIES)); + assert_eq!(properties.owner().unwrap().as_deref(), Some("iceberg")); + assert_eq!(properties.format().unwrap(), "orc"); + assert!(!properties.fanout_enabled().unwrap()); + assert_eq!(properties.dimensions().unwrap(), (1920, 1080, 720)); +} + +#[test] +fn property_view_uses_defaults_and_supports_custom_parsers() { + let raw = HashMap::from([("location".to_string(), " path ".to_string())]); + let properties = TestPropertiesView::new(&raw); + + assert_eq!(properties.retries().unwrap(), 4); + assert_eq!(properties.owner().unwrap(), None); + assert_eq!(properties.format().unwrap(), "parquet"); + assert!(properties.fanout_enabled().unwrap()); + assert!(properties.column_fpp().unwrap().is_empty()); + assert_eq!(properties.location().unwrap(), "path"); +} + +#[test] +fn nested_property_views_borrow_the_same_source_map() { + let raw = HashMap::from([(RETRIES.to_string(), "9".to_string())]); + let commit = { + let properties = NestedPropertiesView::new(&raw); + properties.commit().unwrap() + }; + + assert_eq!(commit.retries().unwrap(), 9); +} + +#[test] +fn property_view_reports_errors_when_the_corresponding_getter_is_called() { + let prefixed_key = format!("{COLUMN_FPP_PREFIX}id"); + let raw = HashMap::from([ + (prefixed_key.clone(), "low".to_string()), + (format!("{COLUMN_FPP_PREFIX}category"), "0.01".to_string()), + ("location".to_string(), " ".to_string()), + ]); + let properties = TestPropertiesView::new(&raw); + + let prefix_error = properties.column_fpp().unwrap_err(); + assert!(prefix_error.message().contains(&prefixed_key)); + + let location_error = properties.location().unwrap_err(); + assert_eq!(location_error.message(), "value must not be empty"); + assert!(format!("{location_error}").contains("property: location")); +} diff --git a/crates/sqllogictest/Cargo.toml b/crates/sqllogictest/Cargo.toml deleted file mode 100644 index e335de433e..0000000000 --- a/crates/sqllogictest/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -edition = { workspace = true } -homepage = { workspace = true } -license = { workspace = true } -name = "iceberg-sqllogictest" -repository = { workspace = true } -rust-version = { workspace = true } -version = { workspace = true } - -[dependencies] -anyhow = { workspace = true } -async-trait = { workspace = true } -datafusion = { workspace = true } -datafusion-sqllogictest = { workspace = true } -enum-ordinalize = { workspace = true } -indicatif = { workspace = true } -sqllogictest = { workspace = true } -toml = { workspace = true } -serde = { workspace = true } -tracing = { workspace = true } -tokio = { workspace = true } - -[package.metadata.cargo-machete] -# These dependencies are added to ensure minimal dependency version -ignored = ["enum-ordinalize"] diff --git a/crates/sqllogictest/DEPENDENCIES.rust.tsv b/crates/sqllogictest/DEPENDENCIES.rust.tsv deleted file mode 100644 index e8af062ea3..0000000000 --- a/crates/sqllogictest/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,398 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib -addr2line@0.24.2 X X -adler2@2.0.1 X X X -adler32@1.2.0 X -ahash@0.8.12 X X -aho-corasick@1.1.3 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.2 X -allocator-api2@0.2.21 X X -android_system_properties@0.1.5 X X -anstream@0.6.20 X X -anstyle@1.0.11 X X -anstyle-parse@0.2.7 X X -anstyle-query@1.1.4 X X -anstyle-wincon@3.0.10 X X -anyhow@1.0.99 X X -apache-avro@0.17.0 X -arrayref@0.3.9 X -arrayvec@0.7.6 X X -arrow@55.2.0 X -arrow-arith@55.2.0 X -arrow-array@55.2.0 X -arrow-buffer@55.2.0 X -arrow-cast@55.2.0 X -arrow-csv@55.2.0 X -arrow-data@55.2.0 X -arrow-ipc@55.2.0 X -arrow-json@55.2.0 X -arrow-ord@55.2.0 X -arrow-row@55.2.0 X -arrow-schema@55.2.0 X -arrow-select@55.2.0 X -arrow-string@55.2.0 X -async-compression@0.4.19 X X -async-recursion@1.1.1 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -autocfg@1.5.0 X X -backtrace@0.3.75 X X -base64@0.21.7 X X -base64@0.22.1 X X -bigdecimal@0.4.8 X X -bitflags@2.9.4 X X -blake2@0.10.6 X X -blake3@1.8.2 X X X -block-buffer@0.10.4 X X -brotli@8.0.2 X X -brotli-decompressor@5.0.0 X X -bumpalo@3.19.0 X X -byteorder@1.5.0 X X -bytes@1.10.1 X -bzip2@0.4.4 X X -bzip2@0.5.2 X X -bzip2-sys@0.1.13+1.0.8 X X -cc@1.2.36 X X -cfg-if@1.0.3 X X -chrono@0.4.42 X X -chrono-tz@0.10.4 X X -clap@4.5.47 X X -clap_builder@4.5.47 X X -clap_derive@4.5.47 X X -clap_lex@0.7.5 X X -colorchoice@1.0.4 X X -comfy-table@7.2.0 X -console@0.15.11 X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -constant_time_eq@0.3.1 X X X -core-foundation-sys@0.8.7 X X -core2@0.4.0 X X -cpufeatures@0.2.17 X X -crc32fast@1.5.0 X X -crossbeam-utils@0.8.21 X X -crunchy@0.2.4 X -crypto-common@0.1.6 X X -csv@1.3.1 X X -csv-core@0.1.12 X X -dary_heap@0.3.7 X X -dashmap@6.1.0 X -datafusion@48.0.1 X -datafusion-catalog@48.0.1 X -datafusion-catalog-listing@48.0.1 X -datafusion-common@48.0.1 X -datafusion-common-runtime@48.0.1 X -datafusion-datasource@48.0.1 X -datafusion-datasource-avro@48.0.1 X -datafusion-datasource-csv@48.0.1 X -datafusion-datasource-json@48.0.1 X -datafusion-datasource-parquet@48.0.1 X -datafusion-doc@48.0.1 X -datafusion-execution@48.0.1 X -datafusion-expr@48.0.1 X -datafusion-expr-common@48.0.1 X -datafusion-functions@48.0.1 X -datafusion-functions-aggregate@48.0.1 X -datafusion-functions-aggregate-common@48.0.1 X -datafusion-functions-nested@48.0.1 X -datafusion-functions-table@48.0.1 X -datafusion-functions-window@48.0.1 X -datafusion-functions-window-common@48.0.1 X -datafusion-macros@48.0.1 X -datafusion-optimizer@48.0.1 X -datafusion-physical-expr@48.0.1 X -datafusion-physical-expr-common@48.0.1 X -datafusion-physical-optimizer@48.0.1 X -datafusion-physical-plan@48.0.1 X -datafusion-session@48.0.1 X -datafusion-spark@48.0.1 X -datafusion-sql@48.0.1 X -datafusion-sqllogictest@48.0.1 X -datafusion-substrait@48.0.1 X -digest@0.10.7 X X -displaydoc@0.2.5 X X -dyn-clone@1.0.20 X X -educe@0.6.0 X -either@1.15.0 X X -encode_unicode@1.0.0 X X -enum-ordinalize@4.3.0 X -enum-ordinalize-derive@4.3.1 X -equivalent@1.0.2 X X -errno@0.3.13 X X -escape8259@0.5.3 X -fallible-iterator@0.2.0 X X -fastrand@2.3.0 X X -find-msvc-tools@0.1.1 X X -fixedbitset@0.5.7 X X -flatbuffers@25.2.10 X -flate2@1.1.2 X X -fnv@1.0.7 X X -foldhash@0.1.5 X -form_urlencoded@1.2.2 X X -fs-err@3.1.1 X X -futures@0.3.31 X X -futures-channel@0.3.31 X X -futures-core@0.3.31 X X -futures-executor@0.3.31 X X -futures-io@0.3.31 X X -futures-macro@0.3.31 X X -futures-sink@0.3.31 X X -futures-task@0.3.31 X X -futures-util@0.3.31 X X -generic-array@0.14.7 X -getrandom@0.2.16 X X -getrandom@0.3.3 X X -gimli@0.31.1 X X -glob@0.3.3 X X -half@2.6.0 X X -hashbrown@0.14.5 X X -hashbrown@0.15.5 X X -heck@0.5.0 X X -hex@0.4.3 X X -hmac@0.12.1 X X -http@1.3.1 X X -humantime@2.2.0 X X -iana-time-zone@0.1.63 X X -iana-time-zone-haiku@0.1.2 X X -iceberg-sqllogictest@0.7.0 X -icu_collections@2.0.0 X -icu_locale_core@2.0.0 X -icu_normalizer@2.0.0 X -icu_normalizer_data@2.0.0 X -icu_properties@2.0.1 X -icu_properties_data@2.0.1 X -icu_provider@2.0.0 X -idna@1.1.0 X X -idna_adapter@1.2.1 X X -indexmap@2.11.0 X X -indicatif@0.17.11 X -integer-encoding@3.0.4 X -io-uring@0.7.10 X X -is_terminal_polyfill@1.70.1 X X -itertools@0.13.0 X X -itertools@0.14.0 X X -itoa@1.0.15 X X -jobserver@0.1.34 X X -js-sys@0.3.78 X X -lexical-core@1.0.5 X X -lexical-parse-float@1.0.5 X X -lexical-parse-integer@1.0.5 X X -lexical-util@1.0.6 X X -lexical-write-float@1.0.5 X X -lexical-write-integer@1.0.5 X X -libc@0.2.175 X X -libflate@2.1.0 X -libflate_lz77@2.1.0 X -libm@0.2.15 X -libtest-mimic@0.8.1 X X -libz-rs-sys@0.5.2 X -linux-raw-sys@0.9.4 X X X -litemap@0.8.0 X -lock_api@0.4.13 X X -log@0.4.28 X X -lz4_flex@0.11.5 X -lzma-sys@0.1.20 X X -md-5@0.10.6 X X -memchr@2.7.5 X X -miniz_oxide@0.8.9 X X X -mio@1.0.4 X -multimap@0.10.1 X X -num@0.4.3 X X -num-bigint@0.4.6 X X -num-complex@0.4.6 X X -num-integer@0.1.46 X X -num-iter@0.1.45 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -number_prefix@0.4.0 X -object@0.36.7 X X -object_store@0.12.3 X X -once_cell@1.21.3 X X -once_cell_polyfill@1.70.1 X X -ordered-float@2.10.1 X -owo-colors@4.2.2 X -parking_lot@0.12.4 X X -parking_lot_core@0.9.11 X X -parquet@55.2.0 X -paste@1.0.15 X X -pbjson@0.7.0 X -pbjson-build@0.7.0 X -pbjson-types@0.7.0 X -percent-encoding@2.3.2 X X -petgraph@0.7.1 X X -petgraph@0.8.2 X X -phf@0.12.1 X -phf_shared@0.12.1 X -pin-project-lite@0.2.16 X X -pin-utils@0.1.0 X X -pkg-config@0.3.32 X X -portable-atomic@1.11.1 X X -postgres-protocol@0.6.8 X X -postgres-types@0.2.9 X X -potential_utf@0.1.3 X -ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.101 X X -prost@0.13.5 X -prost-build@0.13.5 X -prost-derive@0.13.5 X -prost-types@0.13.5 X -psm@0.1.26 X X -quad-rand@0.2.3 X -quote@1.0.40 X X -r-efi@5.3.0 X X X -rand@0.8.5 X X -rand@0.9.2 X X -rand_chacha@0.3.1 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.3 X X -recursive@0.1.1 X -recursive-proc-macro-impl@0.1.1 X -redox_syscall@0.5.17 X -regex@1.11.2 X X -regex-automata@0.4.10 X X -regex-lite@0.1.7 X X -regex-syntax@0.8.6 X X -regress@0.10.4 X X -rle-decode-fast@1.0.3 X X -rust_decimal@1.38.0 X -rustc-demangle@0.1.26 X X -rustc_version@0.4.1 X X -rustix@1.0.8 X X X -rustversion@1.0.22 X X -ryu@1.0.20 X X -same-file@1.0.6 X X -schemars@0.8.22 X -schemars_derive@0.8.22 X -scopeguard@1.2.0 X X -semver@1.0.26 X X -seq-macro@0.3.6 X X -serde@1.0.219 X X -serde_bytes@0.11.17 X X -serde_derive@1.0.219 X X -serde_derive_internals@0.29.1 X X -serde_json@1.0.143 X X -serde_spanned@0.6.9 X X -serde_tokenstream@0.2.2 X -serde_yaml@0.9.34+deprecated X X -sha2@0.10.9 X X -shlex@1.3.0 X X -simdutf8@0.1.5 X X -similar@2.7.0 X -siphasher@1.0.1 X X -slab@0.4.11 X -smallvec@1.15.1 X X -snap@1.1.1 X -sqllogictest@0.28.4 X X -sqlparser@0.55.0 X -sqlparser_derive@0.3.0 X -stable_deref_trait@1.2.0 X X -stacker@0.1.21 X X -static_assertions@1.1.0 X X -stringprep@0.1.5 X X -strsim@0.11.1 X -strum@0.26.3 X -strum_macros@0.26.4 X -subst@0.3.8 X X -substrait@0.56.0 X -subtle@2.6.1 X -syn@2.0.106 X X -synstructure@0.13.2 X -tempfile@3.22.0 X X -thiserror@1.0.69 X X -thiserror@2.0.16 X X -thiserror-impl@1.0.69 X X -thiserror-impl@2.0.16 X X -thrift@0.17.0 X -tiny-keccak@2.0.2 X -tinystr@0.8.1 X -tinyvec@1.10.0 X X X -tinyvec_macros@0.1.1 X X X -tokio@1.47.1 X -tokio-macros@2.5.0 X -tokio-util@0.7.16 X -toml@0.8.23 X X -toml_datetime@0.6.11 X X -toml_edit@0.22.27 X X -toml_write@0.1.2 X X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -twox-hash@2.1.2 X -typed-builder@0.19.1 X X -typed-builder-macro@0.19.1 X X -typenum@1.18.0 X X -typify@0.4.3 X -typify-impl@0.4.3 X -typify-macro@0.4.3 X -unicode-bidi@0.3.18 X X -unicode-ident@1.0.18 X X X -unicode-normalization@0.1.24 X X -unicode-properties@0.1.3 X X -unicode-segmentation@1.12.0 X X -unicode-width@0.1.14 X X -unicode-width@0.2.1 X X -unsafe-libyaml@0.2.11 X -url@2.5.7 X X -utf8_iter@1.0.4 X X -utf8parse@0.2.2 X X -uuid@1.18.1 X X -version_check@0.9.5 X X -walkdir@2.5.0 X X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasi@0.14.4+wasi-0.2.4 X X X -wasm-bindgen@0.2.101 X X -wasm-bindgen-backend@0.2.101 X X -wasm-bindgen-futures@0.4.51 X X -wasm-bindgen-macro@0.2.101 X X -wasm-bindgen-macro-support@0.2.101 X X -wasm-bindgen-shared@0.2.101 X X -web-sys@0.3.78 X X -web-time@1.1.0 X X -winapi-util@0.1.11 X X -windows-core@0.61.2 X X -windows-implement@0.60.0 X X -windows-interface@0.59.1 X X -windows-link@0.1.3 X X -windows-link@0.2.0 X X -windows-result@0.3.4 X X -windows-strings@0.4.2 X X -windows-sys@0.52.0 X X -windows-sys@0.59.0 X X -windows-sys@0.60.2 X X -windows-sys@0.61.0 X X -windows-targets@0.52.6 X X -windows-targets@0.53.3 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_gnullvm@0.53.0 X X -windows_aarch64_msvc@0.52.6 X X -windows_aarch64_msvc@0.53.0 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnu@0.53.0 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_gnullvm@0.53.0 X X -windows_i686_msvc@0.52.6 X X -windows_i686_msvc@0.53.0 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnu@0.53.0 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_gnullvm@0.53.0 X X -windows_x86_64_msvc@0.52.6 X X -windows_x86_64_msvc@0.53.0 X X -winnow@0.7.13 X -wit-bindgen@0.45.1 X X X -writeable@0.6.1 X -xz2@0.1.7 X X -yoke@0.8.0 X -yoke-derive@0.8.0 X -zerocopy@0.8.27 X X X -zerofrom@0.1.6 X -zerofrom-derive@0.1.6 X -zerotrie@0.2.2 X -zerovec@0.11.4 X -zerovec-derive@0.11.1 X -zlib-rs@0.5.2 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/sqllogictest/README.md b/crates/sqllogictest/README.md deleted file mode 100644 index ddcfe851c5..0000000000 --- a/crates/sqllogictest/README.md +++ /dev/null @@ -1,35 +0,0 @@ - - -This crate contains a suite of [sqllogictest](https://crates.io/crates/sqllogictest) tests that are used to validate [iceberg-rust](https://github.com/apache/iceberg-rust). - -## Running the tests - -Just run the following command: - -```bash -cargo test -``` - -## Sql Engines - -The tests are run against the following sql engines: - -* [Apache datafusion](https://crates.io/crates/datafusion) -* [Apache spark](https://github.com/apache/spark) \ No newline at end of file diff --git a/crates/sqllogictest/src/engine/datafusion.rs b/crates/sqllogictest/src/engine/datafusion.rs deleted file mode 100644 index 49d7273d60..0000000000 --- a/crates/sqllogictest/src/engine/datafusion.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use anyhow::{Context, anyhow}; -use datafusion::catalog::CatalogProvider; -use datafusion::prelude::{SessionConfig, SessionContext}; -use datafusion_sqllogictest::DataFusion; -use indicatif::ProgressBar; -use sqllogictest::runner::AsyncDB; -use toml::Table as TomlTable; - -use crate::engine::EngineRunner; -use crate::error::Result; - -pub struct DataFusionEngine { - datafusion: DataFusion, -} - -#[async_trait::async_trait] -impl EngineRunner for DataFusionEngine { - async fn run_slt_file(&mut self, path: &Path) -> Result<()> { - let content = std::fs::read_to_string(path) - .with_context(|| format!("Failed to read slt file {:?}", path)) - .map_err(|e| anyhow!(e))?; - - self.datafusion - .run(content.as_str()) - .await - .with_context(|| format!("Failed to run slt file {:?}", path)) - .map_err(|e| anyhow!(e))?; - - Ok(()) - } -} - -impl DataFusionEngine { - pub async fn new(config: TomlTable) -> Result { - let session_config = SessionConfig::new().with_target_partitions(4); - let ctx = SessionContext::new_with_config(session_config); - ctx.register_catalog("default", Self::create_catalog(&config).await?); - - Ok(Self { - datafusion: DataFusion::new(ctx, PathBuf::from("testdata"), ProgressBar::new(100)), - }) - } - - async fn create_catalog(_: &TomlTable) -> anyhow::Result> { - todo!() - } -} diff --git a/crates/sqllogictest/src/engine/mod.rs b/crates/sqllogictest/src/engine/mod.rs deleted file mode 100644 index a1d34dd9bc..0000000000 --- a/crates/sqllogictest/src/engine/mod.rs +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod datafusion; - -use std::path::Path; - -use toml::Table as TomlTable; - -use crate::engine::datafusion::DataFusionEngine; -use crate::error::Result; - -const KEY_TYPE: &str = "type"; -const TYPE_DATAFUSION: &str = "datafusion"; - -#[async_trait::async_trait] -pub trait EngineRunner: Sized { - async fn run_slt_file(&mut self, path: &Path) -> Result<()>; -} - -pub enum Engine { - DataFusion(DataFusionEngine), -} - -impl Engine { - pub async fn new(config: TomlTable) -> Result { - let engine_type = config - .get(KEY_TYPE) - .ok_or_else(|| anyhow::anyhow!("Missing required key: {KEY_TYPE}"))? - .as_str() - .ok_or_else(|| anyhow::anyhow!("Config value for {KEY_TYPE} must be a string"))?; - - match engine_type { - TYPE_DATAFUSION => { - let engine = DataFusionEngine::new(config).await?; - Ok(Engine::DataFusion(engine)) - } - _ => Err(anyhow::anyhow!("Unsupported engine type: {engine_type}").into()), - } - } - - pub async fn run_slt_file(&mut self, path: &Path) -> Result<()> { - match self { - Engine::DataFusion(engine) => engine.run_slt_file(path).await, - } - } -} - -#[cfg(test)] -mod tests { - use toml::Table as TomlTable; - - use crate::engine::Engine; - - #[tokio::test] - async fn test_engine_new_missing_type_key() { - let config = TomlTable::new(); - let result = Engine::new(config).await; - - assert!(result.is_err()); - } - - #[tokio::test] - async fn test_engine_invalid_type() { - let input = r#" - [engines] - random = { type = "random_engine", url = "http://localhost:8181" } - "#; - let tbl = toml::from_str(input).unwrap(); - let result = Engine::new(tbl).await; - - assert!(result.is_err()); - } -} diff --git a/crates/sqllogictest/src/lib.rs b/crates/sqllogictest/src/lib.rs deleted file mode 100644 index 7b17727d1a..0000000000 --- a/crates/sqllogictest/src/lib.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// This lib contains codes copied from -// [Apache Datafusion](https://github.com/apache/datafusion/tree/main/datafusion/sqllogictest) - -#[allow(dead_code)] -mod engine; -#[allow(dead_code)] -mod error; -#[allow(dead_code)] -mod schedule; diff --git a/crates/sqllogictest/src/schedule.rs b/crates/sqllogictest/src/schedule.rs deleted file mode 100644 index d3284949bc..0000000000 --- a/crates/sqllogictest/src/schedule.rs +++ /dev/null @@ -1,197 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::collections::HashMap; -use std::fs::read_to_string; -use std::path::{Path, PathBuf}; - -use anyhow::{Context, anyhow}; -use serde::{Deserialize, Serialize}; -use toml::{Table as TomlTable, Value}; -use tracing::info; - -use crate::engine::Engine; - -pub struct Schedule { - /// Engine names to engine instances - engines: HashMap, - /// List of test steps to run - steps: Vec, - /// Path of the schedule file - schedule_file: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Step { - /// Engine name - engine: String, - /// Stl file path - slt: String, -} - -impl Schedule { - pub fn new(engines: HashMap, steps: Vec, schedule_file: String) -> Self { - Self { - engines, - steps, - schedule_file, - } - } - - pub async fn from_file>(path: P) -> anyhow::Result { - let path_str = path.as_ref().to_string_lossy().to_string(); - let content = read_to_string(path)?; - let toml_value = content.parse::()?; - let toml_table = toml_value - .as_table() - .ok_or_else(|| anyhow!("Schedule file must be a TOML table"))?; - - let engines = Schedule::parse_engines(toml_table).await?; - let steps = Schedule::parse_steps(toml_table)?; - - Ok(Self::new(engines, steps, path_str)) - } - - pub async fn run(mut self) -> anyhow::Result<()> { - info!("Starting test run with schedule: {}", self.schedule_file); - - for (idx, step) in self.steps.iter().enumerate() { - info!( - "Running step {}/{}, using engine {}, slt file path: {}", - idx + 1, - self.steps.len(), - &step.engine, - &step.slt - ); - - let engine = self - .engines - .get_mut(&step.engine) - .ok_or_else(|| anyhow!("Engine {} not found", step.engine))?; - - let step_sql_path = PathBuf::from(format!( - "{}/testdata/slts/{}", - env!("CARGO_MANIFEST_DIR"), - &step.slt - )); - - engine.run_slt_file(&step_sql_path).await?; - - info!( - "Completed step {}/{}, engine {}, slt file path: {}", - idx + 1, - self.steps.len(), - &step.engine, - &step.slt - ); - } - Ok(()) - } - - async fn parse_engines(table: &TomlTable) -> anyhow::Result> { - let engines_tbl = table - .get("engines") - .with_context(|| "Schedule file must have an 'engines' table")? - .as_table() - .ok_or_else(|| anyhow!("'engines' must be a table"))?; - - let mut engines = HashMap::new(); - - for (name, engine_val) in engines_tbl { - let cfg_tbl = engine_val - .as_table() - .ok_or_else(|| anyhow!("Config of engine '{name}' is not a table"))? - .clone(); - - let engine = Engine::new(cfg_tbl) - .await - .with_context(|| format!("Failed to construct engine '{name}'"))?; - - if engines.insert(name.clone(), engine).is_some() { - return Err(anyhow!("Duplicate engine '{name}'")); - } - } - - Ok(engines) - } - - fn parse_steps(table: &TomlTable) -> anyhow::Result> { - let steps_val = table - .get("steps") - .with_context(|| "Schedule file must have a 'steps' array")?; - - let steps: Vec = steps_val - .clone() - .try_into() - .with_context(|| "Failed to deserialize steps")?; - - Ok(steps) - } -} - -#[cfg(test)] -mod tests { - use toml::Table as TomlTable; - - use crate::schedule::Schedule; - - #[test] - fn test_parse_steps() { - let input = r#" - [[steps]] - engine = "datafusion" - slt = "test.slt" - - [[steps]] - engine = "spark" - slt = "test2.slt" - "#; - - let tbl: TomlTable = toml::from_str(input).unwrap(); - let steps = Schedule::parse_steps(&tbl).unwrap(); - - assert_eq!(steps.len(), 2); - assert_eq!(steps[0].engine, "datafusion"); - assert_eq!(steps[0].slt, "test.slt"); - assert_eq!(steps[1].engine, "spark"); - assert_eq!(steps[1].slt, "test2.slt"); - } - - #[test] - fn test_parse_steps_empty() { - let input = r#" - [[steps]] - "#; - - let tbl: TomlTable = toml::from_str(input).unwrap(); - let steps = Schedule::parse_steps(&tbl); - - assert!(steps.is_err()); - } - - #[tokio::test] - async fn test_parse_engines_invalid_table() { - let toml_content = r#" - engines = "not_a_table" - "#; - - let table: TomlTable = toml::from_str(toml_content).unwrap(); - let result = Schedule::parse_engines(&table).await; - - assert!(result.is_err()); - } -} diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml new file mode 100644 index 0000000000..83541e89c5 --- /dev/null +++ b/crates/storage/opendal/Cargo.toml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = { workspace = true } +license = { workspace = true } +name = "iceberg-storage-opendal" +publish = true +repository = { workspace = true } +version = { workspace = true } + +categories = ["database"] +description = "Apache Iceberg OpenDAL storage implementation" +keywords = ["iceberg", "opendal", "storage"] + +[features] +default = ["opendal-memory", "opendal-fs", "opendal-s3"] +opendal-all = [ + "opendal-memory", + "opendal-fs", + "opendal-s3", + "opendal-gcs", + "opendal-oss", + "opendal-azdls", + "opendal-hf", +] + +opendal-azdls = ["opendal/services-azdls"] +opendal-fs = ["opendal/services-fs"] +opendal-gcs = ["opendal/services-gcs"] +opendal-hdfs-native = [ + "opendal/services-hdfs-native", + "dep:tokio", + "dep:tracing", +] +opendal-hf = ["opendal/services-hf"] +opendal-memory = ["opendal/services-memory"] +opendal-oss = ["opendal/services-oss"] +opendal-s3 = ["opendal/services-s3", "reqsign-aws-v4", "reqsign-core"] + +[dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } +bytes = { workspace = true } +cfg-if = { workspace = true } +futures = { workspace = true } +iceberg = { workspace = true } +iceberg-property-macro = { workspace = true } +opendal = { workspace = true } +reqsign-aws-v4 = { version = "3.0.0", optional = true } +reqsign-core = { version = "3.0.0", optional = true } +serde = { workspace = true } +tokio = { workspace = true, optional = true } +tracing = { workspace = true, optional = true } +typetag = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +async-trait = { workspace = true } +iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } +reqwest = { workspace = true } +serde_json = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["macros", "test-util"] } + +[lints] +workspace = true diff --git a/crates/storage/opendal/LICENSE b/crates/storage/opendal/LICENSE new file mode 120000 index 0000000000..5853aaea53 --- /dev/null +++ b/crates/storage/opendal/LICENSE @@ -0,0 +1 @@ +../../../LICENSE \ No newline at end of file diff --git a/crates/storage/opendal/NOTICE b/crates/storage/opendal/NOTICE new file mode 120000 index 0000000000..295f6bdb3a --- /dev/null +++ b/crates/storage/opendal/NOTICE @@ -0,0 +1 @@ +../../../NOTICE \ No newline at end of file diff --git a/crates/storage/opendal/README.md b/crates/storage/opendal/README.md new file mode 100644 index 0000000000..87ebdbe171 --- /dev/null +++ b/crates/storage/opendal/README.md @@ -0,0 +1,87 @@ + + +# iceberg-storage-opendal + +OpenDAL-based storage backend implementations for [Apache Iceberg Rust](https://rust.iceberg.apache.org/). + +## Supported Storage Backends + +| Storage Backend | Feature Flag | Status | Description | +| -------------------- | ---------------- | --------------- | --------------------------------------------- | +| Memory | `opendal-memory` | ✅ Stable | In-memory storage for testing and development | +| Local Filesystem | `opendal-fs` | ✅ Stable | Local filesystem storage | +| Amazon S3 | `opendal-s3` | ✅ Stable | Amazon S3 storage | +| Google Cloud Storage | `opendal-gcs` | ✅ Stable | Google Cloud Storage | +| Hugging Face | `opendal-hf` | ✅ Stable | Hugging Face buckets and repositories | +| Alibaba Cloud OSS | `opendal-oss` | 🧪 Experimental | Alibaba Cloud Object Storage Service | +| Azure Datalake | `opendal-azdls` | 🧪 Experimental | Azure Datalake Storage v2 | +| HDFS | `opendal-hdfs-native` | 🧪 Experimental | HDFS via the native Rust client (`hdfs-native`) | + +You can enable all stable storage backends at once using the `opendal-all` feature flag. + +> Note that `opendal-oss` and `opendal-azdls` are currently experimental and not included in `opendal-all`. + +> `opendal-hdfs-native` is experimental and not included in `opendal-all`. A portless path authority such as `hdfs://nameservice1/warehouse` must be declared in properties, e.g. `hdfs.name-node.nameservice1 = nn1:8020,nn2:8020`: unlike Hadoop, nameservices defined only in `$HADOOP_CONF_DIR` are not resolved, and a portless host is not dialed on the default port 8020. + +## Usage + +Add the crate to your `Cargo.toml` with the feature flags for the backends you need: + +```toml +[dependencies] +iceberg = { version = "x.y.z" } +iceberg-storage-opendal = { version = "x.y.z", features = ["opendal-s3"] } +iceberg-catalog-rest = { version = "x.y.z" } +``` + +Then pass an `OpenDalStorageFactory` to your catalog builder: + +```rust +use std::collections::HashMap; +use std::sync::Arc; + +use iceberg::{Catalog, CatalogBuilder, TableIdent}; +use iceberg_catalog_rest::{RestCatalogBuilder, REST_CATALOG_PROP_URI}; +use iceberg_storage_opendal::OpenDalStorageFactory; + +#[tokio::main] +async fn main() -> iceberg::Result<()> { + let catalog = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .load( + "my_catalog", + HashMap::from([ + (REST_CATALOG_PROP_URI.to_string(), "http://localhost:8181".to_string()), + ]), + ) + .await?; + + let table = catalog + .load_table(&TableIdent::from_strs(["my_namespace", "my_table"])?) + .await?; + + let scan = table.scan().select_all().build()?; + let stream = scan.to_arrow(Default::default()).await?; + + Ok(()) +} +``` diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt new file mode 100644 index 0000000000..ef24c0f820 --- /dev/null +++ b/crates/storage/opendal/public-api.txt @@ -0,0 +1,135 @@ +pub mod iceberg_storage_opendal +pub use iceberg_storage_opendal::AwsCredential +pub use iceberg_storage_opendal::ProvideCredential +pub enum iceberg_storage_opendal::OpenDalStorage +pub iceberg_storage_opendal::OpenDalStorage::Azdls +pub iceberg_storage_opendal::OpenDalStorage::Azdls::client_config: iceberg_storage_opendal::OpenDalClientConfig +pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::Gcs +pub iceberg_storage_opendal::OpenDalStorage::Gcs::client_config: iceberg_storage_opendal::OpenDalClientConfig +pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::HdfsNative(iceberg_storage_opendal::HdfsNativeStorage) +pub iceberg_storage_opendal::OpenDalStorage::Hf +pub iceberg_storage_opendal::OpenDalStorage::Hf::client_config: iceberg_storage_opendal::OpenDalClientConfig +pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::LocalFs +pub iceberg_storage_opendal::OpenDalStorage::LocalFs::client_config: iceberg_storage_opendal::OpenDalClientConfig +pub iceberg_storage_opendal::OpenDalStorage::Memory +pub iceberg_storage_opendal::OpenDalStorage::Memory::client_config: iceberg_storage_opendal::OpenDalClientConfig +pub iceberg_storage_opendal::OpenDalStorage::Memory::operator: opendal_core::types::operator::operator::Operator +pub iceberg_storage_opendal::OpenDalStorage::Oss +pub iceberg_storage_opendal::OpenDalStorage::Oss::client_config: iceberg_storage_opendal::OpenDalClientConfig +pub iceberg_storage_opendal::OpenDalStorage::Oss::config: alloc::sync::Arc +#[non_exhaustive] pub iceberg_storage_opendal::OpenDalStorage::S3 +pub iceberg_storage_opendal::OpenDalStorage::S3::client_config: iceberg_storage_opendal::OpenDalClientConfig +pub iceberg_storage_opendal::OpenDalStorage::S3::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::S3::customized_credential_load: core::option::Option +pub iceberg_storage_opendal::OpenDalStorage::S3::multipart_part_size: u64 +impl core::clone::Clone for iceberg_storage_opendal::OpenDalStorage +pub fn iceberg_storage_opendal::OpenDalStorage::clone(&self) -> iceberg_storage_opendal::OpenDalStorage +impl core::fmt::Debug for iceberg_storage_opendal::OpenDalStorage +pub fn iceberg_storage_opendal::OpenDalStorage::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::io::storage::Storage for iceberg_storage_opendal::OpenDalStorage +pub fn iceberg_storage_opendal::OpenDalStorage::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalStorage::delete_prefix<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalStorage::delete_stream<'life0, 'async_trait>(&'life0 self, paths: futures_core::stream::BoxStream<'static, alloc::string::String>) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg_storage_opendal::OpenDalStorage::exists<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalStorage::metadata<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalStorage::new_input(&self, path: &str) -> iceberg::error::Result +pub fn iceberg_storage_opendal::OpenDalStorage::new_output(&self, path: &str) -> iceberg::error::Result +pub fn iceberg_storage_opendal::OpenDalStorage::read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalStorage::reader<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalStorage::write<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalStorage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalStorage +pub fn iceberg_storage_opendal::OpenDalStorage::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalStorage +pub fn iceberg_storage_opendal::OpenDalStorage::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub enum iceberg_storage_opendal::OpenDalStorageFactory +pub iceberg_storage_opendal::OpenDalStorageFactory::Azdls +pub iceberg_storage_opendal::OpenDalStorageFactory::Fs +pub iceberg_storage_opendal::OpenDalStorageFactory::Gcs +pub iceberg_storage_opendal::OpenDalStorageFactory::HdfsNative +pub iceberg_storage_opendal::OpenDalStorageFactory::Hf +pub iceberg_storage_opendal::OpenDalStorageFactory::Memory +pub iceberg_storage_opendal::OpenDalStorageFactory::Oss +pub iceberg_storage_opendal::OpenDalStorageFactory::S3 +pub iceberg_storage_opendal::OpenDalStorageFactory::S3::customized_credential_load: core::option::Option +impl core::clone::Clone for iceberg_storage_opendal::OpenDalStorageFactory +pub fn iceberg_storage_opendal::OpenDalStorageFactory::clone(&self) -> iceberg_storage_opendal::OpenDalStorageFactory +impl core::fmt::Debug for iceberg_storage_opendal::OpenDalStorageFactory +pub fn iceberg_storage_opendal::OpenDalStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::io::storage::StorageFactory for iceberg_storage_opendal::OpenDalStorageFactory +pub fn iceberg_storage_opendal::OpenDalStorageFactory::build(&self, config: &iceberg::io::storage::config::StorageConfig) -> iceberg::error::Result> +impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalStorageFactory +pub fn iceberg_storage_opendal::OpenDalStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalStorageFactory +pub fn iceberg_storage_opendal::OpenDalStorageFactory::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_storage_opendal::CustomAwsCredentialLoader(_) +impl iceberg_storage_opendal::CustomAwsCredentialLoader +pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::new(provider: impl reqsign_core::api::ProvideCredential + 'static) -> Self +impl core::clone::Clone for iceberg_storage_opendal::CustomAwsCredentialLoader +pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::clone(&self) -> Self +impl core::fmt::Debug for iceberg_storage_opendal::CustomAwsCredentialLoader +pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_storage_opendal::HdfsNativeStorage +impl core::clone::Clone for iceberg_storage_opendal::HdfsNativeStorage +pub fn iceberg_storage_opendal::HdfsNativeStorage::clone(&self) -> iceberg_storage_opendal::HdfsNativeStorage +impl core::fmt::Debug for iceberg_storage_opendal::HdfsNativeStorage +pub fn iceberg_storage_opendal::HdfsNativeStorage::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg_storage_opendal::HdfsNativeStorage +pub fn iceberg_storage_opendal::HdfsNativeStorage::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::HdfsNativeStorage +pub fn iceberg_storage_opendal::HdfsNativeStorage::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_storage_opendal::OpenDalClientConfig +impl iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::from_properties(properties: &std::collections::hash::map::HashMap) -> iceberg::error::Result +impl core::clone::Clone for iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::clone(&self) -> iceberg_storage_opendal::OpenDalClientConfig +impl core::default::Default for iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::default() -> Self +impl core::fmt::Debug for iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalClientConfig +pub fn iceberg_storage_opendal::OpenDalClientConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalClientConfig where iceberg_storage_opendal::OpenDalClientConfig: core::default::Default +pub fn iceberg_storage_opendal::OpenDalClientConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_storage_opendal::OpenDalResolvingStorage +impl core::fmt::Debug for iceberg_storage_opendal::OpenDalResolvingStorage +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::io::storage::Storage for iceberg_storage_opendal::OpenDalResolvingStorage +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::delete<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::delete_prefix<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::delete_stream<'life0, 'async_trait>(&'life0 self, paths: futures_core::stream::BoxStream<'static, alloc::string::String>) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::exists<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::metadata<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::new_input(&self, path: &str) -> iceberg::error::Result +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::new_output(&self, path: &str) -> iceberg::error::Result +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::read<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::reader<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::write<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str, bs: bytes::bytes::Bytes) -> core::pin::Pin> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::writer<'life0, 'life1, 'async_trait>(&'life0 self, path: &'life1 str) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalResolvingStorage +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalResolvingStorage +pub fn iceberg_storage_opendal::OpenDalResolvingStorage::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg_storage_opendal::OpenDalResolvingStorageFactory +impl iceberg_storage_opendal::OpenDalResolvingStorageFactory +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::new() -> Self +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::with_s3_credential_loader(self, loader: iceberg_storage_opendal::CustomAwsCredentialLoader) -> Self +impl core::clone::Clone for iceberg_storage_opendal::OpenDalResolvingStorageFactory +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::clone(&self) -> iceberg_storage_opendal::OpenDalResolvingStorageFactory +impl core::default::Default for iceberg_storage_opendal::OpenDalResolvingStorageFactory +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::default() -> Self +impl core::fmt::Debug for iceberg_storage_opendal::OpenDalResolvingStorageFactory +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::io::storage::StorageFactory for iceberg_storage_opendal::OpenDalResolvingStorageFactory +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::build(&self, config: &iceberg::io::storage::config::StorageConfig) -> iceberg::error::Result> +impl serde_core::ser::Serialize for iceberg_storage_opendal::OpenDalResolvingStorageFactory +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalResolvingStorageFactory +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub const iceberg_storage_opendal::OPENDAL_IO_TIMEOUT_MS: &str +pub const iceberg_storage_opendal::OPENDAL_RETRY_MAX_WAIT_MS: &str +pub const iceberg_storage_opendal::OPENDAL_RETRY_MIN_WAIT_MS: &str +pub const iceberg_storage_opendal::OPENDAL_RETRY_NUM_RETRIES: &str diff --git a/crates/iceberg/src/io/storage_azdls.rs b/crates/storage/opendal/src/azdls.rs similarity index 84% rename from crates/iceberg/src/io/storage_azdls.rs rename to crates/storage/opendal/src/azdls.rs index fe12167f6f..88a197fea8 100644 --- a/crates/iceberg/src/io/storage_azdls.rs +++ b/crates/storage/opendal/src/azdls.rs @@ -19,40 +19,28 @@ use std::collections::HashMap; use std::fmt::Display; use std::str::FromStr; +use iceberg::io::{ + ADLS_ACCOUNT_KEY, ADLS_ACCOUNT_NAME, ADLS_AUTHORITY_HOST, ADLS_CLIENT_ID, ADLS_CLIENT_SECRET, + ADLS_CONNECTION_STRING, ADLS_SAS_TOKEN, ADLS_TENANT_ID, +}; +use iceberg::{Error, ErrorKind, Result}; use opendal::Configurator; use opendal::services::AzdlsConfig; +use serde::{Deserialize, Serialize}; use url::Url; -use crate::{Error, ErrorKind, Result, ensure_data_valid}; +use crate::utils::from_opendal_error; -/// A connection string. -/// -/// Note, this string is parsed first, and any other passed adls.* properties -/// will override values from the connection string. -const ADLS_CONNECTION_STRING: &str = "adls.connection-string"; - -/// The account that you want to connect to. -pub const ADLS_ACCOUNT_NAME: &str = "adls.account-name"; - -/// The key to authentication against the account. -pub const ADLS_ACCOUNT_KEY: &str = "adls.account-key"; - -/// The shared access signature. -pub const ADLS_SAS_TOKEN: &str = "adls.sas-token"; - -/// The tenant-id. -pub const ADLS_TENANT_ID: &str = "adls.tenant-id"; - -/// The client-id. -pub const ADLS_CLIENT_ID: &str = "adls.client-id"; - -/// The client-secret. -pub const ADLS_CLIENT_SECRET: &str = "adls.client-secret"; - -/// The authority host of the service principal. -/// - required for client_credentials authentication -/// - default value: `https://login.microsoftonline.com` -pub const ADLS_AUTHORITY_HOST: &str = "adls.authority-host"; +/// Local version of `ensure_data_valid` macro since the iceberg crate's macro +/// uses `$crate::error::Error` paths that don't resolve from external crates +/// (the `error` module is private). +macro_rules! ensure_data_valid { + ($cond:expr, $fmt:literal, $($arg:tt)*) => { + if !$cond { + return Err(Error::new(ErrorKind::DataInvalid, format!($fmt, $($arg)*))); + } + }; +} /// Parses adls.* prefixed configuration properties. pub(crate) fn azdls_config_parse(mut properties: HashMap) -> Result { @@ -103,10 +91,9 @@ pub(crate) fn azdls_config_parse(mut properties: HashMap) -> Res pub(crate) fn azdls_create_operator<'a>( absolute_path: &'a str, config: &AzdlsConfig, - configured_scheme: &AzureStorageScheme, ) -> Result<(opendal::Operator, &'a str)> { let path = absolute_path.parse::()?; - match_path_with_config(&path, config, configured_scheme)?; + match_path_with_config(&path, config)?; let op = azdls_config_build(config, &path)?; @@ -114,8 +101,7 @@ pub(crate) fn azdls_create_operator<'a>( // including their filesystem and account name. // OpenDAL's operator methods expect only the relative path, so we split it // off and save it for later use. - let relative_path_len = path.path.len(); - let (_, relative_path) = absolute_path.split_at(absolute_path.len() - relative_path_len); + let relative_path = path.relative_path(absolute_path); Ok((op, relative_path)) } @@ -125,8 +111,8 @@ pub(crate) fn azdls_create_operator<'a>( /// paths are expected to contain the `dfs` storage service. /// - `wasb[s]` is used to refer to files in Blob Storage directly; paths are /// expected to contain the `blob` storage service. -#[derive(Debug, PartialEq)] -pub(crate) enum AzureStorageScheme { +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub enum AzureStorageScheme { Abfs, Abfss, Wasb, @@ -165,25 +151,14 @@ impl FromStr for AzureStorageScheme { "wasbs" => Ok(AzureStorageScheme::Wasbs), _ => Err(Error::new( ErrorKind::DataInvalid, - format!("Unexpected Azure Storage scheme: {}", s), + format!("Unexpected Azure Storage scheme: {s}"), )), } } } /// Validates whether the given path matches what's configured for the backend. -fn match_path_with_config( - path: &AzureStoragePath, - config: &AzdlsConfig, - configured_scheme: &AzureStorageScheme, -) -> Result<()> { - ensure_data_valid!( - &path.scheme == configured_scheme, - "Storage::Azdls: Scheme mismatch: configured {}, passed {}", - configured_scheme, - path.scheme - ); - +pub(crate) fn match_path_with_config(path: &AzureStoragePath, config: &AzdlsConfig) -> Result<()> { if let Some(ref configured_account_name) = config.account_name { ensure_data_valid!( &path.account_name == configured_account_name, @@ -225,12 +200,12 @@ fn azdls_config_build(config: &AzdlsConfig, path: &AzureStoragePath) -> Result(&self, absolute_path: &'a str) -> &'a str { + let path = &absolute_path[absolute_path.len() - self.path.len()..]; + path.strip_prefix('/').unwrap_or(path) + } } impl FromStr for AzureStoragePath { @@ -345,8 +327,7 @@ mod tests { use opendal::services::AzdlsConfig; - use super::{AzureStoragePath, AzureStorageScheme, azdls_create_operator}; - use crate::io::azdls_config_parse; + use super::{AzureStoragePath, AzureStorageScheme, azdls_config_parse, azdls_create_operator}; #[test] fn test_azdls_config_parse() { @@ -397,11 +378,11 @@ mod tests { let config = azdls_config_parse(properties); match expected { Some(expected_config) => { - assert!(config.is_ok(), "Test case {} failed: {:?}", name, config); - assert_eq!(config.unwrap(), expected_config, "Test case: {}", name); + assert!(config.is_ok(), "Test case {name} failed: {config:?}"); + assert_eq!(config.unwrap(), expected_config, "Test case: {name}"); } None => { - assert!(config.is_err(), "Test case {} expected error.", name); + assert!(config.is_err(), "Test case {name} expected error."); } } } @@ -419,46 +400,43 @@ mod tests { endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), ..Default::default() }, - AzureStorageScheme::Abfss, ), - Some(("myfs", "/path/to/file.parquet")), + Some(("myfs", "path/to/file.parquet")), ), ( - "different account", + "filesystem root", ( - "abfss://myfs@anotheraccount.dfs.core.windows.net/path/to/file.parquet", + "abfss://myfs@myaccount.dfs.core.windows.net/", AzdlsConfig { account_name: Some("myaccount".to_string()), endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), ..Default::default() }, - AzureStorageScheme::Abfss, ), - None, + Some(("myfs", "")), ), ( - "different scheme", + "different account", ( - "wasbs://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet", + "abfss://myfs@anotheraccount.dfs.core.windows.net/path/to/file.parquet", AzdlsConfig { account_name: Some("myaccount".to_string()), endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), ..Default::default() }, - AzureStorageScheme::Abfss, ), None, ), ( "incompatible scheme for endpoint", ( - "abfs://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet", + // `abfss` implies https; configured endpoint is plain http. + "abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet", AzdlsConfig { account_name: Some("myaccount".to_string()), endpoint: Some("http://myaccount.dfs.core.windows.net".to_string()), ..Default::default() }, - AzureStorageScheme::Abfss, ), None, ), @@ -471,7 +449,6 @@ mod tests { endpoint: Some("https://myaccount.dfs.core.chinacloudapi.cn".to_string()), ..Default::default() }, - AzureStorageScheme::Abfss, ), None, ), @@ -485,24 +462,37 @@ mod tests { endpoint: None, ..Default::default() }, - AzureStorageScheme::Abfs, ), - Some(("myfs", "/path/to/file.parquet")), + Some(("myfs", "path/to/file.parquet")), + ), + ( + "scheme differs from a previously-configured one is accepted", + ( + // No configured scheme exists anymore; both abfss and wasbs + // should be accepted by the same storage. + "wasbs://myfs@myaccount.blob.core.windows.net/path/to/file.parquet", + AzdlsConfig { + account_name: Some("myaccount".to_string()), + endpoint: Some("https://myaccount.blob.core.windows.net".to_string()), + ..Default::default() + }, + ), + Some(("myfs", "path/to/file.parquet")), ), ]; for (name, input, expected) in test_cases { - let result = azdls_create_operator(input.0, &input.1, &input.2); + let result = azdls_create_operator(input.0, &input.1); match expected { Some((expected_filesystem, expected_path)) => { - assert!(result.is_ok(), "Test case {} failed: {:?}", name, result); + assert!(result.is_ok(), "Test case {name} failed: {result:?}"); let (op, relative_path) = result.unwrap(); assert_eq!(op.info().name(), expected_filesystem); assert_eq!(relative_path, expected_path); } None => { - assert!(result.is_err(), "Test case {} expected error.", name); + assert!(result.is_err(), "Test case {name} expected error."); } } } @@ -543,11 +533,11 @@ mod tests { let result = input.parse::(); match expected { Some(expected_path) => { - assert!(result.is_ok(), "Test case {} failed: {:?}", name, result); - assert_eq!(result.unwrap(), expected_path, "Test case: {}", name); + assert!(result.is_ok(), "Test case {name} failed: {result:?}"); + assert_eq!(result.unwrap(), expected_path, "Test case: {name}"); } None => { - assert!(result.is_err(), "Test case {} expected error.", name); + assert!(result.is_err(), "Test case {name} expected error."); } } } @@ -593,7 +583,7 @@ mod tests { for (name, path, expected) in test_cases { let endpoint = path.as_endpoint(); - assert_eq!(endpoint, expected, "Test case: {}", name); + assert_eq!(endpoint, expected, "Test case: {name}"); } } } diff --git a/crates/iceberg/src/io/storage_fs.rs b/crates/storage/opendal/src/fs.rs similarity index 89% rename from crates/iceberg/src/io/storage_fs.rs rename to crates/storage/opendal/src/fs.rs index d3e121a085..1d830605a8 100644 --- a/crates/iceberg/src/io/storage_fs.rs +++ b/crates/storage/opendal/src/fs.rs @@ -15,15 +15,16 @@ // specific language governing permissions and limitations // under the License. +use iceberg::Result; use opendal::Operator; use opendal::services::FsConfig; -use crate::Result; +use crate::utils::from_opendal_error; /// Build new opendal operator from give path. pub(crate) fn fs_config_build() -> Result { let mut cfg = FsConfig::default(); cfg.root = Some("/".to_string()); - Ok(Operator::from_config(cfg)?.finish()) + Operator::from_config(cfg).map_err(from_opendal_error) } diff --git a/crates/iceberg/src/io/storage_gcs.rs b/crates/storage/opendal/src/gcs.rs similarity index 52% rename from crates/iceberg/src/io/storage_gcs.rs rename to crates/storage/opendal/src/gcs.rs index 8c3d914c86..a00282cc22 100644 --- a/crates/iceberg/src/io/storage_gcs.rs +++ b/crates/storage/opendal/src/gcs.rs @@ -18,36 +18,16 @@ use std::collections::HashMap; +use iceberg::io::{ + GCS_ALLOW_ANONYMOUS, GCS_CREDENTIALS_JSON, GCS_DISABLE_CONFIG_LOAD, GCS_DISABLE_VM_METADATA, + GCS_NO_AUTH, GCS_SERVICE_HOST, GCS_TOKEN, +}; +use iceberg::{Error, ErrorKind, Result}; use opendal::Operator; use opendal::services::GcsConfig; use url::Url; -use crate::io::is_truthy; -use crate::{Error, ErrorKind, Result}; - -// Reference: https://github.com/apache/iceberg/blob/main/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java - -/// Google Cloud Project ID -pub const GCS_PROJECT_ID: &str = "gcs.project-id"; -/// Google Cloud Storage endpoint -pub const GCS_SERVICE_PATH: &str = "gcs.service.path"; -/// Google Cloud user project -pub const GCS_USER_PROJECT: &str = "gcs.user-project"; -/// Allow unauthenticated requests -pub const GCS_NO_AUTH: &str = "gcs.no-auth"; -/// Google Cloud Storage credentials JSON string, base64 encoded. -/// -/// E.g. base64::prelude::BASE64_STANDARD.encode(serde_json::to_string(credential).as_bytes()) -pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json"; -/// Google Cloud Storage token -pub const GCS_TOKEN: &str = "gcs.oauth2.token"; - -/// Option to skip signing requests (e.g. for public buckets/folders). -pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous"; -/// Option to skip loading the credential from GCE metadata server (typically used in conjunction with `GCS_ALLOW_ANONYMOUS`). -pub const GCS_DISABLE_VM_METADATA: &str = "gcs.disable-vm-metadata"; -/// Option to skip loading configuration from config file and the env. -pub const GCS_DISABLE_CONFIG_LOAD: &str = "gcs.disable-config-load"; +use crate::utils::{from_opendal_error, is_truthy}; /// Parse iceberg properties to [`GcsConfig`]. pub(crate) fn gcs_config_parse(mut m: HashMap) -> Result { @@ -61,30 +41,30 @@ pub(crate) fn gcs_config_parse(mut m: HashMap) -> Result Result let bucket = url.host_str().ok_or_else(|| { Error::new( ErrorKind::DataInvalid, - format!("Invalid gcs url: {}, bucket is required", path), + format!("Invalid gcs url: {path}, bucket is required"), ) })?; let mut cfg = cfg.clone(); cfg.bucket = bucket.to_string(); - Ok(Operator::from_config(cfg)?.finish()) + Operator::from_config(cfg).map_err(from_opendal_error) } diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs new file mode 100644 index 0000000000..ef54d86423 --- /dev/null +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -0,0 +1,1355 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HDFS storage backend via OpenDAL's `services-hdfs-native` (pure Rust, no JNI). + +use std::collections::HashMap; +use std::sync::{Arc, LazyLock, PoisonError, RwLock}; + +use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; +use iceberg::{Error, ErrorKind, Result}; +use iceberg_property_macro::Properties; +use opendal::Operator; +use opendal::services::HdfsNativeConfig; +use serde::{Deserialize, Serialize}; +use tokio::runtime::Runtime; +use url::Url; + +use crate::OpenDalClientConfig; +use crate::utils::from_opendal_error; + +/// Hadoop's default filesystem, which serves authority-less paths. +const FS_DEFAULT_FS: &str = "fs.defaultFS"; +const HDFS_DEFAULT_PORT: u16 = 8020; +/// PyIceberg keys with no equivalent in opendal's config. +const HDFS_UNSUPPORTED_KEYS: [&str; 2] = ["hdfs.user", "hdfs.kerberos_ticket"]; +/// Hadoop's keys declaring an HA nameservice: `hdfs.name-node.` +/// expands to them, and they are honored as well when passed through `hadoop.`. +const HA_NAMENODES_PREFIX: &str = "dfs.ha.namenodes"; +const HA_NAMENODE_RPC_ADDRESS_PREFIX: &str = "dfs.namenode.rpc-address"; + +/// Normalizes one NameNode spelling to `hdfs://host:port`, the form path +/// authorities take, so every source shares cache keys. `hdfs-native` dials +/// a socket address and has no default port, so anything else is `None`: +/// another scheme, a logical name, port 0, userinfo, a path. +fn hdfs_native_name_node(entry: &str) -> Option { + let rest = entry.trim().trim_end_matches('/'); + let rest = rest.strip_prefix("hdfs://").unwrap_or(rest); + if rest.is_empty() || rest.contains("://") { + return None; + } + let url = Url::parse(&format!("hdfs://{rest}")).ok()?; + let (host, port) = (url.host_str()?, url.port()?); + let plain = host.is_empty() + || port == 0 + || !url.username().is_empty() + || url.password().is_some() + || !matches!(url.path(), "" | "/") + || url.query().is_some() + || url.fragment().is_some(); + (!plain).then(|| hdfs_native_endpoint(host, port)) +} + +/// An endpoint as every source spells it, the host lowercased because host +/// names are case-insensitive: `NN1:8020` and `nn1:8020` share one client, +/// as they share one `FileSystem` in Hadoop's cache. +fn hdfs_native_endpoint(host: &str, port: u16) -> String { + format!("hdfs://{}:{port}", host.to_ascii_lowercase()) +} + +/// Parses a comma-separated NameNode list, each entry normalized; an empty +/// list is `Ok` and empty. +fn hdfs_native_name_node_list(property: &str, value: &str) -> Result> { + value + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(|entry| { + hdfs_native_name_node(entry).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{property}` entry: {}, expected host:port (hdfs:// optional)", + hdfs_native_redact(entry) + ), + ) + }) + }) + .collect() +} + +/// Prefix of `hdfs.name-node.`, which declares one HA nameservice. +const HDFS_NAME_NODE_DECLARATION_PREFIX: &str = "hdfs.name-node."; + +/// The properties this backend reads; unknown keys are ignored. Blank values +/// count as unset, as in PyIceberg. +#[derive(Properties)] +struct HdfsNativeProperties { + /// `hdfs.name-node`, each entry normalized. + #[property( + key = HDFS_NAME_NODE, + default = Vec::new(), + parse_with = parse_name_node_list + )] + name_node: Vec, + /// `hdfs.name-node.` declarations, keyed by nameservice. + #[property(prefix = HDFS_NAME_NODE_DECLARATION_PREFIX)] + nameservices: HashMap, + #[property(key = HDFS_HOST, default = "", parse_with = parse_trimmed)] + host: String, + #[property(key = HDFS_PORT, default = "", parse_with = parse_trimmed)] + port: String, + /// `hadoop.*` keys with the prefix stripped, forwarded to the client. + #[property(prefix = HDFS_HADOOP_CONF_PREFIX)] + hadoop: HashMap, +} + +// Entries are trimmed one by one: opendal splits the list on `,` as is, so a +// space after a comma would break failover to that NameNode. An empty list +// stays unset because `Operator::from_config` bypasses the builder's +// empty-string guard and `Some("")` would shadow the path-authority fallback. +fn parse_name_node_list(value: &str) -> Result> { + hdfs_native_name_node_list(HDFS_NAME_NODE, value) +} + +fn parse_trimmed(value: &str) -> Result { + Ok(value.trim().to_string()) +} + +/// Parse iceberg properties to [`HdfsNativeConfig`]; dropped properties are +/// logged as warnings. +pub(crate) fn hdfs_native_config_parse(m: HashMap) -> Result { + hdfs_native_config_parse_with(m, |warning| tracing::warn!("{warning}")) +} + +/// The parser proper, with the warning sink injected so tests can see what +/// was dropped without a tracing subscriber. +fn hdfs_native_config_parse_with( + m: HashMap, + mut warn: impl FnMut(String), +) -> Result { + let props = HdfsNativeProperties::from_properties(&m)?; + let mut options = props.hadoop; + + // `hdfs.name-node.` is sugar for Hadoop's own declaration of + // an HA nameservice, which the resolver reads back from the options. + // Explicit `hadoop.` keys win over the sugar. + for (nameservice, value) in props.nameservices { + let key = format!("{HDFS_NAME_NODE_DECLARATION_PREFIX}{nameservice}"); + if nameservice.is_empty() || nameservice.chars().any(char::is_whitespace) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid property `{key}`: a nameservice name must follow `{HDFS_NAME_NODE_DECLARATION_PREFIX}`" + ), + )); + } + let entries = hdfs_native_name_node_list(&key, &value)?; + if entries.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid property `{key}`: no NameNodes"), + )); + } + let ids: Vec = (0..entries.len()).map(|i| format!("nn{i}")).collect(); + options + .entry(format!("{HA_NAMENODES_PREFIX}.{nameservice}")) + .or_insert_with(|| ids.join(",")); + for (id, entry) in ids.iter().zip(&entries) { + options + .entry(format!( + "{HA_NAMENODE_RPC_ADDRESS_PREFIX}.{nameservice}.{id}" + )) + .or_insert_with(|| entry.trim_start_matches("hdfs://").to_string()); + } + } + + // A config carried over from PyIceberg would otherwise change identity + // silently; the client reads `HADOOP_USER_NAME` and the Kerberos cache. + for key in HDFS_UNSUPPORTED_KEYS { + if m.contains_key(key) { + warn(format!( + "`{key}` is not supported by the hdfs-native backend and is ignored" + )); + } + } + if m.contains_key(HDFS_HADOOP_CONF_PREFIX) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid property `{HDFS_HADOOP_CONF_PREFIX}`: a Hadoop key must follow the prefix" + ), + )); + } + + let port = match props.port.as_str() { + "" => None, + port => Some(port.parse::().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid `{HDFS_PORT}`: {port}"), + ) + .with_source(e) + })?), + }; + // PyIceberg's `hdfs.host`/`hdfs.port` name the filesystem for + // authority-less paths, which is what Hadoop's `fs.defaultFS` means; an + // explicit `hadoop.fs.defaultFS` wins. + match props.host.as_str() { + "" if port.is_some() => { + warn(format!( + "`{HDFS_PORT}` has no effect without `{HDFS_HOST}` and is ignored" + )); + } + "" => {} + host => { + // An IPv6 literal needs brackets in a URI authority. + let bracketed = if host.contains(':') && !host.starts_with('[') { + format!("[{host}]") + } else { + host.to_string() + }; + let port = port.unwrap_or(HDFS_DEFAULT_PORT); + // Validated like every other NameNode spelling, so a host that + // carries a scheme or port fails here and not at the first I/O. + let default_fs = hdfs_native_name_node(&format!("hdfs://{bracketed}:{port}")) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{HDFS_HOST}`/`{HDFS_PORT}`: {}, expected a host name or IP and a port", + hdfs_native_redact(&format!("{host}:{port}")) + ), + ) + })?; + options + .entry(FS_DEFAULT_FS.to_string()) + .or_insert(default_fs); + } + } + + let mut cfg = HdfsNativeConfig::default(); + if !props.name_node.is_empty() { + cfg.name_node = Some(props.name_node.join(",")); + } + if !options.is_empty() { + cfg.options = Some(options); + } + + // Every declaration, sugar or passed through, and the default filesystem + // fail here rather than at the first path that uses them. + for nameservice in cfg.options.iter().flat_map(|options| { + options.keys().filter_map(|key| { + key.strip_prefix(HA_NAMENODES_PREFIX) + .and_then(|rest| rest.strip_prefix('.')) + }) + }) { + hdfs_native_nameservice(&cfg, nameservice)?; + } + hdfs_native_default_name_node(&cfg)?; + Ok(cfg) +} + +/// `s` with everything between its scheme and its last `@` masked, so an +/// error message never echoes userinfo, even a password holding a raw `/`, +/// `?`, `#` or `://` that a URL parser would split elsewhere. An `@` further +/// along the path is masked the same way, which only costs context. +fn hdfs_native_redact(s: &str) -> String { + // Only a real scheme is kept; anything else before `://` may be userinfo. + let (scheme, rest) = match s.split_once("://") { + Some((scheme, rest)) + if scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || "+-.".contains(c)) => + { + (format!("{scheme}://"), rest) + } + _ => (String::new(), s), + }; + match rest.rsplit_once('@') { + Some((_, after)) => format!("{scheme}***@{after}"), + None => s.to_string(), + } +} + +/// A `DataInvalid` error for an HDFS path, its userinfo masked. +fn hdfs_native_invalid_path(path: &str, reason: impl std::fmt::Display) -> Error { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid hdfs path: {}, {reason}", hdfs_native_redact(path)), + ) +} + +/// A path's authority, as the resolver reads it. +#[derive(Debug, PartialEq)] +pub(crate) enum HdfsNativeAuthority { + /// `hdfs://host:port`, normalized: dialed as is. + NameNode(String), + /// `hdfs://name` without a port: a nameservice that must be declared. + Nameservice(String), + /// `hdfs:///path`: served by `hdfs.name-node`, else `fs.defaultFS`. + Default, +} + +/// Parse an HDFS path into its authority and the relative path (no leading +/// `/`, opendal style). +pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(HdfsNativeAuthority, &str)> { + let url = Url::parse(path).map_err(|e| hdfs_native_invalid_path(path, e))?; + // Non-special schemes parse even without `//` (e.g. `hdfs:x` is a valid + // non-hierarchical URL), so require the literal prefix before slicing. + let (Some(after_scheme), "hdfs") = (path.strip_prefix("hdfs://"), url.scheme()) else { + return Err(hdfs_native_invalid_path(path, "expected scheme `hdfs://`")); + }; + // Userinfo has nowhere to go and port 0 cannot be dialed; silently + // dropping the one or reading the other as a logical name would mislead. + if !url.username().is_empty() || url.password().is_some() { + return Err(hdfs_native_invalid_path(path, "userinfo is not supported")); + } + if url.port() == Some(0) { + return Err(hdfs_native_invalid_path(path, "port 0 cannot be dialed")); + } + + let authority = match (url.host_str().filter(|h| !h.is_empty()), url.port()) { + (Some(host), Some(port)) => HdfsNativeAuthority::NameNode(hdfs_native_endpoint(host, port)), + (Some(host), None) => HdfsNativeAuthority::Nameservice(host.to_string()), + (None, _) => HdfsNativeAuthority::Default, + }; + + // `url.path()` borrows from `url` and can't be returned with the input's + // lifetime. Take the path from the original input instead: everything + // after the first `/` following the `hdfs://` prefix. Opendal paths must + // not start with `/` (`Deleter::delete` rejects them). + let rel = after_scheme + .split_once('/') + .map_or("", |(_, path)| path.trim_start_matches('/')); + + Ok((authority, rel)) +} + +/// Resolves the effective NameNode for a path, plus the relative path. An +/// authority with a port is used as is, a portless one must be a declared +/// nameservice, and an authority-less path uses `hdfs.name-node`, else +/// `fs.defaultFS`. The operator cache, `delete_stream` batching and +/// `relativize_path` all go through this, so they cannot drift apart. +pub(crate) fn hdfs_native_effective_name_node<'a>( + config: &HdfsNativeConfig, + path: &'a str, +) -> Result<(String, &'a str)> { + let (authority, relative_path) = hdfs_native_parse_path(path)?; + let name_node = match authority { + HdfsNativeAuthority::NameNode(name_node) => name_node, + HdfsNativeAuthority::Nameservice(nameservice) => { + hdfs_native_nameservice(config, &nameservice)?.ok_or_else(|| { + hdfs_native_invalid_path( + path, + format!( + "`{nameservice}` has no port and is not a declared nameservice; add the port or set `{HDFS_NAME_NODE}.{nameservice}`" + ), + ) + })? + } + HdfsNativeAuthority::Default => hdfs_native_default_name_node(config)?.ok_or_else(|| { + hdfs_native_invalid_path( + path, + format!("authority-less paths require `{HDFS_NAME_NODE}` or `{HDFS_HOST}`"), + ) + })?, + }; + Ok((name_node, relative_path)) +} + +/// The NameNode that serves authority-less paths: `hdfs.name-node`, else +/// `fs.defaultFS`, which must then be an HDFS `host:port`. +fn hdfs_native_default_name_node(config: &HdfsNativeConfig) -> Result> { + if let Some(name_node) = &config.name_node { + return Ok(Some(name_node.clone())); + } + let Some(default_fs) = config + .options + .as_ref() + .and_then(|options| options.get(FS_DEFAULT_FS)) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + else { + return Ok(None); + }; + hdfs_native_name_node(default_fs).map(Some).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{FS_DEFAULT_FS}`: {} is not an HDFS host:port, a logical nameservice requires `{HDFS_NAME_NODE}`", + hdfs_native_redact(default_fs) + ), + ) + }) +} + +/// The NameNodes that Hadoop's keys in the forwarded options declare for a +/// nameservice, if any. A declaration with a missing or malformed address is +/// an error rather than a silent fallback. +fn hdfs_native_nameservice(config: &HdfsNativeConfig, nameservice: &str) -> Result> { + let Some(options) = config.options.as_ref() else { + return Ok(None); + }; + let Some(ids) = options.get(&format!("{HA_NAMENODES_PREFIX}.{nameservice}")) else { + return Ok(None); + }; + let mut name_nodes = Vec::new(); + for id in ids.split(',').map(str::trim).filter(|id| !id.is_empty()) { + let key = format!("{HA_NAMENODE_RPC_ADDRESS_PREFIX}.{nameservice}.{id}"); + let Some(name_node) = options + .get(&key) + .and_then(|value| hdfs_native_name_node(value)) + else { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Nameservice `{nameservice}` declares NameNode `{id}` but `{key}` is missing or not host:port" + ), + )); + }; + name_nodes.push(name_node); + } + if name_nodes.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Nameservice `{nameservice}` declares no NameNodes"), + )); + } + Ok(Some(name_nodes.join(","))) +} + +/// State of [`OpenDalStorage::HdfsNative`](crate::OpenDalStorage::HdfsNative): +/// the parsed configuration and the per-NameNode operator cache. Only the +/// storage factories build it. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct HdfsNativeStorage { + pub(crate) config: Arc, + #[serde(skip, default)] + pub(crate) operators: HdfsNativeOperatorCache, + #[serde(default)] + pub(crate) client_config: OpenDalClientConfig, +} + +impl HdfsNativeStorage { + pub(crate) fn new(config: HdfsNativeConfig, client_config: OpenDalClientConfig) -> Self { + Self { + config: Arc::new(config), + operators: HdfsNativeOperatorCache::default(), + client_config, + } + } +} + +/// Operators cached per effective NameNode: one per endpoint or nameservice +/// the storage's paths reach, which in practice is a handful of clusters, so +/// it is not bounded. The cache lives as long as the storage that owns it +/// (clones share it). +#[derive(Clone, Debug, Default)] +pub(crate) struct HdfsNativeOperatorCache(Arc>>); + +// A poisoned lock is recovered rather than reported: writes only insert +// whole entries, so a panic elsewhere cannot leave the map broken. +impl HdfsNativeOperatorCache { + pub(crate) fn get(&self, name_node: &str) -> Option { + self.0 + .read() + .unwrap_or_else(PoisonError::into_inner) + .get(name_node) + .cloned() + } + + /// Inserts `op` unless a concurrent caller got there first, returning + /// whichever operator the cache now holds. + fn insert(&self, name_node: String, op: Operator) -> Operator { + self.0 + .write() + .unwrap_or_else(PoisonError::into_inner) + .entry(name_node) + .or_insert(op) + .clone() + } + + #[cfg(test)] + fn len(&self) -> usize { + self.0.read().unwrap().len() + } +} + +/// The runtime `hdfs-native` clients are built in. A client spawns its +/// NameNode connections on the runtime current when it is built and panics +/// once that runtime is gone, so building it here keeps a cached operator +/// usable from any caller's runtime. Started on first use. +// TODO(#3509): build clients on the catalog's runtime once storage factories +// receive it, and drop this one. +static HDFS_NATIVE_RUNTIME: LazyLock = LazyLock::new(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name("iceberg-hdfs-native") + .build() + .expect("failed to start the HDFS client runtime") +}); + +/// Creates an operator for the path, reusing the cached one for its +/// effective NameNode. +pub(crate) fn hdfs_native_create_operator<'a>( + path: &'a str, + config: &HdfsNativeConfig, + operators: &HdfsNativeOperatorCache, +) -> Result<(Operator, &'a str)> { + let (name_node, relative_path) = hdfs_native_effective_name_node(config, path)?; + + if let Some(op) = operators.get(&name_node) { + return Ok((op, relative_path)); + } + + // Built outside the lock, so a racing first caller may build too; the + // loser is dropped before opening any connection. The build reads the + // Hadoop XML config synchronously (~0.1 ms locally), once per NameNode + // per storage, which is accepted here rather than offloaded. + let op = { + let _runtime = HDFS_NATIVE_RUNTIME.enter(); + hdfs_native_operator_build(config, &name_node)? + }; + Ok((operators.insert(name_node, op), relative_path)) +} + +/// Returns the `delete_stream` grouping key for a path: the effective +/// NameNode, so paths that resolve to different operators never share a +/// deleter. Unresolvable paths key on themselves (as `hf_batch_key` does); +/// `create_operator` then reports the real error. +pub(crate) fn hdfs_native_batch_key(config: &HdfsNativeConfig, path: &str) -> String { + hdfs_native_effective_name_node(config, path) + .map(|(name_node, _)| name_node) + .unwrap_or_else(|_| path.to_string()) +} + +/// Build a new OpenDAL [`Operator`]: OpenDAL splits `name_node` on commas +/// into a synthetic HA name service; `$HADOOP_CONF_DIR` XML still merges in. +fn hdfs_native_operator_build(config: &HdfsNativeConfig, name_node: &str) -> Result { + let mut cfg = config.clone(); + cfg.name_node = Some(name_node.to_string()); + Operator::from_config(cfg).map_err(from_opendal_error) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hdfs_native_config_parse_name_node_and_options() { + let props = HashMap::from([ + ( + HDFS_NAME_NODE.to_string(), + "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), + ), + ( + "hadoop.dfs.client.failover.random.order".to_string(), + "true".to_string(), + ), + ("unrelated.key".to_string(), "ignored".to_string()), + ]); + + let cfg = hdfs_native_config_parse(props).unwrap(); + + assert_eq!( + cfg.name_node.as_deref(), + Some("hdfs://nn1:8020,hdfs://nn2:8020") + ); + let options = cfg.options.unwrap(); + assert_eq!( + options.get("dfs.client.failover.random.order"), + Some(&"true".to_string()) + ); + assert!(!options.contains_key("unrelated.key")); + } + + #[test] + fn test_hdfs_native_config_parse_empty() { + let cfg = hdfs_native_config_parse(HashMap::new()).unwrap(); + + assert_eq!(cfg.name_node, None); + assert_eq!(cfg.options, None); + } + + #[test] + fn test_hdfs_native_config_parse_normalizes_name_node() { + let parse = |value: &str| { + hdfs_native_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + value.to_string(), + )])) + .unwrap() + .name_node + }; + + // Empty must not shadow the path-authority fallback. + assert_eq!(parse(""), None); + assert_eq!(parse(" "), None); + assert_eq!(parse(" , "), None); + // `hdfs://` is optional (Hadoop's own rpc-address format is bare) and + // every entry is normalized to it, so it keys the cache like a path + // authority does; a space after a comma would otherwise break + // failover to the entry behind it. + for (value, normalized) in [ + (" hdfs://nn:8020/ ", "hdfs://nn:8020"), + ("nn:8020", "hdfs://nn:8020"), + ("[::1]:8020", "hdfs://[::1]:8020"), + ("NN:8020", "hdfs://nn:8020"), + ( + "hdfs://nn1:8020/, nn2:8020,", + "hdfs://nn1:8020,hdfs://nn2:8020", + ), + ] { + assert_eq!(parse(value).as_deref(), Some(normalized), "{value}"); + } + } + + #[test] + fn test_hdfs_native_config_parse_host_port_as_default_fs() { + let parse = |props: &[(&str, &str)]| { + let props = props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + hdfs_native_config_parse(props) + .map(|cfg| cfg.options.and_then(|o| o.get(FS_DEFAULT_FS).cloned())) + }; + + assert_eq!( + parse(&[(HDFS_HOST, "nn"), (HDFS_PORT, " 9000 ")]) + .unwrap() + .as_deref(), + Some("hdfs://nn:9000") + ); + assert_eq!( + parse(&[(HDFS_HOST, " nn ")]).unwrap().as_deref(), + Some("hdfs://nn:8020") + ); + assert_eq!( + parse(&[(HDFS_HOST, "::1")]).unwrap().as_deref(), + Some("hdfs://[::1]:8020") + ); + // An explicit `hadoop.fs.defaultFS` wins. + assert_eq!( + parse(&[ + (HDFS_HOST, "nn"), + ("hadoop.fs.defaultFS", "hdfs://explicit:8020") + ]) + .unwrap() + .as_deref(), + Some("hdfs://explicit:8020") + ); + // An empty host is unset; a port without a host has nothing to apply to. + assert_eq!( + parse(&[(HDFS_HOST, " "), (HDFS_PORT, "9000")]).unwrap(), + None + ); + assert_eq!(parse(&[(HDFS_PORT, "9000")]).unwrap(), None); + // A bad port is rejected whether or not a host accompanies it. + for props in [ + &[(HDFS_HOST, "nn"), (HDFS_PORT, "x")][..], + &[(HDFS_PORT, "x")][..], + &[(HDFS_PORT, "70000")][..], + ] { + let err = parse(props).unwrap_err(); + assert!(err.to_string().contains(HDFS_PORT), "{props:?}: {err}"); + // The parse error rides along as the source, not in the message. + assert!( + std::error::Error::source(&err).is_some(), + "{props:?}: {err}" + ); + } + // The composed value is validated like any other NameNode spelling. + for props in [ + &[(HDFS_HOST, "nn:8020")][..], + &[(HDFS_HOST, "hdfs://nn")][..], + &[(HDFS_HOST, "nn"), (HDFS_PORT, "0")][..], + &[(HDFS_HOST, "nn/path")][..], + ] { + let err = parse(props).unwrap_err(); + assert!(err.to_string().contains(HDFS_HOST), "{props:?}: {err}"); + } + } + + #[test] + fn test_hdfs_native_config_parse_rejects_bare_prefix() { + let err = hdfs_native_config_parse(HashMap::from([( + HDFS_HADOOP_CONF_PREFIX.to_string(), + "x".to_string(), + )])) + .unwrap_err(); + assert!(err.to_string().contains(HDFS_HADOOP_CONF_PREFIX), "{err}"); + } + + /// PyIceberg's identity keys and a port without a host are dropped, but + /// visibly: each leaves a warning naming the key, and none is forwarded. + #[test] + fn test_hdfs_native_config_parse_warns_about_ignored_keys() { + let mut warnings = Vec::new(); + let cfg = hdfs_native_config_parse_with( + HashMap::from([ + ("hdfs.user".to_string(), "alice".to_string()), + ( + "hdfs.kerberos_ticket".to_string(), + "/tmp/krb5cc".to_string(), + ), + (HDFS_PORT.to_string(), "9000".to_string()), + ]), + |warning| warnings.push(warning), + ) + .unwrap(); + + assert_eq!(cfg.options, None, "ignored keys must not be forwarded"); + assert_eq!(warnings, [ + "`hdfs.user` is not supported by the hdfs-native backend and is ignored", + "`hdfs.kerberos_ticket` is not supported by the hdfs-native backend and is ignored", + "`hdfs.port` has no effect without `hdfs.host` and is ignored", + ]); + } + + #[test] + fn test_hdfs_native_config_parse_rejects_invalid_name_node_entries() { + let parse = |value: &str| { + hdfs_native_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + value.to_string(), + )])) + }; + + // Logical names, other schemes, userinfo, paths, port 0, unbracketed + // IPv6 and malformed ports: every entry must be a dialable host:port. + for value in [ + "hdfs://ns1", + "nn", + "nn:x", + "nn:0", + "nn:+80", + "::1", + "hdfs://[::1]", + "viewfs://nn:8020", + "foo://nn:8020", + "user@nn:8020", + "nn:8020/path", + "hdfs://nn1:8020,nn2", + ] { + let err = parse(value).unwrap_err().to_string(); + assert!(err.contains(HDFS_NAME_NODE), "{value}: {err}"); + } + } + + #[test] + fn test_hdfs_native_default_fs_must_be_hdfs() { + let parse = |props: &[(&str, &str)]| { + hdfs_native_config_parse( + props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) + }; + let default_fs = |value| parse(&[("hadoop.fs.defaultFS", value)]); + + let (nn, _) = + hdfs_native_effective_name_node(&default_fs("hdfs://nn:8020/").unwrap(), "hdfs:///a") + .unwrap(); + assert_eq!(nn, "hdfs://nn:8020"); + // Not an HDFS `host:port`, including the portless logical-name form, + // which only a declaration resolves: rejected when the config is + // parsed, since it serves every authority-less path. + for value in ["viewfs://cluster/", "hdfs://", "hdfs://ns1"] { + let err = default_fs(value).unwrap_err(); + assert!( + err.to_string().contains(FS_DEFAULT_FS) && err.to_string().contains(HDFS_NAME_NODE), + "{value}: {err}" + ); + } + // Not checked when `hdfs.name-node` serves those paths instead. + parse(&[ + ("hadoop.fs.defaultFS", "viewfs://cluster/"), + (HDFS_NAME_NODE, "nn:8020"), + ]) + .unwrap(); + // Empty is unset, so the authority-less path has nothing to resolve to. + let err = + hdfs_native_effective_name_node(&default_fs("").unwrap(), "hdfs:///a").unwrap_err(); + assert!(err.to_string().contains(HDFS_HOST), "{err}"); + // Normalized like every other source. + let (nn, _) = + hdfs_native_effective_name_node(&default_fs("nn:9000").unwrap(), "hdfs:///a").unwrap(); + assert_eq!(nn, "hdfs://nn:9000"); + } + + /// A bare configured NameNode and the same NameNode as a path authority + /// share one cache entry. + #[test] + fn test_hdfs_native_create_operator_dedupes_spellings() { + let config = Arc::new( + hdfs_native_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + "nn:8020".to_string(), + )])) + .unwrap(), + ); + let operators = HdfsNativeOperatorCache::default(); + + for path in ["hdfs:///a", "hdfs://nn:8020/b", "hdfs://NN:8020/c"] { + hdfs_native_create_operator(path, &config, &operators).unwrap(); + } + + assert_eq!(operators.len(), 1); + assert!(operators.get("hdfs://nn:8020").is_some()); + } + + #[test] + fn test_hdfs_native_config_parse_declares_nameservices() { + let cfg = hdfs_native_config_parse(HashMap::from([ + ( + format!("{HDFS_NAME_NODE}.ns-a"), + "nn1:8020, hdfs://nn2:8020/".to_string(), + ), + ("hadoop.dfs.ha.namenodes.ns-b".to_string(), "x".to_string()), + ( + "hadoop.dfs.namenode.rpc-address.ns-b.x".to_string(), + "nn3:8020".to_string(), + ), + ])) + .unwrap(); + let options = cfg.options.clone().unwrap(); + + // The sugar expands to Hadoop's keys... + assert_eq!( + options.get("dfs.ha.namenodes.ns-a").map(String::as_str), + Some("nn0,nn1") + ); + assert_eq!( + options + .get("dfs.namenode.rpc-address.ns-a.nn0") + .map(String::as_str), + Some("nn1:8020") + ); + assert_eq!( + options + .get("dfs.namenode.rpc-address.ns-a.nn1") + .map(String::as_str), + Some("nn2:8020") + ); + // ...and the resolver reads both forms back. + assert_eq!( + hdfs_native_nameservice(&cfg, "ns-a").unwrap().as_deref(), + Some("hdfs://nn1:8020,hdfs://nn2:8020") + ); + assert_eq!( + hdfs_native_nameservice(&cfg, "ns-b").unwrap().as_deref(), + Some("hdfs://nn3:8020") + ); + assert_eq!(hdfs_native_nameservice(&cfg, "ns-c").unwrap(), None); + + for (key, value) in [ + (format!("{HDFS_NAME_NODE}."), "nn:8020"), + (format!("{HDFS_NAME_NODE}.ns"), "nn"), + (format!("{HDFS_NAME_NODE}.ns"), " "), + ] { + let err = hdfs_native_config_parse(HashMap::from([(key.clone(), value.to_string())])) + .unwrap_err(); + assert!(err.to_string().contains(&key), "{key}={value}: {err}"); + } + // A passed-through declaration with a missing address fails when the + // config is parsed, not at the first path that names it. + let err = hdfs_native_config_parse(HashMap::from([( + "hadoop.dfs.ha.namenodes.ns-d".to_string(), + "a".to_string(), + )])) + .unwrap_err(); + assert!( + err.to_string().contains("dfs.namenode.rpc-address.ns-d.a"), + "{err}" + ); + } + + #[test] + fn test_hdfs_native_effective_name_node_declared_nameservices() { + let parse = |props: &[(String, &str)]| { + hdfs_native_config_parse( + props + .iter() + .map(|(k, v)| (k.clone(), v.to_string())) + .collect(), + ) + .unwrap() + }; + let ns_a = ( + format!("{HDFS_NAME_NODE}.ns-a"), + "hdfs://a1:8020,hdfs://a2:8020", + ); + let declared_only = parse(std::slice::from_ref(&ns_a)); + let with_default = parse(&[ns_a.clone(), (HDFS_NAME_NODE.to_string(), "hdfs://d:8020")]); + + // A declared nameservice resolves to its own list, ahead of the plain key. + for config in [&declared_only, &with_default] { + let (nn, rel) = hdfs_native_effective_name_node(config, "hdfs://ns-a/x").unwrap(); + assert_eq!((nn.as_str(), rel), ("hdfs://a1:8020,hdfs://a2:8020", "x")); + } + // An undeclared one is an error, with or without the plain key. + for config in [&declared_only, &with_default] { + let err = hdfs_native_effective_name_node(config, "hdfs://ns-b/x").unwrap_err(); + assert!(err.to_string().contains("`hdfs.name-node.ns-b`"), "{err}"); + } + // The plain key serves authority-less paths only. + let (nn, _) = hdfs_native_effective_name_node(&with_default, "hdfs:///x").unwrap(); + assert_eq!(nn, "hdfs://d:8020"); + // A concrete authority is never redirected. + let (nn, _) = + hdfs_native_effective_name_node(&with_default, "hdfs://other:9000/x").unwrap(); + assert_eq!(nn, "hdfs://other:9000"); + } + + #[test] + fn test_hdfs_native_create_operator_per_nameservice() { + let config = Arc::new( + hdfs_native_config_parse(HashMap::from([ + ( + format!("{HDFS_NAME_NODE}.ns-a"), + "hdfs://a:8020".to_string(), + ), + ( + format!("{HDFS_NAME_NODE}.ns-b"), + "hdfs://b:8020".to_string(), + ), + (HDFS_NAME_NODE.to_string(), "hdfs://d:8020".to_string()), + ])) + .unwrap(), + ); + let operators = HdfsNativeOperatorCache::default(); + + for path in ["hdfs://ns-a/x", "hdfs://ns-b/y", "hdfs:///w"] { + hdfs_native_create_operator(path, &config, &operators).unwrap(); + } + // ns-a, ns-b, and the plain key for the authority-less path; an + // undeclared nameservice never reaches an operator. + assert_eq!(operators.len(), 3); + assert_eq!( + hdfs_native_batch_key(&config, "hdfs://ns-b/y"), + "hdfs://b:8020" + ); + let err = hdfs_native_create_operator("hdfs://ns-c/z", &config, &operators).unwrap_err(); + assert!(err.to_string().contains("`hdfs.name-node.ns-c`"), "{err}"); + assert_eq!(operators.len(), 3); + } + + #[test] + fn test_hdfs_native_effective_name_node_precedence() { + let configured = hdfs_native_config_parse(HashMap::from([ + ( + HDFS_NAME_NODE.to_string(), + "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), + ), + (HDFS_HOST.to_string(), "ignored".to_string()), + ])) + .unwrap(); + let default_fs = + hdfs_native_config_parse(HashMap::from([(HDFS_HOST.to_string(), "nn".to_string())])) + .unwrap(); + let unconfigured = HdfsNativeConfig::default(); + + // An authority with a port is used as is, whatever is configured. + for config in [&configured, &default_fs, &unconfigured] { + let (nn, rel) = + hdfs_native_effective_name_node(config, "hdfs://other:9000/a/b").unwrap(); + assert_eq!((nn.as_str(), rel), ("hdfs://other:9000", "a/b")); + } + // An authority-less path uses the configured list, ahead of `hdfs.host`. + let (nn, _) = hdfs_native_effective_name_node(&configured, "hdfs:///y").unwrap(); + assert_eq!(nn, "hdfs://nn1:8020,hdfs://nn2:8020"); + // A logical nameservice never falls back to it: it must be declared. + let err = hdfs_native_effective_name_node(&configured, "hdfs://ns-a/x").unwrap_err(); + assert!(err.to_string().contains("`hdfs.name-node.ns-a`"), "{err}"); + // Only an authority-less path falls back to `fs.defaultFS`. + let (nn, rel) = hdfs_native_effective_name_node(&default_fs, "hdfs:///y").unwrap(); + assert_eq!((nn.as_str(), rel), ("hdfs://nn:8020", "y")); + let err = hdfs_native_effective_name_node(&default_fs, "hdfs://ns-a/x").unwrap_err(); + assert!(err.to_string().contains("`hdfs.name-node.ns-a`"), "{err}"); + // Nothing applicable: pointed errors. + for path in ["hdfs:///a", "hdfs://ns-a/x"] { + let err = hdfs_native_effective_name_node(&unconfigured, path).unwrap_err(); + assert!(err.to_string().contains(HDFS_NAME_NODE), "{err}"); + } + } + + #[test] + fn test_hdfs_native_parse_path_with_authority_and_rel() { + let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1/a/b").unwrap(); + + assert_eq!( + nn, + HdfsNativeAuthority::Nameservice("nameservice1".to_string()) + ); + assert_eq!(rel, "a/b"); + } + + #[test] + fn test_hdfs_native_parse_path_with_authority_and_port() { + let (nn, rel) = hdfs_native_parse_path("hdfs://nn:8020/foo").unwrap(); + // Extra leading slashes go too: opendal rejects paths starting with `/`. + assert_eq!( + hdfs_native_parse_path("hdfs://nn:8020//foo").unwrap().1, + "foo" + ); + + assert_eq!( + nn, + HdfsNativeAuthority::NameNode("hdfs://nn:8020".to_string()) + ); + assert_eq!(rel, "foo"); + } + + #[test] + fn test_hdfs_native_parse_path_ipv6_authority_keeps_brackets() { + let (nn, rel) = hdfs_native_parse_path("hdfs://[::1]:8020/a/b").unwrap(); + assert_eq!( + nn, + HdfsNativeAuthority::NameNode("hdfs://[::1]:8020".to_string()) + ); + assert_eq!(rel, "a/b"); + } + + #[test] + fn test_hdfs_native_parse_path_rejects_userinfo_and_port_zero() { + for (path, reason) in [ + ("hdfs://alice@nn:8020/a", "userinfo"), + ("hdfs://alice:secret@nn:8020/a", "userinfo"), + ("hdfs://nn:0/a", "port 0"), + ] { + let err = hdfs_native_parse_path(path).unwrap_err().to_string(); + assert!(err.contains(reason), "{path}: {err}"); + // A password must never reach a log. + assert!(!err.contains("alice") && !err.contains("secret"), "{err}"); + } + } + + #[test] + fn test_hdfs_native_errors_mask_userinfo() { + let path_err = |path: &str| { + hdfs_native_effective_name_node(&HdfsNativeConfig::default(), path) + .unwrap_err() + .to_string() + }; + let config_err = |key: &str, value: &str| { + let props = HashMap::from([(key.to_string(), value.to_string())]); + hdfs_native_config_parse(props) + .and_then(|config| hdfs_native_effective_name_node(&config, "hdfs:///x")) + .unwrap_err() + .to_string() + }; + // Every error that echoes a path or a NameNode value, including the + // ones that fail before the userinfo check can run. + for (err, masked) in [ + ( + path_err("hdfs://alice:s3cret@nn:bad/x"), + "hdfs://***@nn:bad/x", + ), + (path_err("s3://alice:s3cret@b/x"), "s3://***@b/x"), + ( + path_err("hdfs://alice:s3cret@nn:8020/x"), + "hdfs://***@nn:8020/x", + ), + ( + config_err(HDFS_NAME_NODE, "hdfs://alice:s3cret@nn:8020"), + "hdfs://***@nn:8020", + ), + ( + config_err("hdfs.name-node.ns", "hdfs://alice:s3cret@nn:8020"), + "hdfs://***@nn:8020", + ), + (config_err(HDFS_HOST, "alice:s3cret@nn"), "***@nn:8020"), + ( + config_err("hadoop.fs.defaultFS", "hdfs://alice:s3cret@nn:8020"), + "hdfs://***@nn:8020", + ), + // A raw `/`, `#` or `://` in the password, or a second `@`, must + // not move where the masking stops. + ( + path_err("hdfs://alice:s3/cret@nn:8020/x"), + "hdfs://***@nn:8020/x", + ), + (path_err("s3://alice:12#cret@b/x"), "s3://***@b/x"), + ( + path_err("hdfs://alice@corp:s3cret@nn:bad/x"), + "hdfs://***@nn:bad/x", + ), + ( + config_err(HDFS_NAME_NODE, "hdfs://alice:s3/cret@nn:8020"), + "hdfs://***@nn:8020", + ), + (config_err(HDFS_HOST, "alice:s3://cret@nn"), "***@nn:8020"), + ] { + assert!(err.contains(masked), "{err}"); + assert!(!err.contains("alice") && !err.contains("cret"), "{err}"); + } + } + + #[test] + fn test_hdfs_native_parse_path_with_authority_no_path() { + let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1").unwrap(); + + assert_eq!( + nn, + HdfsNativeAuthority::Nameservice("nameservice1".to_string()) + ); + assert_eq!(rel, ""); + } + + #[test] + fn test_hdfs_native_parse_path_with_authority_trailing_slash() { + let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1/").unwrap(); + + assert_eq!( + nn, + HdfsNativeAuthority::Nameservice("nameservice1".to_string()) + ); + assert_eq!(rel, ""); + } + + #[test] + fn test_hdfs_native_parse_path_authority_less_returns_none() { + let (nn, rel) = hdfs_native_parse_path("hdfs:///a/b").unwrap(); + + assert_eq!(nn, HdfsNativeAuthority::Default); + assert_eq!(rel, "a/b"); + } + + #[test] + fn test_hdfs_native_parse_path_wrong_scheme_errors() { + let err = hdfs_native_parse_path("file:///tmp/x").unwrap_err(); + + assert!(err.to_string().contains("expected scheme `hdfs://`")); + } + + #[test] + fn test_hdfs_native_parse_path_invalid_url_errors() { + let err = hdfs_native_parse_path("not-a-url").unwrap_err().to_string(); + + // The URL parser's reason is forwarded; this is not the scheme check. + assert!( + err.contains("Invalid hdfs path") && !err.contains("expected scheme"), + "{err}" + ); + } + + #[test] + fn test_hdfs_native_parse_path_non_hierarchical_errors() { + // `hdfs:x` parses as a valid non-hierarchical URL; it must be + // rejected rather than panic on slicing. + for path in ["hdfs:x", "hdfs:/x", "hdfs:"] { + let err = hdfs_native_parse_path(path).unwrap_err(); + assert!(err.to_string().contains("expected scheme `hdfs://`")); + } + } + + #[test] + fn test_hdfs_native_batch_key_distinguishes_ports() { + let config = HdfsNativeConfig::default(); + + assert_eq!( + hdfs_native_batch_key(&config, "hdfs://namenode:8020/a"), + "hdfs://namenode:8020" + ); + assert_eq!( + hdfs_native_batch_key(&config, "hdfs://namenode:9000/b"), + "hdfs://namenode:9000" + ); + } + + #[test] + fn test_hdfs_native_batch_key_invalid_path_keys_on_itself() { + let config = HdfsNativeConfig::default(); + + // Unresolvable paths must not collapse onto a shared "" key. + for path in ["not-a-url", "hdfs:///a", "hdfs://nn:0/a"] { + assert_eq!(hdfs_native_batch_key(&config, path), path); + } + } + + // Creating an operator offloads the build to a blocking thread, so these + // run under tokio. + /// `relativize_path` follows the same resolution rule as `create_operator`. + #[test] + fn test_hdfs_native_relativize_path() { + let build = |key: String| { + let props = HashMap::from([(key, "hdfs://nn:8020".to_string())]); + crate::OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + hdfs_native_config_parse(props).unwrap(), + OpenDalClientConfig::default(), + )) + }; + let unconfigured = crate::OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + HdfsNativeConfig::default(), + OpenDalClientConfig::default(), + )); + let declared = build(format!("{HDFS_NAME_NODE}.nameservice1")); + let plain = build(HDFS_NAME_NODE.to_string()); + + // A concrete authority needs nothing configured. + assert_eq!( + unconfigured + .relativize_path("hdfs://nn:8020/warehouse/db/t") + .unwrap(), + "warehouse/db/t" + ); + // A logical nameservice needs its declaration, an authority-less path + // the plain key. + for (storage, path, rel) in [ + (&declared, "hdfs://nameservice1/a/b.parquet", "a/b.parquet"), + (&plain, "hdfs:///a/b", "a/b"), + ] { + let err = unconfigured.relativize_path(path).unwrap_err(); + assert!(err.to_string().contains(HDFS_NAME_NODE), "{err}"); + assert_eq!(storage.relativize_path(path).unwrap(), rel); + } + // Another scheme is rejected before any resolution. + let err = unconfigured.relativize_path("s3://bucket/x").unwrap_err(); + assert!( + err.to_string().contains("expected scheme `hdfs://`"), + "{err}" + ); + } + + #[test] + fn test_hdfs_native_create_operator_plain_key_serves_authority_less_paths() { + let config = Arc::new( + hdfs_native_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + "hdfs://nn1:8020/,hdfs://nn2:8020/".to_string(), + )])) + .unwrap(), + ); + let operators = HdfsNativeOperatorCache::default(); + + // Authority-less paths share the configured list's operator. + let (_, rel) = hdfs_native_create_operator("hdfs:///a/b", &config, &operators).unwrap(); + assert_eq!(rel, "a/b"); + assert_eq!(operators.len(), 1); + assert!(operators.get("hdfs://nn1:8020,hdfs://nn2:8020").is_some()); + + // A logical nameservice must be declared; a concrete authority is + // another cluster, never the configured one. + let err = hdfs_native_create_operator("hdfs://ns-a/c", &config, &operators).unwrap_err(); + assert!(err.to_string().contains("`hdfs.name-node.ns-a`"), "{err}"); + hdfs_native_create_operator("hdfs://other:9000/d", &config, &operators).unwrap(); + assert_eq!(operators.len(), 2); + assert!(operators.get("hdfs://other:9000").is_some()); + } + + #[test] + fn test_hdfs_native_create_operator_caches_per_name_node() { + let config = Arc::new(HdfsNativeConfig::default()); + let operators = HdfsNativeOperatorCache::default(); + + for path in [ + "hdfs://nn1:8020/a", + "hdfs://nn1:8020/b", + "hdfs://nn2:8020/c", + ] { + hdfs_native_create_operator(path, &config, &operators).unwrap(); + } + + assert_eq!(operators.len(), 2); + } + + #[tokio::test] + async fn test_hdfs_native_operator_cache_keeps_first_insert() { + let operators = HdfsNativeOperatorCache::default(); + let build = |root: &str| { + let mut config = hdfs_native_config_parse(HashMap::new()).unwrap(); + config.root = Some(root.to_string()); + hdfs_native_operator_build(&config, "hdfs://nn:8020").unwrap() + }; + + // Two callers racing for one NameNode: the first insert wins and both + // get the cached operator. + let first = operators.insert("hdfs://nn:8020".to_string(), build("/first")); + let second = operators.insert("hdfs://nn:8020".to_string(), build("/second")); + assert_eq!(first.info().root(), "/first/"); + assert_eq!(second.info().root(), "/first/"); + assert_eq!(operators.len(), 1); + } + + #[test] + fn test_hdfs_native_operator_cache_survives_a_poisoned_lock() { + let operators = HdfsNativeOperatorCache::default(); + // A panic elsewhere while the lock is held must not break HDFS I/O + // for good: every write inserts a whole entry, so the map is intact. + let _ = std::panic::catch_unwind(|| { + let _guard = operators.0.write().unwrap(); + panic!("poisoning the operator cache lock"); + }); + assert!(operators.0.is_poisoned()); + + let config = Arc::new(HdfsNativeConfig::default()); + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators).unwrap(); + assert!(operators.get("hdfs://nn:8020").is_some()); + } + + /// The configuration round-trips through serde; the operator cache does + /// not and starts empty. + #[test] + fn test_hdfs_native_storage_serde_round_trip() { + let props = HashMap::from([ + (HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string()), + ( + "hadoop.dfs.client.use.datanode.hostname".to_string(), + "true".to_string(), + ), + ]); + let storage = crate::OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + hdfs_native_config_parse(props).unwrap(), + OpenDalClientConfig::from_properties(&HashMap::from([( + crate::OPENDAL_IO_TIMEOUT_MS.to_string(), + "45000".to_string(), + )])) + .unwrap(), + )); + // Populate the cache so that its absence after the round trip means + // something. + let crate::OpenDalStorage::HdfsNative(hdfs) = &storage else { + panic!("expected the HdfsNative variant"); + }; + hdfs_native_create_operator("hdfs:///x", &hdfs.config, &hdfs.operators).unwrap(); + assert!(hdfs.operators.get("hdfs://nn:8020").is_some()); + + let value = serde_json::to_value(&storage).unwrap(); + assert!(value["HdfsNative"].get("operators").is_none()); + let crate::OpenDalStorage::HdfsNative(restored) = serde_json::from_value(value).unwrap() + else { + panic!("expected the HdfsNative variant"); + }; + assert_eq!(restored.config.name_node.as_deref(), Some("hdfs://nn:8020")); + assert_eq!( + restored + .config + .options + .as_ref() + .and_then(|o| o.get("dfs.client.use.datanode.hostname")), + Some(&"true".to_string()) + ); + assert_eq!( + restored.client_config.io_timeout(), + std::time::Duration::from_secs(45) + ); + assert!(restored.operators.get("hdfs://nn:8020").is_none()); + } +} diff --git a/crates/storage/opendal/src/hf.rs b/crates/storage/opendal/src/hf.rs new file mode 100644 index 0000000000..d38229cedf --- /dev/null +++ b/crates/storage/opendal/src/hf.rs @@ -0,0 +1,346 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HuggingFace Hub storage backend. + +use std::collections::HashMap; + +use iceberg::io::{HF_ENDPOINT, HF_REVISION, HF_TOKEN}; +use iceberg::{Error, ErrorKind, Result}; +use opendal::{Configurator, Operator, OperatorUri}; + +use crate::utils::from_opendal_error; + +// --------------------------------------------------------------------------- +// Minimal URI parser — extracts only what the caller needs. +// TODO: remove once opendal-service-hf exports its URI parser publicly. +// --------------------------------------------------------------------------- + +/// Repository type of a HuggingFace Hub repository. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HfRepoType { + /// Model repository (`models/` prefix). + Model, + /// Dataset repository (`datasets/` prefix). + Dataset, + /// Spaces application repository (`spaces/` prefix). + Space, + /// XET-backed object-storage bucket (`buckets/` prefix). + Bucket, +} + +impl HfRepoType { + /// Parse a repo-type keyword (singular or plural) into the corresponding variant. + fn parse(s: &str) -> Option { + match s.to_lowercase().replace(' ', "").as_str() { + "model" | "models" => Some(Self::Model), + "dataset" | "datasets" => Some(Self::Dataset), + "space" | "spaces" => Some(Self::Space), + "bucket" | "buckets" => Some(Self::Bucket), + _ => None, + } + } + + fn canonical(self) -> &'static str { + match self { + Self::Model => "models", + Self::Dataset => "datasets", + Self::Space => "spaces", + Self::Bucket => "buckets", + } + } +} + +/// Parsed HuggingFace URI: `hf:///[@][/]`. +/// +/// `repo_type` must be explicitly specified — there is no implicit default. +/// Only the fields required by this crate are stored; revision is consumed +/// during parsing but not retained. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct HfUri { + pub repo_type: HfRepoType, + /// e.g. `"user/my-repo"`. + pub repo_id: String, + /// Path within the repository, e.g. `"train/data.parquet"`. Empty at repo root. + pub path: String, +} + +impl HfUri { + /// Parse a full `hf://…` URI or the bare path portion (without scheme). + /// Returns `None` if the URI does not begin with a recognized repo-type prefix + /// (`models/`, `datasets/`, `spaces/`, or `buckets/`). + pub(crate) fn parse(full_uri: &str) -> Option { + let s = full_uri.strip_prefix("hf://").unwrap_or(full_uri); + if s.is_empty() { + return None; + } + + // Require an explicit repo_type prefix — no implicit default. + let (first, rest) = s.split_once('/')?; + let repo_type = HfRepoType::parse(first)?; + let s = rest; + + // Remaining: `[@][/]` + let (repo_id, path) = if s.contains('/') { + // Check if `@` appears in the first two slash-segments (the repo_id portion). + // This distinguishes "user/repo@rev/file" from "user/repo/path/@file". + let first_two = s.splitn(3, '/').take(2).collect::>().join("/"); + if first_two.contains('@') { + let (repo_id, rev_and_path) = s.split_once('@').unwrap(); + let rev_and_path = rev_and_path.replace("%2F", "/"); + (repo_id.to_string(), path_after_revision(&rev_and_path)) + } else { + let segs: Vec<_> = s.splitn(3, '/').collect(); + let repo_id = format!("{}/{}", segs[0], segs[1]); + let path = segs.get(2).copied().unwrap_or("").to_string(); + (repo_id, path) + } + } else if let Some((repo_id, _)) = s.split_once('@') { + (repo_id.to_string(), String::new()) + } else { + (s.to_string(), String::new()) + }; + + Some(Self { + repo_type, + repo_id, + path, + }) + } +} + +/// Given the string after `@`, extract the path-in-repo, correctly skipping +/// multi-segment special refs (`refs/convert/parquet`, `refs/pr/N`). +/// These are the only two multi-segment special ref prefixes in HF's git model. +fn path_after_revision(rev_and_path: &str) -> String { + if !rev_and_path.contains('/') { + return String::new(); + } + if let Some(rest) = rev_and_path.strip_prefix("refs/convert/") { + return rest + .find('/') + .map_or(String::new(), |i| rest[i + 1..].to_string()); + } + if let Some(rest) = rev_and_path.strip_prefix("refs/pr/") { + return rest + .find('/') + .map_or(String::new(), |i| rest[i + 1..].to_string()); + } + rev_and_path + .split_once('/') + .map(|(_, path)| path.to_string()) + .unwrap_or_default() +} + +// --------------------------------------------------------------------------- +// Public helpers used by lib.rs +// --------------------------------------------------------------------------- + +/// Parse iceberg `StorageConfig` properties into an opendal [`opendal::services::HfConfig`]. +pub(crate) fn hf_config_parse(m: HashMap) -> Result { + let mut cfg = opendal::services::HfConfig::default(); + if let Some(token) = m.get(HF_TOKEN) { + cfg.token = Some(token.clone()); + } + if let Some(endpoint) = m.get(HF_ENDPOINT) { + cfg.endpoint = Some(endpoint.clone()); + } + if let Some(revision) = m.get(HF_REVISION) { + cfg.revision = Some(revision.clone()); + } + Ok(cfg) +} + +/// Build an [`Operator`] for the given `hf://…` path and return it together with +/// the relative path-in-repo. +/// +/// URI parsing is delegated to opendal's [`HfConfig::from_uri`]. The base config +/// provides fallback values for `revision` and `endpoint`; the `token` is always +/// taken from the base config and never from the URI. +pub(crate) fn hf_config_build<'a>( + cfg: &opendal::services::HfConfig, + path: &'a str, +) -> Result<(Operator, &'a str)> { + let uri = OperatorUri::new(path, Vec::<(String, String)>::new()).map_err(|e| { + Error::new(ErrorKind::DataInvalid, format!("Invalid hf url: {path}")).with_source(e) + })?; + + let mut hf_cfg = opendal::services::HfConfig::from_uri(&uri).map_err(|e| { + Error::new(ErrorKind::DataInvalid, format!("Invalid hf url: {path}")).with_source(e) + })?; + + // Token must come from config only, never from the URI. + hf_cfg.token = cfg.token.clone(); + + if hf_cfg.endpoint.is_none() { + hf_cfg.endpoint = cfg.endpoint.clone(); + } + if hf_cfg.revision.is_none() { + hf_cfg.revision = cfg.revision.clone(); + } + + let parsed = HfUri::parse(path) + .ok_or_else(|| Error::new(ErrorKind::DataInvalid, format!("Invalid hf url: {path}")))?; + let relative_path = &path[path.len() - parsed.path.len()..]; + + let op = Operator::from_config(hf_cfg).map_err(from_opendal_error)?; + Ok((op, relative_path)) +} + +/// Returns a stable cache key for `delete_stream` batching: `"/"` +/// (e.g. `"buckets/user/my-repo"`), without revision. +/// Repo type is included so bucket and dataset paths to the same repo use separate operators. +/// Falls back to the full path so that unparsable paths never share an operator accidentally. +pub(crate) fn hf_batch_key(path: &str) -> String { + HfUri::parse(path) + .map(|u| format!("{}/{}", u.repo_type.canonical(), u.repo_id)) + .unwrap_or_else(|| path.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(uri: &str) -> HfUri { + HfUri::parse(uri).unwrap_or_else(|| panic!("parse failed for {uri:?}")) + } + + #[test] + fn test_model_prefix() { + let u = parse("hf://models/user/my-model/path/to/file.parquet"); + assert_eq!(u.repo_type, HfRepoType::Model); + assert_eq!(u.repo_id, "user/my-model"); + assert_eq!(u.path, "path/to/file.parquet"); + } + + #[test] + fn test_dataset_prefix() { + let u = parse("hf://datasets/user/my-dataset/train/data.parquet"); + assert_eq!(u.repo_type, HfRepoType::Dataset); + assert_eq!(u.repo_id, "user/my-dataset"); + assert_eq!(u.path, "train/data.parquet"); + } + + #[test] + fn test_bucket_prefix() { + let u = parse("hf://buckets/myorg/my-bucket/iceberg/metadata/v1.json"); + assert_eq!(u.repo_type, HfRepoType::Bucket); + assert_eq!(u.repo_id, "myorg/my-bucket"); + assert_eq!(u.path, "iceberg/metadata/v1.json"); + } + + #[test] + fn test_revision() { + let u = parse("hf://datasets/user/my-dataset@main/train/data.parquet"); + assert_eq!(u.repo_type, HfRepoType::Dataset); + assert_eq!(u.repo_id, "user/my-dataset"); + assert_eq!(u.path, "train/data.parquet"); + } + + #[test] + fn test_refs_convert_revision() { + let u = parse("hf://datasets/squad@refs/convert/parquet/data.parquet"); + assert_eq!(u.path, "data.parquet"); + } + + #[test] + fn test_refs_pr_revision() { + let u = parse("hf://models/user/repo@refs/pr/10/file.txt"); + assert_eq!(u.path, "file.txt"); + } + + #[test] + fn test_encoded_revision() { + let u = parse("hf://models/user/repo@refs%2Fpr%2F10/file.txt"); + assert_eq!(u.path, "file.txt"); + } + + #[test] + fn test_no_path() { + let u = parse("hf://models/user/my-model"); + assert_eq!(u.repo_id, "user/my-model"); + assert_eq!(u.path, ""); + } + + #[test] + fn test_at_in_path_not_revision() { + let u = parse("hf://models/user/repo/path/@not-a-revision.txt"); + assert_eq!(u.path, "path/@not-a-revision.txt"); + } + + #[test] + fn test_single_segment_repo_id() { + // Without revision and path: unambiguous. + let u = parse("hf://models/gpt2"); + assert_eq!(u.repo_type, HfRepoType::Model); + assert_eq!(u.repo_id, "gpt2"); + assert_eq!(u.path, ""); + + // With explicit revision: single-segment repos with paths are parsed correctly. + let u = parse("hf://models/gpt2@main/config.json"); + assert_eq!(u.repo_type, HfRepoType::Model); + assert_eq!(u.repo_id, "gpt2"); + assert_eq!(u.path, "config.json"); + } + + #[test] + fn test_batch_key() { + assert_eq!( + hf_batch_key("hf://datasets/user/repo@main/path/file.parquet"), + "datasets/user/repo" + ); + assert_eq!( + hf_batch_key("hf://buckets/org/bucket/data/file.parquet"), + "buckets/org/bucket" + ); + // Same repo_id, different repo_type → different keys. + assert_ne!( + hf_batch_key("hf://buckets/user/repo/file"), + hf_batch_key("hf://datasets/user/repo/file"), + ); + } + + #[test] + fn test_invalid_uri() { + assert!(HfUri::parse("hf://").is_none()); + // bare repo-type, no repo_id + assert!(HfUri::parse("hf://datasets").is_none()); + // missing repo-type prefix + assert!(HfUri::parse("hf://user/my-model").is_none()); + assert!(HfUri::parse("hf://gpt2").is_none()); + // unrecognized repo-type prefix + assert!(HfUri::parse("hf://repos/user/repo/file").is_none()); + } + + #[test] + fn test_hf_config_build_relative_path() { + let cfg = opendal::services::HfConfig::default(); + + let (_, rel) = hf_config_build( + &cfg, + "hf://datasets/user/my-dataset@main/train/data.parquet", + ) + .unwrap(); + assert_eq!(rel, "train/data.parquet"); + + let (_, rel) = hf_config_build(&cfg, "hf://models/user/my-model/config.json").unwrap(); + assert_eq!(rel, "config.json"); + + let (_, rel) = hf_config_build(&cfg, "hf://models/user/my-model").unwrap(); + assert_eq!(rel, ""); + } +} diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs new file mode 100644 index 0000000000..55b651ae1f --- /dev/null +++ b/crates/storage/opendal/src/lib.rs @@ -0,0 +1,1370 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! OpenDAL-based storage implementation for Apache Iceberg. +//! +//! This crate provides [`OpenDalStorage`] and [`OpenDalStorageFactory`], +//! which implement the [`Storage`] and +//! [`StorageFactory`] traits from the `iceberg` crate +//! using [OpenDAL](https://opendal.apache.org/) as the backend. + +mod utils; + +use std::collections::HashMap; +use std::collections::hash_map::Entry; +use std::num::NonZeroU64; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use cfg_if::cfg_if; +use futures::StreamExt; +use futures::stream::BoxStream; +use iceberg::io::{ + FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, + StorageFactory, +}; +use iceberg::{Error, ErrorKind, Result}; +use iceberg_property_macro::Properties; +use opendal::Operator; +use opendal::layers::{RetryLayer, TimeoutLayer}; +use opendal::options::WriteOptions; +use serde::{Deserialize, Serialize}; +use utils::from_opendal_error; + +cfg_if! { + if #[cfg(feature = "opendal-azdls")] { + mod azdls; + use azdls::*; + use opendal::services::AzdlsConfig; + } +} + +cfg_if! { + if #[cfg(feature = "opendal-hf")] { + mod hf; + use hf::*; + use opendal::services::HfConfig; + } +} + +cfg_if! { + if #[cfg(feature = "opendal-fs")] { + mod fs; + use fs::*; + } +} + +cfg_if! { + if #[cfg(feature = "opendal-gcs")] { + mod gcs; + use gcs::*; + use opendal::services::GcsConfig; + } +} + +cfg_if! { + if #[cfg(feature = "opendal-hdfs-native")] { + mod hdfs_native; + use hdfs_native::*; + pub use hdfs_native::HdfsNativeStorage; + } +} + +cfg_if! { + if #[cfg(feature = "opendal-memory")] { + mod memory; + use memory::*; + } +} + +cfg_if! { + if #[cfg(feature = "opendal-oss")] { + mod oss; + use opendal::services::OssConfig; + use oss::*; + } +} + +cfg_if! { + if #[cfg(feature = "opendal-s3")] { + mod s3; + use opendal::services::S3Config; + pub use s3::*; + } +} + +mod resolving; +pub use resolving::{OpenDalResolvingStorage, OpenDalResolvingStorageFactory}; + +/// Timeout in milliseconds for each IO call on a reader, writer, lister or deleter, applied per +/// retry attempt via OpenDAL's `TimeoutLayer::with_io_timeout`. Defaults to 10 seconds. +/// +/// OpenDAL's separate `timeout`, which bounds whole control operations such as `stat`, is not +/// affected and stays at its 60-second default. +pub const OPENDAL_IO_TIMEOUT_MS: &str = "opendal.io-timeout-ms"; + +/// Number of times to retry a failed operation on any backend, `0` to disable retrying. Unset keeps +/// OpenDAL's default. For S3, [`S3_RETRY_NUM_RETRIES`](iceberg::io::S3_RETRY_NUM_RETRIES) takes precedence. +pub const OPENDAL_RETRY_NUM_RETRIES: &str = "opendal.retry.num-retries"; + +/// Minimum wait in milliseconds before retrying a failed operation on any backend. Unset keeps +/// OpenDAL's default. For S3, [`S3_RETRY_MIN_WAIT_MS`](iceberg::io::S3_RETRY_MIN_WAIT_MS) takes precedence. +pub const OPENDAL_RETRY_MIN_WAIT_MS: &str = "opendal.retry.min-wait-ms"; + +/// Maximum wait in milliseconds between two retries on any backend. Unset keeps OpenDAL's default. +/// For S3, [`S3_RETRY_MAX_WAIT_MS`](iceberg::io::S3_RETRY_MAX_WAIT_MS) takes precedence. +pub const OPENDAL_RETRY_MAX_WAIT_MS: &str = "opendal.retry.max-wait-ms"; + +/// Matches OpenDAL's `TimeoutLayer` default. +const DEFAULT_IO_TIMEOUT_MS: NonZeroU64 = NonZeroU64::new(10_000).unwrap(); + +/// Backend-independent client settings shared by every [`OpenDalStorage`] variant. +#[derive(Clone, Debug, Properties, Serialize, Deserialize)] +#[serde(default)] +pub struct OpenDalClientConfig { + /// IO timeout in milliseconds. + #[property( + key = OPENDAL_IO_TIMEOUT_MS, + default = DEFAULT_IO_TIMEOUT_MS, + parse_with = parse_positive_ms + )] + io_timeout_ms: NonZeroU64, + /// Retries on any backend. + #[property(key = OPENDAL_RETRY_NUM_RETRIES, default = None)] + retry_num_retries: Option, + /// Delay before the first retry on any backend in milliseconds. + #[property( + key = OPENDAL_RETRY_MIN_WAIT_MS, + default = None, + parse_with = parse_positive_ms + )] + retry_min_wait_ms: Option, + /// Upper bound of the delay between two retries on any backend in milliseconds. + #[property( + key = OPENDAL_RETRY_MAX_WAIT_MS, + default = None, + parse_with = parse_positive_ms + )] + retry_max_wait_ms: Option, +} + +impl Default for OpenDalClientConfig { + fn default() -> Self { + Self { + io_timeout_ms: DEFAULT_IO_TIMEOUT_MS, + retry_num_retries: None, + retry_min_wait_ms: None, + retry_max_wait_ms: None, + } + } +} + +impl OpenDalClientConfig { + pub(crate) fn io_timeout(&self) -> Duration { + Duration::from_millis(self.io_timeout_ms.get()) + } + + /// The retry policy, with jitter; OpenDAL's default when no key is set. + pub(crate) fn retry_layer(&self) -> RetryLayer { + let mut layer = RetryLayer::new(); + if self.retry_num_retries.is_none() + && self.retry_min_wait_ms.is_none() + && self.retry_max_wait_ms.is_none() + { + return layer; + } + if let Some(times) = self.retry_num_retries { + layer = layer.with_max_times(times); + } + if let Some(ms) = self.retry_min_wait_ms { + layer = layer.with_min_delay(Duration::from_millis(ms.get())); + } + if let Some(ms) = self.retry_max_wait_ms { + layer = layer.with_max_delay(Duration::from_millis(ms.get())); + } + layer.with_jitter() + } +} + +fn parse_positive_ms(value: &str) -> Result { + value.parse().map_err(|error| { + Error::new( + ErrorKind::DataInvalid, + "Expected a positive integer number of milliseconds", + ) + .with_context("value", format!("{value:?}")) + .with_source(error) + }) +} + +/// OpenDAL-based storage factory. +/// +/// Maps scheme to the corresponding OpenDalStorage storage variant. +/// Use this factory with `FileIOBuilder::new(factory)` to create FileIO instances. +/// +/// # Serialization +/// +/// The receiving binary must enable the feature corresponding to the serialized backend variant. +/// For example, deserializing `OpenDalStorageFactory::S3` requires the `opendal-s3` feature. +/// +/// Serialization fails when the `OpenDalStorageFactory::S3` variant contains a custom AWS +/// credential loader because the loader holds process-local state that cannot be reconstructed in +/// another process. Construct the factory without a custom loader before serializing it. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum OpenDalStorageFactory { + /// Memory storage factory. + #[cfg(feature = "opendal-memory")] + Memory, + /// Local filesystem storage factory. + #[cfg(feature = "opendal-fs")] + Fs, + /// S3 storage factory. + #[cfg(feature = "opendal-s3")] + S3 { + /// Custom AWS credential loader. + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_custom_credential_loader" + )] + customized_credential_load: Option, + }, + /// GCS storage factory. + #[cfg(feature = "opendal-gcs")] + Gcs, + /// HDFS storage factory. + #[cfg(feature = "opendal-hdfs-native")] + HdfsNative, + /// OSS storage factory. + #[cfg(feature = "opendal-oss")] + Oss, + /// Azure Data Lake Storage factory. + #[cfg(feature = "opendal-azdls")] + Azdls, + /// HuggingFace Hub storage factory. + #[cfg(feature = "opendal-hf")] + Hf, +} + +#[cfg(feature = "opendal-s3")] +pub(crate) fn serialize_custom_credential_loader( + _loader: &Option, + _serializer: S, +) -> std::result::Result +where + S: serde::Serializer, +{ + Err(serde::ser::Error::custom( + "custom AWS credential loaders cannot be serialized", + )) +} + +#[typetag::serde(name = "OpenDalStorageFactory")] +impl StorageFactory for OpenDalStorageFactory { + #[allow(unused_variables)] + fn build(&self, config: &StorageConfig) -> Result> { + let client_config = OpenDalClientConfig::from_properties(config.props())?; + match self { + #[cfg(feature = "opendal-memory")] + OpenDalStorageFactory::Memory => Ok(Arc::new(OpenDalStorage::Memory { + operator: memory_config_build()?, + client_config, + })), + #[cfg(feature = "opendal-fs")] + OpenDalStorageFactory::Fs => Ok(Arc::new(OpenDalStorage::LocalFs { client_config })), + #[cfg(feature = "opendal-s3")] + OpenDalStorageFactory::S3 { + customized_credential_load, + } => Ok(Arc::new(OpenDalStorage::S3 { + config: s3_config_parse(config.props().clone())?.into(), + multipart_part_size: s3_multipart_part_size_parse(config.props())?, + customized_credential_load: customized_credential_load.clone(), + client_config: s3_client_config(config.props(), client_config)?, + })), + #[cfg(feature = "opendal-gcs")] + OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs { + config: gcs_config_parse(config.props().clone())?.into(), + client_config, + })), + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorageFactory::HdfsNative => Ok(Arc::new(OpenDalStorage::HdfsNative( + HdfsNativeStorage::new( + hdfs_native_config_parse(config.props().clone())?, + client_config, + ), + ))), + #[cfg(feature = "opendal-oss")] + OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss { + config: oss_config_parse(config.props().clone())?.into(), + client_config, + })), + #[cfg(feature = "opendal-azdls")] + OpenDalStorageFactory::Azdls => Ok(Arc::new(OpenDalStorage::Azdls { + config: azdls_config_parse(config.props().clone())?.into(), + client_config, + })), + #[cfg(feature = "opendal-hf")] + OpenDalStorageFactory::Hf => Ok(Arc::new(OpenDalStorage::Hf { + config: hf_config_parse(config.props().clone())?.into(), + client_config, + })), + #[cfg(all( + not(feature = "opendal-memory"), + not(feature = "opendal-fs"), + not(feature = "opendal-s3"), + not(feature = "opendal-gcs"), + not(feature = "opendal-oss"), + not(feature = "opendal-azdls"), + not(feature = "opendal-hf"), + not(feature = "opendal-hdfs-native"), + ))] + _ => Err(Error::new( + ErrorKind::FeatureUnsupported, + "No storage service has been enabled", + )), + } + } +} + +/// Default memory operator for serde deserialization. +#[cfg(feature = "opendal-memory")] +fn default_memory_operator() -> Operator { + memory_config_build().expect("Failed to create default memory operator") +} + +/// OpenDAL-based storage implementation. +/// +/// The serialized representation is not a stable format and may change between crate versions. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum OpenDalStorage { + /// Memory storage variant. + #[cfg(feature = "opendal-memory")] + Memory { + /// Pre-built memory operator. + #[serde(skip, default = "self::default_memory_operator")] + operator: Operator, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, + }, + /// Local filesystem storage variant. + #[cfg(feature = "opendal-fs")] + LocalFs { + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, + }, + /// S3 storage variant. + /// + /// Accepts any S3-family URL (`s3://`, `s3a://`, `s3n://`); the scheme is + /// derived from the path at call time. + #[cfg(feature = "opendal-s3")] + #[non_exhaustive] + S3 { + /// S3 configuration. + config: Arc, + /// Bytes carried by one multipart upload request. + #[serde(default = "default_multipart_part_size")] + multipart_part_size: u64, + /// Custom AWS credential loader. + #[serde(skip)] + customized_credential_load: Option, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, + }, + /// GCS storage variant. + #[cfg(feature = "opendal-gcs")] + Gcs { + /// GCS configuration. + config: Arc, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, + }, + /// HDFS storage variant; see [`HdfsNativeStorage`]. + #[cfg(feature = "opendal-hdfs-native")] + HdfsNative(HdfsNativeStorage), + /// OSS storage variant. + #[cfg(feature = "opendal-oss")] + Oss { + /// OSS configuration. + config: Arc, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, + }, + /// Azure Data Lake Storage variant. + /// + /// Accepts paths of the form + /// `abfs[s]://@.dfs./` or + /// `wasb[s]://@.blob./`. + /// The scheme is derived from the path at call time. + #[cfg(feature = "opendal-azdls")] + Azdls { + /// Azure DLS configuration. + config: Arc, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, + }, + /// HuggingFace Hub storage variant. + /// + /// Accepts paths of the form + /// `hf:////[@]/`, + /// where `` must be one of `models`, `datasets`, `spaces`, or `buckets`. + #[cfg(feature = "opendal-hf")] + Hf { + /// HuggingFace Hub configuration (token + endpoint). + config: Arc, + /// Backend-independent client settings. + #[serde(default)] + client_config: OpenDalClientConfig, + }, +} + +impl OpenDalStorage { + /// Creates operator from path. + /// + /// # Arguments + /// + /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`](iceberg::io::FileIO). + /// + /// # Returns + /// + /// The return value consists of two parts: + /// + /// * An [`opendal::Operator`] instance used to operate on file. + /// * Relative path to the root uri of [`opendal::Operator`]. + #[allow(unreachable_code, unused_variables)] + pub(crate) fn create_operator<'a>( + &self, + path: &'a impl AsRef, + ) -> Result<(Operator, &'a str)> { + let path = path.as_ref(); + let (operator, relative_path): (Operator, &str) = match self { + #[cfg(feature = "opendal-memory")] + OpenDalStorage::Memory { operator: op, .. } => { + if let Some(stripped) = path.strip_prefix("memory:/") { + (op.clone(), stripped) + } else { + (op.clone(), &path[1..]) + } + } + #[cfg(feature = "opendal-fs")] + OpenDalStorage::LocalFs { .. } => { + let op = fs_config_build()?; + if let Some(stripped) = path.strip_prefix("file:/") { + (op, stripped) + } else { + (op, &path[1..]) + } + } + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { + config, + customized_credential_load, + .. + } => { + let op = s3_config_build(config, customized_credential_load, path)?; + let op_info = op.info(); + + // Use the URL scheme in the path for prefix matching. This enables + // use of S3-compatible storage backends using custom schemes (e.g., `minio://`, `r2://`). + let url = url::Url::parse(path).map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid s3 url: {path}: {e}"), + ) + })?; + let prefix = format!("{}://{}/", url.scheme(), op_info.name()); + if path.starts_with(&prefix) { + (op, &path[prefix.len()..]) + } else { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid s3 url: {path}, should start with {prefix}"), + )); + } + } + #[cfg(feature = "opendal-gcs")] + OpenDalStorage::Gcs { config, .. } => { + let operator = gcs_config_build(config, path)?; + let prefix = format!("gs://{}/", operator.info().name()); + if path.starts_with(&prefix) { + (operator, &path[prefix.len()..]) + } else { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid gcs url: {path}, should start with {prefix}"), + )); + } + } + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorage::HdfsNative(storage) => { + hdfs_native_create_operator(path, &storage.config, &storage.operators)? + } + #[cfg(feature = "opendal-oss")] + OpenDalStorage::Oss { config, .. } => { + let op = oss_config_build(config, path)?; + let prefix = format!("oss://{}/", op.info().name()); + if path.starts_with(&prefix) { + (op, &path[prefix.len()..]) + } else { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid oss url: {path}, should start with {prefix}"), + )); + } + } + #[cfg(feature = "opendal-azdls")] + OpenDalStorage::Azdls { config, .. } => azdls_create_operator(path, config)?, + #[cfg(feature = "opendal-hf")] + OpenDalStorage::Hf { config, .. } => hf_config_build(config, path)?, + #[cfg(all( + not(feature = "opendal-s3"), + not(feature = "opendal-fs"), + not(feature = "opendal-gcs"), + not(feature = "opendal-oss"), + not(feature = "opendal-azdls"), + not(feature = "opendal-hf"), + not(feature = "opendal-hdfs-native"), + ))] + _ => { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "No storage service has been enabled", + )); + } + }; + + // Apply observability/resilience layers. TimeoutLayer must be + // inside RetryLayer so each retry attempt is independently + // bounded — without a per-attempt timeout, a future parked on a + // silently dropped TCP connection never produces an `Err` and + // RetryLayer cannot retry, leaving the caller hung indefinitely. + // See: https://opendal.apache.org/docs/rust/opendal/layers/struct.TimeoutLayer.html + // + // Transient errors are common for object stores; we retry temporary + // failures with exponential backoff. The retry behavior also + // benefits non-object-store backends. + let operator = operator + .layer(TimeoutLayer::new().with_io_timeout(self.client_config().io_timeout())) + .layer(self.client_config().retry_layer()); + Ok((operator, relative_path)) + } + + pub(crate) fn client_config(&self) -> &OpenDalClientConfig { + match self { + #[cfg(feature = "opendal-memory")] + OpenDalStorage::Memory { client_config, .. } => client_config, + #[cfg(feature = "opendal-fs")] + OpenDalStorage::LocalFs { client_config } => client_config, + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { client_config, .. } => client_config, + #[cfg(feature = "opendal-gcs")] + OpenDalStorage::Gcs { client_config, .. } => client_config, + #[cfg(feature = "opendal-oss")] + OpenDalStorage::Oss { client_config, .. } => client_config, + #[cfg(feature = "opendal-azdls")] + OpenDalStorage::Azdls { client_config, .. } => client_config, + #[cfg(feature = "opendal-hf")] + OpenDalStorage::Hf { client_config, .. } => client_config, + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorage::HdfsNative(storage) => &storage.client_config, + #[cfg(all( + not(feature = "opendal-memory"), + not(feature = "opendal-s3"), + not(feature = "opendal-fs"), + not(feature = "opendal-gcs"), + not(feature = "opendal-oss"), + not(feature = "opendal-azdls"), + not(feature = "opendal-hf"), + not(feature = "opendal-hdfs-native"), + ))] + _ => unreachable!(), + } + } + + /// Bounds the S3 request size. Without a chunk size OpenDAL turns each caller + /// buffer into one request, and `ParquetWriter` hands over a whole row group + /// at a time. Other backends keep OpenDAL's defaults. + #[allow(unreachable_patterns)] + fn write_options(&self) -> WriteOptions { + match self { + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { + multipart_part_size, + .. + } => WriteOptions { + chunk: Some(usize::try_from(*multipart_part_size).unwrap_or(usize::MAX)), + ..WriteOptions::default() + }, + _ => WriteOptions::default(), + } + } + + /// Returns a cache key used by `delete_stream` to group paths by storage operator. + /// + /// For most backends the URL host (bucket name) is sufficient. For HF the host + /// encodes the repo type, not the repo identity, so a more specific key is used. + fn batch_key_for_path(&self, path: &str) -> String { + match self { + #[cfg(feature = "opendal-hf")] + OpenDalStorage::Hf { .. } => hf_batch_key(path), + // The URL host alone would merge distinct NameNodes that differ + // only by port; key by the effective NameNode instead. + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorage::HdfsNative(storage) => hdfs_native_batch_key(&storage.config, path), + _ => url::Url::parse(path) + .ok() + .and_then(|u| u.host_str().map(|s| s.to_string())) + .unwrap_or_default(), + } + } + + /// Extracts the relative path from an absolute path without building an operator. + /// + /// This is a lightweight alternative to [`create_operator`](Self::create_operator) for cases + /// where only the relative path is needed (e.g. bulk deletes where the operator is already + /// available). + #[allow(unreachable_code, unused_variables)] + pub(crate) fn relativize_path<'a>(&self, path: &'a str) -> Result<&'a str> { + match self { + #[cfg(feature = "opendal-memory")] + OpenDalStorage::Memory { .. } => { + Ok(path.strip_prefix("memory:/").unwrap_or(&path[1..])) + } + #[cfg(feature = "opendal-fs")] + OpenDalStorage::LocalFs { .. } => Ok(path.strip_prefix("file:/").unwrap_or(&path[1..])), + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { .. } => { + let url = url::Url::parse(path)?; + let bucket = url.host_str().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid s3 url: {path}, missing bucket"), + ) + })?; + let prefix = format!("{}://{}/", url.scheme(), bucket); + if path.starts_with(&prefix) { + Ok(&path[prefix.len()..]) + } else { + Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid s3 url: {path}, should start with {prefix}"), + )) + } + } + #[cfg(feature = "opendal-gcs")] + OpenDalStorage::Gcs { .. } => { + let url = url::Url::parse(path)?; + let bucket = url.host_str().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid gcs url: {path}, missing bucket"), + ) + })?; + let prefix = format!("gs://{}/", bucket); + if path.starts_with(&prefix) { + Ok(&path[prefix.len()..]) + } else { + Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid gcs url: {path}, should start with {prefix}"), + )) + } + } + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorage::HdfsNative(storage) => { + let (_, relative_path) = hdfs_native_effective_name_node(&storage.config, path)?; + Ok(relative_path) + } + #[cfg(feature = "opendal-oss")] + OpenDalStorage::Oss { .. } => { + let url = url::Url::parse(path)?; + let bucket = url.host_str().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid oss url: {path}, missing bucket"), + ) + })?; + let prefix = format!("oss://{}/", bucket); + if path.starts_with(&prefix) { + Ok(&path[prefix.len()..]) + } else { + Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid oss url: {path}, should start with {prefix}"), + )) + } + } + #[cfg(feature = "opendal-azdls")] + OpenDalStorage::Azdls { config, .. } => { + let azure_path = path.parse::()?; + match_path_with_config(&azure_path, config)?; + Ok(azure_path.relative_path(path)) + } + #[cfg(feature = "opendal-hf")] + OpenDalStorage::Hf { .. } => { + let parsed = HfUri::parse(path).ok_or_else(|| { + Error::new(ErrorKind::DataInvalid, format!("Invalid hf url: {path}")) + })?; + Ok(&path[path.len() - parsed.path.len()..]) + } + #[cfg(all( + not(feature = "opendal-s3"), + not(feature = "opendal-fs"), + not(feature = "opendal-gcs"), + not(feature = "opendal-oss"), + not(feature = "opendal-azdls"), + not(feature = "opendal-hf"), + not(feature = "opendal-hdfs-native"), + ))] + _ => Err(Error::new( + ErrorKind::FeatureUnsupported, + "No storage service has been enabled", + )), + } + } +} + +#[typetag::serde(name = "OpenDalStorage")] +#[async_trait] +impl Storage for OpenDalStorage { + async fn exists(&self, path: &str) -> Result { + let (op, relative_path) = self.create_operator(&path)?; + Ok(op.exists(relative_path).await.map_err(from_opendal_error)?) + } + + async fn metadata(&self, path: &str) -> Result { + let (op, relative_path) = self.create_operator(&path)?; + let meta = op.stat(relative_path).await.map_err(from_opendal_error)?; + Ok(FileMetadata { + size: meta.content_length(), + }) + } + + async fn read(&self, path: &str) -> Result { + let (op, relative_path) = self.create_operator(&path)?; + Ok(op + .read(relative_path) + .await + .map_err(from_opendal_error)? + .to_bytes()) + } + + async fn reader(&self, path: &str) -> Result> { + let (op, relative_path) = self.create_operator(&path)?; + Ok(Box::new(OpenDalReader( + op.reader(relative_path).await.map_err(from_opendal_error)?, + ))) + } + + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { + let (op, relative_path) = self.create_operator(&path)?; + op.write_options(relative_path, bs, self.write_options()) + .await + .map_err(from_opendal_error)?; + Ok(()) + } + + async fn writer(&self, path: &str) -> Result> { + let (op, relative_path) = self.create_operator(&path)?; + Ok(Box::new(OpenDalWriter::new( + op.writer_options(relative_path, self.write_options()) + .await + .map_err(from_opendal_error)?, + ))) + } + + async fn delete(&self, path: &str) -> Result<()> { + let (op, relative_path) = self.create_operator(&path)?; + Ok(op.delete(relative_path).await.map_err(from_opendal_error)?) + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + let (op, relative_path) = self.create_operator(&path)?; + let path = if relative_path.ends_with('/') { + relative_path.to_string() + } else { + format!("{relative_path}/") + }; + Ok(op + .delete_with(&path) + .recursive(true) + .await + .map_err(from_opendal_error)?) + } + + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + let mut deleters: HashMap = HashMap::new(); + + while let Some(path) = paths.next().await { + let bucket = self.batch_key_for_path(&path); + + let (relative_path, deleter) = match deleters.entry(bucket) { + Entry::Occupied(entry) => { + (self.relativize_path(&path)?.to_string(), entry.into_mut()) + } + Entry::Vacant(entry) => { + let (op, rel) = self.create_operator(&path)?; + let rel = rel.to_string(); + let deleter = op.deleter().await.map_err(from_opendal_error)?; + (rel, entry.insert(deleter)) + } + }; + + deleter + .delete(relative_path) + .await + .map_err(from_opendal_error)?; + } + + for (_, mut deleter) in deleters { + deleter.close().await.map_err(from_opendal_error)?; + } + + Ok(()) + } + + #[allow(unreachable_code, unused_variables)] + fn new_input(&self, path: &str) -> Result { + Ok(InputFile::new(Arc::new(self.clone()), path.to_string())) + } + + #[allow(unreachable_code, unused_variables)] + fn new_output(&self, path: &str) -> Result { + Ok(OutputFile::new(Arc::new(self.clone()), path.to_string())) + } +} + +// Newtype wrappers for opendal types to satisfy orphan rules. +// We can't implement iceberg's FileRead/FileWrite traits directly on opendal's +// Reader/Writer since neither trait nor type is defined in this crate. + +/// Wrapper around `opendal::Reader` that implements `FileRead`. +pub(crate) struct OpenDalReader(pub(crate) opendal::Reader); + +#[async_trait] +impl FileRead for OpenDalReader { + async fn read(&self, range: std::ops::Range) -> Result { + Ok(opendal::Reader::read(&self.0, range) + .await + .map_err(from_opendal_error)? + .to_bytes()) + } +} + +/// Wrapper around `opendal::Writer` that implements `FileWrite`. +pub(crate) struct OpenDalWriter { + inner: opendal::Writer, + bytes_written: u64, +} + +impl OpenDalWriter { + pub(crate) fn new(inner: opendal::Writer) -> Self { + Self { + inner, + bytes_written: 0, + } + } +} + +#[async_trait] +impl FileWrite for OpenDalWriter { + async fn write(&mut self, bs: Bytes) -> Result<()> { + let len = bs.len() as u64; + opendal::Writer::write(&mut self.inner, bs) + .await + .map_err(from_opendal_error)?; + self.bytes_written += len; + Ok(()) + } + + async fn close(&mut self) -> Result { + let metadata = opendal::Writer::close(&mut self.inner) + .await + .map_err(from_opendal_error)?; + + // Object stores may omit the size (reported as 0); validate only a reported nonzero size. + let reported_size = metadata.content_length(); + if reported_size != 0 && reported_size != self.bytes_written { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "Wrote {} bytes but storage reports {reported_size}", + self.bytes_written + ), + )); + } + + Ok(FileMetadata { + size: self.bytes_written, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn client_config(value: &str) -> Result { + OpenDalClientConfig::from_properties(&HashMap::from([( + OPENDAL_IO_TIMEOUT_MS.to_string(), + value.to_string(), + )])) + } + + #[test] + fn test_io_timeout_parsing() { + let unset = OpenDalClientConfig::from_properties(&HashMap::new()).unwrap(); + assert_eq!( + unset.io_timeout(), + Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get()) + ); + + let max = u64::MAX.to_string(); + for (valid, ms) in [("45000", 45_000), ("1", 1), (max.as_str(), u64::MAX)] { + assert_eq!( + client_config(valid).unwrap().io_timeout(), + Duration::from_millis(ms), + "{valid}" + ); + } + + for invalid in ["0", "-1", "12.5", "abc", "", "18446744073709551616"] { + let err = client_config(invalid).unwrap_err().to_string(); + assert!(err.contains(OPENDAL_IO_TIMEOUT_MS), "{invalid}"); + assert!(err.contains(&format!("value: {invalid:?}")), "{err}"); + let reason = invalid.parse::().unwrap_err().to_string(); + assert!(err.contains(&reason), "{err}"); + } + } + + #[test] + fn test_default_timeouts_match_opendal() { + // `TimeoutLayer` has no getters, so compare through `Debug`. + let opendal_default = format!("{:?}", TimeoutLayer::new()); + let layer = |timeout, io_timeout| { + format!( + "{:?}", + TimeoutLayer::new() + .with_timeout(timeout) + .with_io_timeout(io_timeout) + ) + }; + let control = Duration::from_secs(60); + let io = Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get()); + assert_eq!(opendal_default, layer(control, io)); + assert_ne!(opendal_default, layer(control + Duration::from_secs(1), io)); + assert_ne!( + opendal_default, + layer(control, io + Duration::from_millis(1)) + ); + } + + fn retry_config(props: &[(&str, &str)]) -> Result { + OpenDalClientConfig::from_properties( + &props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(), + ) + } + + fn retry_layer_debug(props: &[(&str, &str)]) -> Result { + Ok(format!("{:?}", retry_config(props)?.retry_layer())) + } + + #[test] + fn test_retry_defaults_match_opendal() { + // `RetryLayer` has no getters, so compare through `Debug`. + assert_eq!( + retry_layer_debug(&[]).unwrap(), + format!("{:?}", RetryLayer::new()) + ); + assert_ne!( + format!("{:?}", RetryLayer::new()), + format!("{:?}", RetryLayer::new().with_jitter()) + ); + } + + #[test] + fn test_retry_properties_reach_retry_layer() { + let expected = RetryLayer::new() + .with_max_times(5) + .with_min_delay(Duration::from_millis(100)) + .with_max_delay(Duration::from_millis(200)) + .with_jitter(); + assert_eq!( + retry_layer_debug(&[ + (OPENDAL_RETRY_NUM_RETRIES, "5"), + (OPENDAL_RETRY_MIN_WAIT_MS, "100"), + (OPENDAL_RETRY_MAX_WAIT_MS, "200"), + ]) + .unwrap(), + format!("{expected:?}") + ); + + // any one property switches jitter on and leaves the others at OpenDAL's defaults + assert_eq!( + retry_layer_debug(&[(OPENDAL_RETRY_NUM_RETRIES, "0")]).unwrap(), + format!("{:?}", RetryLayer::new().with_max_times(0).with_jitter()) + ); + } + + #[test] + fn test_retry_property_parsing_rejects_invalid_values() { + for (key, invalid) in [ + (OPENDAL_RETRY_NUM_RETRIES, "-1"), + (OPENDAL_RETRY_NUM_RETRIES, "abc"), + (OPENDAL_RETRY_NUM_RETRIES, ""), + (OPENDAL_RETRY_MIN_WAIT_MS, "0"), + (OPENDAL_RETRY_MIN_WAIT_MS, "abc"), + (OPENDAL_RETRY_MAX_WAIT_MS, "0"), + (OPENDAL_RETRY_MAX_WAIT_MS, "12.5"), + ] { + let err = retry_config(&[(key, invalid)]).unwrap_err().to_string(); + assert!(err.contains(key), "{key}={invalid}: {err}"); + } + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_client_config_serde_round_trip() { + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + multipart_part_size: default_multipart_part_size(), + customized_credential_load: None, + client_config: client_config("45000").unwrap(), + }; + + let mut value = serde_json::to_value(&storage).unwrap(); + let restored: OpenDalStorage = serde_json::from_value(value.clone()).unwrap(); + assert_eq!( + restored.client_config().io_timeout(), + Duration::from_secs(45) + ); + + let mut zero = value.clone(); + zero["S3"]["client_config"]["io_timeout_ms"] = 0.into(); + assert!(serde_json::from_value::(zero).is_err()); + + value["S3"].as_object_mut().unwrap().remove("client_config"); + let restored: OpenDalStorage = serde_json::from_value(value).unwrap(); + assert_eq!( + restored.client_config().io_timeout(), + Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get()) + ); + } + + #[cfg(all(feature = "opendal-fs", feature = "opendal-memory"))] + #[test] + fn test_old_unit_variant_forms_are_rejected() { + for old in [r#""LocalFs""#, r#""Memory""#, r#"{"Memory":null}"#] { + assert!( + serde_json::from_str::(old).is_err(), + "{old}" + ); + } + } + + #[cfg(feature = "opendal-memory")] + #[test] + fn test_factory_rejects_invalid_io_timeout() { + let config = StorageConfig::new().with_prop(OPENDAL_IO_TIMEOUT_MS, "nope"); + + let err = OpenDalStorageFactory::Memory.build(&config).unwrap_err(); + assert!(err.to_string().contains(OPENDAL_IO_TIMEOUT_MS)); + } + + #[test] + fn test_factory_propagates_io_timeout() { + let config = StorageConfig::new().with_prop(OPENDAL_IO_TIMEOUT_MS, "45000"); + let factories = [ + #[cfg(feature = "opendal-memory")] + OpenDalStorageFactory::Memory, + #[cfg(feature = "opendal-fs")] + OpenDalStorageFactory::Fs, + #[cfg(feature = "opendal-s3")] + OpenDalStorageFactory::S3 { + customized_credential_load: None, + }, + #[cfg(feature = "opendal-gcs")] + OpenDalStorageFactory::Gcs, + #[cfg(feature = "opendal-oss")] + OpenDalStorageFactory::Oss, + #[cfg(feature = "opendal-azdls")] + OpenDalStorageFactory::Azdls, + #[cfg(feature = "opendal-hf")] + OpenDalStorageFactory::Hf, + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorageFactory::HdfsNative, + ]; + for factory in factories { + // `build` returns `dyn Storage`, so read the config back from its serialized form. + let storage = factory.build(&config).unwrap(); + let json = serde_json::to_value(&*storage).unwrap(); + let client_config = json + .as_object() + .unwrap() + .values() + .find_map(|variant| variant.get("client_config")) + .unwrap_or_else(|| panic!("{json}")); + assert_eq!(client_config["io_timeout_ms"], 45_000, "{factory:?}"); + } + } + + #[cfg(feature = "opendal-memory")] + #[tokio::test(start_paused = true)] + async fn test_io_timeout_reaches_timeout_layer() { + use opendal::layers::ConcurrentLimitLayer; + + // A zero concurrency limit stalls every IO call; paused time skips the waits. + let storage = OpenDalStorage::Memory { + operator: default_memory_operator().layer(ConcurrentLimitLayer::new(0)), + client_config: client_config("45000").unwrap(), + }; + + let err = storage + .read("memory:/stalled") + .await + .unwrap_err() + .to_string(); + assert!(err.contains("io operation timeout reached"), "{err}"); + assert!(err.contains("{ timeout: 45 }"), "{err}"); + } + + #[cfg(feature = "opendal-s3")] + #[derive(Debug)] + struct EmptyCredentialLoader; + + #[cfg(feature = "opendal-s3")] + impl ProvideCredential for EmptyCredentialLoader { + type Credential = AwsCredential; + + async fn provide_credential( + &self, + _ctx: &reqsign_core::Context, + ) -> reqsign_core::Result> { + Ok(None) + } + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_s3_factory_custom_credential_loader_serialization_fails() { + let file_io = iceberg::io::FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: Some(CustomAwsCredentialLoader::new(EmptyCredentialLoader)), + })) + .build(); + + let err = file_io.serialize_all().unwrap_err(); + assert!( + err.to_string() + .contains("custom AWS credential loaders cannot be serialized") + ); + } + + #[cfg(feature = "opendal-memory")] + #[test] + fn test_default_memory_operator() { + let op = default_memory_operator(); + assert_eq!(op.info().scheme().to_string(), "memory"); + } + + #[cfg(feature = "opendal-memory")] + #[tokio::test] + async fn test_writer_close_returns_stored_size() { + use iceberg::encryption::{EncryptedOutputFile, StandardKeyMetadata}; + + // Note: the memory service does report a content length, so this only pins the happy + // path. The counter in `OpenDalWriter` is what covers services that don't, such as S3. + let storage = Arc::new(OpenDalStorage::Memory { + operator: default_memory_operator(), + client_config: OpenDalClientConfig::default(), + }); + let path = "memory:///stored-size"; + for plaintext in [ + Bytes::new(), + Bytes::from_static(b"test data"), + Bytes::from(vec![7; 3 * 1024]), + ] { + let mut writer = storage.writer(path).await.unwrap(); + for chunk in plaintext.chunks(1024) { + writer.write(Bytes::copy_from_slice(chunk)).await.unwrap(); + } + let metadata = writer.close().await.unwrap(); + assert_eq!(metadata.size, plaintext.len() as u64); + assert_eq!(metadata.size, storage.metadata(path).await.unwrap().size); + + let output = EncryptedOutputFile::new( + OutputFile::new(storage.clone(), path.to_string()), + StandardKeyMetadata::try_new(b"0123456789abcdef").unwrap(), + ); + let metadata = output.write(plaintext.clone()).await.unwrap(); + assert!(metadata.size > plaintext.len() as u64); + assert_eq!(metadata.size, storage.metadata(path).await.unwrap().size); + } + } + + #[cfg(feature = "opendal-memory")] + #[test] + fn test_relativize_path_memory() { + let storage = OpenDalStorage::Memory { + operator: default_memory_operator(), + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage.relativize_path("memory:/path/to/file").unwrap(), + "path/to/file" + ); + // Without the scheme prefix, falls back to stripping the leading slash + assert_eq!( + storage.relativize_path("/path/to/file").unwrap(), + "path/to/file" + ); + } + + #[cfg(feature = "opendal-fs")] + #[test] + fn test_relativize_path_fs() { + let storage = OpenDalStorage::LocalFs { + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage + .relativize_path("file:/tmp/data/file.parquet") + .unwrap(), + "tmp/data/file.parquet" + ); + assert_eq!( + storage.relativize_path("/tmp/data/file.parquet").unwrap(), + "tmp/data/file.parquet" + ); + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_relativize_path_s3() { + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + multipart_part_size: default_multipart_part_size(), + customized_credential_load: None, + client_config: OpenDalClientConfig::default(), + }; + + // All S3-family schemes are accepted by the same storage instance. + // Custom schemes for S3-compatible stores (e.g., `minio://`) are also + // accepted because the path's scheme is used as-is for prefix matching. + for scheme in ["s3", "s3a", "s3n", "minio"] { + assert_eq!( + storage + .relativize_path(&format!("{scheme}://my-bucket/path/to/file.parquet")) + .unwrap(), + "path/to/file.parquet" + ); + } + } + + #[cfg(feature = "opendal-gcs")] + #[test] + fn test_relativize_path_gcs() { + let storage = OpenDalStorage::Gcs { + config: Arc::new(GcsConfig::default()), + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage + .relativize_path("gs://my-bucket/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + } + + #[cfg(feature = "opendal-gcs")] + #[test] + fn test_relativize_path_gcs_invalid_scheme() { + let storage = OpenDalStorage::Gcs { + config: Arc::new(GcsConfig::default()), + client_config: OpenDalClientConfig::default(), + }; + + assert!( + storage + .relativize_path("s3://my-bucket/path/to/file.parquet") + .is_err() + ); + } + + #[cfg(feature = "opendal-oss")] + #[test] + fn test_relativize_path_oss() { + let storage = OpenDalStorage::Oss { + config: Arc::new(OssConfig::default()), + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage + .relativize_path("oss://my-bucket/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + } + + #[cfg(feature = "opendal-oss")] + #[test] + fn test_relativize_path_oss_invalid_scheme() { + let storage = OpenDalStorage::Oss { + config: Arc::new(OssConfig::default()), + client_config: OpenDalClientConfig::default(), + }; + + assert!( + storage + .relativize_path("s3://my-bucket/path/to/file.parquet") + .is_err() + ); + } + + #[cfg(feature = "opendal-azdls")] + #[test] + fn test_relativize_path_azdls() { + let storage = OpenDalStorage::Azdls { + config: Arc::new(AzdlsConfig { + account_name: Some("myaccount".to_string()), + endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), + ..Default::default() + }), + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage + .relativize_path("abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + } +} diff --git a/crates/iceberg/src/io/storage_memory.rs b/crates/storage/opendal/src/memory.rs similarity index 87% rename from crates/iceberg/src/io/storage_memory.rs rename to crates/storage/opendal/src/memory.rs index b8023717b6..746e927161 100644 --- a/crates/iceberg/src/io/storage_memory.rs +++ b/crates/storage/opendal/src/memory.rs @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. +use iceberg::Result; use opendal::Operator; use opendal::services::MemoryConfig; -use crate::Result; +use crate::utils::from_opendal_error; pub(crate) fn memory_config_build() -> Result { - Ok(Operator::from_config(MemoryConfig::default())?.finish()) + Operator::from_config(MemoryConfig::default()).map_err(from_opendal_error) } diff --git a/crates/iceberg/src/io/storage_oss.rs b/crates/storage/opendal/src/oss.rs similarity index 72% rename from crates/iceberg/src/io/storage_oss.rs rename to crates/storage/opendal/src/oss.rs index 8bfffc6ca8..a1cf44549c 100644 --- a/crates/iceberg/src/io/storage_oss.rs +++ b/crates/storage/opendal/src/oss.rs @@ -17,22 +17,13 @@ use std::collections::HashMap; +use iceberg::io::{OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET, OSS_ENDPOINT}; +use iceberg::{Error, ErrorKind, Result}; use opendal::services::OssConfig; use opendal::{Configurator, Operator}; use url::Url; -use crate::{Error, ErrorKind, Result}; - -/// Required configuration arguments for creating an Aliyun OSS Operator with OpenDAL: -/// - `oss.endpoint`: The OSS service endpoint URL -/// - `oss.access-key-id`: The access key ID for authentication -/// - `oss.access-key-secret`: The access key secret for authentication -/// Aliyun oss endpoint. -pub const OSS_ENDPOINT: &str = "oss.endpoint"; -/// Aliyun oss access key id. -pub const OSS_ACCESS_KEY_ID: &str = "oss.access-key-id"; -/// Aliyun oss access key secret. -pub const OSS_ACCESS_KEY_SECRET: &str = "oss.access-key-secret"; +use crate::utils::from_opendal_error; /// Parse iceberg props to oss config. pub(crate) fn oss_config_parse(mut m: HashMap) -> Result { @@ -56,11 +47,11 @@ pub(crate) fn oss_config_build(cfg: &OssConfig, path: &str) -> Result let bucket = url.host_str().ok_or_else(|| { Error::new( ErrorKind::DataInvalid, - format!("Invalid oss url: {}, missing bucket", path), + format!("Invalid oss url: {path}, missing bucket"), ) })?; let builder = cfg.clone().into_builder().bucket(bucket); - Ok(Operator::new(builder)?.finish()) + Operator::new(builder).map_err(from_opendal_error) } diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs new file mode 100644 index 0000000000..47459c42c8 --- /dev/null +++ b/crates/storage/opendal/src/resolving.rs @@ -0,0 +1,497 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Resolving storage that auto-detects the scheme from a path and delegates +//! to the appropriate [`OpenDalStorage`] variant. + +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::StreamExt; +use futures::stream::BoxStream; +use iceberg::io::{ + FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, + StorageFactory, +}; +use iceberg::{Error, ErrorKind, Result}; +use serde::{Deserialize, Serialize}; +use url::Url; + +#[cfg(feature = "opendal-s3")] +use crate::s3::CustomAwsCredentialLoader; +use crate::{OpenDalClientConfig, OpenDalStorage}; + +/// Schemes supported by OpenDalResolvingStorage +pub const SCHEME_MEMORY: &str = "memory"; +pub const SCHEME_FILE: &str = "file"; +pub const SCHEME_S3: &str = "s3"; +pub const SCHEME_S3A: &str = "s3a"; +pub const SCHEME_S3N: &str = "s3n"; +pub const SCHEME_GS: &str = "gs"; +pub const SCHEME_GCS: &str = "gcs"; +pub const SCHEME_OSS: &str = "oss"; +pub const SCHEME_ABFSS: &str = "abfss"; +pub const SCHEME_ABFS: &str = "abfs"; +pub const SCHEME_WASBS: &str = "wasbs"; +pub const SCHEME_WASB: &str = "wasb"; +pub const SCHEME_HF: &str = "hf"; +pub const SCHEME_HDFS: &str = "hdfs"; + +/// Parse a URL scheme string. +fn parse_scheme(scheme: &str) -> Result<&'static str> { + match scheme { + SCHEME_MEMORY => Ok("memory"), + SCHEME_FILE | "" => Ok("file"), + SCHEME_S3 | SCHEME_S3A | SCHEME_S3N => Ok("s3"), + SCHEME_GS | SCHEME_GCS => Ok("gcs"), + SCHEME_OSS => Ok("oss"), + SCHEME_ABFSS | SCHEME_ABFS | SCHEME_WASBS | SCHEME_WASB => Ok("azdls"), + SCHEME_HF => Ok("hf"), + SCHEME_HDFS => Ok("hdfs"), + s => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported storage scheme: {s}"), + )), + } +} + +/// Extract the scheme from a path URL. +fn extract_scheme(path: &str) -> Result<&'static str> { + let url = Url::parse(path).map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid path: {path}, failed to parse URL: {e}"), + ) + })?; + parse_scheme(url.scheme()) +} + +/// Build an [`OpenDalStorage`] variant for the given scheme and config properties. +fn build_storage_for_scheme( + scheme: &'static str, + props: &HashMap, + #[cfg(feature = "opendal-s3")] customized_credential_load: &Option, +) -> Result { + let client_config = OpenDalClientConfig::from_properties(props)?; + match scheme { + #[cfg(feature = "opendal-s3")] + "s3" => { + let config = crate::s3::s3_config_parse(props.clone())?; + Ok(OpenDalStorage::S3 { + config: Arc::new(config), + multipart_part_size: crate::s3::s3_multipart_part_size_parse(props)?, + customized_credential_load: customized_credential_load.clone(), + client_config: crate::s3::s3_client_config(props, client_config)?, + }) + } + #[cfg(feature = "opendal-gcs")] + "gcs" => { + let config = crate::gcs::gcs_config_parse(props.clone())?; + Ok(OpenDalStorage::Gcs { + config: Arc::new(config), + client_config, + }) + } + #[cfg(feature = "opendal-oss")] + "oss" => { + let config = crate::oss::oss_config_parse(props.clone())?; + Ok(OpenDalStorage::Oss { + config: Arc::new(config), + client_config, + }) + } + #[cfg(feature = "opendal-azdls")] + "azdls" => { + let config = crate::azdls::azdls_config_parse(props.clone())?; + Ok(OpenDalStorage::Azdls { + config: Arc::new(config), + client_config, + }) + } + #[cfg(feature = "opendal-fs")] + "file" => Ok(OpenDalStorage::LocalFs { client_config }), + #[cfg(feature = "opendal-memory")] + "memory" => Ok(OpenDalStorage::Memory { + operator: crate::memory::memory_config_build()?, + client_config, + }), + #[cfg(feature = "opendal-hf")] + "hf" => { + let config = crate::hf::hf_config_parse(props.clone())?; + Ok(OpenDalStorage::Hf { + config: Arc::new(config), + client_config, + }) + } + #[cfg(feature = "opendal-hdfs-native")] + "hdfs" => { + let config = crate::hdfs_native::hdfs_native_config_parse(props.clone())?; + Ok(OpenDalStorage::HdfsNative(crate::HdfsNativeStorage::new( + config, + client_config, + ))) + } + unsupported => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported storage scheme: {unsupported}"), + )), + } +} + +/// A resolving storage factory that creates [`OpenDalResolvingStorage`] instances. +/// +/// This factory accepts paths from any supported storage system and dynamically +/// delegates operations to the appropriate [`OpenDalStorage`] variant based on +/// the path scheme. +/// +/// # Serialization +/// +/// Serialization fails when a custom S3 credential loader is configured because the loader holds +/// process-local state that cannot be reconstructed in another process. Construct the factory +/// without a custom loader before serializing it. +/// +/// # Example +/// +/// ```rust,ignore +/// use std::sync::Arc; +/// use iceberg::io::FileIOBuilder; +/// use iceberg_storage_opendal::OpenDalResolvingStorageFactory; +/// +/// let factory = OpenDalResolvingStorageFactory::new(); +/// let file_io = FileIOBuilder::new(Arc::new(factory)) +/// .with_prop("s3.region", "us-east-1") +/// .build(); +/// ``` +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct OpenDalResolvingStorageFactory { + /// Custom AWS credential loader for S3 storage. + #[cfg(feature = "opendal-s3")] + #[serde( + skip_deserializing, + skip_serializing_if = "Option::is_none", + serialize_with = "crate::serialize_custom_credential_loader" + )] + customized_credential_load: Option, +} + +impl Default for OpenDalResolvingStorageFactory { + fn default() -> Self { + Self::new() + } +} + +impl OpenDalResolvingStorageFactory { + /// Create a new resolving storage factory. + pub fn new() -> Self { + Self { + #[cfg(feature = "opendal-s3")] + customized_credential_load: None, + } + } + + /// Set a custom AWS credential loader for S3 storage. + #[cfg(feature = "opendal-s3")] + pub fn with_s3_credential_loader(mut self, loader: CustomAwsCredentialLoader) -> Self { + self.customized_credential_load = Some(loader); + self + } +} + +#[typetag::serde] +impl StorageFactory for OpenDalResolvingStorageFactory { + fn build(&self, config: &StorageConfig) -> Result> { + Ok(Arc::new(OpenDalResolvingStorage { + props: config.props().clone(), + storages: RwLock::new(HashMap::new()), + #[cfg(feature = "opendal-s3")] + customized_credential_load: self.customized_credential_load.clone(), + })) + } +} + +/// A resolving storage that auto-detects the scheme from a path and delegates +/// to the appropriate [`OpenDalStorage`] variant. +/// +/// Sub-storages are lazily created on first use for each scheme and cached +/// for subsequent operations. Scheme aliases like `s3`/`s3a`/`s3n` map to +/// the same canonical scheme, so they share a storage instance. +#[derive(Debug, Serialize, Deserialize)] +pub struct OpenDalResolvingStorage { + /// Configuration properties shared across all backends. + props: HashMap, + /// Cache of canonical scheme to storage mappings. + #[serde(skip, default)] + storages: RwLock>>, + /// Custom AWS credential loader for S3 storage. + #[cfg(feature = "opendal-s3")] + #[serde(skip)] + customized_credential_load: Option, +} + +impl OpenDalResolvingStorage { + /// Resolve the storage for the given path by extracting the canonical scheme and + /// returning the cached or newly-created [`OpenDalStorage`]. + fn resolve(&self, path: &str) -> Result> { + let scheme = extract_scheme(path)?; + + // Fast path: check read lock first. + { + let cache = self + .storages + .read() + .map_err(|_| Error::new(ErrorKind::Unexpected, "Storage cache lock poisoned"))?; + if let Some(storage) = cache.get(&scheme) { + return Ok(storage.clone()); + } + } + + // Slow path: build and insert under write lock. + let mut cache = self + .storages + .write() + .map_err(|_| Error::new(ErrorKind::Unexpected, "Storage cache lock poisoned"))?; + + // Double-check after acquiring write lock. + if let Some(storage) = cache.get(&scheme) { + return Ok(storage.clone()); + } + + let storage = build_storage_for_scheme( + scheme, + &self.props, + #[cfg(feature = "opendal-s3")] + &self.customized_credential_load, + )?; + let storage = Arc::new(storage); + cache.insert(scheme, storage.clone()); + Ok(storage) + } +} + +#[async_trait] +#[typetag::serde] +impl Storage for OpenDalResolvingStorage { + async fn exists(&self, path: &str) -> Result { + self.resolve(path)?.exists(path).await + } + + async fn metadata(&self, path: &str) -> Result { + self.resolve(path)?.metadata(path).await + } + + async fn read(&self, path: &str) -> Result { + self.resolve(path)?.read(path).await + } + + async fn reader(&self, path: &str) -> Result> { + self.resolve(path)?.reader(path).await + } + + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { + self.resolve(path)?.write(path, bs).await + } + + async fn writer(&self, path: &str) -> Result> { + self.resolve(path)?.writer(path).await + } + + async fn delete(&self, path: &str) -> Result<()> { + self.resolve(path)?.delete(path).await + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + self.resolve(path)?.delete_prefix(path).await + } + + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + // Group paths by canonical scheme so each resolved storage receives a batch, + // avoiding repeated operator creation per path. + let mut grouped: HashMap<&'static str, Vec> = HashMap::new(); + while let Some(path) = paths.next().await { + let scheme = extract_scheme(&path)?; + grouped.entry(scheme).or_default().push(path); + } + + for (_, paths) in grouped { + let storage = self.resolve(&paths[0])?; + storage + .delete_stream(futures::stream::iter(paths).boxed()) + .await?; + } + Ok(()) + } + + fn new_input(&self, path: &str) -> Result { + Ok(InputFile::new( + Arc::new(self.resolve(path)?.as_ref().clone()), + path.to_string(), + )) + } + + fn new_output(&self, path: &str) -> Result { + Ok(OutputFile::new( + Arc::new(self.resolve(path)?.as_ref().clone()), + path.to_string(), + )) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::OPENDAL_IO_TIMEOUT_MS; + + #[cfg(feature = "opendal-s3")] + #[derive(Debug)] + struct EmptyCredentialLoader; + + #[cfg(feature = "opendal-s3")] + impl crate::s3::ProvideCredential for EmptyCredentialLoader { + type Credential = crate::s3::AwsCredential; + + async fn provide_credential( + &self, + _ctx: &reqsign_core::Context, + ) -> reqsign_core::Result> { + Ok(None) + } + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_custom_credential_loader_serialization_fails() { + let factory = OpenDalResolvingStorageFactory::new() + .with_s3_credential_loader(CustomAwsCredentialLoader::new(EmptyCredentialLoader)); + let file_io = iceberg::io::FileIOBuilder::new(Arc::new(factory)).build(); + + let err = file_io.serialize_all().unwrap_err(); + assert!( + err.to_string() + .contains("custom AWS credential loaders cannot be serialized") + ); + } + + /// Builds a resolving storage with empty props, suitable for `resolve()` + /// calls that don't actually hit any backend. + fn empty_resolving_storage() -> OpenDalResolvingStorage { + OpenDalResolvingStorage { + props: HashMap::new(), + storages: RwLock::new(HashMap::new()), + #[cfg(feature = "opendal-s3")] + customized_credential_load: None, + } + } + + #[test] + fn test_resolve_propagates_io_timeout() { + let mut storage = empty_resolving_storage(); + storage + .props + .insert(OPENDAL_IO_TIMEOUT_MS.to_string(), "45000".to_string()); + + let paths: &[&str] = &[ + #[cfg(feature = "opendal-memory")] + "memory:/key", + #[cfg(feature = "opendal-fs")] + "file:/key", + #[cfg(feature = "opendal-s3")] + "s3://bucket/key", + #[cfg(feature = "opendal-gcs")] + "gs://bucket/key", + #[cfg(feature = "opendal-oss")] + "oss://bucket/key", + #[cfg(feature = "opendal-azdls")] + "abfss://myfs@myaccount.dfs.core.windows.net/key", + #[cfg(feature = "opendal-hf")] + "hf://datasets/user/repo/key", + ]; + for path in paths { + let resolved = storage.resolve(path).unwrap(); + assert_eq!( + resolved.client_config().io_timeout(), + Duration::from_secs(45), + "{path}" + ); + } + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_resolve_s3_aliases_share_instance() { + let storage = empty_resolving_storage(); + + // All three S3-family schemes must collapse to a single cached + // `Arc` so that catalogs handing the resolver a mix + // of `s3://`, `s3a://`, `s3n://` paths don't rebuild operators. + let a = storage.resolve("s3://bucket/key").unwrap(); + let b = storage.resolve("s3a://bucket/key").unwrap(); + let c = storage.resolve("s3n://bucket/key").unwrap(); + + assert!(Arc::ptr_eq(&a, &b), "s3 and s3a should share one instance"); + assert!(Arc::ptr_eq(&a, &c), "s3 and s3n should share one instance"); + } + + #[cfg(feature = "opendal-hdfs-native")] + #[test] + fn test_resolve_hdfs_native_returns_hdfs_variant() { + let storage = empty_resolving_storage(); + + let resolved = storage.resolve("hdfs://nameservice1/a/b").unwrap(); + + assert!( + matches!(&*resolved, OpenDalStorage::HdfsNative(_)), + "expected HdfsNative variant, got {resolved:?}" + ); + } + + #[cfg(feature = "opendal-hdfs-native")] + #[test] + fn test_resolve_hdfs_native_distinct_authorities_share_instance() { + let storage = empty_resolving_storage(); + + let a = storage.resolve("hdfs://ns1/a").unwrap(); + let b = storage.resolve("hdfs://ns2/b").unwrap(); + + assert!( + Arc::ptr_eq(&a, &b), + "different authorities should share the OpenDalStorage::HdfsNative instance (operator cache is internal)" + ); + } + + #[cfg(feature = "opendal-azdls")] + #[test] + fn test_resolve_azdls_aliases_share_instance() { + let storage = empty_resolving_storage(); + + let path_for = |scheme: &str| { + format!("{scheme}://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet") + }; + + // All Azure schemes collapse onto one cached instance. + let abfss = storage.resolve(&path_for("abfss")).unwrap(); + let abfs = storage.resolve(&path_for("abfs")).unwrap(); + + assert!( + Arc::ptr_eq(&abfss, &abfs), + "abfss and abfs should share one instance" + ); + } +} diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs new file mode 100644 index 0000000000..b7e0166185 --- /dev/null +++ b/crates/storage/opendal/src/s3.rs @@ -0,0 +1,422 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::num::NonZeroU64; +use std::sync::Arc; + +use iceberg::io::{ + CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN, + S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME, S3_DISABLE_CONFIG_LOAD, + S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_MULTIPART_PART_SIZE_BYTES, S3_PATH_STYLE_ACCESS, + S3_REGION, S3_RETRY_MAX_WAIT_MS, S3_RETRY_MIN_WAIT_MS, S3_RETRY_NUM_RETRIES, + S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE, +}; +use iceberg::{Error, ErrorKind, Result}; +use iceberg_property_macro::Properties; +use opendal::services::S3Config; +use opendal::{Configurator, Operator}; +/// AWS credentials: access key ID, secret access key, and optional session token. +pub use reqsign_aws_v4::Credential as AwsCredential; +/// Trait for types that can asynchronously supply [`AwsCredential`] to a [`CustomAwsCredentialLoader`]. +pub use reqsign_core::ProvideCredential; +use reqsign_core::{ProvideCredentialChain, ProvideCredentialDyn}; +use url::Url; + +use crate::utils::{from_opendal_error, is_truthy}; +use crate::{OpenDalClientConfig, parse_positive_ms}; + +#[derive(Properties)] +struct S3RetryConfig { + #[property(key = S3_RETRY_NUM_RETRIES, default = None)] + num_retries: Option, + #[property(key = S3_RETRY_MIN_WAIT_MS, default = None, parse_with = parse_positive_ms)] + min_wait_ms: Option, + #[property(key = S3_RETRY_MAX_WAIT_MS, default = None, parse_with = parse_positive_ms)] + max_wait_ms: Option, +} + +/// Overlays the `s3.retry.*` properties onto the `opendal.retry.*` settings of `client_config`, per key. +pub(crate) fn s3_client_config( + props: &HashMap, + mut client_config: OpenDalClientConfig, +) -> Result { + let retry = S3RetryConfig::from_properties(props)?; + client_config.retry_num_retries = retry.num_retries.or(client_config.retry_num_retries); + client_config.retry_min_wait_ms = retry.min_wait_ms.or(client_config.retry_min_wait_ms); + client_config.retry_max_wait_ms = retry.max_wait_ms.or(client_config.retry_max_wait_ms); + Ok(client_config) +} + +/// S3 rejects a non-final part smaller than this. +const MULTIPART_PART_SIZE_MIN: u64 = 5 * 1024 * 1024; + +/// Java reads this property as an `int`, so a larger value fails there. +const MULTIPART_PART_SIZE_MAX: u64 = i32::MAX as u64; + +/// Matches Java `S3FileIOProperties.MULTIPART_SIZE_DEFAULT`. +const MULTIPART_PART_SIZE_DEFAULT: u64 = 32 * 1024 * 1024; + +pub(crate) fn default_multipart_part_size() -> u64 { + MULTIPART_PART_SIZE_DEFAULT +} + +/// Parse iceberg props to s3 multipart upload part size. +pub(crate) fn s3_multipart_part_size_parse(m: &HashMap) -> Result { + let Some(value) = m.get(S3_MULTIPART_PART_SIZE_BYTES) else { + return Ok(default_multipart_part_size()); + }; + let part_size = value.parse::().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid {S3_MULTIPART_PART_SIZE_BYTES}: {value}: {e}"), + ) + })?; + if part_size < MULTIPART_PART_SIZE_MIN { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid {S3_MULTIPART_PART_SIZE_BYTES}: {part_size} bytes is below the S3 minimum part size of {} MiB", + MULTIPART_PART_SIZE_MIN / (1024 * 1024) + ), + )); + } + if part_size > MULTIPART_PART_SIZE_MAX { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid {S3_MULTIPART_PART_SIZE_BYTES}: {part_size} bytes is above {MULTIPART_PART_SIZE_MAX} bytes, the largest value Java accepts" + ), + )); + } + Ok(part_size) +} + +/// Parse iceberg props to s3 config. +pub(crate) fn s3_config_parse(mut m: HashMap) -> Result { + let mut cfg = S3Config::default(); + // Match Iceberg `S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT = false`: + // virtual-host-style addressing is the spec default. opendal's own + // default is path-style, which disagrees with the Java SDK and breaks + // S3-compatible stores that only accept virtual-hosted-style URLs. + // Any explicit `s3.path-style-access` property below overrides this. + cfg.enable_virtual_host_style = true; + if let Some(endpoint) = m.remove(S3_ENDPOINT) { + cfg.endpoint = Some(endpoint); + }; + if let Some(access_key_id) = m.remove(S3_ACCESS_KEY_ID) { + cfg.access_key_id = Some(access_key_id); + }; + if let Some(secret_access_key) = m.remove(S3_SECRET_ACCESS_KEY) { + cfg.secret_access_key = Some(secret_access_key); + }; + if let Some(session_token) = m.remove(S3_SESSION_TOKEN) { + cfg.session_token = Some(session_token); + }; + if let Some(region) = m.remove(S3_REGION) { + cfg.region = Some(region); + }; + if let Some(region) = m.remove(CLIENT_REGION) { + cfg.region = Some(region); + }; + if let Some(path_style_access) = m.remove(S3_PATH_STYLE_ACCESS) { + cfg.enable_virtual_host_style = !is_truthy(path_style_access.to_lowercase().as_str()); + }; + if let Some(arn) = m.remove(S3_ASSUME_ROLE_ARN) { + cfg.role_arn = Some(arn); + } + if let Some(external_id) = m.remove(S3_ASSUME_ROLE_EXTERNAL_ID) { + cfg.external_id = Some(external_id); + }; + if let Some(session_name) = m.remove(S3_ASSUME_ROLE_SESSION_NAME) { + cfg.role_session_name = Some(session_name); + }; + let s3_sse_key = m.remove(S3_SSE_KEY); + if let Some(sse_type) = m.remove(S3_SSE_TYPE) { + match sse_type.to_lowercase().as_str() { + // No Server Side Encryption + "none" => {} + // S3 SSE-S3 encryption (S3 managed keys). https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html + "s3" => { + cfg.server_side_encryption = Some("AES256".to_string()); + } + // S3 SSE KMS, either using default or custom KMS key. https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html + "kms" => { + cfg.server_side_encryption = Some("aws:kms".to_string()); + cfg.server_side_encryption_aws_kms_key_id = s3_sse_key; + } + // S3 SSE-C, using customer managed keys. https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html + "custom" => { + cfg.server_side_encryption_customer_algorithm = Some("AES256".to_string()); + cfg.server_side_encryption_customer_key = s3_sse_key; + cfg.server_side_encryption_customer_key_md5 = m.remove(S3_SSE_MD5); + } + _ => { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid {S3_SSE_TYPE}: {sse_type}. Expected one of (custom, kms, s3, none)" + ), + )); + } + } + }; + + if let Some(allow_anonymous) = m.remove(S3_ALLOW_ANONYMOUS) + && is_truthy(allow_anonymous.to_lowercase().as_str()) + { + cfg.skip_signature = true; + } + if let Some(disable_ec2_metadata) = m.remove(S3_DISABLE_EC2_METADATA) + && is_truthy(disable_ec2_metadata.to_lowercase().as_str()) + { + cfg.disable_ec2_metadata = true; + }; + if let Some(disable_config_load) = m.remove(S3_DISABLE_CONFIG_LOAD) + && is_truthy(disable_config_load.to_lowercase().as_str()) + { + cfg.disable_config_load = true; + }; + + Ok(cfg) +} + +/// Build new opendal operator from give path. +pub(crate) fn s3_config_build( + cfg: &S3Config, + customized_credential_load: &Option, + path: &str, +) -> Result { + let url = Url::parse(path)?; + let bucket = url.host_str().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid s3 url: {path}, missing bucket"), + ) + })?; + + let mut builder = cfg + .clone() + .into_builder() + // Set bucket name. + .bucket(bucket); + + if let Some(loader) = customized_credential_load { + let chain = ProvideCredentialChain::new().push(Arc::clone(&loader.0)); + builder = builder.credential_provider_chain(chain); + } + + Operator::new(builder).map_err(from_opendal_error) +} + +/// Custom AWS credential loader. +/// +/// Wraps any [`ProvideCredential`] implementation for use with the S3 storage backend. +/// Use [`CustomAwsCredentialLoader::new`] to create one, then pass it to +/// [`OpenDalStorageFactory::S3`](crate::OpenDalStorageFactory). +pub struct CustomAwsCredentialLoader(Arc>); + +impl Clone for CustomAwsCredentialLoader { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +impl std::fmt::Debug for CustomAwsCredentialLoader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CustomAwsCredentialLoader") + .finish_non_exhaustive() + } +} + +impl CustomAwsCredentialLoader { + /// Create a new custom AWS credential loader from any [`ProvideCredential`] implementation. + pub fn new(provider: impl ProvideCredential + 'static) -> Self { + Self(Arc::new(provider) as Arc>) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::time::Duration; + + use iceberg::Result; + use iceberg::io::{ + S3_MULTIPART_PART_SIZE_BYTES, S3_PATH_STYLE_ACCESS, S3_RETRY_MAX_WAIT_MS, + S3_RETRY_MIN_WAIT_MS, S3_RETRY_NUM_RETRIES, + }; + use opendal::layers::RetryLayer; + + use super::{ + MULTIPART_PART_SIZE_DEFAULT, MULTIPART_PART_SIZE_MAX, MULTIPART_PART_SIZE_MIN, + s3_client_config, s3_config_parse, s3_multipart_part_size_parse, + }; + use crate::{ + OPENDAL_RETRY_MAX_WAIT_MS, OPENDAL_RETRY_MIN_WAIT_MS, OPENDAL_RETRY_NUM_RETRIES, + OpenDalClientConfig, + }; + + fn parse_with(prop: Option<&str>) -> bool { + let mut props = HashMap::new(); + if let Some(v) = prop { + props.insert(S3_PATH_STYLE_ACCESS.to_string(), v.to_string()); + } + s3_config_parse(props).unwrap().enable_virtual_host_style + } + + #[test] + fn s3_config_parse_path_style_access() { + // Match Iceberg S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT = false. + assert!(parse_with(None)); + assert!(parse_with(Some("false"))); + assert!(!parse_with(Some("true"))); + } + + fn parse_part_size(prop: Option<&str>) -> Result { + let mut props = HashMap::new(); + if let Some(v) = prop { + props.insert(S3_MULTIPART_PART_SIZE_BYTES.to_string(), v.to_string()); + } + s3_multipart_part_size_parse(&props) + } + + #[test] + fn s3_multipart_part_size_parse_defaults_and_overrides() { + // Match Iceberg S3FileIOProperties.MULTIPART_SIZE_DEFAULT = 32 MiB. + assert_eq!(MULTIPART_PART_SIZE_DEFAULT, 32 * 1024 * 1024); + assert_eq!(parse_part_size(None).unwrap(), MULTIPART_PART_SIZE_DEFAULT); + assert_eq!(parse_part_size(Some("8388608")).unwrap(), 8 * 1024 * 1024); + } + + #[test] + fn s3_multipart_part_size_parse_rejects_invalid() { + // Match Iceberg S3FileIOProperties.MULTIPART_SIZE_MIN = 5 MiB. + assert_eq!(MULTIPART_PART_SIZE_MIN, 5 * 1024 * 1024); + assert!(parse_part_size(Some(&MULTIPART_PART_SIZE_MIN.to_string())).is_ok()); + assert!(parse_part_size(Some(&(MULTIPART_PART_SIZE_MIN - 1).to_string())).is_err()); + assert!(parse_part_size(Some(&MULTIPART_PART_SIZE_MAX.to_string())).is_ok()); + assert!(parse_part_size(Some(&(MULTIPART_PART_SIZE_MAX + 1).to_string())).is_err()); + assert!(parse_part_size(Some("not-a-number")).is_err()); + } + + fn s3_retry_layer(props: &[(&str, &str)]) -> Result { + let props = props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(); + let client_config = + s3_client_config(&props, OpenDalClientConfig::from_properties(&props)?)?; + Ok(format!("{:?}", client_config.retry_layer())) + } + + #[test] + fn test_s3_retry_properties_reach_retry_layer() { + let expected = RetryLayer::new() + .with_max_times(10) + .with_min_delay(Duration::from_secs(2)) + .with_max_delay(Duration::from_secs(20)) + .with_jitter(); + assert_eq!( + s3_retry_layer(&[ + (S3_RETRY_NUM_RETRIES, "10"), + (S3_RETRY_MIN_WAIT_MS, "2000"), + (S3_RETRY_MAX_WAIT_MS, "20000"), + ]) + .unwrap(), + format!("{expected:?}") + ); + assert_eq!( + s3_retry_layer(&[(S3_RETRY_NUM_RETRIES, "0")]).unwrap(), + format!("{:?}", RetryLayer::new().with_max_times(0).with_jitter()) + ); + assert_eq!( + s3_retry_layer(&[]).unwrap(), + format!("{:?}", RetryLayer::new()) + ); + } + + #[test] + fn test_s3_retry_properties_win_over_the_generic_ones_per_key() { + let expected = RetryLayer::new() + .with_max_times(10) + .with_min_delay(Duration::from_secs(2)) + .with_max_delay(Duration::from_secs(20)) + .with_jitter(); + assert_eq!( + s3_retry_layer(&[ + (OPENDAL_RETRY_NUM_RETRIES, "5"), + (OPENDAL_RETRY_MIN_WAIT_MS, "100"), + (OPENDAL_RETRY_MAX_WAIT_MS, "200"), + (S3_RETRY_NUM_RETRIES, "10"), + (S3_RETRY_MIN_WAIT_MS, "2000"), + (S3_RETRY_MAX_WAIT_MS, "20000"), + ]) + .unwrap(), + format!("{expected:?}") + ); + + // a key set at one layer only is taken from that layer + let mixed = RetryLayer::new() + .with_max_times(10) + .with_min_delay(Duration::from_millis(100)) + .with_max_delay(Duration::from_secs(20)) + .with_jitter(); + assert_eq!( + s3_retry_layer(&[ + (S3_RETRY_NUM_RETRIES, "10"), + (OPENDAL_RETRY_MIN_WAIT_MS, "100"), + (OPENDAL_RETRY_MAX_WAIT_MS, "200"), + (S3_RETRY_MAX_WAIT_MS, "20000"), + ]) + .unwrap(), + format!("{mixed:?}") + ); + } + + #[test] + fn test_s3_retry_property_parsing_rejects_invalid_values() { + for (key, invalid) in [ + (S3_RETRY_NUM_RETRIES, "-1"), + (S3_RETRY_NUM_RETRIES, "abc"), + (S3_RETRY_NUM_RETRIES, ""), + (S3_RETRY_MIN_WAIT_MS, "0"), + (S3_RETRY_MIN_WAIT_MS, "abc"), + (S3_RETRY_MAX_WAIT_MS, "0"), + (S3_RETRY_MAX_WAIT_MS, "12.5"), + ] { + let err = s3_retry_layer(&[(key, invalid)]).unwrap_err().to_string(); + assert!(err.contains(key), "{key}={invalid}: {err}"); + } + } + + #[cfg(feature = "opendal-memory")] + #[test] + fn test_other_backends_ignore_the_s3_retry_properties() { + use iceberg::io::{StorageConfig, StorageFactory}; + + use crate::OpenDalStorageFactory; + + let config = StorageConfig::new().with_props([ + (S3_RETRY_NUM_RETRIES.to_string(), "abc".to_string()), + (OPENDAL_RETRY_NUM_RETRIES.to_string(), "5".to_string()), + ]); + let storage = OpenDalStorageFactory::Memory.build(&config).unwrap(); + let json = serde_json::to_string(&storage).unwrap(); + assert!(json.contains(r#""retry_num_retries":5"#), "{json}"); + } +} diff --git a/bindings/python/src/runtime.rs b/crates/storage/opendal/src/utils.rs similarity index 68% rename from bindings/python/src/runtime.rs rename to crates/storage/opendal/src/utils.rs index 81d5b6dd6f..56f8c18059 100644 --- a/bindings/python/src/runtime.rs +++ b/crates/storage/opendal/src/utils.rs @@ -15,18 +15,15 @@ // specific language governing permissions and limitations // under the License. -use std::sync::OnceLock; - -use tokio::runtime::{Handle, Runtime}; - -static RUNTIME: OnceLock = OnceLock::new(); +pub(crate) fn is_truthy(value: &str) -> bool { + ["true", "t", "1", "on"].contains(&value.to_lowercase().as_str()) +} -pub fn runtime() -> Handle { - match Handle::try_current() { - Ok(h) => h.clone(), - _ => { - let rt = RUNTIME.get_or_init(|| Runtime::new().unwrap()); - rt.handle().clone() - } - } +/// Convert an opendal error into an iceberg error. +pub(crate) fn from_opendal_error(e: opendal::Error) -> iceberg::Error { + iceberg::Error::new( + iceberg::ErrorKind::Unexpected, + "Failure in doing io operation", + ) + .with_source(e) } diff --git a/crates/storage/opendal/tests/file_io_fs_test.rs b/crates/storage/opendal/tests/file_io_fs_test.rs new file mode 100644 index 0000000000..f9bb82db7e --- /dev/null +++ b/crates/storage/opendal/tests/file_io_fs_test.rs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for FileIO OpenDAL filesystem storage. + +#[cfg(feature = "opendal-fs")] +mod tests { + use std::sync::Arc; + + use bytes::Bytes; + use iceberg::io::{FileIO, FileIOBuilder}; + use iceberg_storage_opendal::OpenDalStorageFactory; + use tempfile::TempDir; + + #[tokio::test] + async fn test_file_io_fs_serialization_roundtrip() { + let temp_dir = TempDir::new().unwrap(); + let path = temp_dir.path().join("serialization-roundtrip"); + let path = format!("file:/{}", path.display()); + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Fs)).build(); + let serialized = file_io.serialize_all().unwrap(); + let file_io = FileIO::deserialize_all(&serialized).unwrap(); + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"roundtrip")) + .await + .unwrap(); + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"roundtrip") + ); + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); + } +} diff --git a/crates/iceberg/tests/file_io_gcs_test.rs b/crates/storage/opendal/tests/file_io_gcs_test.rs similarity index 63% rename from crates/iceberg/tests/file_io_gcs_test.rs rename to crates/storage/opendal/tests/file_io_gcs_test.rs index 161285ae6f..c6ba770b17 100644 --- a/crates/iceberg/tests/file_io_gcs_test.rs +++ b/crates/storage/opendal/tests/file_io_gcs_test.rs @@ -16,78 +16,75 @@ // under the License. //! Integration tests for FileIO Google Cloud Storage (GCS). +//! +//! These tests assume Docker containers are started externally via `make docker-up`. -#[cfg(all(test, feature = "storage-gcs"))] +#[cfg(feature = "opendal-gcs")] mod tests { use std::collections::HashMap; - use std::net::SocketAddr; - use std::sync::RwLock; + use std::sync::Arc; use bytes::Bytes; - use ctor::{ctor, dtor}; - use iceberg::io::{FileIO, FileIOBuilder, GCS_NO_AUTH, GCS_SERVICE_PATH}; - use iceberg_test_utils::docker::DockerCompose; - use iceberg_test_utils::{normalize_test_name, set_up}; + use iceberg::io::{FileIO, FileIOBuilder, GCS_NO_AUTH, GCS_SERVICE_HOST}; + use iceberg_storage_opendal::OpenDalStorageFactory; + use iceberg_test_utils::{get_gcs_endpoint, set_up}; - static DOCKER_COMPOSE_ENV: RwLock> = RwLock::new(None); - static FAKE_GCS_PORT: u16 = 4443; static FAKE_GCS_BUCKET: &str = "test-bucket"; - #[ctor] - fn before_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - let docker_compose = DockerCompose::new( - normalize_test_name(module_path!()), - format!("{}/testdata/file_io_gcs", env!("CARGO_MANIFEST_DIR")), - ); - docker_compose.up(); - guard.replace(docker_compose); - } - - #[dtor] - fn after_all() { - let mut guard = DOCKER_COMPOSE_ENV.write().unwrap(); - guard.take(); - } - async fn get_file_io_gcs() -> FileIO { set_up(); - let ip = DOCKER_COMPOSE_ENV - .read() - .unwrap() - .as_ref() - .unwrap() - .get_container_ip("gcs-server"); - let addr = SocketAddr::new(ip, FAKE_GCS_PORT); + let gcs_endpoint = get_gcs_endpoint(); // A bucket must exist for FileIO - create_bucket(FAKE_GCS_BUCKET, addr.to_string()) - .await - .unwrap(); + create_bucket(FAKE_GCS_BUCKET, &gcs_endpoint).await.unwrap(); - FileIOBuilder::new("gcs") + FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Gcs)) .with_props(vec![ - (GCS_SERVICE_PATH, format!("http://{}", addr)), + (GCS_SERVICE_HOST, gcs_endpoint), (GCS_NO_AUTH, "true".to_string()), ]) .build() + } + + fn roundtrip_file_io(file_io: &FileIO) -> FileIO { + let serialized = file_io.serialize_all().unwrap(); + FileIO::deserialize_all(&serialized).unwrap() + } + + #[tokio::test] + async fn test_file_io_gcs_serialization_roundtrip() { + let file_io = roundtrip_file_io(&get_file_io_gcs().await); + let path = format!("{}/serialization-roundtrip", get_gs_path()); + + let _ = file_io.delete(&path).await; + file_io + .new_output(&path) .unwrap() + .write(Bytes::from_static(b"roundtrip")) + .await + .unwrap(); + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"roundtrip") + ); + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); } // Create a bucket against the emulated GCS storage server. - async fn create_bucket(name: &str, server_addr: String) -> anyhow::Result<()> { + async fn create_bucket(name: &str, server_endpoint: &str) -> anyhow::Result<()> { let mut bucket_data = HashMap::new(); bucket_data.insert("name", name); let client = reqwest::Client::new(); - let endpoint = format!("http://{}/storage/v1/b", server_addr); + let endpoint = format!("{server_endpoint}/storage/v1/b"); client.post(endpoint).json(&bucket_data).send().await?; Ok(()) } fn get_gs_path() -> String { - format!("gs://{}", FAKE_GCS_BUCKET) + format!("gs://{FAKE_GCS_BUCKET}") } #[tokio::test] @@ -102,7 +99,7 @@ mod tests { let file_io = get_file_io_gcs().await; let output = file_io.new_output(&gs_file).unwrap(); output - .write(bytes::Bytes::from_static(b"iceberg-gcs!")) + .write(Bytes::from_static(b"iceberg-gcs!")) .await .expect("Write to test output file"); assert!(file_io.exists(gs_file).await.unwrap()) @@ -114,7 +111,7 @@ mod tests { let file_io = get_file_io_gcs().await; let output = file_io.new_output(&gs_file).unwrap(); output - .write(bytes::Bytes::from_static(b"iceberg!")) + .write(Bytes::from_static(b"iceberg!")) .await .expect("Write to test output file"); assert!(file_io.exists(&gs_file).await.unwrap()); diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs new file mode 100644 index 0000000000..ef9da20a13 --- /dev/null +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -0,0 +1,400 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for HDFS FileIO via OpenDAL `services-hdfs-native`. +//! +//! These tests need the HDFS fixture in `dev/docker-compose.yaml`, which +//! `make docker-up` starts with the other fixtures. + +#[cfg(feature = "opendal-hdfs-native")] +mod tests { + use std::net::ToSocketAddrs; + use std::sync::Arc; + + use bytes::Bytes; + use futures::StreamExt; + use iceberg::io::{ + FileIO, FileIOBuilder, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT, StorageFactory, + }; + use iceberg_storage_opendal::{OpenDalResolvingStorageFactory, OpenDalStorageFactory}; + use iceberg_test_utils::{get_hdfs_endpoint, normalize_test_name_with_parts, set_up}; + + /// The fixture's DataNode advertises `127.0.0.1`; without this the client + /// dials its container IP, which the host cannot reach on macOS or Windows. + fn file_io_builder(factory: Arc) -> FileIOBuilder { + set_up(); + FileIOBuilder::new(factory).with_prop("hadoop.dfs.client.use.datanode.hostname", "true") + } + + fn get_file_io() -> FileIO { + file_io_builder(Arc::new(OpenDalStorageFactory::HdfsNative)).build() + } + + fn test_path(suffix: &str) -> String { + format!( + "{}/{}", + get_hdfs_endpoint(), + normalize_test_name_with_parts!(suffix) + ) + } + + /// The endpoint with its host replaced by that host's IPv4 address: a + /// second spelling of the same NameNode, so a second batch key. `None` + /// when the endpoint already uses an IP literal. + fn alternate_endpoint(endpoint: &str) -> Option { + let url = url::Url::parse(endpoint).ok()?; + let host = url.host_str()?; + if host.parse::().is_ok() { + return None; + } + let addr = (host, url.port()?) + .to_socket_addrs() + .ok()? + .find(std::net::SocketAddr::is_ipv4)?; + Some(format!("hdfs://{addr}")) + } + + #[tokio::test] + async fn test_file_io_hdfs_exists() { + let file_io = get_file_io(); + let dir = test_path("test_file_io_hdfs_exists"); + let present = format!("{dir}/present"); + let _ = file_io.delete(&present).await; + file_io + .new_output(&present) + .unwrap() + .write(Bytes::from_static(b"x")) + .await + .unwrap(); + + assert!(file_io.exists(&present).await.unwrap()); + assert!(!file_io.exists(&format!("{dir}/absent")).await.unwrap()); + } + + #[tokio::test] + async fn test_file_io_hdfs_write_and_read() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_write_and_read"); + let _ = file_io.delete(&path).await; + + let output = file_io.new_output(&path).unwrap(); + output + .write(Bytes::from_static(b"hello hdfs")) + .await + .unwrap(); + + assert!(file_io.exists(&path).await.unwrap()); + let input = file_io.new_input(&path).unwrap(); + assert_eq!( + input.read().await.unwrap(), + Bytes::from_static(b"hello hdfs") + ); + } + + /// The HA flow: table locations carry a logical authority, and + /// `hdfs.name-node.` declares its comma-separated endpoints. + /// The first one refuses connections, so every call must fail over to + /// the fixture through opendal's synthetic nameservice. + #[tokio::test] + async fn test_file_io_hdfs_declared_nameservice() { + let endpoint = get_hdfs_endpoint(); + // A local port nothing listens on once the listener is dropped. + let dead = std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + let file_io = file_io_builder(Arc::new(OpenDalStorageFactory::HdfsNative)) + .with_prop( + format!("{HDFS_NAME_NODE}.logical-nameservice"), + format!("hdfs://{dead},{endpoint}"), + ) + .build(); + + // The path authority is a logical name resolved by its declaration. + let path = format!( + "hdfs://logical-nameservice/{}", + normalize_test_name_with_parts!("test_file_io_hdfs_declared_nameservice") + ); + let _ = file_io.delete(&path).await; + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"via configured name node")) + .await + .unwrap(); + + assert!(file_io.exists(&path).await.unwrap()); + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"via configured name node") + ); + } + + /// Authority-less paths resolve through plain `hdfs.name-node`, or through + /// PyIceberg's `hdfs.host`/`hdfs.port`; both must reach the same cluster. + #[tokio::test] + async fn test_file_io_hdfs_authority_less_paths() { + let endpoint = get_hdfs_endpoint(); + let url = url::Url::parse(&endpoint).unwrap(); + let by_name_node = file_io_builder(Arc::new(OpenDalStorageFactory::HdfsNative)) + .with_prop(HDFS_NAME_NODE, &endpoint) + .build(); + let by_host_port = file_io_builder(Arc::new(OpenDalStorageFactory::HdfsNative)) + .with_prop(HDFS_HOST, url.host_str().unwrap()) + .with_prop(HDFS_PORT, url.port().unwrap().to_string()) + .build(); + + let path = format!( + "hdfs:///{}", + normalize_test_name_with_parts!("test_file_io_hdfs_authority_less_paths") + ); + let _ = by_name_node.delete(&path).await; + by_name_node + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"authority-less")) + .await + .unwrap(); + + assert_eq!( + by_host_port.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"authority-less") + ); + by_host_port.delete(&path).await.unwrap(); + assert!(!by_name_node.exists(&path).await.unwrap()); + } + + #[tokio::test] + async fn test_file_io_hdfs_overwrite() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_overwrite"); + let _ = file_io.delete(&path).await; + + for content in [b"first".as_slice(), b"second, longer".as_slice()] { + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(content)) + .await + .unwrap(); + } + + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"second, longer") + ); + } + + #[tokio::test] + async fn test_file_io_hdfs_delete_stream() { + let file_io = get_file_io(); + + let paths: Vec = (0..5) + .map(|i| format!("{}/file-{i}", test_path("test_file_io_hdfs_delete_stream"))) + .collect(); + for path in &paths { + let _ = file_io.delete(path).await; + file_io + .new_output(path) + .unwrap() + .write("delete-me".into()) + .await + .unwrap(); + assert!(file_io.exists(path).await.unwrap()); + } + + let stream = futures::stream::iter(paths.clone()).boxed(); + file_io.delete_stream(stream).await.unwrap(); + + for path in &paths { + assert!(!file_io.exists(path).await.unwrap()); + } + } + + /// Paths are batched per effective NameNode; two spellings of the fixture's + /// NameNode drive two operators and two deleters through one stream. The + /// fixture is one cluster, so the keying itself is pinned by the unit + /// tests; this proves the multi-operator path completes and cleans up. + #[tokio::test] + async fn test_file_io_hdfs_delete_stream_two_name_nodes() { + let endpoint = get_hdfs_endpoint(); + let Some(alternate) = alternate_endpoint(&endpoint) else { + eprintln!("Skipping HDFS test: {endpoint} has no second spelling"); + return; + }; + let file_io = get_file_io(); + let dir = test_path("test_file_io_hdfs_delete_stream_two_name_nodes"); + let alt_dir = dir.replacen(&endpoint, &alternate, 1); + assert_ne!(dir, alt_dir); + + let paths = vec![format!("{dir}/a"), format!("{alt_dir}/b")]; + for path in &paths { + let _ = file_io.delete(path).await; + file_io + .new_output(path) + .unwrap() + .write("delete-me".into()) + .await + .unwrap(); + } + // Both spellings reach the same cluster. + assert!(file_io.exists(&format!("{alt_dir}/a")).await.unwrap()); + assert!(file_io.exists(&format!("{dir}/b")).await.unwrap()); + + let stream = futures::stream::iter(paths.clone()).boxed(); + file_io.delete_stream(stream).await.unwrap(); + + // Everything is gone, through whichever deleter. + for path in [ + format!("{dir}/a"), + format!("{dir}/b"), + format!("{alt_dir}/a"), + format!("{alt_dir}/b"), + ] { + assert!(!file_io.exists(&path).await.unwrap(), "{path}"); + } + } + + #[tokio::test] + async fn test_file_io_hdfs_delete_stream_empty() { + let file_io = get_file_io(); + let stream = futures::stream::empty().boxed(); + file_io.delete_stream(stream).await.unwrap(); + } + + #[tokio::test] + async fn test_file_io_hdfs_resolving_storage() { + let file_io = file_io_builder(Arc::new(OpenDalResolvingStorageFactory::new())).build(); + let path = test_path("test_file_io_hdfs_resolving_storage"); + let _ = file_io.delete(&path).await; + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"resolving")) + .await + .unwrap(); + + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"resolving") + ); + + file_io.delete(&path).await.unwrap(); + } + + #[tokio::test] + async fn test_file_io_hdfs_metadata() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_metadata"); + let _ = file_io.delete(&path).await; + let content = Bytes::from_static(b"0123456789"); + + file_io + .new_output(&path) + .unwrap() + .write(content.clone()) + .await + .unwrap(); + + let metadata = file_io.new_input(&path).unwrap().metadata().await.unwrap(); + assert_eq!(metadata.size, content.len() as u64); + } + + #[tokio::test] + async fn test_file_io_hdfs_delete() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_delete"); + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"x")) + .await + .unwrap(); + assert!(file_io.exists(&path).await.unwrap()); + + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); + } + + #[tokio::test] + async fn test_file_io_hdfs_delete_prefix() { + let file_io = get_file_io(); + let dir = test_path("test_file_io_hdfs_delete_prefix"); + let _ = file_io.delete_prefix(&dir).await; + + for i in 0..3 { + let path = format!("{dir}/file_{i}"); + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from(format!("payload {i}"))) + .await + .unwrap(); + } + assert!(file_io.exists(&format!("{dir}/file_0")).await.unwrap()); + + file_io.delete_prefix(&dir).await.unwrap(); + + for i in 0..3 { + assert!(!file_io.exists(&format!("{dir}/file_{i}")).await.unwrap()); + } + } + + #[tokio::test] + async fn test_file_io_hdfs_reader_range() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_reader_range"); + let _ = file_io.delete(&path).await; + let content = Bytes::from_static(b"abcdefghij"); + + file_io + .new_output(&path) + .unwrap() + .write(content.clone()) + .await + .unwrap(); + + let reader = file_io.new_input(&path).unwrap().reader().await.unwrap(); + assert_eq!( + reader.read(0..5).await.unwrap(), + Bytes::from_static(b"abcde") + ); + assert_eq!( + reader.read(5..10).await.unwrap(), + Bytes::from_static(b"fghij") + ); + } + + #[tokio::test] + async fn test_file_io_hdfs_streaming_writer() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_streaming_writer"); + let _ = file_io.delete(&path).await; + + let output = file_io.new_output(&path).unwrap(); + let mut writer = output.writer().await.unwrap(); + writer.write(Bytes::from_static(b"part1 ")).await.unwrap(); + writer.write(Bytes::from_static(b"part2")).await.unwrap(); + writer.close().await.unwrap(); + + let read = file_io.new_input(&path).unwrap().read().await.unwrap(); + assert_eq!(read, Bytes::from_static(b"part1 part2")); + } +} diff --git a/crates/storage/opendal/tests/file_io_hf_test.rs b/crates/storage/opendal/tests/file_io_hf_test.rs new file mode 100644 index 0000000000..f8d17e2ad1 --- /dev/null +++ b/crates/storage/opendal/tests/file_io_hf_test.rs @@ -0,0 +1,384 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for FileIO HuggingFace Hub. +//! +//! These tests require a real HuggingFace token and are skipped when +//! `HF_TOKEN` is not set in the environment. +//! +//! The following environment variables are used: +//! - `HF_TOKEN` — HuggingFace API token (required) +//! - `HF_BUCKET` — `owner/repo` for a bucket-type repo (required when running bucket tests) +//! - `HF_DATASET` — `owner/repo` for a dataset-type repo (required when running dataset tests) + +#[cfg(feature = "opendal-hf")] +mod tests { + use std::sync::Arc; + + use bytes::Bytes; + use futures::StreamExt; + use iceberg::io::{FileIO, FileIOBuilder, HF_REVISION, HF_TOKEN}; + use iceberg_storage_opendal::{OpenDalResolvingStorageFactory, OpenDalStorageFactory}; + use iceberg_test_utils::{normalize_test_name_with_parts, set_up}; + + const ENV_HF_TOKEN: &str = "HF_TOKEN"; + const ENV_HF_BUCKET: &str = "HF_BUCKET"; + const ENV_HF_DATASET: &str = "HF_DATASET"; + + macro_rules! require_env { + ($var:expr) => { + match std::env::var($var) { + Ok(v) if !v.is_empty() => v, + // Treat an unset *or* empty var as "not provided": CI exposes + // these via `env: FOO: ${{ secrets.FOO }}`, which sets the var + // to an empty string when the secret is absent (e.g. fork PRs). + _ => { + eprintln!("Skipping HF test: {} not set", $var); + return; + } + } + }; + } + + fn get_file_io(token: &str) -> FileIO { + set_up(); + FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Hf)) + .with_props(vec![(HF_TOKEN, token.to_string())]) + .build() + } + + fn get_resolving_file_io(token: &str) -> FileIO { + set_up(); + FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) + .with_props(vec![(HF_TOKEN, token.to_string())]) + .build() + } + + fn roundtrip_file_io(file_io: &FileIO) -> FileIO { + let serialized = file_io.serialize_all().unwrap(); + FileIO::deserialize_all(&serialized).unwrap() + } + + // --- bucket tests --- + + #[tokio::test] + async fn test_hf_bucket_write_read_delete() { + let token = require_env!(ENV_HF_TOKEN); + let bucket = require_env!(ENV_HF_BUCKET); + let file_io = roundtrip_file_io(&get_file_io(&token)); + let path = format!( + "hf://buckets/{}/{}", + bucket, + normalize_test_name_with_parts!("test_hf_bucket_write_read_delete") + ); + + let _ = file_io.delete(&path).await; + assert!(!file_io.exists(&path).await.unwrap()); + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"iceberg-hf-bucket")) + .await + .unwrap(); + assert!(file_io.exists(&path).await.unwrap()); + + let data = file_io.new_input(&path).unwrap().read().await.unwrap(); + assert_eq!(data, Bytes::from_static(b"iceberg-hf-bucket")); + + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); + } + + #[tokio::test] + async fn test_hf_bucket_overwrite() { + let token = require_env!(ENV_HF_TOKEN); + let bucket = require_env!(ENV_HF_BUCKET); + let file_io = get_file_io(&token); + let path = format!( + "hf://buckets/{}/{}", + bucket, + normalize_test_name_with_parts!("test_hf_bucket_overwrite") + ); + + let _ = file_io.delete(&path).await; + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"first")) + .await + .unwrap(); + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"second")) + .await + .unwrap(); + + let data = file_io.new_input(&path).unwrap().read().await.unwrap(); + assert_eq!(data, Bytes::from_static(b"second")); + + file_io.delete(&path).await.unwrap(); + } + + #[tokio::test] + async fn test_hf_bucket_range_read() { + let token = require_env!(ENV_HF_TOKEN); + let bucket = require_env!(ENV_HF_BUCKET); + let file_io = get_file_io(&token); + let path = format!( + "hf://buckets/{}/{}", + bucket, + normalize_test_name_with_parts!("test_hf_bucket_range_read") + ); + + let _ = file_io.delete(&path).await; + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"hello world")) + .await + .unwrap(); + + let reader = file_io.new_input(&path).unwrap().reader().await.unwrap(); + let chunk = reader.read(6..11).await.unwrap(); + assert_eq!(chunk, Bytes::from_static(b"world")); + + file_io.delete(&path).await.unwrap(); + } + + #[tokio::test] + async fn test_hf_bucket_metadata() { + let token = require_env!(ENV_HF_TOKEN); + let bucket = require_env!(ENV_HF_BUCKET); + let file_io = get_file_io(&token); + let path = format!( + "hf://buckets/{}/{}", + bucket, + normalize_test_name_with_parts!("test_hf_bucket_metadata") + ); + + let _ = file_io.delete(&path).await; + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"metadata-test")) + .await + .unwrap(); + + let meta = file_io.new_input(&path).unwrap().metadata().await.unwrap(); + assert_eq!(meta.size, b"metadata-test".len() as u64); + + file_io.delete(&path).await.unwrap(); + } + + #[tokio::test] + async fn test_hf_bucket_delete_stream() { + let token = require_env!(ENV_HF_TOKEN); + let bucket = require_env!(ENV_HF_BUCKET); + let file_io = get_file_io(&token); + + let paths: Vec = (0..3) + .map(|i| { + format!( + "hf://buckets/{}/{}/file-{i}", + bucket, + normalize_test_name_with_parts!("test_hf_bucket_delete_stream") + ) + }) + .collect(); + + for path in &paths { + let _ = file_io.delete(path).await; + file_io + .new_output(path) + .unwrap() + .write(Bytes::from_static(b"x")) + .await + .unwrap(); + assert!(file_io.exists(path).await.unwrap()); + } + + let stream = futures::stream::iter(paths.clone()).boxed(); + file_io.delete_stream(stream).await.unwrap(); + + for path in &paths { + assert!(!file_io.exists(path).await.unwrap()); + } + } + + #[tokio::test] + async fn test_hf_bucket_delete_stream_empty() { + let token = require_env!(ENV_HF_TOKEN); + let file_io = get_file_io(&token); + file_io + .delete_stream(futures::stream::empty().boxed()) + .await + .unwrap(); + } + + // --- dataset tests --- + + #[tokio::test] + async fn test_hf_dataset_write_read_delete() { + let token = require_env!(ENV_HF_TOKEN); + let dataset = require_env!(ENV_HF_DATASET); + let file_io = get_file_io(&token); + let path = format!( + "hf://datasets/{}/{}", + dataset, + normalize_test_name_with_parts!("test_hf_dataset_write_read_delete") + ); + + let _ = file_io.delete(&path).await; + assert!(!file_io.exists(&path).await.unwrap()); + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"iceberg-hf-dataset")) + .await + .unwrap(); + assert!(file_io.exists(&path).await.unwrap()); + + let data = file_io.new_input(&path).unwrap().read().await.unwrap(); + assert_eq!(data, Bytes::from_static(b"iceberg-hf-dataset")); + + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); + } + + // --- revision tests --- + + #[tokio::test] + async fn test_hf_explicit_revision_in_uri() { + let token = require_env!(ENV_HF_TOKEN); + let file_io = get_file_io(&token); + let name = normalize_test_name_with_parts!("test_hf_explicit_revision_in_uri"); + + let bucket = require_env!(ENV_HF_BUCKET); + // Write without revision, read back with explicit @main. + let write_path = format!("hf://buckets/{}/{}", bucket, name); + let read_path = format!("hf://buckets/{}@main/{}", bucket, name); + + let _ = file_io.delete(&write_path).await; + file_io + .new_output(&write_path) + .unwrap() + .write(Bytes::from_static(b"revision-test")) + .await + .unwrap(); + + let data = file_io.new_input(&read_path).unwrap().read().await.unwrap(); + assert_eq!(data, Bytes::from_static(b"revision-test")); + + file_io.delete(&write_path).await.unwrap(); + } + + #[tokio::test] + async fn test_hf_revision_from_config() { + let token = require_env!(ENV_HF_TOKEN); + set_up(); + + // Build FileIO with HF_REVISION set in config — paths without @revision use it. + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Hf)) + .with_props(vec![ + (HF_TOKEN, token.to_string()), + (HF_REVISION, "main".to_string()), + ]) + .build(); + + let bucket = require_env!(ENV_HF_BUCKET); + let path = format!( + "hf://buckets/{}/{}", + bucket, + normalize_test_name_with_parts!("test_hf_revision_from_config") + ); + + let _ = file_io.delete(&path).await; + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"config-revision")) + .await + .unwrap(); + + let data = file_io.new_input(&path).unwrap().read().await.unwrap(); + assert_eq!(data, Bytes::from_static(b"config-revision")); + + file_io.delete(&path).await.unwrap(); + } + + // --- resolving storage tests --- + + #[tokio::test] + async fn test_hf_resolving_storage() { + let token = require_env!(ENV_HF_TOKEN); + let file_io = roundtrip_file_io(&get_resolving_file_io(&token)); + + let bucket = require_env!(ENV_HF_BUCKET); + let path = format!( + "hf://buckets/{}/{}", + bucket, + normalize_test_name_with_parts!("test_hf_resolving_storage") + ); + + let _ = file_io.delete(&path).await; + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"resolving")) + .await + .unwrap(); + + let data = file_io.new_input(&path).unwrap().read().await.unwrap(); + assert_eq!(data, Bytes::from_static(b"resolving")); + + file_io.delete(&path).await.unwrap(); + } + + #[tokio::test] + async fn test_hf_resolving_delete_stream_across_repo_types() { + let token = require_env!(ENV_HF_TOKEN); + let file_io = get_resolving_file_io(&token); + + let bucket = require_env!(ENV_HF_BUCKET); + let dataset = require_env!(ENV_HF_DATASET); + let name = normalize_test_name_with_parts!("test_hf_resolving_delete_stream_across"); + let bucket_path = format!("hf://buckets/{}/{}", bucket, name); + let dataset_path = format!("hf://datasets/{}/{}", dataset, name); + + for path in [&bucket_path, &dataset_path] { + let _ = file_io.delete(path).await; + file_io + .new_output(path) + .unwrap() + .write(Bytes::from_static(b"x")) + .await + .unwrap(); + assert!(file_io.exists(path).await.unwrap()); + } + + let stream = futures::stream::iter(vec![bucket_path.clone(), dataset_path.clone()]).boxed(); + file_io.delete_stream(stream).await.unwrap(); + + assert!(!file_io.exists(&bucket_path).await.unwrap()); + assert!(!file_io.exists(&dataset_path).await.unwrap()); + } +} diff --git a/crates/storage/opendal/tests/file_io_memory_test.rs b/crates/storage/opendal/tests/file_io_memory_test.rs new file mode 100644 index 0000000000..9c3c335a3f --- /dev/null +++ b/crates/storage/opendal/tests/file_io_memory_test.rs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for FileIO OpenDAL memory storage. + +#[cfg(feature = "opendal-memory")] +mod tests { + use std::sync::Arc; + + use bytes::Bytes; + use iceberg::io::{FileIO, FileIOBuilder}; + use iceberg_storage_opendal::OpenDalStorageFactory; + + #[tokio::test] + async fn test_file_io_memory_serialization_roundtrip() { + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Memory)).build(); + let serialized = file_io.serialize_all().unwrap(); + let file_io = FileIO::deserialize_all(&serialized).unwrap(); + let path = "memory://serialization-roundtrip"; + + file_io + .new_output(path) + .unwrap() + .write(Bytes::from_static(b"roundtrip")) + .await + .unwrap(); + assert_eq!( + file_io.new_input(path).unwrap().read().await.unwrap(), + Bytes::from_static(b"roundtrip") + ); + file_io.delete(path).await.unwrap(); + assert!(!file_io.exists(path).await.unwrap()); + } +} diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs new file mode 100644 index 0000000000..14e54dcf59 --- /dev/null +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -0,0 +1,418 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for FileIO S3. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. +//! Each test uses unique file paths based on module path to avoid conflicts. +#[cfg(feature = "opendal-s3")] +mod tests { + use std::sync::Arc; + + use bytes::Bytes; + use futures::StreamExt; + use iceberg::io::{ + FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_MULTIPART_PART_SIZE_BYTES, + S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, + }; + use iceberg_storage_opendal::{ + AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, + }; + use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up}; + use opendal::Configurator; + use reqsign_core::Context; + + const OBJECT_STORE_ACCESS_KEY_ID: &str = "admin"; + const OBJECT_STORE_SECRET_ACCESS_KEY: &str = "password"; + const OBJECT_STORE_REGION: &str = "us-east-1"; + const OBJECT_STORE_BUCKET: &str = "bucket1"; + + async fn get_file_io() -> FileIO { + get_file_io_with_props(vec![]).await + } + + async fn get_file_io_with_props(extra: Vec<(&'static str, String)>) -> FileIO { + set_up(); + + let object_store_endpoint = get_object_store_endpoint(); + + let mut props = vec![ + (S3_ENDPOINT, object_store_endpoint), + (S3_ACCESS_KEY_ID, OBJECT_STORE_ACCESS_KEY_ID.to_string()), + ( + S3_SECRET_ACCESS_KEY, + OBJECT_STORE_SECRET_ACCESS_KEY.to_string(), + ), + (S3_REGION, OBJECT_STORE_REGION.to_string()), + (S3_PATH_STYLE_ACCESS, "true".to_string()), + ]; + props.extend(extra); + + FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .with_props(props) + .build() + } + + /// Number of upload parts S3 recorded for `key`. A multipart upload reports + /// a `-` ETag; a single-request upload reports a bare MD5. + /// + /// A server-side encrypted object can report an ETag without the suffix, so + /// this helper holds only for the object store fixture these tests run against. + async fn upload_part_count(key: &str) -> usize { + let mut config = opendal::services::S3Config::default(); + config.endpoint = Some(get_object_store_endpoint()); + config.access_key_id = Some(OBJECT_STORE_ACCESS_KEY_ID.to_string()); + config.secret_access_key = Some(OBJECT_STORE_SECRET_ACCESS_KEY.to_string()); + config.region = Some(OBJECT_STORE_REGION.to_string()); + config.bucket = OBJECT_STORE_BUCKET.to_string(); + // Each `#[tokio::test]` has its own runtime, and the first request after + // an earlier one went away fails before the client reconnects. The + // storage under test carries the same layer. + let op = opendal::Operator::new(config.into_builder()) + .unwrap() + .layer(opendal::layers::RetryLayer::new()); + let meta = op.stat(key).await.unwrap(); + let etag = meta + .etag() + .expect("object store reports an ETag") + .trim_matches('"'); + match etag.rsplit_once('-') { + Some((_, parts)) => parts + .parse() + .unwrap_or_else(|e| panic!("unexpected ETag format {etag}: {e}")), + None => 1, + } + } + + fn roundtrip_file_io(file_io: &FileIO) -> FileIO { + let serialized = file_io.serialize_all().unwrap(); + FileIO::deserialize_all(&serialized).unwrap() + } + + #[tokio::test] + async fn test_file_io_s3_serialization_roundtrip() { + let file_io = roundtrip_file_io(&get_file_io().await); + let path = format!( + "s3://{OBJECT_STORE_BUCKET}/{}", + normalize_test_name_with_parts!("test_file_io_s3_serialization_roundtrip") + ); + + let _ = file_io.delete(&path).await; + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"roundtrip")) + .await + .unwrap(); + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"roundtrip") + ); + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); + } + + #[tokio::test] + async fn test_file_io_s3_exists() { + let file_io = get_file_io().await; + assert!(!file_io.exists("s3://bucket2/any").await.unwrap()); + assert!( + file_io + .exists(&format!("s3://{OBJECT_STORE_BUCKET}/")) + .await + .unwrap() + ); + } + + #[tokio::test] + async fn test_file_io_s3_output() { + let file_io = get_file_io().await; + // Use unique file path based on module path to avoid conflicts + let output_path = format!( + "s3://{OBJECT_STORE_BUCKET}/{}", + normalize_test_name_with_parts!("test_file_io_s3_output") + ); + // Clean up from any previous test runs + let _ = file_io.delete(&output_path).await; + assert!(!file_io.exists(&output_path).await.unwrap()); + let output_file = file_io.new_output(&output_path).unwrap(); + { + output_file.write("123".into()).await.unwrap(); + } + assert!(file_io.exists(&output_path).await.unwrap()); + } + + /// `ParquetWriter` hands over a whole row group in one `write` call, so an + /// unbounded part size would make the request as large as that row group. + #[tokio::test] + async fn test_file_io_s3_writer_splits_write_into_bounded_parts() { + const PART_SIZE: usize = 5 * 1024 * 1024; + const PARTS: usize = 3; + + let file_io = + get_file_io_with_props(vec![(S3_MULTIPART_PART_SIZE_BYTES, PART_SIZE.to_string())]) + .await; + let key = normalize_test_name_with_parts!( + "test_file_io_s3_writer_splits_write_into_bounded_parts" + ); + let path = format!("s3://{OBJECT_STORE_BUCKET}/{key}"); + + let _ = file_io.delete(&path).await; + let mut writer = file_io.new_output(&path).unwrap().writer().await.unwrap(); + writer + .write(Bytes::from(vec![0u8; PART_SIZE * PARTS])) + .await + .unwrap(); + writer.close().await.unwrap(); + + assert_eq!(upload_part_count(&key).await, PARTS); + file_io.delete(&path).await.unwrap(); + } + + #[tokio::test] + async fn test_file_io_s3_write_splits_buffer_into_bounded_parts() { + const PART_SIZE: usize = 5 * 1024 * 1024; + const PARTS: usize = 3; + + let file_io = + get_file_io_with_props(vec![(S3_MULTIPART_PART_SIZE_BYTES, PART_SIZE.to_string())]) + .await; + let key = normalize_test_name_with_parts!( + "test_file_io_s3_write_splits_buffer_into_bounded_parts" + ); + let path = format!("s3://{OBJECT_STORE_BUCKET}/{key}"); + + let _ = file_io.delete(&path).await; + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from(vec![0u8; PART_SIZE * PARTS])) + .await + .unwrap(); + + assert_eq!(upload_part_count(&key).await, PARTS); + file_io.delete(&path).await.unwrap(); + } + + #[tokio::test] + async fn test_file_io_s3_write_below_part_size_stays_single_request() { + let file_io = get_file_io_with_props(vec![( + S3_MULTIPART_PART_SIZE_BYTES, + (5 * 1024 * 1024).to_string(), + )]) + .await; + let key = normalize_test_name_with_parts!( + "test_file_io_s3_write_below_part_size_stays_single_request" + ); + let path = format!("s3://{OBJECT_STORE_BUCKET}/{key}"); + + let _ = file_io.delete(&path).await; + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"{\"format-version\":2}")) + .await + .unwrap(); + + assert_eq!(upload_part_count(&key).await, 1); + file_io.delete(&path).await.unwrap(); + } + + #[tokio::test] + async fn test_file_io_s3_input() { + let file_io = get_file_io().await; + // Use unique file path based on module path to avoid conflicts + let file_path = format!( + "s3://{OBJECT_STORE_BUCKET}/{}", + normalize_test_name_with_parts!("test_file_io_s3_input") + ); + let output_file = file_io.new_output(&file_path).unwrap(); + { + output_file.write("test_input".into()).await.unwrap(); + } + + let input_file = file_io.new_input(&file_path).unwrap(); + + { + let buffer = input_file.read().await.unwrap(); + assert_eq!(buffer, "test_input".as_bytes()); + } + } + + // Mock credential loader for testing + #[derive(Debug)] + struct MockCredentialLoader { + credential: Option, + } + + impl MockCredentialLoader { + fn new(credential: Option) -> Self { + Self { credential } + } + + fn new_object_store() -> Self { + Self::new(Some(AwsCredential { + access_key_id: "admin".to_string(), + secret_access_key: "password".to_string(), + session_token: None, + expires_in: None, + })) + } + } + + impl ProvideCredential for MockCredentialLoader { + type Credential = AwsCredential; + + async fn provide_credential( + &self, + _ctx: &Context, + ) -> reqsign_core::Result> { + Ok(self.credential.clone()) + } + } + + #[test] + fn test_custom_aws_credential_loader_instantiation() { + // Test creating CustomAwsCredentialLoader with mock loader + let mock_loader = MockCredentialLoader::new_object_store(); + let custom_loader = CustomAwsCredentialLoader::new(mock_loader); + + // Test that the loader can be used in FileIOBuilder with OpenDalStorageFactory + let _builder = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: Some(custom_loader), + })) + .with_props(vec![ + (S3_ENDPOINT, "http://localhost:9000".to_string()), + ("bucket", "test-bucket".to_string()), + (S3_REGION, "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS, "true".to_string()), + ]); + } + + #[tokio::test] + async fn test_s3_with_custom_credential_loader_integration() { + let _file_io = get_file_io().await; + + // Create a mock credential loader + let mock_loader = MockCredentialLoader::new_object_store(); + let custom_loader = CustomAwsCredentialLoader::new(mock_loader); + + let object_store_endpoint = get_object_store_endpoint(); + + // Build FileIO with custom credential loader via OpenDalStorageFactory + let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: Some(custom_loader), + })) + .with_props(vec![ + (S3_ENDPOINT, object_store_endpoint), + (S3_REGION, "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS, "true".to_string()), + ]) + .build(); + + // Test that the FileIO was built successfully with the custom loader + match file_io_with_custom_creds + .exists(&format!("s3://{OBJECT_STORE_BUCKET}/any")) + .await + { + Ok(_) => {} + Err(e) => panic!("Failed to check existence of bucket: {e}"), + } + } + + #[tokio::test] + async fn test_s3_with_custom_credential_loader_integration_failure() { + let _file_io = get_file_io().await; + + // Create a mock credential loader with no credentials + let mock_loader = MockCredentialLoader::new(None); + let custom_loader = CustomAwsCredentialLoader::new(mock_loader); + + let object_store_endpoint = get_object_store_endpoint(); + + // Build FileIO with custom credential loader via OpenDalStorageFactory + let file_io_with_custom_creds = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: Some(custom_loader), + })) + .with_props(vec![ + (S3_ENDPOINT, object_store_endpoint), + (S3_REGION, "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS, "true".to_string()), + ]) + .build(); + + // Test that the FileIO was built successfully with the custom loader + match file_io_with_custom_creds + .exists(&format!("s3://{OBJECT_STORE_BUCKET}/any")) + .await + { + Ok(_) => panic!( + "Expected error, but got Ok - the credential loader should fail to provide valid credentials" + ), + Err(e) => { + assert!( + e.to_string().contains("failed to load signing credential"), + "unexpected error: {e}" + ); + } + } + } + + #[tokio::test] + async fn test_file_io_s3_delete_stream() { + let file_io = get_file_io().await; + + // Write multiple files + let paths: Vec = (0..5) + .map(|i| { + format!( + "s3://{OBJECT_STORE_BUCKET}/{}/file-{i}", + normalize_test_name_with_parts!("test_file_io_s3_delete_stream") + ) + }) + .collect(); + for path in &paths { + let _ = file_io.delete(path).await; + file_io + .new_output(path) + .unwrap() + .write("delete-me".into()) + .await + .unwrap(); + assert!(file_io.exists(path).await.unwrap()); + } + + // Delete via delete_stream + let stream = futures::stream::iter(paths.clone()).boxed(); + file_io.delete_stream(stream).await.unwrap(); + + // Verify all files are gone + for path in &paths { + assert!(!file_io.exists(path).await.unwrap()); + } + } + + #[tokio::test] + async fn test_file_io_s3_delete_stream_empty() { + let file_io = get_file_io().await; + let stream = futures::stream::empty().boxed(); + // Should succeed with no-op + file_io.delete_stream(stream).await.unwrap(); + } +} diff --git a/crates/storage/opendal/tests/hdfs_native_runtime_test.rs b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs new file mode 100644 index 0000000000..da22260d72 --- /dev/null +++ b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A cached HDFS operator must keep working after the runtime that first +//! used it is dropped. Needs no HDFS: the NameNode is a local listener that +//! only counts dials. + +#[cfg(feature = "opendal-hdfs-native")] +mod tests { + use std::io::Read; + use std::net::{TcpListener, TcpStream}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use iceberg::io::{FileIO, FileIOBuilder, HDFS_NAME_NODE}; + use iceberg_storage_opendal::OpenDalStorageFactory; + + /// Accepts and immediately closes connections, counting them. A failed + /// accept (a dial reset while queued) is skipped so the listener lives on. + fn fake_name_node() -> (u16, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let dials = Arc::new(AtomicUsize::new(0)); + let counter = dials.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(_stream) = stream else { continue }; + counter.fetch_add(1, Ordering::SeqCst); + } + }); + (port, dials) + } + + fn stat(runtime: &tokio::runtime::Runtime, file_io: &FileIO) { + runtime.block_on(async { + let input = file_io.new_input("hdfs:///f").unwrap(); + // The fake NameNode never answers, so this fails; only the dial matters. + let _ = tokio::time::timeout(Duration::from_secs(10), input.metadata()).await; + }); + } + + #[test] + fn test_hdfs_operator_outlives_the_runtime_that_first_used_it() { + let (port, dials) = fake_name_node(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) + .with_prop(HDFS_NAME_NODE, format!("hdfs://127.0.0.1:{port}")) + .with_prop("hadoop.dfs.client.failover.max.attempts", "1") + .build(); + + let first = tokio::runtime::Runtime::new().unwrap(); + stat(&first, &file_io); + assert!( + dials.load(Ordering::SeqCst) >= 1, + "the NameNode was never dialed" + ); + drop(first); + + // The listener accepts in order, so once it has closed this marker + // connection, every dial from the first runtime has been counted. + let mut marker = TcpStream::connect(("127.0.0.1", port)).unwrap(); + marker + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + assert_eq!(marker.read(&mut [0]).unwrap(), 0); + let after_first = dials.load(Ordering::SeqCst); + + // Reusing the FileIO from another runtime used to panic inside + // hdfs-native, whose client was bound to the dropped runtime. + let second = tokio::runtime::Runtime::new().unwrap(); + stat(&second, &file_io); + assert!( + dials.load(Ordering::SeqCst) > after_first, + "no dial from the second runtime" + ); + } +} diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs new file mode 100644 index 0000000000..bcc3d51842 --- /dev/null +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -0,0 +1,306 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for OpenDalResolvingStorage. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. +//! Each test uses unique file paths based on module path to avoid conflicts. + +#[cfg(all( + feature = "opendal-s3", + feature = "opendal-fs", + feature = "opendal-memory" +))] +mod tests { + use std::sync::Arc; + + use iceberg::io::{ + FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, + S3_SECRET_ACCESS_KEY, + }; + use iceberg_storage_opendal::OpenDalResolvingStorageFactory; + use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up}; + + fn get_resolving_file_io() -> iceberg::io::FileIO { + set_up(); + + let object_store_endpoint = get_object_store_endpoint(); + + FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) + .with_props(vec![ + (S3_ENDPOINT, object_store_endpoint), + (S3_ACCESS_KEY_ID, "admin".to_string()), + (S3_SECRET_ACCESS_KEY, "password".to_string()), + (S3_REGION, "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS, "true".to_string()), + ]) + .build() + } + + fn roundtrip_file_io(file_io: &iceberg::io::FileIO) -> iceberg::io::FileIO { + let serialized = file_io.serialize_all().unwrap(); + iceberg::io::FileIO::deserialize_all(&serialized).unwrap() + } + + fn temp_fs_path(name: &str) -> String { + let dir = std::env::temp_dir().join("iceberg_resolving_tests"); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(name); + // Clean up from previous runs + let _ = std::fs::remove_file(&path); + format!("file:/{}", path.display()) + } + + #[tokio::test] + async fn test_mixed_scheme_write_and_read() { + let file_io = roundtrip_file_io(&get_resolving_file_io()); + + let s3_path = format!( + "s3://bucket1/{}", + normalize_test_name_with_parts!("test_mixed_scheme_write_and_read") + ); + let fs_path = temp_fs_path("mixed_write_and_read.txt"); + let mem_path = "memory://test_mixed_scheme_write_and_read"; + + // Write to all three schemes + file_io + .new_output(&s3_path) + .unwrap() + .write("from_s3".into()) + .await + .unwrap(); + file_io + .new_output(&fs_path) + .unwrap() + .write("from_fs".into()) + .await + .unwrap(); + file_io + .new_output(mem_path) + .unwrap() + .write("from_memory".into()) + .await + .unwrap(); + + // Read back from all three + assert_eq!( + file_io.new_input(&s3_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("from_s3") + ); + assert_eq!( + file_io.new_input(&fs_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("from_fs") + ); + assert_eq!( + file_io.new_input(mem_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("from_memory") + ); + } + + #[tokio::test] + async fn test_mixed_scheme_exists_independently() { + let file_io = get_resolving_file_io(); + + let s3_path = format!( + "s3://bucket1/{}", + normalize_test_name_with_parts!("test_mixed_scheme_exists_independently") + ); + let fs_path = temp_fs_path("mixed_exists_independently.txt"); + let mem_path = "memory://test_mixed_scheme_exists_independently"; + + // Clean up S3 from previous runs + let _ = file_io.delete(&s3_path).await; + + // None exist initially + assert!(!file_io.exists(&s3_path).await.unwrap()); + assert!(!file_io.exists(&fs_path).await.unwrap()); + assert!(!file_io.exists(mem_path).await.unwrap()); + + // Write only to fs + file_io + .new_output(&fs_path) + .unwrap() + .write("fs_only".into()) + .await + .unwrap(); + + // Only fs exists + assert!(!file_io.exists(&s3_path).await.unwrap()); + assert!(file_io.exists(&fs_path).await.unwrap()); + assert!(!file_io.exists(mem_path).await.unwrap()); + } + + #[tokio::test] + async fn test_mixed_scheme_delete_one_keeps_others() { + let file_io = get_resolving_file_io(); + + let s3_path = format!( + "s3://bucket1/{}", + normalize_test_name_with_parts!("test_mixed_scheme_delete_one_keeps_others") + ); + let fs_path = temp_fs_path("mixed_delete_one_keeps_others.txt"); + let mem_path = "memory://test_mixed_scheme_delete_one_keeps_others"; + + // Write to all three + file_io + .new_output(&s3_path) + .unwrap() + .write("s3".into()) + .await + .unwrap(); + file_io + .new_output(&fs_path) + .unwrap() + .write("fs".into()) + .await + .unwrap(); + file_io + .new_output(mem_path) + .unwrap() + .write("mem".into()) + .await + .unwrap(); + + // Delete only the fs file + file_io.delete(&fs_path).await.unwrap(); + + // fs gone, S3 and memory still there + assert!(file_io.exists(&s3_path).await.unwrap()); + assert!(!file_io.exists(&fs_path).await.unwrap()); + assert!(file_io.exists(mem_path).await.unwrap()); + + assert_eq!( + file_io.new_input(&s3_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("s3") + ); + assert_eq!( + file_io.new_input(mem_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("mem") + ); + } + + #[tokio::test] + async fn test_mixed_scheme_interleaved_operations() { + let file_io = get_resolving_file_io(); + + let s3_path = format!( + "s3://bucket1/{}", + normalize_test_name_with_parts!("test_mixed_scheme_interleaved") + ); + let fs_path = temp_fs_path("mixed_interleaved.txt"); + let mem_path = "memory://test_mixed_scheme_interleaved"; + + // Interleave: write fs, write memory, write s3 + file_io + .new_output(&fs_path) + .unwrap() + .write("fs_data".into()) + .await + .unwrap(); + file_io + .new_output(mem_path) + .unwrap() + .write("mem_data".into()) + .await + .unwrap(); + file_io + .new_output(&s3_path) + .unwrap() + .write("s3_data".into()) + .await + .unwrap(); + + // Read in reverse order: s3, memory, fs + assert_eq!( + file_io.new_input(&s3_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("s3_data") + ); + assert_eq!( + file_io.new_input(mem_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("mem_data") + ); + assert_eq!( + file_io.new_input(&fs_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("fs_data") + ); + } + + #[tokio::test] + async fn test_invalid_scheme() { + let file_io = get_resolving_file_io(); + let result = file_io.exists("unknown://bucket/key").await; + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("Unsupported storage scheme"), + ); + } + + #[tokio::test] + async fn test_missing_scheme() { + let file_io = get_resolving_file_io(); + let result = file_io.exists("no-scheme-path").await; + assert!(result.is_err()); + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_with_custom_credential_loader() { + use iceberg_storage_opendal::{ + AwsCredential, CustomAwsCredentialLoader, ProvideCredential, + }; + use reqsign_core::Context; + + #[derive(Debug)] + struct ObjectStoreCredentialLoader; + + impl ProvideCredential for ObjectStoreCredentialLoader { + type Credential = AwsCredential; + + async fn provide_credential( + &self, + _ctx: &Context, + ) -> reqsign_core::Result> { + Ok(Some(AwsCredential { + access_key_id: "admin".to_string(), + secret_access_key: "password".to_string(), + session_token: None, + expires_in: None, + })) + } + } + + set_up(); + let object_store_endpoint = get_object_store_endpoint(); + + let factory = OpenDalResolvingStorageFactory::new() + .with_s3_credential_loader(CustomAwsCredentialLoader::new(ObjectStoreCredentialLoader)); + + let file_io = FileIOBuilder::new(Arc::new(factory)) + .with_props(vec![ + (S3_ENDPOINT, object_store_endpoint), + (S3_REGION, "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS, "true".to_string()), + ]) + .build(); + + // Should be able to access S3 using the custom credential loader + assert!(file_io.exists("s3://bucket1/").await.unwrap()); + } +} diff --git a/crates/test_utils/Cargo.toml b/crates/test_utils/Cargo.toml index 9a486b15ed..41f4e38dd8 100644 --- a/crates/test_utils/Cargo.toml +++ b/crates/test_utils/Cargo.toml @@ -19,6 +19,7 @@ edition = { workspace = true } homepage = { workspace = true } name = "iceberg_test_utils" +publish = false rust-version = { workspace = true } version = { workspace = true } @@ -26,8 +27,12 @@ license = { workspace = true } repository = { workspace = true } [dependencies] -tracing = { workspace = true } +iceberg = { workspace = true } +tokio = { workspace = true } tracing-subscriber = { workspace = true } [features] tests = [] + +[lints] +workspace = true diff --git a/crates/test_utils/DEPENDENCIES.rust.tsv b/crates/test_utils/DEPENDENCIES.rust.tsv deleted file mode 100644 index ef7c315df7..0000000000 --- a/crates/test_utils/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,30 +0,0 @@ -crate Apache-2.0 MIT Unicode-3.0 -cfg-if@1.0.3 X X -iceberg_test_utils@0.7.0 X -lazy_static@1.5.0 X X -log@0.4.28 X X -nu-ansi-term@0.50.1 X -once_cell@1.21.3 X X -pin-project-lite@0.2.16 X X -proc-macro2@1.0.101 X X -quote@1.0.40 X X -sharded-slab@0.1.7 X -smallvec@1.15.1 X X -syn@2.0.106 X X -thread_local@1.1.9 X X -tracing@0.1.41 X -tracing-attributes@0.1.30 X -tracing-core@0.1.34 X -tracing-log@0.2.0 X -tracing-subscriber@0.3.20 X -unicode-ident@1.0.18 X X X -windows-sys@0.52.0 X X -windows-targets@0.52.6 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X diff --git a/crates/test_utils/src/cmd.rs b/crates/test_utils/src/cmd.rs deleted file mode 100644 index e5e3f09e4b..0000000000 --- a/crates/test_utils/src/cmd.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::process::Command; - -use tracing::{error, info}; - -pub fn run_command(mut cmd: Command, desc: impl ToString) -> bool { - let desc = desc.to_string(); - info!("Starting to {}, command: {:?}", &desc, cmd); - let exit = cmd.status().unwrap(); - if exit.success() { - info!("{} succeed!", desc); - true - } else { - error!("{} failed: {:?}", desc, exit); - false - } -} - -pub fn get_cmd_output_result(mut cmd: Command, desc: impl ToString) -> Result { - let desc = desc.to_string(); - info!("Starting to {}, command: {:?}", &desc, cmd); - let result = cmd.output(); - match result { - Ok(output) => { - if output.status.success() { - info!("{} succeed!", desc); - Ok(String::from_utf8(output.stdout).unwrap()) - } else { - Err(format!("{} failed with rc: {:?}", desc, output.status)) - } - } - Err(err) => Err(format!("{} failed with error: {}", desc, { err })), - } -} - -pub fn get_cmd_output(cmd: Command, desc: impl ToString) -> String { - let result = get_cmd_output_result(cmd, desc); - match result { - Ok(output_str) => output_str, - Err(err) => panic!("{}", err), - } -} diff --git a/crates/test_utils/src/docker.rs b/crates/test_utils/src/docker.rs deleted file mode 100644 index 1a6e883fde..0000000000 --- a/crates/test_utils/src/docker.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::net::IpAddr; -use std::process::Command; - -use tracing::error; - -use crate::cmd::{get_cmd_output, get_cmd_output_result, run_command}; - -/// A utility to manage the lifecycle of `docker compose`. -/// -/// It will start `docker compose` when calling the `run` method and will be stopped via [`Drop`]. -#[derive(Debug)] -pub struct DockerCompose { - project_name: String, - docker_compose_dir: String, -} - -impl DockerCompose { - pub fn new(project_name: impl ToString, docker_compose_dir: impl ToString) -> Self { - Self { - project_name: project_name.to_string(), - docker_compose_dir: docker_compose_dir.to_string(), - } - } - - pub fn project_name(&self) -> &str { - self.project_name.as_str() - } - - fn get_os_arch() -> String { - let mut cmd = Command::new("docker"); - cmd.arg("info") - .arg("--format") - .arg("{{.OSType}}/{{.Architecture}}"); - - let result = get_cmd_output_result(cmd, "Get os arch".to_string()); - match result { - Ok(value) => value.trim().to_string(), - Err(_err) => { - // docker/podman do not consistently place OSArch info in the same json path across OS and versions - // Below tries an alternative path if the above path fails - let mut alt_cmd = Command::new("docker"); - alt_cmd - .arg("info") - .arg("--format") - .arg("{{.Version.OsArch}}"); - get_cmd_output(alt_cmd, "Get os arch".to_string()) - .trim() - .to_string() - } - } - } - - pub fn up(&self) { - let mut cmd = Command::new("docker"); - cmd.current_dir(&self.docker_compose_dir); - - cmd.env("DOCKER_DEFAULT_PLATFORM", Self::get_os_arch()); - - cmd.args(vec![ - "compose", - "-p", - self.project_name.as_str(), - "up", - "-d", - "--wait", - "--timeout", - "1200000", - ]); - - let ret = run_command( - cmd, - format!( - "Starting docker compose in {}, project name: {}", - self.docker_compose_dir, self.project_name - ), - ); - - if !ret { - let mut cmd = Command::new("docker"); - cmd.current_dir(&self.docker_compose_dir); - - cmd.env("DOCKER_DEFAULT_PLATFORM", Self::get_os_arch()); - - cmd.args(vec![ - "compose", - "-p", - self.project_name.as_str(), - "logs", - "spark-iceberg", - ]); - run_command(cmd, "Docker compose logs"); - panic!("Docker compose up failed!") - } - } - - pub fn down(&self) { - let mut cmd = Command::new("docker"); - cmd.current_dir(&self.docker_compose_dir); - - cmd.args(vec![ - "compose", - "-p", - self.project_name.as_str(), - "down", - "-v", - "--remove-orphans", - ]); - - let ret = run_command( - cmd, - format!( - "Stopping docker compose in {}, project name: {}", - self.docker_compose_dir, self.project_name - ), - ); - - if !ret { - panic!("Failed to stop docker compose") - } - } - - pub fn get_container_ip(&self, service_name: impl AsRef) -> IpAddr { - let container_name = format!("{}-{}-1", self.project_name, service_name.as_ref()); - let mut cmd = Command::new("docker"); - cmd.arg("inspect") - .arg("-f") - .arg("{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}") - .arg(&container_name); - - let ip_result = get_cmd_output(cmd, format!("Get container ip of {container_name}")) - .trim() - .parse::(); - match ip_result { - Ok(ip) => ip, - Err(e) => { - error!("Invalid IP, {e}"); - panic!("Failed to parse IP for {container_name}") - } - } - } -} - -impl Drop for DockerCompose { - fn drop(&mut self) { - self.down() - } -} diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index 6d1104a1b9..176767813e 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -20,17 +20,19 @@ //! It's not intended for use outside of `iceberg-rust`. #[cfg(feature = "tests")] -mod cmd; -#[cfg(feature = "tests")] -pub mod docker; +mod runtime; #[cfg(feature = "tests")] pub use common::*; +#[cfg(feature = "tests")] +pub use runtime::test_runtime; #[cfg(feature = "tests")] mod common { use std::sync::Once; + use iceberg::{Catalog, NamespaceIdent}; + static INIT: Once = Once::new(); pub fn set_up() { INIT.call_once(tracing_subscriber::fmt::init); @@ -38,4 +40,92 @@ mod common { pub fn normalize_test_name(s: impl ToString) -> String { s.to_string().replace("::", "__").replace('.', "_") } + + // Environment variable names for service endpoints + pub const ENV_OBJECT_STORE_ENDPOINT: &str = "ICEBERG_TEST_OBJECT_STORE_ENDPOINT"; + pub const ENV_REST_CATALOG_ENDPOINT: &str = "ICEBERG_TEST_REST_ENDPOINT"; + pub const ENV_HMS_ENDPOINT: &str = "ICEBERG_TEST_HMS_ENDPOINT"; + pub const ENV_GLUE_ENDPOINT: &str = "ICEBERG_TEST_GLUE_ENDPOINT"; + pub const ENV_GCS_ENDPOINT: &str = "ICEBERG_TEST_GCS_ENDPOINT"; + pub const ENV_HDFS_ENDPOINT: &str = "ICEBERG_TEST_HDFS_ENDPOINT"; + + // Default ports matching dev/docker-compose.yaml + pub const DEFAULT_OBJECT_STORE_PORT: u16 = 9000; + pub const DEFAULT_REST_CATALOG_PORT: u16 = 8181; + pub const DEFAULT_HMS_PORT: u16 = 9083; + pub const DEFAULT_GLUE_PORT: u16 = 5001; + pub const DEFAULT_GCS_PORT: u16 = 4443; + pub const DEFAULT_HDFS_PORT: u16 = 8020; + + /// Returns the S3-compatible object store endpoint. + /// Checks ICEBERG_TEST_OBJECT_STORE_ENDPOINT env var, otherwise returns localhost default. + pub fn get_object_store_endpoint() -> String { + std::env::var(ENV_OBJECT_STORE_ENDPOINT) + .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_OBJECT_STORE_PORT}")) + } + + /// Returns the REST catalog endpoint. + /// Checks ICEBERG_TEST_REST_ENDPOINT env var, otherwise returns localhost default. + pub fn get_rest_catalog_endpoint() -> String { + std::env::var(ENV_REST_CATALOG_ENDPOINT) + .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_REST_CATALOG_PORT}")) + } + + /// Returns the HMS (Hive Metastore) endpoint. + /// Checks ICEBERG_TEST_HMS_ENDPOINT env var, otherwise returns localhost default. + pub fn get_hms_endpoint() -> String { + std::env::var(ENV_HMS_ENDPOINT).unwrap_or_else(|_| format!("localhost:{DEFAULT_HMS_PORT}")) + } + + /// Returns the Glue (Moto mock) endpoint. + /// Checks ICEBERG_TEST_GLUE_ENDPOINT env var, otherwise returns localhost default. + pub fn get_glue_endpoint() -> String { + std::env::var(ENV_GLUE_ENDPOINT) + .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_GLUE_PORT}")) + } + + /// Returns the GCS (fake-gcs-server) endpoint. + /// Checks ICEBERG_TEST_GCS_ENDPOINT env var, otherwise returns localhost default. + pub fn get_gcs_endpoint() -> String { + std::env::var(ENV_GCS_ENDPOINT) + .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_GCS_PORT}")) + } + + /// Returns the HDFS NameNode endpoint (e.g. `hdfs://localhost:8020`). + /// Checks ICEBERG_TEST_HDFS_ENDPOINT env var, otherwise returns localhost default. + pub fn get_hdfs_endpoint() -> String { + std::env::var(ENV_HDFS_ENDPOINT) + .unwrap_or_else(|_| format!("hdfs://localhost:{DEFAULT_HDFS_PORT}")) + } + + /// Helper to clean up a namespace and its tables before a test runs. + /// This handles the case where previous test runs left data in the persistent database. + pub async fn cleanup_namespace(catalog: &C, ns: &NamespaceIdent) { + // Try to drop all tables in the namespace first + if let Ok(tables) = catalog.list_tables(ns).await { + for table in tables { + let _ = catalog.drop_table(&table).await; + } + } + // Then try to drop the namespace itself + let _ = catalog.drop_namespace(ns).await; + } + + /// Macro to generate a normalized test name with module path prefix. + /// Takes one or more string parts and joins them with the module path. + /// + /// Example: + /// ```ignore + /// // Returns something like "rest_catalog_test__test_create_table" + /// let name = normalize_test_name_with_parts!("test_create_table"); + /// + /// // Returns something like "rest_catalog_test__apple__ios" + /// let name = normalize_test_name_with_parts!("apple", "ios"); + /// ``` + #[macro_export] + macro_rules! normalize_test_name_with_parts { + ($($part:expr),+) => { + $crate::normalize_test_name([module_path!(), $($part),+].join("_")) + }; + } } diff --git a/crates/test_utils/src/runtime.rs b/crates/test_utils/src/runtime.rs new file mode 100644 index 0000000000..28568b67fd --- /dev/null +++ b/crates/test_utils/src/runtime.rs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A shared tokio runtime for tests that build an [`iceberg::table::Table`]. + +use std::sync::OnceLock; + +use iceberg::Runtime; + +/// Returns a process-wide [`Runtime`] suitable for tests that need to construct +/// a [`Table`](iceberg::table::Table) outside a tokio context. +/// +/// The returned [`Runtime`] wraps a single shared multi-thread tokio runtime +/// that is lazily built on first call and lives until process exit. Cloning is +/// cheap, so test code can call this every time it needs a runtime to feed +/// into [`TableBuilder::runtime`](iceberg::table::TableBuilder::runtime). +pub fn test_runtime() -> Runtime { + static TOKIO_RT: OnceLock = OnceLock::new(); + let tokio_rt = TOKIO_RT.get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build test tokio runtime") + }); + Runtime::new(tokio_rt) +} diff --git a/deny.toml b/deny.toml index 6d75c5d219..351d8fa471 100644 --- a/deny.toml +++ b/deny.toml @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +[graph] +# Include dependencies from optional features. +all-features = true + [licenses] allow = [ "Apache-2.0", @@ -25,18 +29,20 @@ allow = [ "ISC", "CC0-1.0", "Zlib", + # Category-A: https://issues.apache.org/jira/browse/LEGAL-732 "CDLA-Permissive-2.0", # Category-A: https://issues.apache.org/jira/browse/LEGAL-660 "Unicode-3.0", # Boost Software License Version 1.0 is allowed (Category-A): # https://www.apache.org/legal/resolved.html#category-a - "BSL-1.0" + "BSL-1.0", ] +unused-allowed-license = "deny" +unused-license-exception = "deny" exceptions = [ # The MPL license is allowed (binary-only): # https://www.apache.org/legal/resolved.html#category-b - { allow = ["MPL-2.0"], crate = "webpki-roots" }, - { allow = ["MPL-2.0"], crate = "generational-arena" }, + { allow = ["MPL-2.0"], crate = "colored" }, { allow = ["MPL-2.0"], crate = "option-ext" }, -] \ No newline at end of file +] diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml new file mode 100644 index 0000000000..b4b37505ab --- /dev/null +++ b/dev/docker-compose.yaml @@ -0,0 +1,245 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Unified Docker Compose for all integration tests. +# This file combines services needed for REST, HMS, Glue, S3, and GCS tests. +# Start once with: docker compose -f dev/docker-compose.yaml up -d --wait +# Stop with: docker compose -f dev/docker-compose.yaml down -v + +networks: + iceberg_test: + +services: + # ============================================================================= + # Object store - S3-compatible storage backed by RustFS (shared by all tests) + # ============================================================================= + object-store: + image: rustfs/rustfs:1.0.3 + environment: + - RUSTFS_ACCESS_KEY=admin + - RUSTFS_SECRET_KEY=password + - RUSTFS_OBS_LOG_STDOUT_ENABLED=true + - RUSTFS_CONSOLE_ENABLE=false + - RUSTFS_SERVER_DOMAINS=object-store + hostname: object-store + networks: + iceberg_test: + # Add aliases for virtual-hosted style bucket access + aliases: + - icebergdata.object-store + - warehouse.object-store + - bucket1.object-store + ports: + - "9000:9000" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/health/ready"] + interval: 5s + timeout: 5s + retries: 5 + + # One-shot job that creates buckets for tests. Re-creating an existing bucket + # returns 200, so this is safe to re-run. + create-buckets: + depends_on: + object-store: + condition: service_healthy + image: rustfs/rustfs:1.0.3 + entrypoint: > + /bin/sh -c " + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/icebergdata && + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/warehouse && + curl -fsS --aws-sigv4 aws:amz:us-east-1:s3 -u admin:password -X PUT http://object-store:9000/bucket1 + " + networks: + iceberg_test: + + # ============================================================================= + # REST Catalog - Apache Iceberg REST Catalog + # ============================================================================= + rest: + image: apache/iceberg-rest-fixture:1.10.1 + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + - CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog + - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest.db?journal_mode=WAL + - CATALOG_WAREHOUSE=s3://icebergdata/demo + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://object-store:9000 + depends_on: + object-store: + condition: service_healthy + networks: + iceberg_test: + ports: + - "8181:8181" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8181/v1/config"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + + # ============================================================================= + # Hive Metastore - HMS Catalog + # ============================================================================= + hive-metastore: + build: + context: ./hms + dockerfile: Dockerfile + platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} + depends_on: + object-store: + condition: service_healthy + networks: + iceberg_test: + ports: + - "9083:9083" + environment: + SERVICE_NAME: "metastore" + SERVICE_OPTS: "-Dmetastore.warehouse.dir=s3a://warehouse/hive/" + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/localhost/9083"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 30s + + # ============================================================================= + # Moto - AWS Glue Mock for Glue Catalog tests + # ============================================================================= + moto: + image: motoserver/moto:5.0.3 + networks: + iceberg_test: + ports: + - "5001:5001" + environment: + # Use port 5001 instead of default 5000 to avoid conflict with macOS AirPlay Receiver. + - MOTO_PORT=5001 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5001/moto-api/"] + interval: 5s + timeout: 5s + retries: 5 + + # ============================================================================= + # HDFS - single-node NameNode + DataNode for HDFS tests + # ============================================================================= + # The DataNode advertises 127.0.0.1, so clients that set + # `dfs.client.use.datanode.hostname` reach it through its published port. + hdfs-namenode: + image: apache/hadoop:3.5.0 + networks: + iceberg_test: + command: ["hdfs", "namenode"] + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + ports: + - "8020:8020" + volumes: + - ./hdfs/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro + - ./hdfs/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml:ro + healthcheck: + test: ["CMD-SHELL", "hdfs dfsadmin -safemode get | grep -q OFF"] + interval: 5s + timeout: 15s + retries: 30 + start_period: 30s + + hdfs-datanode: + image: apache/hadoop:3.5.0 + networks: + iceberg_test: + command: ["hdfs", "datanode"] + depends_on: + hdfs-namenode: + condition: service_healthy + ports: + - "9866:9866" + volumes: + - ./hdfs/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro + - ./hdfs/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml:ro + # Healthy only once registered with the NameNode — writes fail until then. + healthcheck: + test: ["CMD-SHELL", "hdfs dfsadmin -report | grep -q 'Live datanodes (1)'"] + interval: 5s + timeout: 15s + retries: 30 + start_period: 10s + + # ============================================================================= + # Fake GCS Server - GCS emulator for GCS tests + # ============================================================================= + gcs-server: + image: fsouza/fake-gcs-server@sha256:36b0116fae5236e8def76ccb07761a9ca323e476f366a5f4bf449cac19deaf2d + networks: + iceberg_test: + ports: + - "4443:4443" + command: --scheme http + + # ============================================================================= + # Spark - Spark with Iceberg for integration tests + # ============================================================================= + spark-iceberg: + build: + context: ./spark + dockerfile: Dockerfile + networks: + iceberg_test: + depends_on: + rest: + condition: service_healthy + create-buckets: + condition: service_completed_successfully + environment: + - AWS_ACCESS_KEY_ID=admin + - AWS_SECRET_ACCESS_KEY=password + - AWS_REGION=us-east-1 + ports: + - "15002:15002" + - "4040:4040" + healthcheck: + test: ["CMD", "sh", "-c", "netstat -an | grep 15002 | grep LISTEN"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 90s + + # Provision service - creates test data via Spark + provision: + image: ghcr.io/astral-sh/uv:python3.12-bookworm-slim + networks: + iceberg_test: + depends_on: + spark-iceberg: + condition: service_healthy + entrypoint: ["/bin/sh", "-c", "uv run /opt/spark/provision.py && touch /tmp/provision_complete && tail -f /dev/null"] + volumes: + - ./spark/provision.py:/opt/spark/provision.py:ro + - uv-cache:/root/.cache/uv + healthcheck: + test: ["CMD-SHELL", "[ -f /tmp/provision_complete ]"] + interval: 2s + timeout: 2s + retries: 90 + start_period: 20s + +volumes: + uv-cache: diff --git a/dev/hdfs/core-site.xml b/dev/hdfs/core-site.xml new file mode 100644 index 0000000000..273676bba2 --- /dev/null +++ b/dev/hdfs/core-site.xml @@ -0,0 +1,27 @@ + + + + + fs.defaultFS + hdfs://hdfs-namenode:8020 + + + hadoop.http.staticuser.user + root + + diff --git a/dev/hdfs/hdfs-site.xml b/dev/hdfs/hdfs-site.xml new file mode 100644 index 0000000000..b9bf0c63c1 --- /dev/null +++ b/dev/hdfs/hdfs-site.xml @@ -0,0 +1,42 @@ + + + + + dfs.replication + 1 + + + dfs.permissions.enabled + false + + + dfs.namenode.name.dir + file:///tmp/hadoop-root/dfs/name + + + dfs.datanode.data.dir + file:///tmp/hadoop-root/dfs/data + + + + dfs.datanode.hostname + 127.0.0.1 + + diff --git a/dev/hms/Dockerfile b/dev/hms/Dockerfile new file mode 100644 index 0000000000..5e6cfdbfef --- /dev/null +++ b/dev/hms/Dockerfile @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM apache/hive:4.2.1 + +USER root + +# Link the hadoop-aws and AWS SDK jars that the image ships into the metastore classpath +RUN ln -s /opt/hadoop/share/hadoop/tools/lib/hadoop-aws-*.jar /opt/hive/lib/ && \ + ln -s /opt/hadoop/share/hadoop/tools/lib/bundle-*.jar /opt/hive/lib/ + +# The entrypoint links this directory into the Hive config directory, over its own core-site.xml +ENV HIVE_CUSTOM_CONF_DIR=/opt/hive/custom-conf +COPY core-site.xml ${HIVE_CUSTOM_CONF_DIR}/core-site.xml + +USER hive diff --git a/crates/catalog/hms/testdata/hms_catalog/core-site.xml b/dev/hms/core-site.xml similarity index 93% rename from crates/catalog/hms/testdata/hms_catalog/core-site.xml rename to dev/hms/core-site.xml index f0583a0bc0..d88ef93570 100644 --- a/crates/catalog/hms/testdata/hms_catalog/core-site.xml +++ b/dev/hms/core-site.xml @@ -1,51 +1,51 @@ - - - - - fs.defaultFS - s3a://warehouse/hive - - - fs.s3a.impl - org.apache.hadoop.fs.s3a.S3AFileSystem - - - fs.s3a.fast.upload - true - - - fs.s3a.endpoint - http://minio:9000 - - - fs.s3a.access.key - admin - - - fs.s3a.secret.key - password - - - fs.s3a.connection.ssl.enabled - false - - - fs.s3a.path.style.access - true - - \ No newline at end of file + + + + + fs.defaultFS + s3a://warehouse/hive + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3a.fast.upload + true + + + fs.s3a.endpoint + http://object-store:9000 + + + fs.s3a.access.key + admin + + + fs.s3a.secret.key + password + + + fs.s3a.connection.ssl.enabled + false + + + fs.s3a.path.style.access + true + + diff --git a/dev/release/README.md b/dev/release/README.md new file mode 100644 index 0000000000..5979149746 --- /dev/null +++ b/dev/release/README.md @@ -0,0 +1,145 @@ + + +# Release Helpers + +These scripts support local Apache Iceberg Rust release candidate creation and verification. + +## Create an RC + +```shell +dev/release/create_rc.sh 0.9.1 2 +``` + +This creates: + +- source archive `dist/apache-iceberg-rust-0.9.1-rc2/apache-iceberg-rust-0.9.1.tar.gz` +- detached signature `.asc` +- SHA-512 checksum `.sha512` +- signed annotated tag `v0.9.1-rc.2` + +The license header check runs against the generated source archive, not the live Git worktree. The optional SVN upload runs after local artifact verification and before RC tag creation. The signed RC tag is created as the final release step, then the script prints a draft VOTE email for `dev@iceberg.apache.org`. The script logs every step before it runs and after it succeeds. If a step fails, it prints the failed step and stops. + +The script always archives and tags `HEAD`, so check out the exact commit to release before running it. + +Common options: + +```shell +dev/release/create_rc.sh 0.9.1 2 --create_rc_tag 0 --sign 0 +dev/release/create_rc.sh 0.9.1 2 --upload_svn 1 +dev/release/create_rc.sh 0.9.1 2 --check_headers 0 --check_deps 0 --check_publish 0 +``` + +Defaults: + +- `--dist_dir dist`: artifact output root. +- `--create_rc_tag 1`: create the signed annotated RC tag as the final release step. +- `--check_headers 1`: check Apache license headers against the source archive. +- `--check_deps 1`: run dependency license checks before artifact creation. +- `--check_publish 1`: dry-run publishing every crate to crates.io before artifact creation. +- `--sign 1`: create and verify the detached GPG signature. +- `--upload_svn 0`: upload RC artifacts to the ASF dev dist SVN repository. +- `--svn_dist_url https://dist.apache.org/repos/dist/dev/iceberg`: SVN directory URL where the RC artifact directory will be uploaded. + +`--sign 1` or `--create_rc_tag 1` requires a local GPG secret key. See the website's GPG setup guide before creating a real release candidate. + +`--check_deps 1` requires `cargo-deny`. Install it with: + +```shell +make install-cargo-deny +``` + +`--check_publish 1` runs `cargo publish --workspace --dry-run`, which packages and compiles every crate and needs network access to crates.io. +Pass `--check_publish 0` to skip it when offline or when iterating on the script. + +## Verify an RC + +```shell +dev/release/verify_rc.sh 0.9.1 2 +``` + +By default this downloads from `https://dist.apache.org/repos/dist/dev/iceberg/apache-iceberg-rust-0.9.1-rc2/`, verifies checksum and signature with the local GPG keyring, checks source headers and that every publishable crate packages the root `LICENSE` and `NOTICE`, and runs Rust and Python build/tests. + +To verify artifacts already created under local `dist/`: + +```shell +dev/release/verify_rc.sh 0.9.1 2 --download 0 +``` + +Common options: + +```shell +dev/release/verify_rc.sh 0.9.1 2 --verify_signature 0 +dev/release/verify_rc.sh 0.9.1 2 --import_gpg_keys 1 +dev/release/verify_rc.sh 0.9.1 2 --build 0 --python 0 --check_headers 0 +dev/release/verify_rc.sh 0.9.1 2 --check_crate_licenses 0 +``` + +Defaults: + +- `--dist_dir dist`: local artifact root used when `--download 0`. +- `--download 1`: download artifacts from ASF dev dist. +- `--verify_signature 1`: verify the `.asc` signature with the local GPG keyring. +- `--import_gpg_keys 0`: download and import Apache Iceberg release keys before signature verification. +- `--check_headers 1`: check Apache license headers against the extracted source archive. +- `--check_crate_licenses 1`: run `dev/release/licenses.py crates check` against the extracted source archive, so every publishable crate must package the root `LICENSE` and `NOTICE`. Requires `cargo` and `python3`. +- `--build 1`: build and test the Rust source distribution. +- `--python 1`: build and test pyiceberg-core. +- `--tmp_dir `: verification sandbox; auto-created and deleted on success when omitted. + +## Promote an RC to a Release + +After the VOTE passes, convert the approved RC to the official release: + +```shell +dev/release/release.sh 0.9.1 2 +``` + +This creates the signed annotated final release tag `v0.9.1` from the RC tag `v0.9.1-rc.2`, then moves the ASF SVN artifacts from `dev/iceberg/apache-iceberg-rust-0.9.1-rc2` to `release/iceberg/apache-iceberg-rust-0.9.1`. + +Common options: + +```shell +dev/release/release.sh 0.9.1 2 --create_release_tag 0 +dev/release/release.sh 0.9.1 2 --move_svn 0 +``` + +Defaults: + +- `--create_release_tag 1`: create the signed annotated final release git tag. +- `--move_svn 1`: move the RC artifacts from ASF dev dist to ASF release dist. +- `--tag_ref `: git commit-ish to tag as the final release. +- `--dev_dist_url https://dist.apache.org/repos/dist/dev/iceberg`: SVN directory URL containing RC artifact directories. +- `--release_dist_url https://dist.apache.org/repos/dist/release/iceberg`: SVN directory URL where final release artifact directories are published. + +The script does not push the final release tag. Push it manually after reviewing the output: + +```shell +git push origin "v0.9.1" +``` + +## Dependencies + +Verify every dependency's license is one the project allows: + +```shell +make check-dependency-licenses +``` + +CI also runs this check on every pull request. diff --git a/dev/release/create_rc.sh b/dev/release/create_rc.sh new file mode 100755 index 0000000000..890318513e --- /dev/null +++ b/dev/release/create_rc.sh @@ -0,0 +1,569 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -Eeuo pipefail + +# bash 3.2 (macOS default) does not run the ERR trap for a failing subshell: +# the script still exits on failure but does not report which step failed. +if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 +fi + +CURRENT_STEP="" + +on_error() { + local status=$? + if [ -n "${CURRENT_STEP}" ]; then + echo "FAILED: ${CURRENT_STEP}" >&2 + else + echo "FAILED" >&2 + fi + exit "${status}" +} + +trap on_error ERR + +usage() { + cat < [options] + +Arguments: + + Release version in .. format. + Example: 0.9.1 + + + Numeric release candidate round. + Example: 2 creates tag v0.9.1-rc.2 and dist dir apache-iceberg-rust-0.9.1-rc2. + +The script always archives and tags HEAD, so check out the exact commit to +release before running it. + +Options: + --dist_dir + Directory where RC artifacts are written. + Relative paths are resolved from the repository root. + Default: dist + + --create_rc_tag <0|1> + Whether to create the signed annotated RC git tag as the final release step. + Default: 1 + + --check_headers <0|1> + Whether to check Apache license headers against the generated source archive. + Default: 1 + + --check_deps <0|1> + Whether to run the dependency license check before creating artifacts. + Default: 1 + + --check_publish <0|1> + Whether to dry-run publishing every crate to crates.io before creating artifacts. + Default: 1 + + --sign <0|1> + Whether to create and verify the detached GPG signature for the source archive. + Default: 1 + + --upload_svn <0|1> + Whether to upload RC artifacts to the ASF dev dist SVN repository. + Default: 0 + + --svn_dist_url + SVN directory URL where the RC artifact directory will be uploaded. + Default: https://dist.apache.org/repos/dist/dev/iceberg + + -h, --help + Show this help message. + +Examples: + $0 0.9.1 2 + $0 0.9.1 2 --create_rc_tag 0 --sign 0 + $0 0.9.1 2 --upload_svn 1 + $0 0.9.1 2 --dist_dir /tmp/iceberg-rust-dist +USAGE +} + +show_help_if_requested() { + while [ "$#" -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + esac + shift + done +} + +start_step() { + CURRENT_STEP="$1" + echo "==> ${CURRENT_STEP}" +} + +finish_step() { + echo "OK: ${CURRENT_STEP}" + CURRENT_STEP="" +} + +run_step() { + local step="$1" + shift + start_step "${step}" + "$@" + finish_step +} + +skip_step() { + local step="$1" + start_step "${step}" + echo "OK: ${CURRENT_STEP} (skipped)" + CURRENT_STEP="" +} + +enabled() { + [ "$1" != "0" ] +} + +require_command() { + local command_name="$1" + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "This step requires '${command_name}', but it is not installed." >&2 + return 1 + fi +} + +require_checksum_command() { + if ! command -v shasum >/dev/null 2>&1 && ! command -v sha512sum >/dev/null 2>&1; then + echo "This step requires either 'shasum' or 'sha512sum'." >&2 + return 1 + fi +} + +require_gpg_secret_key() { + require_command gpg + if ! gpg --list-secret-keys --with-colons 2>/dev/null | grep -q '^sec'; then + echo "This step requires a GPG secret key for signing release artifacts or tags." >&2 + echo "Set up a release signing key first: https://rust.iceberg.apache.org/reference/setup_gpg.html" >&2 + echo "To skip signing locally, pass: --sign 0 --create_rc_tag 0" >&2 + return 1 + fi +} + +require_option_value() { + local option_name="$1" + local option_value="${2:-}" + if [ -z "${option_value}" ]; then + echo "Missing value for ${option_name}." >&2 + usage >&2 + return 1 + fi +} + +validate_bool_option() { + local option_name="$1" + local option_value="$2" + case "${option_value}" in + 0 | 1) ;; + *) + echo "${option_name} must be 0 or 1, got '${option_value}'." >&2 + return 1 + ;; + esac +} + +parse_args() { + local positional=() + + while [ "$#" -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + --dist_dir | --dist-dir) + require_option_value "$1" "${2:-}" + RELEASE_DIST_DIR="$2" + shift 2 + ;; + --create_rc_tag | --create-rc-tag) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + CREATE_RC_TAG="$2" + shift 2 + ;; + --check_headers | --check-headers) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + CHECK_HEADERS="$2" + shift 2 + ;; + --check_deps | --check-deps) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + CHECK_DEPS="$2" + shift 2 + ;; + --check_publish | --check-publish) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + CHECK_PUBLISH="$2" + shift 2 + ;; + --sign) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + SIGN_ARCHIVE="$2" + shift 2 + ;; + --upload_svn | --upload-svn) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + UPLOAD_SVN="$2" + shift 2 + ;; + --svn_dist_url | --svn-dist-url) + require_option_value "$1" "${2:-}" + SVN_DIST_URL="$2" + shift 2 + ;; + --*) + echo "Unknown option: $1" >&2 + usage >&2 + return 1 + ;; + *) + positional+=("$1") + shift + ;; + esac + done + + if [ "${#positional[@]}" -ne 2 ]; then + usage >&2 + return 1 + fi + + VERSION="${positional[0]}" + RC="${positional[1]}" +} + +validate_args() { + if [[ ! "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Version '${VERSION}' must use format ..." >&2 + return 1 + fi + + if [[ ! "${RC}" =~ ^[0-9]+$ ]]; then + echo "RC '${RC}' must be a number, for example: 0, 1, or 2." >&2 + return 1 + fi +} + +derive_release_names() { + RC_TAG="v${VERSION}-rc.${RC}" + RC_ID="apache-iceberg-rust-${VERSION}-rc${RC}" + ARCHIVE_BASE_NAME="apache-iceberg-rust-${VERSION}" + ARCHIVE_FILE_NAME="${ARCHIVE_BASE_NAME}.tar.gz" + + if [[ "${RELEASE_DIST_DIR}" = /* ]]; then + DIST_ROOT="${RELEASE_DIST_DIR}" + else + DIST_ROOT="${REPO_ROOT}/${RELEASE_DIST_DIR}" + fi + RC_DIR="${DIST_ROOT}/${RC_ID}" + SVN_DIST_URL="${SVN_DIST_URL%/}" + SVN_RC_URL="${SVN_DIST_URL}/${RC_ID}" +} + +check_release_ref() { + require_command git + git -C "${REPO_ROOT}" rev-parse --verify "HEAD^{commit}" >/dev/null +} + +check_rc_tag_available() { + require_command git + if git -C "${REPO_ROOT}" rev-parse --verify --quiet "refs/tags/${RC_TAG}" >/dev/null; then + echo "Tag '${RC_TAG}' already exists." >&2 + return 1 + fi +} + +check_dependency_licenses() { + require_command make + require_command python3 + make -C "${REPO_ROOT}" check-dependency-licenses +} + +# Packages and builds every crate as `cargo publish` would, without uploading, +# so crates that cannot be published fail RC creation. +check_crates_publishable() { + require_command cargo + cargo publish --workspace --all-features --dry-run --manifest-path "${REPO_ROOT}/Cargo.toml" +} + +prepare_output_directory() { + rm -rf "${RC_DIR}" + mkdir -p "${RC_DIR}" +} + +create_source_archive() { + require_command git + git -C "${REPO_ROOT}" archive \ + --format=tar.gz \ + --output="${RC_DIR}/${ARCHIVE_FILE_NAME}" \ + --prefix="${ARCHIVE_BASE_NAME}/" \ + HEAD +} + +check_license_headers() { + require_command tar + require_command docker + require_command mktemp + + ( + trap - ERR + local header_tmp_dir + header_tmp_dir="$(mktemp -d -t "iceberg-rust-headers.XXXXXX")" + cleanup() { + rm -rf "${header_tmp_dir}" + } + trap cleanup EXIT + + tar -xzf "${RC_DIR}/${ARCHIVE_FILE_NAME}" -C "${header_tmp_dir}" + docker run --rm -v "${header_tmp_dir}/${ARCHIVE_BASE_NAME}:/github/workspace" apache/skywalking-eyes header check + ) +} + +sign_source_archive() { + require_gpg_secret_key + ( + trap - ERR + cd "${RC_DIR}" + gpg --armor --output "${ARCHIVE_FILE_NAME}.asc" --detach-sig "${ARCHIVE_FILE_NAME}" + ) +} + +verify_source_signature() { + require_command gpg + ( + trap - ERR + cd "${RC_DIR}" + gpg --verify "${ARCHIVE_FILE_NAME}.asc" "${ARCHIVE_FILE_NAME}" + ) +} + +generate_checksum() { + require_checksum_command + ( + trap - ERR + cd "${RC_DIR}" + if command -v shasum >/dev/null 2>&1; then + shasum -a 512 "${ARCHIVE_FILE_NAME}" > "${ARCHIVE_FILE_NAME}.sha512" + else + sha512sum "${ARCHIVE_FILE_NAME}" > "${ARCHIVE_FILE_NAME}.sha512" + fi + ) +} + +verify_checksum() { + require_checksum_command + ( + trap - ERR + cd "${RC_DIR}" + if command -v shasum >/dev/null 2>&1; then + shasum -a 512 -c "${ARCHIVE_FILE_NAME}.sha512" + else + sha512sum -c "${ARCHIVE_FILE_NAME}.sha512" + fi + ) +} + +require_release_artifact() { + local artifact_path="$1" + if [ ! -f "${artifact_path}" ]; then + echo "Required release artifact is missing: ${artifact_path}" >&2 + return 1 + fi +} + +upload_to_svn() { + require_command svn + require_release_artifact "${RC_DIR}/${ARCHIVE_FILE_NAME}" + require_release_artifact "${RC_DIR}/${ARCHIVE_FILE_NAME}.asc" + require_release_artifact "${RC_DIR}/${ARCHIVE_FILE_NAME}.sha512" + + if svn info "${SVN_RC_URL}" >/dev/null 2>&1; then + echo "SVN target already exists: ${SVN_RC_URL}" >&2 + return 1 + fi + + svn import "${RC_DIR}" "${SVN_RC_URL}" -m "Prepare Apache Iceberg Rust ${VERSION} RC${RC}" +} + +create_rc_tag() { + require_command git + require_gpg_secret_key + git -C "${REPO_ROOT}" tag -s "${RC_TAG}" HEAD -m "Apache Iceberg Rust ${VERSION} RC${RC}" +} + +print_summary() { + echo "Created Apache Iceberg Rust RC artifacts:" + echo " Tag: ${RC_TAG}" + echo " Directory: ${RC_DIR}" + echo " Archive: ${RC_DIR}/${ARCHIVE_FILE_NAME}" + if enabled "${SIGN_ARCHIVE}"; then + echo " Signature: ${RC_DIR}/${ARCHIVE_FILE_NAME}.asc" + fi + echo " Checksum: ${RC_DIR}/${ARCHIVE_FILE_NAME}.sha512" + if enabled "${UPLOAD_SVN}"; then + echo " SVN URL: ${SVN_RC_URL}" + fi +} + +print_vote_email() { + cat < +--------------------------------------------------------- +MAIL +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +VERSION="" +RC="" +RELEASE_DIST_DIR="dist" +CREATE_RC_TAG="1" +CHECK_HEADERS="1" +CHECK_DEPS="1" +CHECK_PUBLISH="1" +SIGN_ARCHIVE="1" +UPLOAD_SVN="0" +SVN_DIST_URL="https://dist.apache.org/repos/dist/dev/iceberg" + +show_help_if_requested "$@" +run_step "Parse command arguments" parse_args "$@" +run_step "Validate release arguments" validate_args +derive_release_names + +run_step "Check release reference HEAD" check_release_ref + +if enabled "${CREATE_RC_TAG}"; then + run_step "Check RC tag ${RC_TAG} is available" check_rc_tag_available +else + skip_step "Check RC tag ${RC_TAG} is available" +fi + +if enabled "${CHECK_DEPS}"; then + run_step "Check dependency licenses" check_dependency_licenses +else + skip_step "Check dependency licenses" +fi + +if enabled "${CHECK_PUBLISH}"; then + run_step "Check crates can be published" check_crates_publishable +else + skip_step "Check crates can be published" +fi + +run_step "Prepare output directory ${RC_DIR}" prepare_output_directory +run_step "Create source archive ${ARCHIVE_FILE_NAME}" create_source_archive + +if enabled "${CHECK_HEADERS}"; then + run_step "Check source archive license headers" check_license_headers +else + skip_step "Check source archive license headers" +fi + +if enabled "${SIGN_ARCHIVE}"; then + run_step "Sign source archive ${ARCHIVE_FILE_NAME}" sign_source_archive + run_step "Verify source archive signature" verify_source_signature +else + skip_step "Sign source archive ${ARCHIVE_FILE_NAME}" + skip_step "Verify source archive signature" +fi + +run_step "Generate SHA-512 checksum" generate_checksum +run_step "Verify SHA-512 checksum" verify_checksum + +if enabled "${UPLOAD_SVN}"; then + run_step "Upload artifacts to SVN dist ${SVN_RC_URL}" upload_to_svn +else + skip_step "Upload artifacts to SVN dist ${SVN_RC_URL}" +fi + +if enabled "${CREATE_RC_TAG}"; then + run_step "Create signed RC tag ${RC_TAG}" create_rc_tag +else + skip_step "Create signed RC tag ${RC_TAG}" +fi + +run_step "Print artifact summary" print_summary +run_step "Print draft vote email" print_vote_email diff --git a/dev/release/licenses.py b/dev/release/licenses.py new file mode 100755 index 0000000000..a0bc83c193 --- /dev/null +++ b/dev/release/licenses.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +CLI for assisting maintenance of the project's license checks. +""" + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +# Every distributed artifact must ship its own copy of these files. +REQUIRED_CRATE_FILES = ("LICENSE", "NOTICE") + + +class Failure(Exception): + """A user-facing error.""" + + +def step(message: str) -> None: + print(f"==> {message}", flush=True) + + +def ok(message: str) -> None: + print(f"OK: {message}", flush=True) + + +def report_error(message: str) -> None: + # GitHub Actions turns `::error::` lines into annotations on the job summary. + if os.environ.get("GITHUB_ACTIONS") == "true": + print(f"::error::{message}", flush=True) + else: + print(f"ERROR: {message}", file=sys.stderr, flush=True) + + +def require_cargo() -> None: + """Assert cargo is installed.""" + try: + subprocess.run( + ["cargo", "--version"], + capture_output=True, + check=True, + ) + except (FileNotFoundError, subprocess.CalledProcessError): + raise Failure("This script requires 'cargo', but it is not installed.") + + +def require_cargo_deny() -> None: + """Assert cargo-deny is installed.""" + try: + subprocess.run( + ["cargo", "deny", "--version"], + capture_output=True, + check=True, + ) + except FileNotFoundError: + raise Failure("This script requires 'cargo', but it is not installed.") + except subprocess.CalledProcessError: + raise Failure( + "This script requires 'cargo-deny' for dependency license checks.\n" + "Install it with: make install-cargo-deny" + ) + + +def is_non_empty_file(path: Path) -> bool: + """Whether the path resolves, following symlinks, to a non-empty regular file.""" + return path.is_file() and path.stat().st_size > 0 + + +def is_same_file(path: Path, other: Path) -> bool: + """Whether both paths exist and resolve to the same file; False if either is missing.""" + try: + return path.samefile(other) + except OSError: + return False + + +def publishable_crates() -> list[tuple[str, Path]]: + """Name and directory of every crate that would be published in a release.""" + result = subprocess.run( + ["cargo", "metadata", "--no-deps", "--format-version", "1"], + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + raise Failure("Could not list the publishable crates.") + + # `publish` is an empty list for `publish = false` crates and null when the + # crate may be published to any registry. + return [ + (package["name"], Path(package["manifest_path"]).parent) + for package in json.loads(result.stdout)["packages"] + if package["publish"] != [] + ] + + +def check_crate_files(crate: str, crate_dir: Path) -> bool: + """Check the packaged crate ships LICENSE and NOTICE resolving to the root files.""" + print(f"Checking {crate}...", flush=True) + result = subprocess.run( + [ + "cargo", + "package", + "--package", + crate, + "--list", + "--locked", + # allows `make check` during development. + "--allow-dirty", + ], + cwd=REPO_ROOT, + stdout=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode != 0: + report_error(f"{crate} could not be listed by cargo; see the error above") + return False + + packaged_paths = set(result.stdout.splitlines()) + passed = True + for file_name in REQUIRED_CRATE_FILES: + repo_path = crate_dir / file_name + if file_name not in packaged_paths: + report_error(f"{crate} crate is missing {file_name} in packaged file list") + passed = False + elif not is_non_empty_file(repo_path): + report_error(f"{crate} {file_name} does not resolve to a non-empty file: {repo_path}") + passed = False + elif not is_same_file(repo_path, REPO_ROOT / file_name): + report_error( + f"{crate} {file_name} does not resolve to the repository root {file_name}: {repo_path}" + ) + passed = False + return passed + + +def command_check_crates(_args: argparse.Namespace) -> None: + # Checking the packaged file list, rather than the crate directory, also + # catches files that exist on disk but are excluded from the published crate. + # Each crate's copy must be the repository root file itself, so the license + # text cannot drift from one crate to the next. + step("Check crate LICENSE and NOTICE files") + require_cargo() + + passed = True + for file_name in REQUIRED_CRATE_FILES: + if not is_non_empty_file(REPO_ROOT / file_name): + report_error( + f"Repository root {file_name} does not resolve to a non-empty file: {REPO_ROOT / file_name}" + ) + passed = False + + crates = publishable_crates() + if not crates: + raise Failure("No publishable crates found in the workspace, expected at least one.") + + for crate, crate_dir in crates: + passed = check_crate_files(crate, crate_dir) and passed + + if not passed: + raise Failure( + "Every publishable crate must package a top-level LICENSE and NOTICE " + "resolving to the repository root copies." + ) + + ok("Check crate LICENSE and NOTICE files") + + +def command_check_dependencies(_args: argparse.Namespace) -> None: + step("Check dependency licenses") + require_cargo_deny() + + result = subprocess.run( + ["cargo", "deny", "check", "license"], + cwd=REPO_ROOT, + check=False, + ) + if result.returncode != 0: + raise Failure("cargo-deny rejected the dependency licenses.") + + ok("Check dependency licenses") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + cmd_groups = parser.add_subparsers(dest="group", required=True) + + dependencies = cmd_groups.add_parser( + "dependencies", + help="license policy for the Rust dependency graph, enforced through deny.toml", + ) + dependencies_commands = dependencies.add_subparsers(dest="command", required=True) + + check = dependencies_commands.add_parser( + "check", + help="assert every dependency's license is one deny.toml allows", + ) + check.set_defaults(func=command_check_dependencies) + + crates = cmd_groups.add_parser( + "crates", + help="Sub-commands for working with licensing on crates", + ) + crates_commands = crates.add_subparsers(dest="command", required=True) + + check = crates_commands.add_parser( + "check", + help="assert every publishable crate packages the repository root LICENSE and NOTICE", + ) + check.set_defaults(func=command_check_crates) + + args = parser.parse_args() + try: + args.func(args) + except Failure as failure: + print(str(failure), file=sys.stderr) + print(f"FAILED: {args.group} {args.command}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev/release/release.sh b/dev/release/release.sh new file mode 100755 index 0000000000..1d1d200666 --- /dev/null +++ b/dev/release/release.sh @@ -0,0 +1,341 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -Eeuo pipefail + +# bash 3.2 (macOS default) does not run the ERR trap for a failing subshell: +# the script still exits on failure but does not report which step failed. +if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 +fi + +CURRENT_STEP="" + +on_error() { + local status=$? + if [ -n "${CURRENT_STEP}" ]; then + echo "FAILED: ${CURRENT_STEP}" >&2 + else + echo "FAILED" >&2 + fi + exit "${status}" +} + +trap on_error ERR + +usage() { + cat < [options] + +Arguments: + + Release version in .. format. + Example: 0.9.1 + + + Numeric release candidate round that passed the vote. + Example: 2 promotes v0.9.1-rc.2 and apache-iceberg-rust-0.9.1-rc2. + +Options: + --create_release_tag <0|1> + Whether to create the signed annotated final release git tag. + Default: 1 + + --move_svn <0|1> + Whether to move the RC artifacts from ASF dev dist to ASF release dist. + Default: 1 + + --tag_ref + Git commit-ish to tag as the final release. + Default: the commit pointed to by the RC tag + + --dev_dist_url + SVN directory URL containing RC artifact directories. + Default: https://dist.apache.org/repos/dist/dev/iceberg + + --release_dist_url + SVN directory URL where final release artifact directories are published. + Default: https://dist.apache.org/repos/dist/release/iceberg + + -h, --help + Show this help message. + +Examples: + $0 0.9.1 2 + $0 0.9.1 2 --create_release_tag 0 + $0 0.9.1 2 --move_svn 0 +USAGE +} + +show_help_if_requested() { + while [ "$#" -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + esac + shift + done +} + +start_step() { + CURRENT_STEP="$1" + echo "==> ${CURRENT_STEP}" +} + +finish_step() { + echo "OK: ${CURRENT_STEP}" + CURRENT_STEP="" +} + +run_step() { + local step="$1" + shift + start_step "${step}" + "$@" + finish_step +} + +skip_step() { + local step="$1" + start_step "${step}" + echo "OK: ${CURRENT_STEP} (skipped)" + CURRENT_STEP="" +} + +enabled() { + [ "$1" != "0" ] +} + +require_command() { + local command_name="$1" + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "This step requires '${command_name}', but it is not installed." >&2 + return 1 + fi +} + +require_gpg_secret_key() { + require_command gpg + if ! gpg --list-secret-keys --with-colons 2>/dev/null | grep -q '^sec'; then + echo "This step requires a GPG secret key for signing the release tag." >&2 + echo "Set up a release signing key first: https://rust.iceberg.apache.org/reference/setup_gpg.html" >&2 + echo "To skip tag creation locally, pass: --create_release_tag 0" >&2 + return 1 + fi +} + +require_option_value() { + local option_name="$1" + local option_value="${2:-}" + if [ -z "${option_value}" ]; then + echo "Missing value for ${option_name}." >&2 + usage >&2 + return 1 + fi +} + +validate_bool_option() { + local option_name="$1" + local option_value="$2" + case "${option_value}" in + 0 | 1) ;; + *) + echo "${option_name} must be 0 or 1, got '${option_value}'." >&2 + return 1 + ;; + esac +} + +parse_args() { + local positional=() + + while [ "$#" -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + --create_release_tag | --create-release-tag) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + CREATE_RELEASE_TAG="$2" + shift 2 + ;; + --move_svn | --move-svn) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + MOVE_SVN="$2" + shift 2 + ;; + --tag_ref | --tag-ref) + require_option_value "$1" "${2:-}" + TAG_REF="$2" + shift 2 + ;; + --dev_dist_url | --dev-dist-url) + require_option_value "$1" "${2:-}" + DEV_DIST_URL="$2" + shift 2 + ;; + --release_dist_url | --release-dist-url) + require_option_value "$1" "${2:-}" + RELEASE_DIST_URL="$2" + shift 2 + ;; + --*) + echo "Unknown option: $1" >&2 + usage >&2 + return 1 + ;; + *) + positional+=("$1") + shift + ;; + esac + done + + if [ "${#positional[@]}" -ne 2 ]; then + usage >&2 + return 1 + fi + + VERSION="${positional[0]}" + RC="${positional[1]}" +} + +validate_args() { + if [[ ! "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Version '${VERSION}' must use format ..." >&2 + return 1 + fi + + if [[ ! "${RC}" =~ ^[0-9]+$ ]]; then + echo "RC '${RC}' must be a number, for example: 0, 1, or 2." >&2 + return 1 + fi +} + +derive_release_names() { + RC_TAG="v${VERSION}-rc.${RC}" + RELEASE_TAG="v${VERSION}" + RC_ID="apache-iceberg-rust-${VERSION}-rc${RC}" + RELEASE_ID="apache-iceberg-rust-${VERSION}" + + if [ -z "${TAG_REF}" ]; then + TAG_REF="${RC_TAG}^{commit}" + fi + + DEV_DIST_URL="${DEV_DIST_URL%/}" + RELEASE_DIST_URL="${RELEASE_DIST_URL%/}" + SVN_RC_URL="${DEV_DIST_URL}/${RC_ID}" + SVN_RELEASE_URL="${RELEASE_DIST_URL}/${RELEASE_ID}" +} + +check_tag_ref() { + require_command git + git -C "${REPO_ROOT}" rev-parse --verify "${TAG_REF}" >/dev/null +} + +check_release_tag_available() { + require_command git + if git -C "${REPO_ROOT}" rev-parse --verify --quiet "refs/tags/${RELEASE_TAG}" >/dev/null; then + echo "Tag '${RELEASE_TAG}' already exists." >&2 + return 1 + fi +} + +create_release_tag() { + require_command git + require_gpg_secret_key + git -C "${REPO_ROOT}" tag -s "${RELEASE_TAG}" "${TAG_REF}" -m "Apache Iceberg Rust ${VERSION}" +} + +check_svn_rc_available() { + require_command svn + if ! svn info "${SVN_RC_URL}" >/dev/null; then + echo "SVN RC source does not exist: ${SVN_RC_URL}" >&2 + return 1 + fi +} + +check_svn_release_available() { + require_command svn + if svn info "${SVN_RELEASE_URL}" >/dev/null 2>&1; then + echo "SVN release target already exists: ${SVN_RELEASE_URL}" >&2 + return 1 + fi +} + +move_svn_release() { + require_command svn + svn mv "${SVN_RC_URL}" "${SVN_RELEASE_URL}" -m "Release Apache Iceberg Rust ${VERSION}" +} + +print_summary() { + echo "Promoted Apache Iceberg Rust RC to release:" + echo " RC tag: ${RC_TAG}" + echo " Release tag: ${RELEASE_TAG}" + echo " Release tag ref: ${TAG_REF}" + echo " SVN RC URL: ${SVN_RC_URL}" + echo " SVN release URL: ${SVN_RELEASE_URL}" + echo + echo "Next step:" + echo " git push origin \"${RELEASE_TAG}\"" +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +VERSION="" +RC="" +CREATE_RELEASE_TAG="1" +MOVE_SVN="1" +TAG_REF="" +DEV_DIST_URL="https://dist.apache.org/repos/dist/dev/iceberg" +RELEASE_DIST_URL="https://dist.apache.org/repos/dist/release/iceberg" + +show_help_if_requested "$@" +run_step "Parse command arguments" parse_args "$@" +run_step "Validate release arguments" validate_args +derive_release_names + +if enabled "${CREATE_RELEASE_TAG}"; then + run_step "Check release tag ref ${TAG_REF}" check_tag_ref + run_step "Check release tag ${RELEASE_TAG} is available" check_release_tag_available + run_step "Create signed release tag ${RELEASE_TAG}" create_release_tag +else + skip_step "Check release tag ref ${TAG_REF}" + skip_step "Check release tag ${RELEASE_TAG} is available" + skip_step "Create signed release tag ${RELEASE_TAG}" +fi + +if enabled "${MOVE_SVN}"; then + run_step "Check SVN RC source ${SVN_RC_URL}" check_svn_rc_available + run_step "Check SVN release target ${SVN_RELEASE_URL} is available" check_svn_release_available + run_step "Move SVN RC artifacts to release dist" move_svn_release +else + skip_step "Check SVN RC source ${SVN_RC_URL}" + skip_step "Check SVN release target ${SVN_RELEASE_URL} is available" + skip_step "Move SVN RC artifacts to release dist" +fi + +run_step "Print release summary" print_summary diff --git a/dev/release/verify_rc.sh b/dev/release/verify_rc.sh new file mode 100755 index 0000000000..326e3a92dd --- /dev/null +++ b/dev/release/verify_rc.sh @@ -0,0 +1,550 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -Eeuo pipefail + +# bash 3.2 (macOS default) does not run the ERR trap for a failing subshell: +# the script still exits on failure but does not report which step failed. +if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "Warning: bash ${BASH_VERSION} will not print which step failed. Use bash 4 or newer to see it." >&2 +fi + +CURRENT_STEP="" +VERIFY_SUCCESS="no" +VERIFY_OWNS_TMPDIR="0" +VERIFY_TMPDIR="" + +on_error() { + local status=$? + if [ -n "${CURRENT_STEP}" ]; then + echo "FAILED: ${CURRENT_STEP}" >&2 + else + echo "FAILED" >&2 + fi + exit "${status}" +} + +cleanup() { + if [ -z "${VERIFY_TMPDIR}" ]; then + return + fi + + if [ "${VERIFY_SUCCESS}" = "yes" ]; then + if [ "${VERIFY_OWNS_TMPDIR}" = "1" ]; then + rm -rf "${VERIFY_TMPDIR}" + fi + else + echo "Failed to verify release candidate. See ${VERIFY_TMPDIR} for details." >&2 + fi +} + +trap on_error ERR +trap cleanup EXIT + +usage() { + cat < [options] + +Arguments: + + Release version in .. format. + Example: 0.9.1 + + + Numeric release candidate round. + Example: 2 verifies apache-iceberg-rust-0.9.1-rc2. + +Options: + --dist_dir + Local artifact root used when --download is 0. + Relative paths are resolved from the repository root. + Default: dist + + --download <0|1> + Whether to download artifacts from ASF dev dist. Use 0 to verify local artifacts. + Default: 1 + + --verify_signature <0|1> + Whether to verify the .asc signature with the local GPG keyring. + Default: 1 + + --import_gpg_keys <0|1> + Whether to download and import Apache Iceberg release keys before signature verification. + Default: 0 + + --verify_git_tag <0|1> + Whether to verify that the source archive matches the git tag v-rc.. + The tag is cloned and re-archived, then compared against the extracted archive. + Default: 1 + + --git_repo_url + Git repository URL used to clone the RC tag for the git tag comparison. + Default: https://github.com/apache/iceberg-rust.git + + --check_headers <0|1> + Whether to check Apache license headers against the extracted source archive. + Default: 1 + + --check_crate_licenses <0|1> + Whether to check that every publishable crate in the extracted source archive + packages the root LICENSE and NOTICE (dev/release/licenses.py crates check). + Requires cargo and python3. + Default: 1 + + --build <0|1> + Whether to build and test the Rust source distribution. + Default: 1 + + --python <0|1> + Whether to build and test pyiceberg-core. + Default: 1 + + --tmp_dir + Verification sandbox. If omitted, a temporary directory is created and deleted on success. + Default: auto-created temporary directory + + -h, --help + Show this help message. + +Examples: + $0 0.9.1 2 + $0 0.9.1 2 --download 0 + $0 0.9.1 2 --import_gpg_keys 1 + $0 0.9.1 2 --verify_git_tag 0 + $0 0.9.1 2 --download 0 --build 0 --python 0 --check_headers 0 --check_crate_licenses 0 --verify_signature 0 +USAGE +} + +show_help_if_requested() { + while [ "$#" -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + esac + shift + done +} + +start_step() { + CURRENT_STEP="$1" + echo "==> ${CURRENT_STEP}" +} + +finish_step() { + echo "OK: ${CURRENT_STEP}" + CURRENT_STEP="" +} + +run_step() { + local step="$1" + shift + start_step "${step}" + "$@" + finish_step +} + +skip_step() { + local step="$1" + start_step "${step}" + echo "OK: ${CURRENT_STEP} (skipped)" + CURRENT_STEP="" +} + +enabled() { + [ "$1" != "0" ] +} + +require_command() { + local command_name="$1" + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "This step requires '${command_name}', but it is not installed." >&2 + return 1 + fi +} + +require_checksum_command() { + if ! command -v shasum >/dev/null 2>&1 && ! command -v sha512sum >/dev/null 2>&1; then + echo "This step requires either 'shasum' or 'sha512sum'." >&2 + return 1 + fi +} + +require_option_value() { + local option_name="$1" + local option_value="${2:-}" + if [ -z "${option_value}" ]; then + echo "Missing value for ${option_name}." >&2 + usage >&2 + return 1 + fi +} + +validate_bool_option() { + local option_name="$1" + local option_value="$2" + case "${option_value}" in + 0 | 1) ;; + *) + echo "${option_name} must be 0 or 1, got '${option_value}'." >&2 + return 1 + ;; + esac +} + +parse_args() { + local positional=() + + while [ "$#" -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + --dist_dir | --dist-dir) + require_option_value "$1" "${2:-}" + RELEASE_DIST_DIR="$2" + shift 2 + ;; + --download) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + DOWNLOAD="$2" + shift 2 + ;; + --verify_signature | --verify-signature) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + VERIFY_SIGNATURE="$2" + shift 2 + ;; + --import_gpg_keys | --import-gpg-keys) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + IMPORT_GPG_KEYS="$2" + shift 2 + ;; + --verify_git_tag | --verify-git-tag) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + VERIFY_GIT_TAG="$2" + shift 2 + ;; + --git_repo_url | --git-repo-url) + require_option_value "$1" "${2:-}" + GIT_REPO_URL="$2" + shift 2 + ;; + --check_headers | --check-headers) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + CHECK_HEADERS="$2" + shift 2 + ;; + --check_crate_licenses | --check-crate-licenses) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + CHECK_CRATE_LICENSES="$2" + shift 2 + ;; + --build) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + BUILD="$2" + shift 2 + ;; + --python) + require_option_value "$1" "${2:-}" + validate_bool_option "$1" "$2" + PYTHON="$2" + shift 2 + ;; + --tmp_dir | --tmp-dir) + require_option_value "$1" "${2:-}" + VERIFY_TMPDIR="$2" + shift 2 + ;; + --*) + echo "Unknown option: $1" >&2 + usage >&2 + return 1 + ;; + *) + positional+=("$1") + shift + ;; + esac + done + + if [ "${#positional[@]}" -ne 2 ]; then + usage >&2 + return 1 + fi + + VERSION="${positional[0]}" + RC="${positional[1]}" +} + +validate_args() { + if [[ ! "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Version '${VERSION}' must use format ..." >&2 + return 1 + fi + + if [[ ! "${RC}" =~ ^[0-9]+$ ]]; then + echo "RC '${RC}' must be a number, for example: 0, 1, or 2." >&2 + return 1 + fi +} + +derive_release_names() { + RC_TAG="v${VERSION}-rc.${RC}" + RC_ID="apache-iceberg-rust-${VERSION}-rc${RC}" + ARCHIVE_BASE_NAME="apache-iceberg-rust-${VERSION}" + ARCHIVE_FILE_NAME="${ARCHIVE_BASE_NAME}.tar.gz" + ICEBERG_DIST_BASE_URL="https://downloads.apache.org/iceberg" + DOWNLOAD_RC_BASE_URL="https://dist.apache.org/repos/dist/dev/iceberg/${RC_ID}" + + if [[ "${RELEASE_DIST_DIR}" = /* ]]; then + LOCAL_DIST_ROOT="${RELEASE_DIST_DIR}" + else + LOCAL_DIST_ROOT="${REPO_ROOT}/${RELEASE_DIST_DIR}" + fi + LOCAL_RC_DIR="${LOCAL_DIST_ROOT}/${RC_ID}" +} + +setup_tmpdir() { + if [ -z "${VERIFY_TMPDIR}" ]; then + require_command mktemp + VERIFY_TMPDIR="$(mktemp -d -t "iceberg-rust-${VERSION}-rc${RC}.XXXXXX")" + VERIFY_OWNS_TMPDIR="1" + else + mkdir -p "${VERIFY_TMPDIR}" + fi +} + +enter_tmpdir() { + cd "${VERIFY_TMPDIR}" +} + +download_file() { + require_command curl + local file_name="$1" + curl \ + --fail \ + --location \ + --remote-name \ + --show-error \ + --silent \ + "${DOWNLOAD_RC_BASE_URL}/${file_name}" +} + +copy_local_file() { + require_command cp + local file_name="$1" + cp "${LOCAL_RC_DIR}/${file_name}" "${file_name}" +} + +fetch_file() { + local file_name="$1" + if enabled "${DOWNLOAD}"; then + download_file "${file_name}" + else + copy_local_file "${file_name}" + fi +} + +import_gpg_keys() { + require_command curl + require_command gpg + curl \ + --fail \ + --location \ + --output KEYS \ + --show-error \ + --silent \ + "${ICEBERG_DIST_BASE_URL}/KEYS" + gpg --import KEYS +} + +verify_source_signature() { + require_command gpg + gpg --verify "${ARCHIVE_FILE_NAME}.asc" "${ARCHIVE_FILE_NAME}" +} + +verify_checksum() { + require_checksum_command + if command -v shasum >/dev/null 2>&1; then + shasum -a 512 -c "${ARCHIVE_FILE_NAME}.sha512" + else + sha512sum -c "${ARCHIVE_FILE_NAME}.sha512" + fi +} + +extract_source_archive() { + require_command tar + tar -xzf "${ARCHIVE_FILE_NAME}" +} + +verify_git_tag_matches_archive() { + require_command git + require_command tar + require_command diff + + local tag_checkout_dir="git-tag-checkout" + local tag_archive_dir="git-tag-archive" + local tag_archive_file="git-tag-source.tar.gz" + + # Clone only the RC tag from the git repository. + git clone --quiet --depth 1 --branch "${RC_TAG}" "${GIT_REPO_URL}" "${tag_checkout_dir}" + + # Recreate the source archive from the tag using the same layout as create_rc.sh + # (git archive honors .gitattributes export-ignore rules, e.g. the website dir). + git -C "${tag_checkout_dir}" archive \ + --format=tar.gz \ + --prefix="${ARCHIVE_BASE_NAME}/" \ + --output="${VERIFY_TMPDIR}/${tag_archive_file}" \ + HEAD + + mkdir -p "${tag_archive_dir}" + tar -xzf "${tag_archive_file}" -C "${tag_archive_dir}" + + # Fail if the released archive differs from what the git tag produces. + if ! diff -r "${tag_archive_dir}/${ARCHIVE_BASE_NAME}" "${ARCHIVE_BASE_NAME}"; then + echo "Source archive does not match git tag ${RC_TAG}." >&2 + return 1 + fi +} + +check_license_headers() { + require_command docker + docker run --rm -v "${VERIFY_TMPDIR}/${ARCHIVE_BASE_NAME}:/github/workspace" apache/skywalking-eyes header check +} + +check_crate_licenses() { + require_command cargo + require_command python3 + ( + trap - ERR + python3 "${ARCHIVE_BASE_NAME}/dev/release/licenses.py" crates check + ) +} + +test_source_distribution() { + require_command make + ( + trap - ERR + cd "${ARCHIVE_BASE_NAME}" + make build + make test + ) +} + +test_python_distribution() { + require_command make + require_command uv + ( + trap - ERR + cd "${ARCHIVE_BASE_NAME}/bindings/python" + make install + make test + ) +} + +mark_success() { + VERIFY_SUCCESS="yes" + echo "RC looks good!" +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +VERSION="" +RC="" +RELEASE_DIST_DIR="dist" +DOWNLOAD="1" +VERIFY_SIGNATURE="1" +IMPORT_GPG_KEYS="0" +VERIFY_GIT_TAG="1" +GIT_REPO_URL="https://github.com/apache/iceberg-rust.git" +CHECK_HEADERS="1" +CHECK_CRATE_LICENSES="1" +BUILD="1" +PYTHON="1" + +show_help_if_requested "$@" +run_step "Parse command arguments" parse_args "$@" +run_step "Validate release arguments" validate_args +derive_release_names + +run_step "Create verification sandbox" setup_tmpdir +run_step "Enter verification sandbox ${VERIFY_TMPDIR}" enter_tmpdir +run_step "Fetch source archive ${ARCHIVE_FILE_NAME}" fetch_file "${ARCHIVE_FILE_NAME}" + +if enabled "${VERIFY_SIGNATURE}"; then + run_step "Fetch source archive signature" fetch_file "${ARCHIVE_FILE_NAME}.asc" + if enabled "${IMPORT_GPG_KEYS}"; then + run_step "Import Apache Iceberg GPG keys" import_gpg_keys + else + skip_step "Import Apache Iceberg GPG keys" + fi + run_step "Verify source archive signature" verify_source_signature +else + skip_step "Fetch source archive signature" + skip_step "Import Apache Iceberg GPG keys" + skip_step "Verify source archive signature" +fi + +run_step "Fetch source archive checksum" fetch_file "${ARCHIVE_FILE_NAME}.sha512" +run_step "Verify source archive checksum" verify_checksum +run_step "Extract source archive" extract_source_archive + +if enabled "${VERIFY_GIT_TAG}"; then + run_step "Verify source archive matches git tag ${RC_TAG}" verify_git_tag_matches_archive +else + skip_step "Verify source archive matches git tag ${RC_TAG}" +fi + +if enabled "${CHECK_HEADERS}"; then + run_step "Check license headers" check_license_headers +else + skip_step "Check license headers" +fi + +if enabled "${CHECK_CRATE_LICENSES}"; then + run_step "Check crate licenses in source archive" check_crate_licenses +else + skip_step "Check crate licenses in source archive" +fi + +if enabled "${BUILD}"; then + run_step "Build and test Rust source distribution" test_source_distribution +else + skip_step "Build and test Rust source distribution" +fi + +if enabled "${PYTHON}"; then + run_step "Build and test pyiceberg-core" test_python_distribution +else + skip_step "Build and test pyiceberg-core" +fi + +run_step "Print verification summary" mark_success diff --git a/dev/spark/Dockerfile b/dev/spark/Dockerfile new file mode 100644 index 0000000000..4b486c9001 --- /dev/null +++ b/dev/spark/Dockerfile @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BASE_IMAGE_SPARK_VERSION=4.0.1 + +FROM apache/spark:${BASE_IMAGE_SPARK_VERSION} + +# Dependency versions - keep these compatible +ARG ICEBERG_VERSION=1.10.1 +ARG ICEBERG_SPARK_RUNTIME_VERSION=4.0_2.13 +ARG HADOOP_VERSION=3.4.1 +ARG SCALA_VERSION=2.13 +ARG AWS_SDK_VERSION=2.24.6 +ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2 + +USER root +WORKDIR ${SPARK_HOME} + +# Install curl for JAR downloads +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl && \ + rm -rf /var/lib/apt/lists/* + +# Copy configuration (early for better caching) +COPY --chown=spark:spark spark-defaults.conf ${SPARK_HOME}/conf/ + +# Create event log directory +RUN mkdir -p /home/iceberg/spark-events && \ + chown -R spark:spark /home/iceberg + +# Required JAR dependencies +ENV JARS_TO_DOWNLOAD="\ + org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \ + org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \ + org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \ + software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar" + +# Download JARs with retry logic +RUN set -e && \ + cd "${SPARK_HOME}/jars" && \ + for jar_path in ${JARS_TO_DOWNLOAD}; do \ + jar_name=$(basename "${jar_path}") && \ + echo "Downloading ${jar_name}..." && \ + curl -fsSL --retry 3 --retry-delay 5 \ + -o "${jar_name}" \ + "${MAVEN_MIRROR}/${jar_path}" && \ + echo "✓ Downloaded ${jar_name}"; \ + done && \ + chown -R spark:spark "${SPARK_HOME}/jars" + +USER spark +WORKDIR ${SPARK_HOME} + +# Start Spark Connect server +CMD ["sh", "-c", "SPARK_NO_DAEMONIZE=true ${SPARK_HOME}/sbin/start-connect-server.sh"] diff --git a/crates/integration_tests/testdata/spark/provision.py b/dev/spark/provision.py old mode 100755 new mode 100644 similarity index 80% rename from crates/integration_tests/testdata/spark/provision.py rename to dev/spark/provision.py index 899a49f6ac..40f9ba0f38 --- a/crates/integration_tests/testdata/spark/provision.py +++ b/dev/spark/provision.py @@ -15,20 +15,18 @@ # specific language governing permissions and limitations # under the License. +# /// script +# requires-python = ">=3.12" +# dependencies = [ +# "pyspark[connect]==4.0.1", +# ] +# /// + from pyspark.sql import SparkSession from pyspark.sql.functions import current_date, date_add, expr -# The configuration is important, otherwise we get many small -# parquet files with a single row. When a positional delete -# hits the Parquet file with one row, the parquet file gets -# dropped instead of having a merge-on-read delete file. -spark = ( - SparkSession - .builder - .config("spark.sql.shuffle.partitions", "1") - .config("spark.default.parallelism", "1") - .getOrCreate() -) +# Create SparkSession against the remote Spark Connect server +spark = SparkSession.builder.remote("sc://spark-iceberg:15002").getOrCreate() spark.sql(f"""CREATE NAMESPACE IF NOT EXISTS rest.default""") @@ -87,24 +85,25 @@ """ ) -spark.sql( - f""" -INSERT INTO rest.default.test_positional_merge_on_read_double_deletes -VALUES - (CAST('2023-03-01' AS date), 1, 'a'), - (CAST('2023-03-02' AS date), 2, 'b'), - (CAST('2023-03-03' AS date), 3, 'c'), - (CAST('2023-03-04' AS date), 4, 'd'), - (CAST('2023-03-05' AS date), 5, 'e'), - (CAST('2023-03-06' AS date), 6, 'f'), - (CAST('2023-03-07' AS date), 7, 'g'), - (CAST('2023-03-08' AS date), 8, 'h'), - (CAST('2023-03-09' AS date), 9, 'i'), - (CAST('2023-03-10' AS date), 10, 'j'), - (CAST('2023-03-11' AS date), 11, 'k'), - (CAST('2023-03-12' AS date), 12, 'l'); -""" -) +# Use coalesce(1) to write all data into a single parquet file. +# This ensures that deleting individual rows creates positional delete files +# instead of dropping entire files (which happens when a file has only one row). +spark.sql(""" + SELECT * FROM VALUES + (CAST('2023-03-01' AS date), 1, 'a'), + (CAST('2023-03-02' AS date), 2, 'b'), + (CAST('2023-03-03' AS date), 3, 'c'), + (CAST('2023-03-04' AS date), 4, 'd'), + (CAST('2023-03-05' AS date), 5, 'e'), + (CAST('2023-03-06' AS date), 6, 'f'), + (CAST('2023-03-07' AS date), 7, 'g'), + (CAST('2023-03-08' AS date), 8, 'h'), + (CAST('2023-03-09' AS date), 9, 'i'), + (CAST('2023-03-10' AS date), 10, 'j'), + (CAST('2023-03-11' AS date), 11, 'k'), + (CAST('2023-03-12' AS date), 12, 'l') + AS t(dt, number, letter) +""").coalesce(1).writeTo("rest.default.test_positional_merge_on_read_double_deletes").append() # Creates two positional deletes that should be merged spark.sql(f"DELETE FROM rest.default.test_positional_merge_on_read_double_deletes WHERE number = 9") diff --git a/crates/integration_tests/testdata/spark/spark-defaults.conf b/dev/spark/spark-defaults.conf similarity index 84% rename from crates/integration_tests/testdata/spark/spark-defaults.conf rename to dev/spark/spark-defaults.conf index 9db0c7da9e..a02640bbdd 100644 --- a/crates/integration_tests/testdata/spark/spark-defaults.conf +++ b/dev/spark/spark-defaults.conf @@ -16,14 +16,23 @@ # spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions + +# Configure Iceberg REST catalog spark.sql.catalog.rest org.apache.iceberg.spark.SparkCatalog spark.sql.catalog.rest.type rest spark.sql.catalog.rest.uri http://rest:8181 spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO spark.sql.catalog.rest.warehouse s3://warehouse/rest/ -spark.sql.catalog.rest.s3.endpoint http://minio:9000 +spark.sql.catalog.rest.s3.endpoint http://object-store:9000 +spark.sql.catalog.rest.cache-enabled false + spark.sql.defaultCatalog rest + +# Configure Spark UI and event logging +spark.ui.enabled true spark.eventLog.enabled true spark.eventLog.dir /home/iceberg/spark-events + +spark.sql.ansi.enabled false spark.history.fs.logDirectory /home/iceberg/spark-events spark.sql.catalogImplementation in-memory diff --git a/docs/rfcs/0001_modularize_iceberg_implementations.md b/docs/rfcs/0001_modularize_iceberg_implementations.md new file mode 100644 index 0000000000..14bd478270 --- /dev/null +++ b/docs/rfcs/0001_modularize_iceberg_implementations.md @@ -0,0 +1,120 @@ + + +# RFC: Modularize `iceberg` Implementations + +## Background + +Issue #1819 highlighted that the current `iceberg` crate mixes the Iceberg protocol abstractions (catalog/table/plan/transaction) with concrete runtime, storage, and execution code (Tokio runtime wrappers, opendal-based `FileIO`, Arrow helpers, DataFusion glue, etc.). This coupling makes the crate heavy and blocks users from composing their own storage or execution stacks. + +Two principles have been agreed: +1. The `iceberg` crate remains the single source of truth for all protocol traits and data structures. We will not create a separate “kernel” crate or facade layer. +2. Concrete integrations (Tokio runtime, opendal `FileIO`, Arrow/DataFusion glue, catalog adapters, etc.) move out into dedicated companion crates. Users needing a ready path can depend on those crates (e.g., `iceberg-datafusion` or `integrations/local`), while custom stacks depend only on `iceberg`. + +This RFC focuses on modularizing implementations; detailed trait signatures (e.g., `FileIO`, `Runtime`) will be handled in separate RFCs. + +## Goals and Scope + +- Keep `iceberg` as the protocol crate (traits + metadata + planning), without bundling runtimes, storage adapters, or execution glue. +- Relocate concrete code into companion crates under `crates/fileio/*`, `crates/runtime/*`, and `crates/integrations/*`. +- Provide a staged plan for extracting Arrow-dependent APIs to avoid destabilizing file-format code. +- Minimize breaking surfaces: traits stay in `iceberg`; downstream crates mainly adjust dependencies. + +Out of scope: changes to the Iceberg table specification or catalog adapter external behavior; detailed trait method design (covered by follow-up RFCs). + +## Architecture Overview + +### Workspace Layout (target) + +``` +crates/ + iceberg/ # core traits, metadata, planning, transactions + fileio/ + opendal/ # e.g. `iceberg-fileio-opendal` + fs/ # other FileIO implementations + runtime/ + tokio/ # e.g. `iceberg-runtime-tokio` + smol/ + catalog/* # catalog adapters (REST, HMS, Glue, etc.) + integrations/ + local/ # simple local/arrow-based helper crate + datafusion/ # combines core + implementations for DF + cache-moka/ + playground/ +``` + +- `crates/iceberg` drops direct deps on opendal, Tokio, Arrow, and DataFusion. +- Implementation crates depend on `iceberg` to implement the traits. +- Higher-level crates (`integrations/local`, `iceberg-datafusion`) assemble the pieces for ready-to-use scenarios. + +### Core Trait Surfaces + +`FileIO`, `Runtime`, `Catalog`, `Table`, `Transaction`, `TableScan` (plan descriptors) all remain hosted in `iceberg`. Precise method signatures are deferred to dedicated RFCs to avoid locking details prematurely. + +### Usage Modes + +- **Custom stacks**: depend on `iceberg` and provide your own implementations. +- **Pre-built stacks**: depend on `integrations/local` or `iceberg-datafusion`, which bundle `iceberg` with selected runtime/FileIO/Arrow helpers. +- `iceberg` does not re-export companion crates; users compose explicitly. + +## Migration Plan (staged, with Arrow extraction phased) + +1. **Phase 1 – Confirm trait hosting, defer details** + - Keep all protocol traits in `iceberg`; move detailed API design (FileIO, Runtime, etc.) to separate RFCs. + - Add temporary shims/deprecations only when traits are finalized. + +2. **Phase 2 – First Arrow step: move `to_arrow()` out** + - Relocate the public `to_arrow()` API to `integrations/local` (or another higher-level crate). Core no longer exposes Arrow entry points. + - Keep internal Arrow-dependent helpers (e.g., `ArrowFileReader`) temporarily in `iceberg` to avoid breaking file-format flows. + +3. **Phase 3 – Gradual Arrow dependency removal** + - Incrementally migrate/replace Arrow-dependent internals (`ArrowFileReader`, format-specific readers) into `integrations/local` or other helper crates. + - Adjust file-format APIs as needed; expect this to be multi-release work. + +4. **Phase 4 – Dependency cleanup** + - Ensure catalog and integration crates depend only on `iceberg` plus the specific runtime/FileIO/helper crates they need. + - Verify build/test pipelines against the new dependency graph. + +5. **Phase 5 – Docs & release** + - Publish migration guides: where `to_arrow()` moved, how to assemble local/DataFusion stacks. + - Schedule deprecation windows for remaining Arrow helpers; target a breaking release once Arrow is fully removed from `iceberg`. + +## Compatibility + +- Short term: users of `Table::scan().to_arrow()` must switch to `integrations/local` (or another crate that rehosts that API). Other Arrow types stay temporarily but will migrate in later phases. +- Long term: `iceberg` will be Arrow-free; companion crates provide Arrow-based helpers. +- Tests/examples move alongside the implementations they exercise. + +## Risks and Mitigations + +| Risk | Description | Mitigation | +| ---- | ----------- | ---------- | +| Arrow dependency unwinding is complex | File-format readers may rely on Arrow types | Phase the work; move `to_arrow()` first, then refactor readers; document interim state | +| Discoverability | Users may not know where Arrow helpers went | Clear docs pointing to `integrations/local` and `iceberg-datafusion`; migration guide | +| Trait churn | Future trait RFCs may break early adopters | Use deprecation shims and communicate timelines | +| Duplicate impls | Multiple helper crates could overlap | Provide recommended combinations and feature guidance | + +## Open Questions + +1. Versioning: align companion crate versions with `iceberg`, or allow independent versions plus compatibility matrix? +2. Deprecation schedule: how long do we keep interim Arrow helpers before full removal from `iceberg`? + +## Conclusion + +We will keep `iceberg` as the protocol crate while modularizing concrete implementations. Arrow removal will be phased: first relocating `to_arrow()` to `integrations/local`, then gradually moving Arrow-dependent readers and helpers. This keeps the core lean, lets users compose their preferred runtime/FileIO stacks, and still offers ready-to-use combinations via companion crates. diff --git a/docs/rfcs/0002_storage_trait.md b/docs/rfcs/0002_storage_trait.md new file mode 100644 index 0000000000..b68141aec4 --- /dev/null +++ b/docs/rfcs/0002_storage_trait.md @@ -0,0 +1,862 @@ + + +# Making Storage a Trait + +## Background + +### Existing Implementation + +The existing code implements storage functionality through a concrete `Storage` enum that handles different storage backends (S3, local filesystem, GCS, etc.). This implementation is tightly coupled with OpenDAL as the underlying storage layer. The `FileIO` struct wraps this `Storage` enum and provides a high-level API for file operations. + +```rust +// Current: Concrete enum with variants for each backend +pub(crate) enum Storage { + #[cfg(feature = "storage-memory")] + Memory(Operator), + #[cfg(feature = "storage-fs")] + LocalFs, + #[cfg(feature = "storage-s3")] + S3 { + configured_scheme: String, + config: Arc, + customized_credential_load: Option, + }, + #[cfg(feature = "storage-gcs")] + Gcs { config: Arc }, + // ... other variants +} +``` + +Current structure: + +- **FileIO:** Main interface for file operations, wraps `Arc` +- **Storage:** Enum with variants for different storage backends +- **InputFile / OutputFile:** Concrete structs that hold an `Operator` and path + +### Problem Statement + +The original design has several limitations: + +- **Tight Coupling** – All storage logic depends on OpenDAL, limiting flexibility. Users cannot easily opt in for other storage implementations like `object_store` +- **Customization Barriers** – Users cannot easily add custom behaviors or optimizations +- **No Extensibility** – Adding new backends requires modifying the core enum in the `iceberg` crate + +As discussed in Issue #1314, making Storage a trait would allow pluggable storage and better integration with existing systems. + +--- + +## High-Level Architecture + +The new design introduces a trait-based storage abstraction with a factory pattern for creating storage instances. This enables pluggable storage backends while maintaining a clean separation between the core Iceberg library and storage implementations. + +### Component Overview + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ User Application │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Catalog │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ CatalogBuilder::with_storage_factory(storage_factory) │ │ +│ │ - Accepts optional StorageFactory injection │ │ +│ │ - Falls back to LocalFsStorageFactory if not provided │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ FileIO / FileIOBuilder │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ FileIOBuilder: │ │ +│ │ - factory: Arc │ │ +│ │ - config: StorageConfig │ │ +│ │ - Methods: new(), with_prop(), with_props(), config(), build() │ │ +│ │ │ │ +│ │ FileIO: │ │ +│ │ - config: StorageConfig (properties only, no scheme) │ │ +│ │ - factory: Arc │ │ +│ │ - storage: OnceCell> (lazy initialization) │ │ +│ │ - Methods: new_with_memory(), new_with_fs(), into_builder() │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ Methods: new_input(), new_output(), delete(), exists(), delete_prefix() │ +└─────────────────────────────────────────────────────────────────────────────┘ + │ + ┌─────────────────┴─────────────────┐ + ▼ ▼ +┌───────────────────────────────┐ ┌───────────────────────────────────────┐ +│ StorageFactory │ │ Storage │ +│ (trait in iceberg crate) │ │ (trait in iceberg crate) │ +│ │ │ │ +│ fn build(&self, config) │────▶│ async fn exists(&self, path) │ +│ -> Arc │ │ async fn read(&self, path) │ +│ │ │ async fn write(&self, path, bytes) │ +│ │ │ async fn delete(&self, path) │ +│ │ │ fn new_input(&self, path) │ +│ │ │ fn new_output(&self, path) │ +└───────────────────────────────┘ └───────────────────────────────────────┘ + │ ▲ + │ │ + ▼ │ +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Storage Implementations │ +│ │ +│ ┌─────────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐ │ +│ │ MemoryStorage │ │ LocalFsStorage │ │ OpenDalStorage │ │ +│ │ (iceberg crate) │ │ (iceberg crate) │ │ (iceberg-storage- │ │ +│ │ │ │ │ │ opendal) │ │ +│ │ - In-memory HashMap│ │ - std::fs ops │ │ - S3, GCS, Azure │ │ +│ │ - For testing │ │ - For local files │ │ - OSS, filesystem │ │ +│ └─────────────────────┘ └─────────────────────┘ └─────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +### Data Flow + +``` +┌──────────────────────────────────────────────────────────────────────────────┐ +│ FileIO Creation Flow │ +└──────────────────────────────────────────────────────────────────────────────┘ + + User Code FileIOBuilder StorageFactory + │ │ │ + │ FileIOBuilder::new() │ │ + │─────────────────────────▶│ │ + │ │ │ + │ .with_prop() │ │ + │─────────────────────────▶│ │ + │ │ │ + │ .build() │ │ + │─────────────────────────▶│ │ + │ │ │ + │◀─────────────────────────│ │ + │ FileIO │ │ + │ │ │ + │ new_input(path) │ │ + │─────────────────────────▶│ │ + │ │ (lazy) factory.build() │ + │ │─────────────────────────────▶│ + │ │ │ + │ │◀─────────────────────────────│ + │ │ Arc │ + │ │ │ + │◀─────────────────────────│ │ + │ InputFile │ │ + + +┌──────────────────────────────────────────────────────────────────────────────┐ +│ Catalog with FileIO Injection │ +└──────────────────────────────────────────────────────────────────────────────┘ + + User Code CatalogBuilder Catalog + │ │ │ + │ ::default() │ │ + │─────────────────────────▶│ │ + │ │ │ + │ .with_storage_factory(factory) │ │ + │─────────────────────────────────▶│ │ + │ │ │ + │ .load(name, props) │ │ + │─────────────────────────▶│ │ + │ │ new(config, Some(file_io)) │ + │ │─────────────────────────────▶│ + │ │ │ + │◀─────────────────────────│◀─────────────────────────────│ + │ Catalog │ │ +``` + +### Crate Structure + +``` +crates/ +├── iceberg/ # Core Iceberg functionality +│ └── src/ +│ └── io/ +│ ├── mod.rs # Re-exports +│ ├── storage.rs # Storage + StorageFactory traits +│ ├── file_io.rs # FileIO, InputFile, OutputFile +│ ├── config/ # StorageConfig and backend configs +│ │ ├── mod.rs # StorageConfig +│ │ ├── s3.rs # S3Config constants +│ │ ├── gcs.rs # GcsConfig constants +│ │ ├── oss.rs # OssConfig constants +│ │ └── azdls.rs # AzdlsConfig constants +│ ├── memory.rs # MemoryStorage (built-in) +│ └── local_fs.rs # LocalFsStorage (built-in) +│ +├── storage/ +│ └── opendal/ # OpenDAL-based implementations +│ └── src/ +│ ├── lib.rs # Re-exports +│ ├── storage.rs # OpenDalStorage + OpenDalStorageFactory +│ ├── storage_s3.rs # S3 support +│ ├── storage_gcs.rs # GCS support +│ ├── storage_oss.rs # OSS support +│ ├── storage_azdls.rs # Azure support +│ └── storage_fs.rs # Filesystem support +│ +└── catalog/ # Catalog implementations + ├── rest/ # Uses with_storage_factory injection + ├── glue/ # Uses with_storage_factory injection + ├── hms/ # Uses with_storage_factory injection + ├── s3tables/ # Uses with_storage_factory injection + └── sql/ # Uses with_storage_factory injection +``` + +--- + +## Design Phase 1: Storage Trait and Core Types + +Phase 1 focuses on converting Storage from an enum to a trait, introducing `StorageFactory` and `StorageConfig`, and updating `FileIO`, `InputFile`, and `OutputFile` to use the trait-based abstraction. + +### Storage Trait + +The `Storage` trait is defined in the `iceberg` crate and provides the interface for all storage operations. It uses `typetag` for serialization support across process boundaries. + +```rust +#[async_trait] +#[typetag::serde(tag = "type")] +pub trait Storage: Debug + Send + Sync { + /// Check if a file exists at the given path + async fn exists(&self, path: &str) -> Result; + + /// Get metadata from an input path + async fn metadata(&self, path: &str) -> Result; + + /// Read bytes from a path + async fn read(&self, path: &str) -> Result; + + /// Get FileRead from a path + async fn reader(&self, path: &str) -> Result>; + + /// Write bytes to an output path + async fn write(&self, path: &str, bs: Bytes) -> Result<()>; + + /// Get FileWrite from a path + async fn writer(&self, path: &str) -> Result>; + + /// Delete a file at the given path + async fn delete(&self, path: &str) -> Result<()>; + + /// Delete all files with the given prefix + async fn delete_prefix(&self, path: &str) -> Result<()>; + + /// Create a new input file for reading + fn new_input(&self, path: &str) -> Result; + + /// Create a new output file for writing + fn new_output(&self, path: &str) -> Result; +} +``` + +### StorageFactory Trait + +The `StorageFactory` trait creates `Storage` instances from configuration. This enables lazy initialization and custom storage injection. + +```rust +#[typetag::serde(tag = "type")] +pub trait StorageFactory: Debug + Send + Sync { + /// Build a new Storage instance from the given configuration. + fn build(&self, config: &StorageConfig) -> Result>; +} +``` + +### StorageConfig + +`StorageConfig` is a pure property container without scheme. The storage type is determined by explicit factory selection: + +```rust +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +pub struct StorageConfig { + /// Configuration properties for the storage backend + props: HashMap, +} + +impl StorageConfig { + pub fn new() -> Self; + pub fn from_props(props: HashMap) -> Self; + pub fn props(&self) -> &HashMap; + pub fn get(&self, key: &str) -> Option<&String>; + pub fn with_prop(self, key: impl Into, value: impl Into) -> Self; + pub fn with_props(self, props: impl IntoIterator, impl Into)>) -> Self; +} +``` + +#### Backend-Specific Configuration Types + +In addition to `StorageConfig`, we provide typed configuration structs for each storage backend. +These can be constructed from `StorageConfig` using `TryFrom` and provide a structured way to access backend-specific settings: + +- `S3Config` - Amazon S3 configuration +- `GcsConfig` - Google Cloud Storage configuration +- `OssConfig` - Alibaba Cloud OSS configuration +- `AzdlsConfig` - Azure Data Lake Storage configuration + +Example of `S3Config`: + +```rust +#[derive(Clone, Debug, Default, Serialize, Deserialize, TypedBuilder)] +pub struct S3Config { + #[builder(default, setter(strip_option, into))] + endpoint: Option, + #[builder(default, setter(strip_option, into))] + access_key_id: Option, + #[builder(default, setter(strip_option, into))] + secret_access_key: Option, + #[builder(default, setter(strip_option, into))] + region: Option, + #[builder(default)] + allow_anonymous: bool, + // ... other S3-specific fields +} + +impl S3Config { + /// Returns the S3 endpoint URL. + pub fn endpoint(&self) -> Option<&str> { + self.endpoint.as_deref() + } + + // ... other getter methods +} + +// Fallible conversion from StorageConfig +impl TryFrom<&StorageConfig> for S3Config { + type Error = iceberg::Error; + + fn try_from(config: &StorageConfig) -> Result {/* ... */} +} +``` + +Usage with the builder pattern: + +```rust +let s3_config = S3Config::builder() + .region("us-east-1") + .access_key_id("my-access-key") + .secret_access_key("my-secret-key") + .build(); + +assert_eq!(s3_config.region(), Some("us-east-1")); +``` + +These typed configs are used internally by storage implementations (e.g., `OpenDalStorage`) to +parse properties from `StorageConfig` into strongly-typed configuration. + +### FileIO and FileIOBuilder + +`FileIO` is redesigned to use lazy storage initialization with a factory pattern. Configuration is done via `FileIOBuilder`: + +```rust +#[derive(Clone)] +pub struct FileIO { + /// Storage configuration containing properties + config: StorageConfig, + /// Factory for creating storage instances + factory: Arc, + /// Cached storage instance (lazily initialized) + storage: Arc>>, +} + +impl FileIO { + /// Create a new FileIO backed by in-memory storage. + pub fn new_with_memory() -> Self; + + /// Create a new FileIO backed by local filesystem storage. + pub fn new_with_fs() -> Self; + + /// Convert this FileIO into a FileIOBuilder for modification. + pub fn into_builder(self) -> FileIOBuilder; + + // File operations delegate to the lazily-initialized storage + pub async fn delete(&self, path: impl AsRef) -> Result<()>; + pub async fn delete_prefix(&self, path: impl AsRef) -> Result<()>; + pub async fn exists(&self, path: impl AsRef) -> Result; + pub fn new_input(&self, path: impl AsRef) -> Result; + pub fn new_output(&self, path: impl AsRef) -> Result; +} + +/// Builder for creating FileIO instances. +pub struct FileIOBuilder { + factory: Arc, + config: StorageConfig, +} + +impl FileIOBuilder { + /// Create a new FileIOBuilder with the given storage factory. + pub fn new(factory: Arc) -> Self; + + /// Add a configuration property. + pub fn with_prop(self, key: impl Into, value: impl Into) -> Self; + + /// Add multiple configuration properties. + pub fn with_props(self, props: impl IntoIterator, impl Into)>) -> Self; + + /// Get the storage configuration. + pub fn config(&self) -> &StorageConfig; + + /// Build the FileIO instance. + pub fn build(self) -> Result; +} +``` + +Key changes from the old design: +- `FileIOBuilder` is used for configuration with explicit factory injection +- `FileIO` has convenience constructors (`new_with_memory()`, `new_with_fs()`) for common cases +- Removed `Extensions` - custom behavior is now provided via `StorageFactory` +- Storage is lazily initialized on first use via `OnceCell` + +### InputFile and OutputFile Changes + +`InputFile` and `OutputFile` now hold a reference to `Arc` instead of an `Operator`: + +```rust +pub struct InputFile { + storage: Arc, + path: String, +} + +impl InputFile { + pub fn new(storage: Arc, path: String) -> Self; + pub fn location(&self) -> &str; + pub async fn exists(&self) -> Result; + pub async fn metadata(&self) -> Result; + pub async fn read(&self) -> Result; + pub async fn reader(&self) -> Result>; +} + +pub struct OutputFile { + storage: Arc, + path: String, +} + +impl OutputFile { + pub fn new(storage: Arc, path: String) -> Self; + pub fn location(&self) -> &str; + pub async fn exists(&self) -> Result; + pub async fn delete(&self) -> Result<()>; + pub fn to_input_file(self) -> InputFile; + pub async fn write(&self, bs: Bytes) -> Result<()>; + pub async fn writer(&self) -> Result>; +} +``` + +### Built-in Storage Implementations + +The `iceberg` crate includes two built-in storage implementations for testing and basic use cases: + +#### MemoryStorage + +In-memory storage using a thread-safe `HashMap`, primarily for testing: + +```rust +#[derive(Debug, Clone, Default)] +pub struct MemoryStorage { + data: Arc>>, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct MemoryStorageFactory; + +#[typetag::serde] +impl StorageFactory for MemoryStorageFactory { + fn build(&self, config: &StorageConfig) -> Result> { + if config.scheme() != "memory" { + return Err(/* error */); + } + Ok(Arc::new(MemoryStorage::new())) + } +} +``` + +#### LocalFsStorage + +Local filesystem storage using standard Rust `std::fs` operations: + +```rust +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct LocalFsStorage; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct LocalFsStorageFactory; + +#[typetag::serde] +impl StorageFactory for LocalFsStorageFactory { + fn build(&self, config: &StorageConfig) -> Result> { + if config.scheme() != "file" { + return Err(/* error */); + } + Ok(Arc::new(LocalFsStorage::new())) + } +} +``` + +### CatalogBuilder Changes + +The `CatalogBuilder` trait is extended with `with_storage_factory()` to allow StorageFactory injection: + +```rust +pub trait CatalogBuilder: Default + Debug + Send + Sync { + type C: Catalog; + + /// Set a custom StorageFactory to use for storage operations. + /// + /// When a StorageFactory is provided, the catalog will use it to build FileIO + /// instances for all storage operations instead of using the default factory. + fn with_storage_factory(self, storage_factory: Arc) -> Self; + + /// Create a new catalog instance. + fn load( + self, + name: impl Into, + props: HashMap, + ) -> impl Future> + Send; +} +``` + +Catalog implementations store the optional `StorageFactory` and use it when provided: + +```rust +pub struct GlueCatalogBuilder { + config: GlueCatalogConfig, + storage_factory: Option>, // New field +} + +impl CatalogBuilder for GlueCatalogBuilder { + fn with_storage_factory(mut self, storage_factory: Arc) -> Self { + self.storage_factory = Some(storage_factory); + self + } + + // In load(): + // Use provided StorageFactory or LocalFsStorageFactory as fallback + let factory = storage_factory.unwrap_or_else(|| Arc::new(LocalFsStorageFactory)); + let file_io = FileIOBuilder::new(factory) + .with_props(file_io_props) + .build()?; +} +``` + +--- + +## Design Part 2: Separate Storage Crate + +Phase 2 moves concrete OpenDAL-based implementations to a separate crate (`iceberg-storage-opendal`). + +### iceberg-storage-opendal Crate + +This crate provides OpenDAL-based storage implementations for cloud storage backends: + +```rust +// crates/storage/opendal/src/storage.rs + +/// Explicit storage factory variants for OpenDAL-based backends. +/// +/// Each variant represents a specific storage backend. Path scheme +/// validation is handled by the underlying Storage implementation +/// when operations are performed. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum OpenDalStorageFactory { + /// Local filesystem storage factory. + #[cfg(feature = "storage-fs")] + Fs, + + /// Amazon S3 storage factory. + #[cfg(feature = "storage-s3")] + S3, + + /// Google Cloud Storage factory. + #[cfg(feature = "storage-gcs")] + Gcs, + + /// Alibaba Cloud OSS storage factory. + #[cfg(feature = "storage-oss")] + Oss, + + /// Azure Data Lake Storage factory. + #[cfg(feature = "storage-azdls")] + Azdls, +} + +#[typetag::serde] +impl StorageFactory for OpenDalStorageFactory { + fn build(&self, config: &StorageConfig) -> Result> { + let storage = match self { + #[cfg(feature = "storage-fs")] + Self::Fs => OpenDalStorage::LocalFs, + + #[cfg(feature = "storage-s3")] + Self::S3 => { + let iceberg_s3_config = S3Config::try_from(config)?; + let opendal_s3_config = s3_config_to_opendal(&iceberg_s3_config); + OpenDalStorage::S3 { + configured_scheme: "s3".to_string(), + config: opendal_s3_config.into(), + customized_credential_load: None, + } + } + // ... other variants + }; + Ok(Arc::new(storage)) + } +} + +/// Unified OpenDAL-based storage implementation. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum OpenDalStorage { + #[cfg(feature = "storage-fs")] + LocalFs, + + #[cfg(feature = "storage-s3")] + S3 { + configured_scheme: String, + config: Arc, + #[serde(skip)] + customized_credential_load: Option, + }, + + #[cfg(feature = "storage-gcs")] + Gcs { config: Arc }, + + #[cfg(feature = "storage-oss")] + Oss { config: Arc }, + + #[cfg(feature = "storage-azdls")] + Azdls { + configured_scheme: AzureStorageScheme, + config: Arc, + }, +} + +impl OpenDalStorage { + /// Creates operator from path. + fn create_operator<'a>(&self, path: &'a str) -> Result<(Operator, &'a str)>; +} + +#[async_trait] +#[typetag::serde] +impl Storage for OpenDalStorage { + // Delegates all operations to the appropriate OpenDAL operator +} +``` + +Feature flags in `iceberg-storage-opendal`: +- `storage-s3` : Enables S3 storage backend +- `storage-gcs`: Enables Google Cloud Storage backend +- `storage-oss`: Enables Alibaba Cloud OSS backend +- `storage-azdls`: Enables Azure Data Lake Storage backend +- `storage-fs`: Enables OpenDAL filesystem backend +- `storage-all`: Enables all storage backends + +--- + +## Example Usage + +### Basic Usage with Memory Storage (Testing) + +```rust +use iceberg::io::FileIO; + +// Create in-memory FileIO for testing +let file_io = FileIO::new_with_memory(); + +// Write and read files +let output = file_io.new_output("memory://test/file.txt")?; +output.write("Hello, World!".into()).await?; + +let input = file_io.new_input("memory://test/file.txt")?; +let content = input.read().await?; +assert_eq!(content, bytes::Bytes::from("Hello, World!")); +``` + +### Using OpenDAL Storage Factory + +```rust +use std::sync::Arc; +use iceberg::io::FileIOBuilder; +use iceberg_storage_opendal::OpenDalStorageFactory; + +// Create FileIO with explicit S3 factory +let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3)) + .with_prop("s3.region", "us-east-1") + .with_prop("s3.access-key-id", "my-access-key") + .with_prop("s3.secret-access-key", "my-secret-key") + .build()?; + +// Use the FileIO +let input = file_io.new_input("s3://my-bucket/warehouse/table/metadata.json")?; +let metadata = input.read().await?; +``` + +### Using Catalogs with Custom Storage + +When using a catalog without injecting a custom `StorageFactory`, the catalog falls back to +`LocalFsStorageFactory`. For cloud storage, inject the appropriate factory: + +```rust +use std::collections::HashMap; +use std::sync::Arc; +use iceberg::CatalogBuilder; +use iceberg_catalog_glue::GlueCatalogBuilder; +use iceberg_storage_opendal::OpenDalStorageFactory; + +// Inject S3 storage factory for cloud storage support +let catalog = GlueCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3)) + .load("my_catalog", HashMap::from([ + ("warehouse".to_string(), "s3://my-bucket/warehouse".to_string()), + ("s3.region".to_string(), "us-east-1".to_string()), + ])) + .await?; + +// Load and scan a table - storage is handled automatically +let table = catalog.load_table(&TableIdent::from_strs(["db", "my_table"])?).await?; +let scan = table.scan().build()?; +``` + +### Injecting Custom StorageFactory into Catalogs + +For advanced use cases, you can inject a custom `StorageFactory` with specific storage configuration: + +```rust +use std::collections::HashMap; +use std::sync::Arc; +use iceberg::CatalogBuilder; +use iceberg::io::FileIOBuilder; +use iceberg_catalog_glue::GlueCatalogBuilder; +use iceberg_storage_opendal::OpenDalStorageFactory; + +// Create a custom StorageFactory +let storage_factory = Arc::new(OpenDalStorageFactory::S3); + +// Inject StorageFactory into catalog +let catalog = GlueCatalogBuilder::default() + .with_storage_factory(storage_factory) + .load("my_catalog", HashMap::from([ + ("warehouse".to_string(), "s3://my-bucket/warehouse".to_string()), + ("s3.region".to_string(), "us-east-1".to_string()), + ])) + .await?; +``` + +### Implementing Custom Storage + +To implement a custom storage backend, implement the `Storage` trait with `#[typetag::serde]`: + +```rust +use std::sync::Arc; +use async_trait::async_trait; +use iceberg::io::{Storage, StorageFactory, StorageConfig, InputFile, OutputFile}; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct MyCustomStorage { /* fields */ } + +#[async_trait] +#[typetag::serde] +impl Storage for MyCustomStorage { + // Implement all required methods: exists, metadata, read, reader, + // write, writer, delete, delete_prefix, new_input, new_output +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct MyCustomStorageFactory; + +#[typetag::serde] +impl StorageFactory for MyCustomStorageFactory { + fn build(&self, config: &StorageConfig) -> iceberg::Result> { + Ok(Arc::new(MyCustomStorage { /* ... */ })) + } +} +``` + +### Routing to Multiple Storage Backends + +To use different storage implementations for different schemes (e.g., a native S3 client +for S3 and OpenDAL for other schemes), implement routing at the `Storage` level: + +```rust +use std::sync::Arc; +use async_trait::async_trait; +use iceberg::io::{Storage, StorageFactory, StorageConfig, InputFile, OutputFile}; + +/// A storage that routes to different backends based on path scheme +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct RoutingStorage { + s3_storage: NativeS3Storage, // Custom S3 implementation + opendal_storage: OpenDalStorage, // OpenDAL for other schemes +} + +#[async_trait] +#[typetag::serde] +impl Storage for RoutingStorage { + async fn read(&self, path: &str) -> iceberg::Result { + if path.starts_with("s3://") || path.starts_with("s3a://") { + self.s3_storage.read(path).await + } else { + self.opendal_storage.read(path).await + } + } + + // Route other methods similarly... +} + +/// Factory that creates RoutingStorage +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct RoutingStorageFactory; + +#[typetag::serde] +impl StorageFactory for RoutingStorageFactory { + fn build(&self, config: &StorageConfig) -> iceberg::Result> { + Ok(Arc::new(RoutingStorage { + s3_storage: NativeS3Storage::new(config)?, + opendal_storage: OpenDalStorage::build_from_config(config)?, + })) + } +} +``` + +--- + + +## Implementation Plan + +### Phase 1: Storage Trait +- Define `Storage` trait in `iceberg` crate +- Define `StorageFactory` trait in `iceberg` crate +- Introduce `StorageConfig` for configuration properties +- Update `FileIO` to use lazy storage initialization with factory pattern +- Update `InputFile`/`OutputFile` to use `Arc` +- Implement `MemoryStorage` and `LocalFsStorage` in `iceberg` crate +- Add `with_storage_factory()` to `CatalogBuilder` trait +- Update all catalog implementations to support StorageFactory injection +- Improve naming: Storage handles locations rather than paths + +### Phase 2: Separate Storage Crate +- Create `iceberg-storage-opendal` crate with `OpenDalStorage` and `OpenDalStorageFactory` +- Move S3, GCS, OSS, Azure implementations to `iceberg-storage-opendal` +- Remove storage feature flags from `iceberg` crate + +### Future Work +- Implement ResolvingStorage backed by OpenDal +- Add `object_store`-based storage implementations +- Consider introducing `IoErrorKind` for storage-specific error handling +- Introduce custom key values in StorageConfigs diff --git a/docs/rfcs/0003_stateful_transaction.md b/docs/rfcs/0003_stateful_transaction.md new file mode 100644 index 0000000000..6e8f5bc3ce --- /dev/null +++ b/docs/rfcs/0003_stateful_transaction.md @@ -0,0 +1,442 @@ + + +# RFC: Stateful Transactions and Snapshot Production + +**Status:** Draft +**Target:** Apache Iceberg Rust (`iceberg-rust`) + +## 1. Motivation and Scope + +Replaying transaction actions after a catalog conflict is sufficient for simple +metadata updates, but snapshot-producing actions benefit from retaining work +across attempts. FastAppend and merging operations such as RowDelta need stable +execution identity, reusable metadata processing, and ownership of generated +files that may require cleanup. + +This RFC establishes a stateful action model and a shared foundation for snapshot +production. The transaction owns catalog refresh and replay; each action execution +owns retry-persistent state; snapshot-producing actions use persistent producers +as that state. + +The proposal fixes lifetimes, ownership, the core action interface, and the +responsibilities of snapshot producers. It does not prescribe cache layouts, +conflict-validation predicates, or the complete semantics of each operation. +Those can be implemented incrementally within these boundaries. + +## 2. Transaction and Action State Model + +### 2.1 Three Lifetimes + +| Lifetime | Responsibility | +| --- | --- | +| Transaction | Catalog refresh, retry policy, action ordering, replay, and the terminal result | +| Action execution | Immutable operation intent and exclusively owned retry-persistent state | +| Attempt | Execution against the current transaction-local table and construction of that attempt's result | + +An action's intent does not change during replay. Its state survives attempts of +one logical execution, while values derived from a particular base are local to +an attempt or reused only under an appropriate validity check. + +### 2.2 Stateful Action Interface + +Each action declares an associated state type. The following interface captures +the lifecycle; the heterogeneous storage adapters are implementation details. + +```rust +#[async_trait] +pub(crate) trait TransactionAction: Clone + Send + Sync + 'static { + type State: Send + Sync + 'static; + + /// Fresh state for one logical execution; infallible and table-independent. + fn new_state(&self) -> Self::State; + + /// One attempt against the current transaction-local table. + async fn commit( + &self, + state: &mut Self::State, + table: &Table, + ) -> Result; + + /// Best-effort cleanup after the transaction reaches a terminal result. + /// Consumes the action and its state: the type system guarantees that + /// no further attempt can run for this entry after cleanup. + async fn cleanup( + self: Box, + state: Self::State, + table: &Table, + status: CommitStatus, + ); +} +``` + +During replay the action is borrowed immutably, its execution state mutably, +and the table per attempt. The terminal `cleanup` instead consumes the action +and its state by value, so replay cannot resume after cleanup. Stateless +actions use `State = ()`. State is created when an action is added to a +transaction. Table-dependent initialization happens during execution; once +resolved, logical snapshot identity remains stable across retries. + +`CommitStatus` describes terminal cleanup safety (§6). Neither the action nor its +state needs to control the transaction's catalog retry loop. + +The action interface and its storage machinery remain `pub(crate)`: external +crates cannot define transaction actions, so the erasure and adapter details +are internal rather than public API commitments. + +### 2.3 Action Entries and Cloning + +A transaction stores one entry per action, pairing immutable intent with its +execution state. Conceptually: + +```rust +struct ActionEntry { + action: Box, + state: A::State, +} +``` + +The transaction must preserve that pairing when storing heterogeneous entries. +A typed entry erased behind an adapter and separately erased action/state values +with a checked pairing are both acceptable implementations. + +Entries own their actions exclusively so that terminal cleanup can consume +them. Retry reuses the same entry state. Cloning a transaction creates a new +execution of the action plan: it duplicates the immutable actions (actions are +`Clone`) and creates fresh state via `new_state()`. Mutable retry state and +generated execution identity are never shared between transaction executions, +and transaction cloning is expected to be rare, so duplicating intent is an +acceptable cost. + +## 3. Retry and Replay Workflow + +`Transaction::commit` owns refresh, retry, and replay. Each attempt starts from a +catalog base, executes actions in application order, and submits their combined +updates and requirements as one `TableCommit`. + +```mermaid +flowchart TD + Start["Begin transaction commit"] --> Refresh["Load latest catalog base"] + Refresh --> Init["Initialize attempt-local table and update accumulators"] + Init --> Action["Execute next action with retained state"] + Action --> Apply["Apply ActionCommit locally and accumulate changes"] + Apply --> More{"More actions?"} + More -->|Yes| Action + More -->|No| Submit["Build TableCommit and submit to catalog"] + Submit -->|Success| Committed["Committed"] + Submit -->|Retryable conflict and budget remains| Retry["Retain state and artifacts; back off"] + Retry --> Refresh + Submit -->|Definitive failure; no retry| Failed["Failed"] + Submit -->|Indeterminate result| Unknown["Unknown"] + Action -->|Definitive execution failure| Failed + Apply -->|Definitive local apply failure| Failed + Committed --> Finish["Call cleanup on every entry"] + Failed --> Finish + Unknown --> Finish +``` + +Later actions observe earlier actions' transaction-local results, even though +none of those results has yet been committed to the catalog. After a conflict, +the next attempt rebuilds the local table from the refreshed base and replays +the complete action sequence. It does not continue accumulating changes on the +previous attempt's local result. + +The diagram omits refresh errors and other error branches for clarity. The +transaction/catalog layer determines retryability and whether a failed request +definitively did not commit. An unresolved catalog outcome is `Unknown`; it must +not be treated as a known conflict or definite failure. + +Replaying an action does not require repeating all of its I/O or metadata +processing. The same persistent state is passed into each attempt, allowing the +action to reuse valid work while producing a new result for the current base. + +## 4. Persistent Snapshot Producers + +### 4.1 Producers as Action State + +Snapshot-producing actions use persistent producers as their associated state: + +| Producer | Role | Initial consumer | +| --- | --- | --- | +| `SimpleSnapshotProducer` | Append-shaped snapshot production | `FastAppendAction` | +| `MergingSnapshotProducer` | Snapshot production involving data/delete file additions and removals | `RowDeltaAction` | + +An action expresses operation intent and selects the checks required by its +semantics and configuration. The producer supplies reusable validation and +metadata-processing capabilities, retains work across retries, and constructs +the snapshot result for an attempt. + +The producers are independent concrete types, not an inheritance hierarchy. +`MergingSnapshotProducer` does not contain `SimpleSnapshotProducer`. Each producer +owns the persistent state needed for its responsibilities; the transaction does +not need to know that an action's state is a producer. + +### 4.2 Shared Snapshot Production + +Common mechanics are shared through stateless helpers: snapshot-ID generation, +metadata path generation, summary construction, manifest-list writing, and +snapshot/update/requirement assembly. These helpers operate on explicit inputs +and do not own retry state or select operation policy. + +This replaces the attempt-scoped producer and generic hook-trait pipeline with +concrete producers and shared functions. Identity, path allocation, reuse, and +artifact ownership remain producer responsibilities. Whether common persistent +fields are extracted into a shared struct is an implementation choice. + +### 4.3 From Action Intent to a Snapshot + +Snapshot production has three logical stages. These describe responsibilities, +not a requirement to put all stages into one `apply()` method. + +1. **The action selects and invokes validation.** For example, a RowDelta action + may require referenced files to remain live or reject conflicting changes + since a starting snapshot. The producer provides shared validation capabilities + and reusable history processing; the action determines which checks apply. + The concrete conflict predicates are outside this RFC. + +2. **The producer prepares the resulting manifests.** It obtains the current + transaction-local manifest set, applies removals through manifest filtering, + generates or reuses manifests for additions, and assembles the complete + resulting set. Filtering updates manifest entries to reflect file removals; + it does not read Parquet files to execute row-level deletes. Unaffected + manifests can be retained, and valid transformation results can be reused. + +3. **The producer constructs the attempt's snapshot.** It determines parent, + sequence number, applicable row-ID allocation, and summary from the current + table and resulting changes. It writes an attempt-specific manifest list and + returns an `ActionCommit`. The transaction applies that result locally and + eventually submits the combined `TableCommit`. + +For example, if the current set is `M1, M2`, an action removes a file described +by `M1` and adds file `X`, the producer may produce `M1′, M2, MX`: `M1′` reflects +the removal, `M2` is unchanged, and `MX` describes the addition. Manifest filter +managers and history-processing helpers support this workflow without defining +the public action semantics themselves. + +## 5. Reuse Across Attempts + +> Retries recompute base-dependent snapshot composition, while reusing previously +> completed work whenever all semantic inputs that produced it remain unchanged. + +### 5.1 Retained Work and Attempt-Specific Results + +| Work or state | Behavior across retry/rebase | +| --- | --- | +| Action intent and resolved logical identity | Preserve for the execution | +| Added manifests and per-manifest transformation results | Reuse while all relevant inputs remain unchanged | +| Metadata reads and processed validation history | Reuse under the same semantic validity rule | +| Parent, sequence number, and row-ID allocation | Determine from the current attempt's table | +| Complete resulting manifest set and snapshot summary | Recompute for the current attempt | +| Manifest list, `ActionCommit`, and `TableCommit` | Construct for the current attempt | + +The main reuse opportunities are expensive reads, decoding, and manifest +materialization. Reconstructing the complete manifest set does not require +discarding those results, nor does it prohibit caching the current snapshot's +input manifests under an appropriate identity. + +An immutable file is a useful reuse boundary, but file identity alone is not +necessarily the full semantic input to a computation. Operation intent, relevant +metadata context, and inherited values must also remain compatible. Each attempt +must establish its required validation coverage and current-state checks; cached +history or an earlier successful check cannot substitute for that obligation. + +This RFC identifies reuse opportunities without requiring a particular cache +inventory, key type, or storage layer. Correct recomputation is an acceptable +fallback whenever reuse cannot be established safely. + +One assumption deserves explicit statement because it is format-dependent. +Reusing a generated added manifest across attempts is valid under current +formats because the attempt-sensitive values — committing snapshot ID, data +sequence number, first-row-id — are inherited through manifest-list metadata +rather than embedded in the manifest file, so a rebase changes what the +manifest inherits without invalidating its content. A future format that +embeds base-dependent values directly into manifests would add those values +to the manifest's semantic inputs and shrink this reuse accordingly. That is +a seam inside producers and their writers; it changes what may be reused, not +the ownership, replay, or cleanup architecture. + +### 5.2 Logical Identity Versus Snapshot Representation + +A stable snapshot ID identifies one logical snapshot-producing execution. It +does not imply that an uncommitted snapshot's representation is stable across +attempts. + +In a multi-action transaction, action B can observe a snapshot produced locally +by action A. After rebase, A retains its snapshot ID but may produce a different +parent, sequence number, manifest set, and manifest-list location. B must use +that new representation. + +Committed snapshots have immutable representations. Transaction-local snapshots +must instead be treated as attempt-dependent until committed. Any reuse involving +them must be guarded by the relevant representation or semantic inputs, or the +work must be recomputed. Snapshot ID alone is insufficient for that purpose. + +### 5.3 Multi-Action Replay Example + +Consider A appending file X and B removing a file described by manifest M1. +Assume the relevant transformation inputs remain compatible across the rebase, +and a concurrent commit adds an unrelated manifest M3. + +| Stage | First attempt | After rebase | +| --- | --- | --- | +| Catalog base | `M1, M2` | `M3, M1, M2` | +| A: append X | Write `MX`; assemble `MX, M1, M2` | Reuse `MX`; assemble `MX, M3, M1, M2` | +| B: remove file | Transform `M1 → M1′`; retain `MX, M2` | Reuse valid results for `M1, MX, M2`; evaluate `M3` | +| B: resulting set | `MX, M1′, M2` | `MX, M3, M1′, M2` if M3 is unaffected | +| Snapshot construction | A and B each write a manifest list | A and B each write a new manifest list | + +A and B replay in full, but neither necessarily rewrites its materialized +manifests. B consumes A's current output, not A's previous attempt's complete +manifest set. If relevant inputs change or validation finds a conflict, the +affected work is recomputed or the action fails according to its semantics. + +## 6. Artifact Ownership and Terminal Cleanup + +Producers create metadata before catalog commit succeeds. Each producer must +track its generated artifacts, including those written by owned helper +components. Caller-provided data or delete files do not become producer-owned +merely because the action references them. + +Generated metadata paths are write-once: a new attempt must not overwrite files +from an earlier attempt. Reused artifacts keep their existing paths. Artifacts +are retained during retry and considered for cleanup only when the transaction +reaches a terminal result. + +```rust +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CommitStatus { + /// The catalog confirmed that the commit was applied. + Committed, + /// The transaction definitively did not commit and will not retry. + Failed, + /// The commit request was submitted, but its outcome could not be + /// resolved: the catalog may or may not have applied it. + /// + /// Example: every action executed and validated successfully, but the + /// connection failed while awaiting the catalog's response to + /// `update_table`. + Unknown, +} +``` + +| Status | Meaning | Cleanup | +| --- | --- | --- | +| `Committed` | The transaction committed successfully | Retain owned artifacts needed by the committed result; clean up the rest | +| `Failed` | The transaction definitively did not commit and will not retry | Clean up owned artifacts | +| `Unknown` | The catalog may have committed, but the result is unresolved | Delete nothing | + +The transaction/catalog layer determines this classification. Producers consume +it rather than independently interpreting catalog errors. Once the retry loop +terminates, every entry receives `cleanup`, including entries that wrote +files before a later action failed. + +For `Committed`, the supplied table reflects the accepted metadata and supports +reachability checks. Cleanup must account for all snapshots and references +retained by the committed transaction, not only its final current snapshot; if +reachability cannot be established safely, cleanup retains the artifact. +For `Failed`, no reachability reasoning is required: the transaction +definitively did not commit, so producer-owned artifacts are deletable. +For `Unknown`, the supplied table is available execution context only — it is +not proof that the commit failed or that a potentially committed artifact is +unreachable — and cleanup deletes nothing. + +`Unknown` arises only at the catalog boundary, after local work has succeeded: +the commit request was submitted, but its result could not be verified — for +example, a network failure while awaiting the catalog's response. The catalog +layer should make a best-effort attempt to resolve the true outcome before +reporting `Unknown` (for example through idempotent request handling or by +re-checking table state), so `Unknown` is a last resort, not a routine result. +When it is reported, the transaction fails with an error and must not be +retried: retrying could commit a second time if the original request landed, +and cleaning up could delete metadata that is now live. + +Cleanup is best-effort and must not change the already-determined transaction +result. Failures are reported without turning a successful commit into a failed +one. Process crashes and abandoned transactions may still leave orphans; orphan +cleanup remains the safety net. Resolving unknown catalog outcomes is deferred. + +## 7. Alternatives Considered + +**Attempt-scoped producer borrowing a separate retry-state struct.** An earlier +draft kept the producer attempt-scoped and had it borrow a long-lived +retry-state bundle owned by the action entry. Field-for-field, that bundle +converges on exactly what a persistent producer holds, while adding a borrow +split and a second struct. The persistent producer expresses the same boundary +more simply: its fields are retry-persistent, and values derived from one base +are locals of one attempt or validity-guarded cache entries. + +**Java-style producer inheritance and hook traits.** The previous attempt-scoped +`SnapshotProducer` used hook traits (`SnapshotProduceOperation`, +`ManifestProcess`) to let operations inject behavior into a generic pipeline — +a template-method pattern standing in for Java's class hierarchy. With concrete +sibling producers whose attempt logic owns the pipeline and takes operation +inputs as arguments, the hooks have no remaining role, and no base type or +internal locking is needed. + +**Producer-typed action state.** `TransactionAction::State` could have been +bounded by a snapshot-producer trait. But many actions are stateless or not +snapshot-producing, and no transaction-level code needs to know that a state is +a producer: replay calls `commit`, terminal cleanup calls `cleanup`, both on +the action. A generic `State` subsumes the producer case without vacuous impls. + +## 8. Adoption Plan and Deferred Decisions + +### 8.1 Implementation Plan + +Implement the architecture incrementally along its dependency boundaries: + +1. **Stateful actions and replay.** Introduce the associated state, action-entry + ownership, and fresh-state cloning. Migrate stateless actions with `State = ()` + and verify that retry retains state while rebuilding attempt-local results. + +2. **Persistent simple snapshot production.** Extract stateless helpers, + introduce `SimpleSnapshotProducer`, and migrate FastAppend. Remove the old + attempt-scoped producer and obsolete hook traits once consumers are migrated. + +3. **Merging and artifact lifecycle.** Introduce `MergingSnapshotProducer`, + manifest filtering, and reusable added-manifest production. Integrate terminal + status reporting and cleanup for both producers. + +4. **Validation capabilities.** Add reusable history processing and shared + validation operations. Specify concrete predicates and their semantics in + companion work. + +5. **End-to-end operations.** Wire RowDelta intent and validation controls into + the producer, then extend to other snapshot operations. Verify multi-action + replay, changed transaction-local snapshot representations, valid manifest + reuse, and cleanup under success, definite failure, and unknown outcomes. + +These are implementation milestones, not mandatory one-to-one PR boundaries. +The goal is to establish and exercise the shared architecture through concrete +operations, without requiring every optimization or operation up front. + +### 8.2 Deferred Implementation Decisions + +The following do not need to be fixed to agree on this architecture: + +- Shared state structs such as `CommitIdentity`, and eager/lazy UUID generation. +- Cache representations, keys, bounds, eviction, and placement in lower layers. +- Manifest-filter internals and per-spec bookkeeping. +- Conflict predicates, isolation levels, and operation-specific validation APIs. +- Manifest merging/organization and reuse of those results. +- Whole-attempt reuse, unknown-outcome resolution, REST idempotency integration, + and cross-process retry state. + +Implementations may evolve these details while preserving the lifecycle, +ownership, replay, reuse, and cleanup boundaries defined above. diff --git a/rust-toolchain.toml b/rust-toolchain.toml index ff7d1f7fbb..359cdaf68d 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -20,5 +20,5 @@ # # The channel is exactly same day for our MSRV. [toolchain] -channel = "nightly-2025-03-28" +channel = "nightly-2026-04-16" components = ["rustfmt", "clippy"] diff --git a/scripts/dependencies.py b/scripts/dependencies.py deleted file mode 100755 index 56c83a1796..0000000000 --- a/scripts/dependencies.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, REMAINDER -import subprocess -import os - - -def get_git_root(): - try: - return ( - subprocess.check_output(["git", "rev-parse", "--show-toplevel"]) - .decode() - .strip() - ) - except subprocess.CalledProcessError: - raise RuntimeError("Not a git repository (or git not installed)") - - -def find_cargo_dirs(): - repo_root = get_git_root() - cargo_dirs = [] - for root, dirs, files in os.walk(repo_root): - # Skip the current directory - if os.path.abspath(root) == os.path.abspath(repo_root): - continue - if "Cargo.toml" in files: - cargo_dirs.append(root) - return cargo_dirs - - -def check_deps(): - cargo_dirs = find_cargo_dirs() - for root in cargo_dirs: - print(f"Checking dependencies of {root}") - subprocess.run(["cargo", "deny", "check", "license"], cwd=root, check=True) - - -def generate_deps(): - cargo_dirs = find_cargo_dirs() - for root in cargo_dirs: - print(f"Generating dependencies {root}") - try: - result = subprocess.run( - ["cargo", "deny", "list", "-f", "tsv", "-t", "0.6"], - cwd=root, - capture_output=True, - text=True, - check=True, - ) - except subprocess.CalledProcessError as e: - raise RuntimeError( - f"Failed to run 'cargo deny' in '{root}'. " - f"Is it installed?\n\nSTDERR:\n{e.stderr}" - ) from e - - with open(f"{root}/DEPENDENCIES.rust.tsv", "w") as f: - f.write(result.stdout) - - -if __name__ == "__main__": - parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) - parser.set_defaults(func=parser.print_help) - subparsers = parser.add_subparsers() - - parser_check = subparsers.add_parser( - "check", description="Check dependencies", help="Check dependencies" - ) - parser_check.set_defaults(func=check_deps) - - parser_generate = subparsers.add_parser( - "generate", description="Generate dependencies", help="Generate dependencies" - ) - parser_generate.set_defaults(func=generate_deps) - - args = parser.parse_args() - arg_dict = dict(vars(args)) - del arg_dict["func"] - args.func(**arg_dict) diff --git a/scripts/release.sh b/scripts/release.sh deleted file mode 100755 index 1f66b35911..0000000000 --- a/scripts/release.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ -z "${ICEBERG_VERSION}" ]; then - echo "ICEBERG_VERSION is unset" - exit 1 -else - echo "var is set to '$ICEBERG_VERSION'" -fi - -# tar source code -release_version=${ICEBERG_VERSION} -# rc versions -rc_version="${ICEBERG_VERSION_RC:-rc.1}" -# Corresponding git repository branch -git_branch=release-${release_version}-${rc_version} - -rm -rf dist -mkdir -p dist/ - -echo "> Checkout version branch" -git checkout -B "${git_branch}" - -# Run a few checks -echo "> Check license" -docker run -it --rm -v $(pwd):/github/workspace apache/skywalking-eyes header check - -echo "> Run dependency license check using cargo-deny" -python3 ./scripts/dependencies.py check - -# Generate and verify artifacts -echo "> Start package" -git archive --format=tar.gz --output="dist/apache-iceberg-rust-$release_version-src.tar.gz" --prefix="apache-iceberg-rust-$release_version-src/" --add-file=Cargo.toml "$git_branch" - -cd dist - -echo "> Generate signature" -for i in *.tar.gz; do - echo "$i" - gpg --armor --output "$i.asc" --detach-sig "$i" -done -echo "> Check signature" -for i in *.tar.gz; do - echo "$i" - gpg --verify "$i.asc" "$i" -done -echo "> Generate sha512sum" -for i in *.tar.gz; do - echo "$i" - sha512sum "$i" >"$i.sha512" -done -echo "> Check sha512sum" -for i in *.tar.gz; do - echo "$i" - sha512sum --check "$i.sha512" -done diff --git a/scripts/verify.py b/scripts/verify.py deleted file mode 100755 index 415bb243a2..0000000000 --- a/scripts/verify.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -import subprocess -import sys -import os - -BASE_DIR = os.getcwd() - - -def check_rust(): - try: - subprocess.run(["cargo", "--version"], check=True) - return True - except FileNotFoundError: - return False - except Exception as e: - raise Exception("Check rust met unexpected error", e) - -def build_core(): - print("Start building iceberg rust") - - subprocess.run(["cargo", "build", "--release"], check=True) - -def main(): - if not check_rust(): - print( - "Cargo is not found, please check if rust development has been setup correctly" - ) - print("Visit https://www.rust-lang.org/tools/install for more information") - sys.exit(1) - - build_core() - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/website/README.md b/website/README.md index 9914b82226..8d0f50345d 100644 --- a/website/README.md +++ b/website/README.md @@ -19,22 +19,10 @@ # Iceberg Rust Website -## Setup - -Install mdbook first - -```shell -cargo install mdbook -``` - ## Preview -```shell -mdbook serve -``` - -## Build +This will automatically install mdbook if needed. ```shell -mdbook build +make site ``` diff --git a/website/src/SUMMARY.md b/website/src/SUMMARY.md index da82364f58..dfe0043ffd 100644 --- a/website/src/SUMMARY.md +++ b/website/src/SUMMARY.md @@ -32,6 +32,5 @@ # Reference -- [Using Podman instead of Docker](./reference/podman.md) -- [OrbStack as a docker alternative](./reference/orbstack.md) +- [Container Runtimes](./reference/container-runtimes.md) - [Setup GPG key](./reference/setup_gpg.md) diff --git a/website/src/api.md b/website/src/api.md index 27a68cbdcf..acf0270413 100644 --- a/website/src/api.md +++ b/website/src/api.md @@ -24,6 +24,9 @@ * Create and list namespaces. * Create, load, and drop tables +`SessionCatalog` offers the same operations, but takes a `SessionContext` with each call instead of +binding one for the lifetime of the catalog. It is currently implemented by the REST catalog only. + There is support for the following catalogs: | Catalog | Description | @@ -37,7 +40,16 @@ There is support for the following catalogs: Not all catalog implementations are complete. -## `RestCatalog` +## REST catalog APIs + +The REST catalog provides two interfaces: + +* `RestCatalog` implements `Catalog` and binds one `SessionContext` to every operation. +* `RestSessionCatalog` implements `SessionCatalog` and accepts a context with each operation. + +REST configuration, authentication, and the HTTP client are shared by the catalog instance. + +### `RestCatalog` Here is an example of how to create a `RestCatalog`: @@ -52,10 +64,32 @@ You can run following code to list all root namespaces: ``` Then you can run following code to create namespace: + ```rust,no_run,noplayground {{#rustdoc_include ../../crates/examples/src/rest_catalog_namespace.rs:create_namespace}} ``` +### `RestSessionCatalog` + +Use `RestSessionCatalog` when the caller supplies a session context for each operation. Build the +catalog and the context it should run under: + +```rust,no_run,noplayground +{{#rustdoc_include ../../crates/examples/src/rest_session_catalog_namespace.rs:create_catalog}} +``` + +Every operation then takes that context. You can run following code to list all root namespaces: + +```rust,no_run,noplayground +{{#rustdoc_include ../../crates/examples/src/rest_session_catalog_namespace.rs:list_all_namespace}} +``` + +Then you can run following code to create namespace: + +```rust,no_run,noplayground +{{#rustdoc_include ../../crates/examples/src/rest_session_catalog_namespace.rs:create_namespace}} +``` + # Table After creating `Catalog`, we can manipulate tables through `Catalog`. diff --git a/website/src/download.md b/website/src/download.md index fa29d98177..5158d7e94e 100644 --- a/website/src/download.md +++ b/website/src/download.md @@ -23,8 +23,7 @@ The official Apache Iceberg-Rust releases are provided as source artifacts. ## Releases -The latest source release is [0.6.0](https://www.apache.org/dyn/closer.lua/iceberg/apache-iceberg-rust-0.6.0/apache-iceberg-rust-0.6.0-src.tar.gz?action=download) ([asc](https://downloads.apache.org/iceberg/apache-iceberg-rust-0.6.0/apache-iceberg-rust-0.6.0-src.tar.gz.asc), -[sha512](https://downloads.apache.org/iceberg/apache-iceberg-rust-0.6.0/apache-iceberg-rust-0.6.0-src.tar.gz.sha512)). +Find the latest source release at the [Apache CDN](https://dlcdn.apache.org/iceberg/). For release notes, see the [GitHub releases page](https://github.com/apache/iceberg-rust/releases). For older releases, please check the [archive](https://archive.apache.org/dist/iceberg/). diff --git a/website/src/reference/podman.md b/website/src/reference/container-runtimes.md similarity index 53% rename from website/src/reference/podman.md rename to website/src/reference/container-runtimes.md index eee93a0b35..412aa70053 100644 --- a/website/src/reference/podman.md +++ b/website/src/reference/container-runtimes.md @@ -17,18 +17,74 @@ ~ under the License. --> -# Using Podman instead of Docker +# Container Runtimes -Iceberg-rust does not require containerization, except for integration tests, where "docker" and "docker-compose" are used to start containers for minio and various catalogs. Below instructions setup "rootful podman" and docker's official docker-compose plugin to run integration tests as an alternative to docker or Orbstack. +Iceberg-rust uses containers for integration tests, where `docker` and `docker compose` start containers for an S3-compatible object store (RustFS) and various catalogs. You can use any of the following container runtimes. + +## Docker Desktop + +[Docker Desktop](https://www.docker.com/products/docker-desktop/) is available for macOS, Windows, and Linux. + +1. Install Docker Desktop by downloading the [installer](https://www.docker.com/products/docker-desktop/) or using Homebrew on macOS. + ```shell + brew install --cask docker + ``` + +2. Launch Docker Desktop and complete the setup. + +3. Verify the installation. + ```shell + docker --version + docker compose version + ``` + +4. Try some integration tests! + ```shell + make test + ``` + +## OrbStack (macOS) + +[OrbStack](https://orbstack.dev/) is a lightweight alternative to Docker Desktop on macOS. + +1. Install OrbStack by downloading the [installer](https://orbstack.dev/download) or using Homebrew. + ```shell + brew install orbstack + ``` + +2. Migrate Docker data (if switching from Docker Desktop). + ```shell + orb migrate docker + ``` + +3. (Optional) Add registry mirrors. + + You can edit the config directly at `~/.orbstack/config/docker.json` and restart the engine with `orb restart docker`. + + ```json + { + "registry-mirrors": [""] + } + ``` + +4. Try some integration tests! + ```shell + make test + ``` + +## Podman + +[Podman](https://podman.io/) is a daemonless container engine. The instructions below set up "rootful podman" with docker's official docker-compose plugin. 1. Have podman v4 or newer. ```shell $ podman --version podman version 4.9.4-rhel ``` + 2. Create a docker wrapper script: - * Create a fresh `/usr/bin/docker` file and add the below contents: + Create a fresh `/usr/bin/docker` file and add the below contents: ```bash #!/bin/sh [ -e /etc/containers/nodocker ] || \ @@ -36,7 +92,7 @@ Iceberg-rust does not require containerization, except for integration tests, wh exec sudo /usr/bin/podman "$@" ``` - * Set new `/usr/bin/docker` file to executable. + Set new `/usr/bin/docker` file to executable. ```shell sudo chmod +x /usr/bin/docker ``` @@ -78,23 +134,24 @@ Iceberg-rust does not require containerization, except for integration tests, wh cargo test -p iceberg --test file_io_s3_test ``` -# References +### Note on rootless containers -* -* +As of podman v4, ["To be succinct and simple, when running rootless containers, the container itself does not have an IP address"](https://www.redhat.com/sysadmin/container-ip-address-podman). This causes issues with iceberg-rust's integration tests, which rely upon IP-addressable containers via docker-compose. As a result, podman "rootful" containers are required to ensure containers have IP addresses. + +### Podman troubleshooting + +**Error:** `short-name "apache/iceberg-rest-fixture" did not resolve to an alias and no unqualified-search registries are defined in "/etc/containers/registries.conf"` -# Note on rootless containers +**Fix:** Add or modify the `/etc/containers/registries.conf` file: +```toml +[[registry]] +prefix = "docker.io" +location = "docker.io" +``` -As of podman v4, ["To be succinct and simple, when running rootless containers, the container itself does not have an IP address"](https://www.redhat.com/sysadmin/container-ip-address-podman) This causes issues with iceberg-rust's integration tests, which rely upon ip-addressable containers via docker-compose. As a result, podman "rootful" containers are required throughout to ensure containers have IP addresses. Perhaps as a future work or with updates to default podman networking, the need for "rootful" podman containers can be eliminated. +### Podman references +* +* * * - -# Debugging Note: - - Fix for error: `Error: short-name "apache/iceberg-rest-fixture" did not resolve to an alias and no unqualified-search registries are defined in "/etc/containers/registries.conf"` - - Add or modify the `/etc/containers/registries.conf` file: - ```toml - [[registry]] - prefix = "docker.io" - location = "docker.io" - ``` diff --git a/website/src/reference/orbstack.md b/website/src/reference/orbstack.md deleted file mode 100644 index cb00849d6b..0000000000 --- a/website/src/reference/orbstack.md +++ /dev/null @@ -1,39 +0,0 @@ - - -# OrbStack as a docker alternative on macOS -1. Install OrbStack by downloading [installer](https://orbstack.dev/download) or using Homebrew. - ```shell - brew install orbstack - ``` - -2. Migrate Docker data - ```shell - orbstack migrate docker - ``` - -3. (Optional) Add registry mirrors - - You can edit the config directly at `~/.orbstack/config/docker.json` and restart the engine with `orb restart docker`. - - ``` - { - "registry-mirrors": [""] - } - ``` \ No newline at end of file diff --git a/website/src/release.md b/website/src/release.md index 28b98f1700..61ede8502f 100644 --- a/website/src/release.md +++ b/website/src/release.md @@ -17,7 +17,7 @@ ~ under the License. --> -This document mainly introduces how the release manager releases a new version in accordance with the Apache requirements. +This document explains how the release process works for Apache Iceberg Rust in accordance with Apache requirements. ## Introduction @@ -31,29 +31,62 @@ This guide complements the foundation-wide policies and guides: - [Release Distribution Policy](https://infra.apache.org/release-distribution) - [Release Creation Process](https://infra.apache.org/release-publishing.html) -## Some Terminology of release +## Terminology -In the context of our release, we use several terms to describe different stages of the release process. +In this guide: -Here's an explanation of these terms: +- `iceberg_version`: the final Iceberg Rust version, like `0.9.1`. +- `iceberg_minor_release_branch`: the release branch for the minor version, like `0.9.x`. +- `rc`: the numeric release candidate voting round, like `2`. +- `rc_tag`: the git tag for a release candidate, like `v0.9.1-rc.2`. +- `rc_dist_dir`: the ASF dev distribution directory, like `apache-iceberg-rust-0.9.1-rc2`. +- `source_archive`: the source tarball, like `apache-iceberg-rust-0.9.1.tar.gz`. -- `iceberg_version`: the version of Iceberg to be released, like `0.2.0`. -- `rc_version`: the minor version for voting round, like `rc.1`. -- `release_version`: the version of release candidate, like `0.2.0-rc.1`. +The RC tag includes `rc.`. The ASF dev distribution directory uses `rc`. The source archive name uses only the final version. + +## Release Manager + +The release manager is the person taking ownership of a particular Iceberg Rust release. + +This is usually a committer for the Apache Iceberg project, however the role can be supported by non-committers in the early stages of coordinating a release. +Where this can be supported by a non-committer, this will be mentioned in the guide. ## Preparation
-This section is the requirements for individuals who are new to the role of release manager. +This section is the requirements for committers or PMC members who are new to the role of release manager.
Refer to [Setup GPG Key](reference/setup_gpg.md) to make sure the GPG key has been set up. +The RC creation script requires a local GPG secret key when artifact signing or tag creation is enabled. + +Install the release tooling used by the local scripts: + +- `cargo-deny` (`make install-cargo-deny`) +- `docker` +- `gpg` +- `svn` + +The local release helpers are under `dev/release/`. They log every step before it runs and after it succeeds. If a step fails, the script prints the failed step and stops. + +## How to propose a new release + +Ultimately, it is up to the community on when to cut a new release. +You can propose this via the [Apache Iceberg dev mailing list](https://lists.apache.org/list.html?dev@iceberg.apache.org) +or simply open a tracking issue, mentioned later in this guide. ## Start a tracking issue about the next release -Start a tracking issue on GitHub for the upcoming release to track all tasks that need to be completed. +Start a tracking issue on GitHub for the upcoming release to track all the changes needed to be merged/addressed for the release. + +If you are acting as release manager, you should own this issue and coordinate with the community when to target a new release and what changes need to be included. +You do not need to be a committer to create and own the tracking issue. + +### Template + +You may use the template below to create the issue. Title: @@ -66,168 +99,265 @@ Content: ```markdown This issue is used to track tasks of the iceberg rust ${iceberg_version} release. -## Tasks - ### Blockers -> Blockers are the tasks that must be completed before the release. +> Blockers are the tasks, such as bugs, that should be completed before the release. + +- TBD + +### Community-desired features / changes + +> These are features or other changes that the community would prefer to go into the next release, +> but should not block it. + +- TBD -### Build Release +### Release Guide -#### GitHub Side +For details on how to run a release, please refer to: https://rust.iceberg.apache.org/release +``` + +## GitHub Side -- [ ] Bump version in project -- [ ] Update docs -- [ ] Generate dependencies list -- [ ] Push release candidate tag to GitHub +The following steps should be followed once the release is ready to begin. -#### ASF Side +### Update dependencies -- [ ] Create an ASF Release -- [ ] Upload artifacts to the SVN dist repo +Dependabot runs monthly to reduce update noise, so dependencies may be out of date by the time of a release. +Before cutting a release candidate, open the repository's [Dependabot page](https://github.com/apache/iceberg-rust/network/updates) and trigger a manual check for each configured ecosystem: `cargo` and `github-actions` at `/`, and `uv` at `/bindings/python`. +Review and merge the applicable dependency update PRs before creating the release branch. +For an existing release branch, backport the applicable updates to that branch before cutting the release candidate. -### Voting +Only a committer with write access to the repository can trigger Dependabot manually. +Work with a committer if you do not have write access. -- [ ] Start VOTE at iceberg community +### Create a minor version release branch, if it doesn't already exist -### Official Release +A committer must ensure that a release branch exists for each minor version. -- [ ] Push the release git tag -- [ ] Publish artifacts to SVN RELEASE branch -- [ ] Change Iceberg Rust Website download link -- [ ] Send the announcement +If the release is for an existing minor version (such as the `.1` patch release in `v0.10.1`), the release branch should already exist with the name `v0.10.x`. -For details of each step, please refer to: https://rust.iceberg.apache.org/release +If the release is for a new minor version (such as the `.10` release in `v0.10.0`), you should create the release branch now. + +``` +git switch -c ${iceberg_minor_release_branch} && git push upstream ${iceberg_minor_release_branch} ``` -## GitHub Side +### Draft GitHub release -### Bump version in project +- [Draft a new GitHub Release using the GitHub web UI](https://github.com/apache/iceberg-rust/releases/new). +- Enter the git tag of this release version, of the form `v0.y.z`. For example, `v0.9.0`. + It is correct that the tag does not exist at this stage, as it will be created later in the release process. +- Use the minor version branch created earlier as the branch target. For example, `v0.9.x`. +- Save the draft. -Bump all components' version in the project to the new iceberg version. -Please note that this version is the exact version of the release, not the release candidate version. +### Update crate versions and changelog -- rust core: bump version in `Cargo.toml` +The following changes can be made in one pull request against the minor version branch (e.g. `v0.9.x`). -### Update docs +#### Bump crate versions -- Update `CHANGELOG.md` by Drafting a new release [note on Github Releases](https://github.com/apache/iceberg-rust/releases/new) +Bump all components' version in the project to the new Iceberg Rust version. +This version is the final version, not the release candidate version. -### Generate dependencies list +- Rust core and Python binding: bump version in root `Cargo.toml` under `[workspace.package]`. -Download and setup `cargo-deny`. You can refer to [cargo-deny](https://embarkstudios.github.io/cargo-deny/cli/index.html). +If you are preparing changes for later release candidates (2+), bumping crate versions should not be necessary. -Run `python3 ./scripts/dependencies.py generate` to update the dependencies list of every package. +#### Update CHANGELOG.md -Run `python3 ./scripts/dependencies.py check` to verify the updated dependencies' license. +Update `CHANGELOG.md` based on the changes since the previous version. +You may use generative AI to assist making this update, but please review the proposed changes for correctness. +The changelog should reflect a summary of each commit in the new release. -### Push release candidate tag +#### Verify dependency licenses -After bump version PR gets merged, we can create a GitHub release for the release candidate: +Run the following command to verify the licenses meet the project's policy. -- Create a tag at `main` branch on the `Bump Version` / `Patch up version` commit: `git tag -s "v${release_version}"`, please correctly check out the corresponding commit instead of directly tagging on the main branch. -- Push tags to GitHub: `git push origin "v${release_version}"`. +```shell +make check-dependency-licenses +``` -## ASF Side +#### Check pyiceberg-core LICENSE and NOTICE + +The `pyiceberg-core` wheels bundle third-party licenses and notices, and some of the entries are maintained by hand. +Check and update them following [`bindings/python/LICENSING.md`](https://github.com/apache/iceberg-rust/blob/main/bindings/python/LICENSING.md#keeping-entries-up-to-date). -If any step in the ASF Release process fails and requires code changes, -we will abandon that version and prepare for the next one. -Our release page will only display ASF releases instead of GitHub Releases. +#### Open pull request -### Create an ASF Release +Open a pull request with all the changes. +The release branch should be used as the base for the pull request. -After GitHub Release has been created, we can start to create ASF Release. +### Create release candidate tag and artifacts -- Checkout to released tag. (e.g. `git checkout v0.2.0-rc.1`, tag is created in the previous step) -- Use the release script to create a new release: `ICEBERG_VERSION= ICEBERG_VERSION_RC= ./scripts/release.sh`(e.g. `ICEBERG_VERSION=0.2.0 ICEBERG_VERSION_RC=rc.1 ./scripts/release.sh`) - - This script will do the following things: - - Create a new branch named by `release-${release_version}` from the tag - - Generate the release candidate artifacts under `dist`, including: - - `apache-iceberg-rust-${release_version}-src.tar.gz` - - `apache-iceberg-rust-${release_version}-src.tar.gz.asc` - - `apache-iceberg-rust-${release_version}-src.tar.gz.sha512` - - Check the header of the source code. This step needs docker to run. +This step must be completed by a committer. -This script will create a new release under `dist`. +After the version bump PR gets merged, check out the exact commit to release and run: + +```shell +dev/release/create_rc.sh ${iceberg_version} ${rc} +``` For example: ```shell -> tree dist -dist -├── apache-iceberg-rust-0.2.0-src.tar.gz -├── apache-iceberg-rust-0.2.0-src.tar.gz.asc -└── apache-iceberg-rust-0.2.0-src.tar.gz.sha512 +dev/release/create_rc.sh 0.9.1 2 +``` + +The script always archives and tags `HEAD`, so make sure the exact commit to release is checked out before running it. + +Useful options include: + +- `--dist_dir dist`: artifact output root. +- `--create_rc_tag 1`: create the signed annotated RC tag as the final release step. +- `--check_headers 1`: check Apache license headers against the source archive. +- `--check_deps 1`: run dependency license checks before artifact creation. +- `--check_publish 1`: dry-run publishing every crate to crates.io before artifact creation. +- `--sign 1`: create and verify the detached GPG signature. +- `--upload_svn 0`: upload RC artifacts to the ASF dev dist SVN repository. +- `--svn_dist_url https://dist.apache.org/repos/dist/dev/iceberg`: SVN directory URL where the RC artifact directory will be uploaded. + +This script creates: + +- Local artifact directory: `dist/apache-iceberg-rust-${iceberg_version}-rc${rc}/` +- Source archive: `apache-iceberg-rust-${iceberg_version}.tar.gz` +- Signature: `apache-iceberg-rust-${iceberg_version}.tar.gz.asc` +- SHA-512 checksum: `apache-iceberg-rust-${iceberg_version}.tar.gz.sha512` +- Signed annotated RC tag: `v${iceberg_version}-rc.${rc}` + +The script runs a number of verifications. +It performs a dry-run packaging step and builds every crate as the final release would, so a crate that cannot be published fails RC creation. +It also checks license headers against the generated source archive. +If enabled, SVN upload runs after local artifact verification and before RC tag creation. The script creates the signed RC tag as the final release step, then prints a draft VOTE email for `dev@iceberg.apache.org`. + +To upload artifacts to ASF dev dist as part of RC creation, pass: + +```shell +dev/release/create_rc.sh ${iceberg_version} ${rc} --upload_svn 1 +``` + +The script does not push the RC tag. Review the output, then push the tag manually: + +```shell +git push origin "v${iceberg_version}-rc.${rc}" +``` + +If an RC has a problem, abandon that RC and increment the RC number. + +### Trigger release candidate PyPI publish + +Python packages based on the release candidate are published to PyPI. +These are published by a workflow that must be triggered manually. + +Trigger this now using the following GitHub CLI command, or the equivalent on the GitHub website. +It must use the published release tag as the reference for the workflow run. + +```shell +gh workflow run --repo apache/iceberg-rust release_python.yml -f release_tag=${rc_tag} --ref refs/tags/${rc_tag} +``` + +### Draft an Apache Iceberg blog post + +The [Apache Iceberg blog](https://iceberg.apache.org/blog/) is one mechanism to share news about the project, +however it is not a strict requirement for any release. +Blog posts are published on the website, and shared via the project's social media. + +A blog post is typically published covering the whole minor release (i.e. `v0.9`), and communicates the changes since the last minor version. +It may be updated for patch versions if deemed appropriate. +For example, the [0.10 blog post was updated](https://iceberg.apache.org/blog/apache-iceberg-rust-0.10.0-release/#patch-releases) to communicate why `0.10.1` was released. + +If someone would like to volunteer to author this blog post, now is a good time to draft it based on the proposed changes to be released. +The blog post should be a curated summary of the biggest changes to go in. +Drafting the initial body of the post by hand is recommended, as this allows for a more authentic contribution and avoids mistakes where generative AI may summarize changes incorrectly. + +Blog posts are created by opening a PR to the main Apache Iceberg repository in the [`site/docs/blog/posts` directory](https://github.com/apache/iceberg/tree/main/site/docs/blog/posts). +It should not be merged until the release has completed. + +## ASF Side + +All ASF-side steps must be performed by a project committer. + +**If any step in the ASF release process fails and requires code changes, abandon that RC and prepare a new RC number.** + +Our release page displays ASF releases instead of GitHub Releases. + +### Verify the release candidate locally + +Before uploading artifacts to ASF dev dist, verify the local artifacts: + +```shell +dev/release/verify_rc.sh ${iceberg_version} ${rc} --download 0 +``` + +To skip expensive build steps during a quick local check: + +```shell +dev/release/verify_rc.sh ${iceberg_version} ${rc} --download 0 --build 0 --python 0 ``` ### Upload artifacts to the SVN dist repo SVN is required for this step. -The svn repository of the dev branch is: +The SVN repository of the dev branch is: -First, checkout Iceberg to local directory: +First, check out Iceberg to a local directory: ```shell -# As this step will copy all the versions, it will take some time. If the network is broken, please use svn cleanup to delete the lock before re-execute it. svn co https://dist.apache.org/repos/dist/dev/iceberg/ /tmp/iceberg-dist-dev ``` -Then, upload the artifacts: - -> The `${release_version}` here should be like `0.2.0-rc.1` +If the artifacts were not uploaded by `dev/release/create_rc.sh --upload_svn 1`, upload them manually: ```shell -# create a directory named by version -mkdir /tmp/iceberg-dist-dev/apache-iceberg-rust-${release_version}/ +rc_dist_dir="apache-iceberg-rust-${iceberg_version}-rc${rc}" -# copy source code and signature package to the versioned directory -cp ./dist/* /tmp/iceberg-dist-dev/apache-iceberg-rust-${release_version}/ +mkdir "/tmp/iceberg-dist-dev/${rc_dist_dir}/" +cp "./dist/${rc_dist_dir}/"* "/tmp/iceberg-dist-dev/${rc_dist_dir}/" -# change dir to the svn folder cd /tmp/iceberg-dist-dev/ - -# check svn status svn status - -# add to svn -svn add apache-iceberg-rust-${release_version} - -# commit to SVN remote server -svn commit -m "Prepare for iceberg-rust ${release_version}" +svn add "${rc_dist_dir}" +svn commit -m "Prepare Apache Iceberg Rust ${iceberg_version} RC${rc}" ``` Visit to make sure the artifacts are uploaded correctly. +### Verify the uploaded release candidate + +After uploading the artifacts, verify them from ASF dev dist: + +```shell +dev/release/verify_rc.sh ${iceberg_version} ${rc} +``` + ### Rescue -If you accidentally published wrong or unexpected artifacts, like wrong signature files, wrong sha256 files, -please cancel the release for the current `release_version`, -_increase th RC counting_ and re-initiate a release with the new `release_version`. -And remember to delete the wrong artifacts from the SVN dist repo. +If you accidentally publish wrong or unexpected artifacts, like wrong signature files or checksum files, cancel the current RC, increment the RC number, and initiate a new release candidate. +Remember to delete the wrong artifacts from the SVN dist repo. ## Voting -Iceberg Community Vote should send email to: : +Send the Iceberg community VOTE email to . Title: ``` -[VOTE] Release Apache Iceberg Rust ${iceberg_version} ${rc_version} +[VOTE] Release Apache Iceberg Rust ${iceberg_version} RC${rc} ``` Content: ``` -Hello, Apache Iceberg Rust Community, +Hello Apache Iceberg Rust Community, -This is a call for a vote to release Apache Iceberg rust version ${iceberg_version}. +This is a call for a vote to release Apache Iceberg Rust version ${iceberg_version}. -The tag to be voted on is v${release_version}. +The tag to be voted on is: v${iceberg_version}-rc.${rc}. The release candidate: -https://dist.apache.org/repos/dist/dev/iceberg/apache-iceberg-rust-${release_version}/ +https://dist.apache.org/repos/dist/dev/iceberg/apache-iceberg-rust-${iceberg_version}-rc${rc}/ Keys to verify the release candidate: @@ -235,51 +365,52 @@ https://downloads.apache.org/iceberg/KEYS Git tag for the release: -https://github.com/apache/iceberg-rust/releases/tag/v${release_version} +https://github.com/apache/iceberg-rust/releases/tag/v${iceberg_version}-rc.${rc} -Please download, verify, and test. +Please download, verify, and test the release candidate. -The VOTE will be open for at least 72 hours and until the necessary -number of votes are reached. +This vote will be open for at least 72 hours and will remain open until the required number of votes is reached. -[ ] +1 approve -[ ] +0 no opinion -[ ] -1 disapprove with the reason +Please vote accordingly: +[ ] +1 Approve +[ ] +0 No opinion +[ ] -1 Disapprove (please provide a reason) -To learn more about Apache Iceberg, please see https://rust.iceberg.apache.org/ +To learn more about Apache Iceberg Rust, please visit: +https://rust.iceberg.apache.org/ Checklist for reference: - -[ ] Download links are valid. -[ ] Checksums and signatures. -[ ] LICENSE/NOTICE files exist -[ ] No unexpected binary files +[ ] Download links are valid +[ ] Checksums and signatures are correct +[ ] LICENSE and NOTICE files are present +[ ] No unexpected binary files are included [ ] All source files have ASF headers -[ ] Can compile from source +[ ] The project builds successfully from source +[ ] pyiceberg-core builds and tests successfully -More details please refer to https://rust.iceberg.apache.org/release.html#how-to-verify-a-release. - -Thanks +For more details, please refer to: +https://rust.iceberg.apache.org/release.html#how-to-verify-a-release +Thanks, ${name} ``` Example: -After at least 3 `+1` binding vote (from Iceberg PMC member), claim the vote result: +After at least 72 hours and 3 `+1` binding votes from Iceberg PMC members, claim the vote result. Title: ``` -[RESULT][VOTE] Release Apache Iceberg Rust ${iceberg_version} ${rc_version} +[RESULT][VOTE] Release Apache Iceberg Rust ${iceberg_version} RC${rc} ``` Content: ``` -Hello, Apache Iceberg Rust Community, +Hello Apache Iceberg Rust Community, -The vote to release Apache Iceberg Rust ${release_version} has passed. +The vote to release Apache Iceberg Rust ${iceberg_version} RC${rc} has passed. The vote PASSED with 3 +1 binding and 1 +1 non-binding votes, no +0 or -1 votes: @@ -295,8 +426,7 @@ Non-Binding votes: Vote thread: ${vote_thread_url} -Thanks - +Thanks, ${name} ``` @@ -304,85 +434,145 @@ Example: ## How to verify a release -### Validating a source release +### Validate with the helper script + +Run: + +```shell +dev/release/verify_rc.sh ${iceberg_version} ${rc} +``` + +The helper downloads the source archive, signature, and checksum from ASF dev dist, verifies the signature with the local GPG keyring, verifies the checksum, extracts the archive, checks source headers, and runs Rust and Python build/tests. + +To import Apache Iceberg release keys before signature verification, run: + +```shell +dev/release/verify_rc.sh ${iceberg_version} ${rc} --import_gpg_keys 1 +``` + +### Validate manually -A release contains links to following things: +A release candidate contains links to following things: -* A source tarball -* A signature(.asc) -* A checksum(.sha512) +- A source tarball +- A signature (`.asc`) +- A checksum (`.sha512`) After downloading them, here are the instructions on how to verify them. -* Import keys: +- Import keys: ```bash curl https://downloads.apache.org/iceberg/KEYS -o KEYS gpg --import KEYS ``` -* Verify the `.asc` file: - + +- Verify the `.asc` file: + ```bash gpg --verify apache-iceberg-rust-*.tar.gz.asc ``` - Expects: `"apache-iceberg-rust-0.7.0-src.tar.gz: OK"` -* Verify the checksums: - + + Expects: `gpg: Good signature from ...` + +- Verify the checksum: + ```bash shasum -a 512 -c apache-iceberg-rust-*.tar.gz.sha512 ``` - Expects: `gpg: Good signature from ...` -* Verify build and test: - + + Expects: `"apache-iceberg-rust-...tar.gz: OK"` + +- Verify build and test: + ```bash tar -xzf apache-iceberg-rust-*.tar.gz cd apache-iceberg-rust-*/ make build && make test ``` -* Verify license headers: - + +- Verify pyiceberg-core build and tests: + + ```bash + ( + cd bindings/python + make install + make test + ) + ``` + +- Verify license headers: + ```bash - docker run -it --rm -v $(pwd):/github/workspace apache/skywalking-eyes header check + docker run --rm -v $(pwd):/github/workspace apache/skywalking-eyes header check ``` - Expects: `INFO Totally checked _ files, valid: _, invalid: 0, ignored: _, fixed: 0 ` + + Expects: `INFO Totally checked _ files, valid: _, invalid: 0, ignored: _, fixed: 0` ## Official Release -### Push the release git tag +All steps in this section must be performed by a project committer, +except the announcement e-mail which must be performed by a PMC member. + +### Promote the RC + +After the VOTE passes, create the final release tag and move the ASF artifacts from dev dist to release dist: + +```shell +dev/release/release.sh ${iceberg_version} ${rc} +``` + +Useful options include: + +- `--create_release_tag 1`: create the signed annotated final release git tag. +- `--move_svn 1`: move the RC artifacts from ASF dev dist to ASF release dist. +- `--tag_ref `: git commit-ish to tag as the final release. +- `--dev_dist_url https://dist.apache.org/repos/dist/dev/iceberg`: SVN directory URL containing RC artifact directories. +- `--release_dist_url https://dist.apache.org/repos/dist/release/iceberg`: SVN directory URL where final release artifact directories are published. + +The release script does not push the final signed release tag. +Review the output and then manually publish the release tag. ```shell -# Checkout the tags that passed VOTE -git checkout "v${release_version}" -# Tag with the iceberg version -git tag -s "v${iceberg_version}" -# Push tags to github to trigger releases git push origin "v${iceberg_version}" ``` -### Publish artifacts to SVN RELEASE branch +The creation of the final release tag triggers the publish workflow for crates. +Python packages are manually triggered later. +Please verify that the triggered workflows for the crates succeeded. + +New crates are reserved on crates.io when they are added, so the workflow should not fail on a new crate. +If it does, see [if the Publish workflow fails on a missing crate](#if-the-publish-workflow-fails-on-a-missing-crate). +Once all the crates are published, Python publishing should start. + +Python publishing is performed by a GitHub workflow, however the trigger is manual. +Trigger this now using the following GitHub CLI command, or the equivalent on the GitHub website. +It must use the published release tag as the reference for the workflow run. ```shell -svn mv https://dist.apache.org/repos/dist/dev/iceberg/apache-iceberg-rust-${release_version} https://dist.apache.org/repos/dist/release/iceberg/apache-iceberg-rust-${iceberg_version} -m "Release Apache Iceberg Rust ${iceberg_version}" +gh workflow run --repo apache/iceberg-rust release_python.yml -f release_tag=v${iceberg_version} --ref refs/tags/v${iceberg_version} ``` -### Change Iceberg Rust Website download link +Verify that the workflow succeeds, indicating that the Python packages are released. +If there is any issue, you may continue with the release however please note in any relevant steps that the Python package publish failed and open a patch release tracking issue to plan addressing the problem. +Note, this should be an exceptional case. -Update the download link in `website/src/download.md` to the new release version. +### Publish the GitHub Release -### Create a GitHub Release +A GitHub release should have been drafted earlier in the release process. -- Click [here](https://github.com/apache/iceberg-rust/releases/new) to create a new release. -- Pick the git tag of this release version from the dropdown menu. -- Make sure the branch target is `main`. -- Generate the release note by clicking the `Generate release notes` button. -- Add the release note from every component's `upgrade.md` if there are breaking changes before the content generated by GitHub. Check them carefully. -- Publish the release. +Open the GitHub release and update the body with the contents of the changelog. +Then, publish the GitHub release. ### Send the announcement Send the release announcement to `dev@iceberg.apache.org` and CC `announce@apache.org`. -Instead of adding breaking changes, let's include the new features as "notable changes" in the announcement. +You must be a PMC member to send e-mails to `announce@apache.org`. +The e-mail must be plain text. +Disable HTML formatting if your e-mail client enables it by default. + +Instead of adding breaking changes, we include the new features as "notable changes" in the announcement. Title: @@ -401,7 +591,7 @@ that Apache Iceberg Rust ${iceberg_version} has been released! Iceberg is a data access layer that allows users to easily and efficiently retrieve data from various storage services in a unified way. -The notable changes since ${iceberg_version} include: +The notable changes since the previous release include: 1. xxxxx 2. yyyyyy 3. zzzzzz @@ -413,7 +603,7 @@ Apache Iceberg Rust website: https://rust.iceberg.apache.org/ Download Links: https://rust.iceberg.apache.org/download -From official ASF distribution: https://dist.apache.org/repos/dist/release/iceberg/apache-iceberg-rust-${release_version}/ +From official ASF distribution: https://dist.apache.org/repos/dist/release/iceberg/apache-iceberg-rust-${iceberg_version}/ Iceberg Resources: - Issue: https://github.com/apache/iceberg-rust/issues @@ -424,3 +614,106 @@ On behalf of Apache Iceberg Community ``` Example: + +### Publish the release blog post + +If a release blog post has been drafted (introduced earlier in this guide), now is the time to ensure it is ready to merge and publish. +It does not need to be published immediately for the release to be considered complete, +however it is in the community's best interest to publish it soon after. + +## Appendix + +### Adding a new crate + +Every publishable crate must exist on crates.io before the pull request that adds it is merged. +CI enforces this with a check that fails for any publishable crate missing from crates.io. + +The release workflow publishes crates with [trusted publishing](https://crates.io/docs/trusted-publishing), which cannot create a crate. +crates.io requires the first version of a crate to be published manually with an API token ([announcement](https://blog.rust-lang.org/2025/07/11/crates-io-development-update-2025-07/)). +Publishing the crate by hand after the release workflow fails does not work either: the workflow runs `cargo publish --workspace`, which refuses to run if any crate already exists at the release version. + +A committer therefore reserves the crate when it is added, in three steps: publish a placeholder version, configure owners, and configure trusted publishing. + +#### Prerequisites + +Any crates.io account can publish an unclaimed name, and the first publisher becomes the crate's only owner. +Reserve the crate promptly once the pull request is open, and have a committer do it, since the reserver must hand ownership to the project. + +The committer needs: + +- A crates.io account linked to their GitHub account. +- Membership in the `apache/iceberg-private` GitHub team, with the `read:org` permission granted to crates.io, in order to add that team as an owner. +- A crates.io API token with the `publish-new` and `change-owners` scopes, used with `cargo login`. + Existing tokens usually lack `publish-new`; create one at [crates.io/settings/tokens](https://crates.io/settings/tokens). + +Committers who already own the existing Iceberg crates meet the first two. +List them with `cargo owner --list iceberg`. + +#### Step 1: Reserve the crate name + +Publish an empty placeholder crate as version `0.0.0`, so it never collides with a real release. +The placeholder is created outside the repository; overriding the version inside the workspace does not work because other workspace crates depend on the new crate at the workspace version. + +```shell +mkdir -p /tmp//src +cd /tmp/ +touch src/lib.rs +cat > Cargo.toml <<'EOF' +[package] +name = "" +version = "0.0.0" +edition = "2024" +license = "Apache-2.0" +description = "Placeholder that reserves the crate name for Apache Iceberg Rust. See the repository for the real crate." +repository = "https://github.com/apache/iceberg-rust" +EOF +cargo publish --dry-run +cargo publish +``` + +A `403 Forbidden: authentication failed` error means the token lacks the `publish-new` scope. + +#### Step 2: Configure crate owners + +Add the GitHub team that owns the existing Iceberg crates. + +```shell +cargo owner --add github:apache:iceberg-private +``` + +Then add two PMC members other than yourself as individual owners. + +```shell +cargo owner --add +``` + +This sends an invitation; ask each invitee to accept it on crates.io. + +Individual owners are required because a team owner can publish and yank versions but cannot add or remove owners. +Without them, the crate could become unmanageable if a PMC member becomes inactive. +See the [Cargo documentation for `cargo owner`](https://doc.rust-lang.org/cargo/reference/publishing.html#cargo-owner). + +#### Step 3: Configure trusted publishing + +As a crate owner, open the crate's settings on the crates.io website and add a trusted publisher, following the [crates.io trusted publishing documentation](https://crates.io/docs/trusted-publishing). +Use an existing Iceberg crate as the reference: + +- Repository: `apache/iceberg-rust` +- Workflow: `publish.yml` +- Environment: `publish` + +Then re-run CI on the pull request. +The next release publishes the crate's first real version along with the rest of the workspace. + +#### If the Publish workflow fails on a missing crate + +Reserve the crate as above, using the `0.0.0` placeholder and never the release version. + +Then check the failed run's log for `Uploaded` lines to see which crates were already published, since cargo uploads crates one at a time in dependency order. + +- If nothing was uploaded, re-run the failed workflow run from the GitHub Actions UI. Re-running keeps the original tag as the workflow ref. +- If some crates were uploaded, publish the rest from a clean checkout of the release tag, excluding each crate that was already published: + +```shell +cargo publish --workspace --all-features --exclude --exclude +```