From 62c8f702c7a3fbd93e393cc2f3a78e314c5ef1bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 10:29:57 +0800 Subject: [PATCH 01/21] feat(bindings): add C++ streaming facade --- Cargo.lock | 112 +- DEPENDENCIES.rust.tsv | 10 +- benchmarks/tpcds/DEPENDENCIES.rust.tsv | 10 +- bindings/c/Cargo.toml | 4 +- bindings/c/DEPENDENCIES.rust.tsv | 10 +- bindings/c/cbindgen.toml | 24 + bindings/c/check-header.sh | 37 + bindings/c/include/paimon.h | 1837 +++++++++++++++++ bindings/c/src/error.rs | 31 +- bindings/c/src/lib.rs | 2 + bindings/c/src/result.rs | 12 + bindings/c/src/stream.rs | 1168 +++++++++++ bindings/c/src/table.rs | 4 + bindings/c/src/tests.rs | 648 ++++++ bindings/c/src/types.rs | 82 + bindings/c/src/version.rs | 46 + bindings/c/src/write.rs | 817 +++++++- bindings/cpp/CMakeLists.txt | 341 +++ bindings/cpp/README.md | 146 ++ bindings/cpp/cmake/PaimonCppConfig.cmake.in | 55 + .../cpp/cmake/PaimonNoRuntimePlugin.cmake | 140 ++ bindings/cpp/examples/batch_read.cpp | 111 + bindings/cpp/examples/stream_read.cpp | 137 ++ bindings/cpp/examples/streaming_write.cpp | 209 ++ bindings/cpp/include/paimon/paimon.hpp | 1489 +++++++++++++ bindings/cpp/scripts/verify_linux_elf.sh | 231 +++ .../cpp/tests/check_incremental_relink.cmake | 49 + bindings/cpp/tests/dlopen_smoke.c | 63 + .../cpp/tests/elf_fixtures/executable_stack.c | 19 + .../cpp/tests/elf_fixtures/needs_libgcc.c | 24 + .../cpp/tests/elf_fixtures/pie_executable.c | 19 + .../tests/elf_fixtures/undefined_host_hook.c | 21 + .../elf_fixtures/undefined_operator_new.c | 24 + bindings/cpp/tests/expect_elf_rejected.cmake | 39 + bindings/cpp/tests/header_smoke.cpp | 130 ++ .../helper_config/paimon_cpp_helper_config.h | 22 + .../install_tree_consumer/CMakeLists.txt | 31 + .../tests/install_tree_consumer/plugin.cpp | 23 + bindings/cpp/tests/no_cpp_runtime_plugin.cpp | 71 + bindings/cpp/tests/paimon_test_stub.h | 314 +++ bindings/cpp/tests/relink_probe.cpp | 21 + .../cpp/tests/run_install_tree_consumer.cmake | 90 + bindings/cpp/tests/run_isolated_load.cmake | 37 + bindings/go/DEPENDENCIES.rust.tsv | 10 +- bindings/python/DEPENDENCIES.rust.tsv | 10 +- .../integration_tests/DEPENDENCIES.rust.tsv | 10 +- .../datafusion/DEPENDENCIES.rust.tsv | 10 +- .../paimon-rest-server/DEPENDENCIES.rust.tsv | 10 +- crates/paimon/Cargo.toml | 10 +- crates/paimon/DEPENDENCIES.rust.tsv | 10 +- crates/paimon/src/io/file_io.rs | 154 +- crates/paimon/src/lib.rs | 3 +- crates/paimon/src/spec/core_options.rs | 41 + crates/paimon/src/table/commit_message.rs | 3 +- crates/paimon/src/table/mod.rs | 4 + crates/paimon/src/table/read_builder.rs | 59 + crates/paimon/src/table/snapshot_manager.rs | 88 +- crates/paimon/src/table/source.rs | 179 ++ crates/paimon/src/table/stream_scan.rs | 685 ++++++ crates/paimon/src/table/table_commit.rs | 365 +++- crates/paimon/src/table/table_scan.rs | 203 +- crates/paimon/tests/stream_scan_test.rs | 524 +++++ docs/src/c-binding.md | 99 +- scripts/release_licenses.py | 26 - scripts/verify_python_wheels.py | 50 +- 65 files changed, 10901 insertions(+), 362 deletions(-) create mode 100644 bindings/c/cbindgen.toml create mode 100755 bindings/c/check-header.sh create mode 100644 bindings/c/include/paimon.h create mode 100644 bindings/c/src/stream.rs create mode 100644 bindings/c/src/version.rs create mode 100644 bindings/cpp/CMakeLists.txt create mode 100644 bindings/cpp/README.md create mode 100644 bindings/cpp/cmake/PaimonCppConfig.cmake.in create mode 100644 bindings/cpp/cmake/PaimonNoRuntimePlugin.cmake create mode 100644 bindings/cpp/examples/batch_read.cpp create mode 100644 bindings/cpp/examples/stream_read.cpp create mode 100644 bindings/cpp/examples/streaming_write.cpp create mode 100644 bindings/cpp/include/paimon/paimon.hpp create mode 100755 bindings/cpp/scripts/verify_linux_elf.sh create mode 100644 bindings/cpp/tests/check_incremental_relink.cmake create mode 100644 bindings/cpp/tests/dlopen_smoke.c create mode 100644 bindings/cpp/tests/elf_fixtures/executable_stack.c create mode 100644 bindings/cpp/tests/elf_fixtures/needs_libgcc.c create mode 100644 bindings/cpp/tests/elf_fixtures/pie_executable.c create mode 100644 bindings/cpp/tests/elf_fixtures/undefined_host_hook.c create mode 100644 bindings/cpp/tests/elf_fixtures/undefined_operator_new.c create mode 100644 bindings/cpp/tests/expect_elf_rejected.cmake create mode 100644 bindings/cpp/tests/header_smoke.cpp create mode 100644 bindings/cpp/tests/helper_config/paimon_cpp_helper_config.h create mode 100644 bindings/cpp/tests/install_tree_consumer/CMakeLists.txt create mode 100644 bindings/cpp/tests/install_tree_consumer/plugin.cpp create mode 100644 bindings/cpp/tests/no_cpp_runtime_plugin.cpp create mode 100644 bindings/cpp/tests/paimon_test_stub.h create mode 100644 bindings/cpp/tests/relink_probe.cpp create mode 100644 bindings/cpp/tests/run_install_tree_consumer.cmake create mode 100644 bindings/cpp/tests/run_isolated_load.cmake create mode 100644 crates/paimon/src/table/stream_scan.rs create mode 100644 crates/paimon/tests/stream_scan_test.rs diff --git a/Cargo.lock b/Cargo.lock index 6af8279c3..a338b39a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2647,21 +2647,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -3160,27 +3145,12 @@ dependencies = [ "hyper", "hyper-util", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.20" @@ -4073,23 +4043,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "native-tls" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "never-say-never" version = "6.6.666" @@ -4536,49 +4489,12 @@ dependencies = [ "url", ] -[[package]] -name = "openssl" -version = "0.10.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" -dependencies = [ - "bitflags 2.13.1", - "cfg-if", - "foreign-types", - "libc", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "openssl-sys" -version = "0.9.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "orc-rust" version = "0.8.0" @@ -4731,7 +4647,9 @@ dependencies = [ "futures", "paimon", "paimon-vindex-core", + "serde", "serde_json", + "sha2 0.10.9", "tempfile", "tokio", "url", @@ -5858,21 +5776,22 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", "mime", - "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-native-tls", + "tokio-rustls", "tower", "tower-http", "tower-service", @@ -6036,6 +5955,7 @@ checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -7121,16 +7041,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" @@ -7415,12 +7325,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.5" diff --git a/DEPENDENCIES.rust.tsv b/DEPENDENCIES.rust.tsv index 98277510c..c0db34674 100644 --- a/DEPENDENCIES.rust.tsv +++ b/DEPENDENCIES.rust.tsv @@ -217,8 +217,6 @@ flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.1.5 X foldhash@0.2.0 X -foreign-types@0.3.2 X X -foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X @@ -266,7 +264,6 @@ humantime@2.4.0 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X -hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -354,7 +351,6 @@ moka@0.12.15 X X murmurhash32@0.3.1 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X -native-tls@0.2.18 X X never-say-never@6.6.666 X X X no_std_io2@0.9.4 X X nom@7.1.3 X @@ -391,10 +387,7 @@ opendal-service-hdfs-native@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X -openssl@0.10.81 X -openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X -openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-float@5.3.0 X @@ -495,6 +488,7 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X +ring@0.17.14 X X rle-decode-fast@1.0.3 X X roaring@0.11.4 X X roxmltree@0.21.1 X X @@ -604,7 +598,6 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X -tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-stream@0.1.18 X tokio-util@0.7.18 X @@ -636,7 +629,6 @@ utf8-ranges@1.0.5 X X utf8_iter@1.0.4 X X utf8parse@0.2.2 X X uuid@1.24.0 X X -vcpkg@0.2.15 X X version_check@0.9.5 X X vortex@0.75.0 X vortex-alp@0.75.0 X diff --git a/benchmarks/tpcds/DEPENDENCIES.rust.tsv b/benchmarks/tpcds/DEPENDENCIES.rust.tsv index 0da960875..099fb4091 100644 --- a/benchmarks/tpcds/DEPENDENCIES.rust.tsv +++ b/benchmarks/tpcds/DEPENDENCIES.rust.tsv @@ -148,8 +148,6 @@ flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.1.5 X foldhash@0.2.0 X -foreign-types@0.3.2 X X -foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -185,7 +183,6 @@ humantime@2.4.0 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X -hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -246,7 +243,6 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X -native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-complex@0.4.6 X X @@ -263,10 +259,7 @@ opendal-http-transport-reqwest@0.58.2 X opendal-layer-retry@0.58.2 X opendal-service-fs@0.58.2 X opendal-service-oss@0.58.2 X -openssl@0.10.81 X -openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X -openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -323,6 +316,7 @@ reqsign-core@3.3.1 X reqsign-file-read-tokio@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X +ring@0.17.14 X X roaring@0.11.4 X X rust-ini@0.21.3 X rustc_version@0.4.1 X X @@ -394,7 +388,6 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X -tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-stream@0.1.18 X tokio-util@0.7.18 X @@ -419,7 +412,6 @@ urlencoding@2.1.3 X utf8_iter@1.0.4 X X utf8parse@0.2.2 X X uuid@1.24.0 X X -vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/bindings/c/Cargo.toml b/bindings/c/Cargo.toml index 949044a0b..944eaa00f 100644 --- a/bindings/c/Cargo.toml +++ b/bindings/c/Cargo.toml @@ -43,7 +43,9 @@ futures = "0.3" arrow = { workspace = true } arrow-array = { workspace = true } arrow-schema = { workspace = true } -serde_json = "1.0.120" +serde_json = { version = "1.0.120", features = ["raw_value"] } +serde = { version = "1.0", features = ["derive"] } +sha2 = "0.10" async-trait = "0.1.81" bytes = "1.7.1" diff --git a/bindings/c/DEPENDENCIES.rust.tsv b/bindings/c/DEPENDENCIES.rust.tsv index f50b9243b..9cd511fcc 100644 --- a/bindings/c/DEPENDENCIES.rust.tsv +++ b/bindings/c/DEPENDENCIES.rust.tsv @@ -102,8 +102,6 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X -foreign-types@0.3.2 X X -foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -136,7 +134,6 @@ httpdate@1.0.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X -hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -193,7 +190,6 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X -native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-bigint-dig@0.8.6 X X @@ -214,10 +210,7 @@ opendal-service-gcs@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X -openssl@0.10.81 X -openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X -openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -276,6 +269,7 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X +ring@0.17.14 X X roaring@0.11.4 X X rsa@0.9.10 X X rust-ini@0.21.3 X @@ -350,7 +344,6 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X -tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -372,7 +365,6 @@ url@2.5.8 X X urlencoding@2.1.3 X utf8_iter@1.0.4 X X uuid@1.24.0 X X -vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/bindings/c/cbindgen.toml b/bindings/c/cbindgen.toml new file mode 100644 index 000000000..d214b2137 --- /dev/null +++ b/bindings/c/cbindgen.toml @@ -0,0 +1,24 @@ +language = "C" +header = """ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +""" +include_guard = "PAIMON_C_H" +cpp_compat = true +documentation = true +usize_is_size_t = true +style = "both" +sort_by = "Name" diff --git a/bindings/c/check-header.sh b/bindings/c/check-header.sh new file mode 100755 index 000000000..9626484c8 --- /dev/null +++ b/bindings/c/check-header.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repo_dir=$(CDPATH= cd -- "$script_dir/../.." && pwd) +generated=$(mktemp) +trap 'rm -f "$generated"' EXIT HUP INT TERM + +cbindgen --quiet --config "$script_dir/cbindgen.toml" \ + "$script_dir" --output "$generated" + +if ! cmp -s "$generated" "$script_dir/include/paimon.h"; then + echo "bindings/c/include/paimon.h is stale; regenerate it with cbindgen" >&2 + diff -u "$script_dir/include/paimon.h" "$generated" || true + exit 1 +fi + +cc -std=c11 -fsyntax-only -x c "$generated" +c++ -std=c++17 -fno-exceptions -fno-rtti -fsyntax-only -x c++ "$generated" + +echo "C header is current and C/C++ compatible in $repo_dir" diff --git a/bindings/c/include/paimon.h b/bindings/c/include/paimon.h new file mode 100644 index 000000000..8aaba187a --- /dev/null +++ b/bindings/c/include/paimon.h @@ -0,0 +1,1837 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +#ifndef PAIMON_C_H +#define PAIMON_C_H + +#include +#include +#include +#include +#include + +#define PAIMON_ERROR_ALREADY_EXISTS 3 + +#define PAIMON_ERROR_INVALID_INPUT 4 + +#define PAIMON_ERROR_IO 5 + +#define PAIMON_ERROR_NOT_FOUND 2 + +#define PAIMON_ERROR_OUT_OF_RANGE 6 + +#define PAIMON_ERROR_UNEXPECTED 0 + +#define PAIMON_ERROR_UNSUPPORTED 1 + +#define PAIMON_STREAM_FOLLOW_UP_AUTO 0 + +#define PAIMON_STREAM_FOLLOW_UP_CHANGELOG 2 + +#define PAIMON_STREAM_FOLLOW_UP_DELTA 1 + +#define PAIMON_STREAM_POLL_DATA 0 + +#define PAIMON_STREAM_POLL_END 2 + +#define PAIMON_STREAM_POLL_WAITING 1 + +#define PAIMON_STREAM_READ_AUDIT_LOG 1 + +#define PAIMON_STREAM_READ_DATA 0 + +#define PAIMON_STREAM_STARTUP_FROM_SNAPSHOT 2 + +#define PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL 3 + +#define PAIMON_STREAM_STARTUP_LATEST 1 + +#define PAIMON_STREAM_STARTUP_LATEST_FULL 0 + +/** + * A single Arrow record batch exported via the Arrow C Data Interface. + * + * `array` and `schema` point to heap-allocated ArrowArray and ArrowSchema + * structs. After importing the data, call `paimon_arrow_batch_free` to free + * the container structs. + */ +typedef struct paimon_arrow_batch { + /** + * Pointer to a heap-allocated ArrowArray. + */ + void *array; + /** + * Pointer to a heap-allocated ArrowSchema. + */ + void *schema; +} paimon_arrow_batch; + +typedef struct paimon_blob_reader { + void *inner; +} paimon_blob_reader; + +/** + * C-compatible byte buffer. + */ +typedef struct paimon_bytes { + uint8_t *data; + size_t len; +} paimon_bytes; + +/** + * C-compatible error type. + */ +typedef struct paimon_error { + int32_t code; + struct paimon_bytes message; +} paimon_error; + +typedef struct paimon_result_blob_reader { + struct paimon_blob_reader *reader; + struct paimon_error *error; +} paimon_result_blob_reader; + +/** + * C-compatible key-value pair for options. + */ +typedef struct paimon_option { + const char *key; + const char *value; +} paimon_option; + +typedef struct paimon_blob_stream { + void *inner; +} paimon_blob_stream; + +typedef struct paimon_result_blob_stream { + struct paimon_blob_stream *stream; + struct paimon_error *error; +} paimon_result_blob_stream; + +typedef struct paimon_bytes_array { + struct paimon_bytes *data; + size_t len; +} paimon_bytes_array; + +typedef struct paimon_result_read_blobs { + struct paimon_bytes_array blobs; + struct paimon_error *error; +} paimon_result_read_blobs; + +typedef struct paimon_byte_slice { + const uint8_t *data; + size_t len; +} paimon_byte_slice; + +typedef struct paimon_result_blob_stream_read { + size_t bytes_read; + struct paimon_error *error; +} paimon_result_blob_stream_read; + +typedef struct paimon_result_blob_stream_seek { + uint64_t position; + struct paimon_error *error; +} paimon_result_blob_stream_seek; + +/** + * Opaque wrapper around a heap-allocated Rust object. + */ +typedef struct paimon_catalog { + void *inner; +} paimon_catalog; + +typedef struct paimon_result_catalog_new { + struct paimon_catalog *catalog; + struct paimon_error *error; +} paimon_result_catalog_new; + +typedef struct paimon_table { + void *inner; +} paimon_table; + +typedef struct paimon_result_get_table { + struct paimon_table *table; + struct paimon_error *error; +} paimon_result_get_table; + +typedef struct paimon_identifier { + void *inner; +} paimon_identifier; + +/** + * Opaque container for commit messages and their originating write context. + */ +typedef struct paimon_commit_messages { + void *inner; +} paimon_commit_messages; + +/** + * Opaque durable prepared-commit handle for a standard table write. + */ +typedef struct paimon_prepared_commit { + void *inner; +} paimon_prepared_commit; + +typedef struct paimon_result_prepared_commit { + struct paimon_prepared_commit *prepared; + struct paimon_error *error; +} paimon_result_prepared_commit; + +/** + * Opaque wrapper around a cloneable Paimon FileIO. + */ +typedef struct paimon_file_io { + void *inner; +} paimon_file_io; + +typedef struct paimon_result_file_io_new { + struct paimon_file_io *file_io; + struct paimon_error *error; +} paimon_result_file_io_new; + +/** + * Version 1 callbacks for an externally managed file-block cache. + * + * Callbacks may run concurrently on arbitrary Rust runtime blocking threads. + * They must not unwind across the C ABI. `get` returns the number of bytes + * copied into `output`; return `-1` for a miss and any value other than the + * requested length for a fail-open miss. All callback buffers and paths are + * borrowed only for the duration of the call. Paths use pointer-plus-length + * because canonical storage keys may contain embedded NUL separators. + */ +typedef struct paimon_file_cache_callbacks_v1 { + void *context; + int64_t (*get)(void *context, + const uint8_t *path_data, + size_t path_length, + uint64_t offset, + size_t length, + uint8_t *output); + int32_t (*put)(void *context, + const uint8_t *path_data, + size_t path_length, + uint64_t offset, + const uint8_t *data, + size_t length); + int32_t (*invalidate_path)(void *context, const uint8_t *path_data, size_t path_length); + int32_t (*invalidate_prefix)(void *context, const uint8_t *prefix_data, size_t prefix_length); + /** + * Releases `context` after the last FileIO/table clone is dropped. + */ + void (*destroy)(void *context); +} paimon_file_cache_callbacks_v1; + +typedef struct paimon_result_identifier_new { + struct paimon_identifier *identifier; + struct paimon_error *error; +} paimon_result_identifier_new; + +typedef struct paimon_plan { + void *inner; +} paimon_plan; + +typedef struct paimon_result_plan { + struct paimon_plan *plan; + struct paimon_error *error; +} paimon_result_plan; + +typedef struct paimon_postpone_fixed_bucket_commit_messages { + void *inner; +} paimon_postpone_fixed_bucket_commit_messages; + +typedef struct paimon_postpone_fixed_bucket_table_commit { + void *inner; +} paimon_postpone_fixed_bucket_table_commit; + +typedef struct paimon_postpone_fixed_bucket_table_write { + void *inner; +} paimon_postpone_fixed_bucket_table_write; + +typedef struct paimon_result_postpone_fixed_bucket_prepare_commit { + struct paimon_postpone_fixed_bucket_commit_messages *messages; + struct paimon_error *error; +} paimon_result_postpone_fixed_bucket_prepare_commit; + +typedef struct paimon_postpone_fixed_bucket_write_builder { + void *inner; +} paimon_postpone_fixed_bucket_write_builder; + +typedef struct paimon_result_postpone_fixed_bucket_table_commit { + struct paimon_postpone_fixed_bucket_table_commit *commit; + struct paimon_error *error; +} paimon_result_postpone_fixed_bucket_table_commit; + +typedef struct paimon_result_postpone_fixed_bucket_table_write { + struct paimon_postpone_fixed_bucket_table_write *write; + struct paimon_error *error; +} paimon_result_postpone_fixed_bucket_table_write; + +/** + * Opaque wrapper around a Predicate. + */ +typedef struct paimon_predicate { + void *inner; +} paimon_predicate; + +typedef struct paimon_result_predicate { + struct paimon_predicate *predicate; + struct paimon_error *error; +} paimon_result_predicate; + +/** + * A typed literal value for predicate comparison, passed across FFI. + * + * # Design + * + * We use a tagged flat struct instead of opaque heap-allocated handles + * (like DuckDB's `duckdb_value`). The trade-off: + * + * - **Pro**: Zero allocation — the entire datum is passed by value on the + * stack, with no heap round-trips or free calls needed. This keeps the + * FFI surface minimal and the Go/C caller simple. + * - **Con**: The struct is larger than any single variant needs, wasting + * some bytes per datum (currently ~56 bytes vs. ~16 for the largest + * single variant). + * + * Since datums are only used for predicate construction (not a hot path), + * the extra size is acceptable. + * + * # Tags + * + * - 0: Bool, 1: TinyInt, 2: SmallInt, 3: Int, 4: Long + * - 5: Float, 6: Double, 7: String, 8: Date, 9: Time + * - 10: Timestamp, 11: LocalZonedTimestamp, 12: Decimal, 13: Bytes + * + * `tag` determines which value fields are valid: + * - `Bool`/`TinyInt`/`SmallInt`/`Int`/`Long`/`Date`/`Time` → `int_val` + * - `Float`/`Double` → `double_val` + * - `String`/`Bytes` → `str_data` + `str_len` + * - `Timestamp`/`LocalZonedTimestamp` → `int_val` (millis) + `int_val2` (nanos) + * - `Decimal` → `int_val` + `int_val2` (unscaled i128) + `uint_val` (precision) + `uint_val2` (scale) + */ +typedef struct paimon_datum { + int32_t tag; + int64_t int_val; + double double_val; + const uint8_t *str_data; + size_t str_len; + int64_t int_val2; + uint32_t uint_val; + uint32_t uint_val2; +} paimon_datum; + +typedef struct paimon_result_bytes { + struct paimon_bytes bytes; + struct paimon_error *error; +} paimon_result_bytes; + +typedef struct paimon_read_builder { + void *inner; +} paimon_read_builder; + +typedef struct paimon_table_read { + void *inner; +} paimon_table_read; + +typedef struct paimon_result_new_read { + struct paimon_table_read *read; + struct paimon_error *error; +} paimon_result_new_read; + +typedef struct paimon_table_scan { + void *inner; +} paimon_table_scan; + +typedef struct paimon_result_table_scan { + struct paimon_table_scan *scan; + struct paimon_error *error; +} paimon_result_table_scan; + +typedef struct paimon_stream_scan { + void *inner; +} paimon_stream_scan; + +typedef struct paimon_result_stream_scan { + struct paimon_stream_scan *scan; + struct paimon_error *error; +} paimon_result_stream_scan; + +/** + * Extensible options for a continuous scan. + * + * Initialize this with `paimon_stream_scan_options_init`; future versions may + * consume fields from `reserved` while preserving this prefix. + */ +typedef struct paimon_stream_scan_options { + uint32_t struct_size; + int32_t startup_mode; + int32_t follow_up_mode; + int64_t snapshot_id; + uint64_t reserved[4]; +} paimon_stream_scan_options; + +typedef struct paimon_record_batch_reader { + void *inner; +} paimon_record_batch_reader; + +typedef struct paimon_result_next_batch { + struct paimon_arrow_batch batch; + struct paimon_error *error; +} paimon_result_next_batch; + +typedef struct paimon_stream_plan { + void *inner; +} paimon_stream_plan; + +typedef struct paimon_result_stream_poll { + int32_t status; + struct paimon_stream_plan *plan; + int64_t snapshot_id; + int64_t next_snapshot_id; + int64_t watermark; + uint8_t has_watermark; + uint8_t reserved[7]; + struct paimon_error *error; +} paimon_result_stream_poll; + +typedef struct paimon_result_record_batch_reader { + struct paimon_record_batch_reader *reader; + struct paimon_error *error; +} paimon_result_record_batch_reader; + +typedef struct paimon_table_commit { + void *inner; +} paimon_table_commit; + +typedef struct paimon_result_postpone_fixed_bucket_write_builder { + struct paimon_postpone_fixed_bucket_write_builder *write_builder; + struct paimon_error *error; +} paimon_result_postpone_fixed_bucket_write_builder; + +typedef struct paimon_result_read_builder { + struct paimon_read_builder *read_builder; + struct paimon_error *error; +} paimon_result_read_builder; + +/** + * Opaque wrapper around a vector-search builder. + */ +typedef struct paimon_vector_search_builder { + void *inner; +} paimon_vector_search_builder; + +typedef struct paimon_result_vector_search_builder { + struct paimon_vector_search_builder *builder; + struct paimon_error *error; +} paimon_result_vector_search_builder; + +typedef struct paimon_write_builder { + void *inner; +} paimon_write_builder; + +typedef struct paimon_result_write_builder { + struct paimon_write_builder *write_builder; + struct paimon_error *error; +} paimon_result_write_builder; + +typedef struct paimon_table_write { + void *inner; +} paimon_table_write; + +typedef struct paimon_result_prepare_commit { + struct paimon_commit_messages *messages; + struct paimon_error *error; +} paimon_result_prepare_commit; + +typedef struct paimon_result_table_commit { + struct paimon_table_commit *commit; + struct paimon_error *error; +} paimon_result_table_commit; + +typedef struct paimon_result_table_write { + struct paimon_table_write *write; + struct paimon_error *error; +} paimon_result_table_write; + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +/** + * ABI version for the native C boundary. + * + * Version 1 is additive: callers must still feature-detect newer symbols when + * loading the shared library dynamically. + */ +uint32_t paimon_abi_version(void); + +/** + * Free the ArrowArray and ArrowSchema container structs for a single batch. + * + * # Safety + * `batch` must contain valid pointers returned by `paimon_record_batch_reader_next`. + */ +void paimon_arrow_batch_free(struct paimon_arrow_batch batch); + +/** + * # Safety + * `reader` is null or was returned by `paimon_blob_reader_new`. + */ +void paimon_blob_reader_free(struct paimon_blob_reader *reader); + +/** + * # Safety + * `options` is null for zero length or points to valid UTF-8 C-string pairs. + */ +struct paimon_result_blob_reader paimon_blob_reader_new(const struct paimon_option *options, + size_t options_len); + +/** + * Open one descriptor for incremental reads. + * + * # Safety + * `reader` is valid and `descriptor` points to `descriptor_len` bytes. + */ +struct paimon_result_blob_stream paimon_blob_reader_open_blob(const struct paimon_blob_reader *reader, + const uint8_t *descriptor, + size_t descriptor_len); + +/** + * # Safety + * The handle and input slices are valid for this call. Free the output with + * `paimon_bytes_array_free`. + */ +struct paimon_result_read_blobs paimon_blob_reader_read_blobs(const struct paimon_blob_reader *reader, + const struct paimon_byte_slice *descriptors, + size_t descriptors_len); + +/** + * # Safety + * `stream` is null or was returned by `paimon_blob_reader_open_blob`. + */ +void paimon_blob_stream_free(struct paimon_blob_stream *stream); + +/** + * Read at most `buffer_len` bytes into caller-owned memory. + * + * A zero `bytes_read` result means end of stream when `buffer_len` is nonzero. + * + * # Safety + * `stream` is valid and `buffer` points to `buffer_len` writable bytes. + */ +struct paimon_result_blob_stream_read paimon_blob_stream_read(struct paimon_blob_stream *stream, + uint8_t *buffer, + size_t buffer_len); + +/** + * Seek within the descriptor's range. `whence` uses the standard 0, 1, 2 values. + * + * # Safety + * `stream` is valid. + */ +struct paimon_result_blob_stream_seek paimon_blob_stream_seek(struct paimon_blob_stream *stream, + int64_t offset, + int32_t whence); + +/** + * # Safety + * `array` was returned by `paimon_blob_reader_read_blobs`. + */ +void paimon_bytes_array_free(struct paimon_bytes_array array); + +/** + * Free a paimon_bytes buffer. + * + * # Safety + * Only call with bytes returned from paimon C functions. + */ +void paimon_bytes_free(struct paimon_bytes bytes); + +/** + * Create a catalog using CatalogFactory with the given options. + * + * # Safety + * `options` must be a valid pointer to an array of `paimon_option` with `options_len` elements. + * Each key and value in the options must be valid null-terminated C strings. + */ +struct paimon_result_catalog_new paimon_catalog_create(const struct paimon_option *options, + size_t options_len); + +/** + * Free a paimon_catalog. + * + * # Safety + * Only call with a catalog returned from `paimon_catalog_create`. + */ +void paimon_catalog_free(struct paimon_catalog *catalog); + +/** + * Get a table from the catalog. + * + * # Safety + * `catalog` and `identifier` must be valid pointers from previous paimon C calls, or null (returns error). + */ +struct paimon_result_get_table paimon_catalog_get_table(const struct paimon_catalog *catalog, + const struct paimon_identifier *identifier); + +/** + * Free standard commit messages. + */ +void paimon_commit_messages_free(struct paimon_commit_messages *msgs); + +/** + * Merge standard commit messages for one logical commit. + */ +struct paimon_error *paimon_commit_messages_merge(struct paimon_commit_messages *target, + const struct paimon_commit_messages *source); + +/** + * Bind standard commit messages to a monotonically increasing streaming + * commit identifier. The returned prepared commit owns a clone of the + * messages, so the source handle remains valid. Valid identifiers are in + * `[0, INT64_MAX)`; `INT64_MAX` is reserved for unidentified batch commits. + */ +struct paimon_result_prepared_commit paimon_commit_messages_prepare(const struct paimon_commit_messages *msgs, + int64_t commit_identifier); + +/** + * Free a paimon_error. + * + * # Safety + * Only call with errors returned from paimon C functions. + */ +void paimon_error_free(struct paimon_error *err); + +/** + * Create a reusable FileIO from a representative storage path and options. + */ +struct paimon_result_file_io_new paimon_file_io_create(const char *path, + const struct paimon_option *options, + size_t options_len); + +/** + * Create a reusable FileIO backed by a caller-managed block cache. + * + * A non-null `callbacks->get` and a non-zero `block_size` are required. + * `whitelist` may be null to use `meta,global-index`. Once validation and + * storage construction succeed, Rust owns `callbacks->context` and invokes + * `destroy` exactly once after the last derived FileIO/table is dropped. + */ +struct paimon_result_file_io_new paimon_file_io_create_with_cache_v1(const char *path, + const struct paimon_option *options, + size_t options_len, + const struct paimon_file_cache_callbacks_v1 *callbacks, + uint64_t block_size, + const char *whitelist); + +/** + * Free a FileIO handle. Tables created from it retain their own clone. + */ +void paimon_file_io_free(struct paimon_file_io *file_io); + +/** + * Free a paimon_identifier. + * + * # Safety + * Only call with an identifier returned from `paimon_identifier_new`. + */ +void paimon_identifier_free(struct paimon_identifier *id); + +/** + * Create a new Identifier. + * + * # Safety + * `database` and `object` must be valid null-terminated C strings, or null (returns error). + */ +struct paimon_result_identifier_new paimon_identifier_new(const char *database, const char *object); + +/** + * Return the paimon-rust package version as an owned UTF-8 byte buffer. + * + * The returned bytes are not NUL terminated and must be released with + * `paimon_bytes_free`. + */ +struct paimon_bytes paimon_library_version(void); + +/** + * Free a paimon_plan. + * + * # Safety + * Only call with a plan returned from `paimon_table_scan_plan`. + * A plan returned from `paimon_plan_from_split_bytes` is also a valid source. + */ +void paimon_plan_free(struct paimon_plan *plan); + +/** + * Build a one-split `paimon_plan` from a serialized Paimon-native `DataSplit` + * byte buffer (the wire form produced by `DataSplit::serialize` / Java + * `DataSplit#serialize`). `data` must be raw bytes (Base64 already decoded by + * the caller). + * + * The returned plan is usable with `paimon_table_read_to_arrow` and must be + * freed with `paimon_plan_free`. + * + * # Safety + * `data` must point to `len` valid bytes, or be null when `len == 0`. + */ +struct paimon_result_plan paimon_plan_from_split_bytes(const uint8_t *data, size_t len); + +/** + * Return the number of data splits in a plan. + * + * # Safety + * `plan` must be a valid pointer from `paimon_table_scan_plan`, or null (returns 0). + * A plan returned from `paimon_plan_from_split_bytes` is also a valid source. + */ +size_t paimon_plan_num_splits(const struct paimon_plan *plan); + +/** + * Free postpone fixed-bucket commit messages. + */ +void paimon_postpone_fixed_bucket_commit_messages_free(struct paimon_postpone_fixed_bucket_commit_messages *msgs); + +/** + * Merge postpone fixed-bucket messages for one logical commit. + */ +struct paimon_error *paimon_postpone_fixed_bucket_commit_messages_merge(struct paimon_postpone_fixed_bucket_commit_messages *target, + const struct paimon_postpone_fixed_bucket_commit_messages *source); + +/** + * Abort postpone fixed-bucket commit messages. + */ +struct paimon_error *paimon_postpone_fixed_bucket_table_commit_abort(const struct paimon_postpone_fixed_bucket_table_commit *tc, + struct paimon_postpone_fixed_bucket_commit_messages *msgs); + +/** + * Commit postpone fixed-bucket messages using the builder's mode. + */ +struct paimon_error *paimon_postpone_fixed_bucket_table_commit_commit(const struct paimon_postpone_fixed_bucket_table_commit *tc, + struct paimon_postpone_fixed_bucket_commit_messages *msgs); + +/** + * Commit postpone fixed-bucket messages with an identifier. + */ +struct paimon_error *paimon_postpone_fixed_bucket_table_commit_commit_with_identifier(const struct paimon_postpone_fixed_bucket_table_commit *tc, + struct paimon_postpone_fixed_bucket_commit_messages *msgs, + int64_t commit_identifier); + +/** + * Filter a committed identifier before committing fixed-bucket messages. + */ +struct paimon_error *paimon_postpone_fixed_bucket_table_commit_filter_and_commit_with_identifier(const struct paimon_postpone_fixed_bucket_table_commit *tc, + struct paimon_postpone_fixed_bucket_commit_messages *msgs, + int64_t commit_identifier); + +/** + * Free a postpone fixed-bucket TableCommit. + */ +void paimon_postpone_fixed_bucket_table_commit_free(struct paimon_postpone_fixed_bucket_table_commit *tc); + +/** + * Truncate a table with a postpone fixed-bucket TableCommit. + */ +struct paimon_error *paimon_postpone_fixed_bucket_table_commit_truncate_table(const struct paimon_postpone_fixed_bucket_table_commit *tc); + +/** + * Truncate a table with a stable identifier. + */ +struct paimon_error *paimon_postpone_fixed_bucket_table_commit_truncate_table_with_identifier(const struct paimon_postpone_fixed_bucket_table_commit *tc, + int64_t commit_identifier); + +/** + * Free a postpone fixed-bucket TableWrite. + * + * # Safety + * Only call with a write returned from + * paimon_postpone_fixed_bucket_write_builder_new_write. + */ +void paimon_postpone_fixed_bucket_table_write_free(struct paimon_postpone_fixed_bucket_table_write *tw); + +/** + * Prepare postpone fixed-bucket commit messages. + * + * The returned handle remains owned by the caller. + */ +struct paimon_result_postpone_fixed_bucket_prepare_commit paimon_postpone_fixed_bucket_table_write_prepare_commit(struct paimon_postpone_fixed_bucket_table_write *tw); + +/** + * Write one Arrow record batch with a postpone fixed-bucket TableWrite. + * + * Ownership of array and schema is transferred once Arrow import starts. + */ +struct paimon_error *paimon_postpone_fixed_bucket_table_write_write_arrow_batch(struct paimon_postpone_fixed_bucket_table_write *tw, + void *array, + void *schema); + +/** + * Free a postpone fixed-bucket write builder. + * + * # Safety + * Only call with a builder returned from + * `paimon_table_new_postpone_fixed_bucket_write_builder`. + */ +void paimon_postpone_fixed_bucket_write_builder_free(struct paimon_postpone_fixed_bucket_write_builder *wb); + +/** + * Create a postpone fixed-bucket TableCommit. + */ +struct paimon_result_postpone_fixed_bucket_table_commit paimon_postpone_fixed_bucket_write_builder_new_commit(const struct paimon_postpone_fixed_bucket_write_builder *wb); + +/** + * Create a postpone fixed-bucket TableWrite. + * + * # Safety + * wb must be a valid fixed-bucket builder, or null (returns error). + */ +struct paimon_result_postpone_fixed_bucket_table_write paimon_postpone_fixed_bucket_write_builder_new_write(const struct paimon_postpone_fixed_bucket_write_builder *wb); + +/** + * Set a shared `partition -> total_buckets` plan. + * The caller retains ownership when pointer or builder validation fails. Once + * Arrow import starts, this call consumes both structs even if plan validation + * returns an error. + * + * # Safety + * `wb` must be a valid postpone fixed-bucket builder. `array` and + * `schema` must point to initialized Arrow C Data structs. + */ +struct paimon_error *paimon_postpone_fixed_bucket_write_builder_with_bucket_plan(struct paimon_postpone_fixed_bucket_write_builder *wb, + void *array, + void *schema); + +/** + * Enable overwrite mode for a postpone fixed-bucket write operation. + * + * # Safety + * `wb` must be a valid fixed-bucket builder, or null (returns error). + */ +struct paimon_error *paimon_postpone_fixed_bucket_write_builder_with_overwrite(struct paimon_postpone_fixed_bucket_write_builder *wb); + +/** + * Combine two predicates with AND. Consumes both inputs. + * + * # Safety + * `a` and `b` must be valid pointers from predicate functions. + */ +struct paimon_predicate *paimon_predicate_and(struct paimon_predicate *a, + struct paimon_predicate *b); + +/** + * Create a BETWEEN predicate: `low <= column <= high` (inclusive, case-sensitive + * column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_between(const struct paimon_table *table, + const char *column, + struct paimon_datum low, + struct paimon_datum high); + +/** + * Create a BETWEEN predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_between_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum low, + struct paimon_datum high, + bool case_sensitive); + +/** + * Create a contains predicate: `column LIKE '%datum%'` (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_contains(const struct paimon_table *table, + const char *column, + struct paimon_datum datum); + +/** + * Create a contains predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_contains_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum datum, + bool case_sensitive); + +/** + * Create an ends-with predicate: `column LIKE '%datum'` (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_ends_with(const struct paimon_table *table, + const char *column, + struct paimon_datum datum); + +/** + * Create an ends-with predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_ends_with_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum datum, + bool case_sensitive); + +/** + * Create an equality predicate: `column = datum` (case-sensitive column match). + * + * For case-insensitive column matching use + * `paimon_predicate_equal_with_case_sensitive`. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_equal(const struct paimon_table *table, + const char *column, + struct paimon_datum datum); + +/** + * Create an equality predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_equal_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum datum, + bool case_sensitive); + +/** + * Free a paimon_predicate. + * + * # Safety + * Only call with a predicate returned from paimon predicate functions. + */ +void paimon_predicate_free(struct paimon_predicate *p); + +/** + * Create a greater-or-equal predicate: `column >= datum` (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_greater_or_equal(const struct paimon_table *table, + const char *column, + struct paimon_datum datum); + +/** + * Create a greater-or-equal predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_greater_or_equal_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum datum, + bool case_sensitive); + +/** + * Create a greater-than predicate: `column > datum` (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_greater_than(const struct paimon_table *table, + const char *column, + struct paimon_datum datum); + +/** + * Create a greater-than predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_greater_than_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum datum, + bool case_sensitive); + +/** + * Create an IN predicate: `column IN (datum1, datum2, ...)` (case-sensitive column match). + * + * # Safety + * `table`, `column`, and `datums` must be valid pointers. `datums_len` must be the length. + */ +struct paimon_result_predicate paimon_predicate_is_in(const struct paimon_table *table, + const char *column, + const struct paimon_datum *datums, + size_t datums_len); + +/** + * Create an IN predicate with configurable column-name case sensitivity. + * + * # Safety + * `table`, `column`, and `datums` must be valid pointers. `datums_len` must be the length. + */ +struct paimon_result_predicate paimon_predicate_is_in_with_case_sensitive(const struct paimon_table *table, + const char *column, + const struct paimon_datum *datums, + size_t datums_len, + bool case_sensitive); + +/** + * Create a NOT IN predicate: `column NOT IN (datum1, datum2, ...)` (case-sensitive column match). + * + * # Safety + * `table`, `column`, and `datums` must be valid pointers. `datums_len` must be the length. + */ +struct paimon_result_predicate paimon_predicate_is_not_in(const struct paimon_table *table, + const char *column, + const struct paimon_datum *datums, + size_t datums_len); + +/** + * Create a NOT IN predicate with configurable column-name case sensitivity. + * + * # Safety + * `table`, `column`, and `datums` must be valid pointers. `datums_len` must be the length. + */ +struct paimon_result_predicate paimon_predicate_is_not_in_with_case_sensitive(const struct paimon_table *table, + const char *column, + const struct paimon_datum *datums, + size_t datums_len, + bool case_sensitive); + +/** + * Create an IS NOT NULL predicate (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_is_not_null(const struct paimon_table *table, + const char *column); + +/** + * Create an IS NOT NULL predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_is_not_null_with_case_sensitive(const struct paimon_table *table, + const char *column, + bool case_sensitive); + +/** + * Create an IS NULL predicate (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_is_null(const struct paimon_table *table, + const char *column); + +/** + * Create an IS NULL predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_is_null_with_case_sensitive(const struct paimon_table *table, + const char *column, + bool case_sensitive); + +/** + * Create a less-or-equal predicate: `column <= datum` (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_less_or_equal(const struct paimon_table *table, + const char *column, + struct paimon_datum datum); + +/** + * Create a less-or-equal predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_less_or_equal_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum datum, + bool case_sensitive); + +/** + * Create a less-than predicate: `column < datum` (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_less_than(const struct paimon_table *table, + const char *column, + struct paimon_datum datum); + +/** + * Create a less-than predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_less_than_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum datum, + bool case_sensitive); + +/** + * Create a LIKE predicate: `column LIKE pattern ESCAPE escape` (case-sensitive + * column match). `escape == 0` uses the default escape character. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_like(const struct paimon_table *table, + const char *column, + struct paimon_datum pattern, + char escape); + +/** + * Create a LIKE predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_like_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum pattern, + char escape, + bool case_sensitive); + +/** + * Negate a predicate with NOT. Consumes the input. + * + * # Safety + * `p` must be a valid pointer from a predicate function. + */ +struct paimon_predicate *paimon_predicate_not(struct paimon_predicate *p); + +/** + * Create a NOT BETWEEN predicate: `column < low OR column > high` + * (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_not_between(const struct paimon_table *table, + const char *column, + struct paimon_datum low, + struct paimon_datum high); + +/** + * Create a NOT BETWEEN predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_not_between_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum low, + struct paimon_datum high, + bool case_sensitive); + +/** + * Create a not-equal predicate: `column != datum` (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_not_equal(const struct paimon_table *table, + const char *column, + struct paimon_datum datum); + +/** + * Create a not-equal predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_not_equal_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum datum, + bool case_sensitive); + +/** + * Combine two predicates with OR. Consumes both inputs. + * + * # Safety + * `a` and `b` must be valid pointers from predicate functions. + */ +struct paimon_predicate *paimon_predicate_or(struct paimon_predicate *a, + struct paimon_predicate *b); + +/** + * Create a starts-with predicate: `column LIKE 'datum%'` (case-sensitive column match). + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_starts_with(const struct paimon_table *table, + const char *column, + struct paimon_datum datum); + +/** + * Create a starts-with predicate with configurable column-name case sensitivity. + * + * # Safety + * `table` and `column` must be valid pointers. + */ +struct paimon_result_predicate paimon_predicate_starts_with_with_case_sensitive(const struct paimon_table *table, + const char *column, + struct paimon_datum datum, + bool case_sensitive); + +/** + * Restore a prepared commit serialized by `paimon_prepared_commit_serialize`. + */ +struct paimon_result_prepared_commit paimon_prepared_commit_deserialize(const uint8_t *data, + size_t len); + +/** + * Free a prepared commit. + */ +void paimon_prepared_commit_free(struct paimon_prepared_commit *prepared); + +/** + * Return the commit identifier carried by a prepared commit, or -1 for null. + */ +int64_t paimon_prepared_commit_identifier(const struct paimon_prepared_commit *prepared); + +/** + * Merge two durable prepared commits produced by parallel writers for the + * same table, commit user, mode and identifier. + */ +struct paimon_error *paimon_prepared_commit_merge(struct paimon_prepared_commit *target, + const struct paimon_prepared_commit *source); + +/** + * Serialize a prepared commit into a process-independent, versioned buffer. + * The bytes must be released with `paimon_bytes_free`. + */ +struct paimon_result_bytes paimon_prepared_commit_serialize(const struct paimon_prepared_commit *prepared); + +/** + * Free a paimon_read_builder. + * + * # Safety + * Only call with a read_builder returned from `paimon_table_new_read_builder`. + */ +void paimon_read_builder_free(struct paimon_read_builder *rb); + +/** + * Create a new TableRead from a ReadBuilder. + * + * # Safety + * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). + */ +struct paimon_result_new_read paimon_read_builder_new_read(const struct paimon_read_builder *rb); + +/** + * Create a new TableScan from a ReadBuilder. + * + * # Safety + * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). + */ +struct paimon_result_table_scan paimon_read_builder_new_scan(const struct paimon_read_builder *rb); + +/** + * Create an owned stream scan from a read builder. + * + * The returned scan clones all required Rust state and remains valid after + * the read builder and table handles are freed. A scan handle is + * single-thread-confined: callers must serialize poll/checkpoint/restore/free. + */ +struct paimon_result_stream_scan paimon_read_builder_new_stream_scan(const struct paimon_read_builder *read_builder, + const struct paimon_stream_scan_options *options); + +/** + * Set whether column-name matching for **projection** is case-sensitive for + * this ReadBuilder. Defaults to `true` (exact match). When `false`, projected + * column names are matched by ASCII case-folding and an ambiguous + * (case-colliding) request errors. + * + * This does **not** affect predicate resolution: a predicate is resolved when + * it is constructed, so its case sensitivity is chosen by which constructor + * you call — `paimon_predicate_*` (case-sensitive) or the additive + * `paimon_predicate_*_with_case_sensitive` variant — independently of this + * setting. + * + * # Safety + * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). + */ +struct paimon_error *paimon_read_builder_with_case_sensitive(struct paimon_read_builder *rb, + bool case_sensitive); + +/** + * Set a filter predicate for scan planning. + * + * The predicate is consumed (ownership transferred to the read builder). + * Pass null to clear any previously set filter. + * + * # Safety + * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). + * `predicate` must be a valid pointer from a `paimon_predicate_*` function, or null. + */ +struct paimon_error *paimon_read_builder_with_filter(struct paimon_read_builder *rb, + struct paimon_predicate *predicate); + +/** + * Set column projection for a ReadBuilder. + * + * The `columns` parameter is a null-terminated array of null-terminated C strings. + * Output order follows the caller-specified order. An empty list is a valid + * zero-column projection. An obvious typo — a name that matches no field under + * any case sensitivity — is rejected by this call. Case-dependent resolution + * (a name that matches only case-insensitively, or a case-fold ambiguity) is + * deferred to `paimon_read_builder_new_read`, which uses the case sensitivity + * effective then, so this stays order-independent with + * `paimon_read_builder_with_case_sensitive`. + * + * # Safety + * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). + * `columns` must be a null-terminated array of null-terminated C strings, or null for no projection. + */ +struct paimon_error *paimon_read_builder_with_projection(struct paimon_read_builder *rb, + const char *const *columns); + +/** + * Free a paimon_record_batch_reader. + * + * # Safety + * Only call with a reader returned from `paimon_table_read_to_arrow` or + * `paimon_vector_search_builder_execute_read`. + */ +void paimon_record_batch_reader_free(struct paimon_record_batch_reader *reader); + +/** + * Get the next Arrow record batch from the reader. + * + * When the stream is exhausted, both `batch.array` and `batch.schema` will + * be null. On error, `error` will be non-null. + * + * After importing each batch, call `paimon_arrow_batch_free` to free the + * ArrowArray and ArrowSchema container structs. + * + * # Safety + * `reader` must be a valid pointer from `paimon_table_read_to_arrow`, or null (returns error). + */ +struct paimon_result_next_batch paimon_record_batch_reader_next(struct paimon_record_batch_reader *reader); + +/** + * Restore a stream plan serialized by `paimon_stream_plan_serialize`. + */ +struct paimon_result_stream_poll paimon_stream_plan_deserialize(const uint8_t *data, size_t len); + +/** + * Free a stream plan. It is valid to pass null. + */ +void paimon_stream_plan_free(struct paimon_stream_plan *plan); + +/** + * Return whether a stream plan is an initial full-snapshot plan. + */ +uint8_t paimon_stream_plan_is_full(const struct paimon_stream_plan *plan); + +/** + * Return the number of work splits in a stream plan. + */ +size_t paimon_stream_plan_num_splits(const struct paimon_stream_plan *plan); + +/** + * Read a contiguous split range from a stream plan. + * + * `read_mode=PAIMON_STREAM_READ_AUDIT_LOG` exposes a stable UTF-8 `rowkind` + * column for incremental plans. Full startup plans currently support data + * mode only; callers requiring one fixed audit schema should start at + * `latest` or `from-snapshot`. + */ +struct paimon_result_record_batch_reader paimon_stream_plan_read_to_arrow(const struct paimon_table_read *read, + const struct paimon_stream_plan *plan, + size_t offset, + size_t length, + int32_t read_mode); + +/** + * Serialize planned-but-not-yet-consumed work for an external checkpoint. + * + * The current format checkpoints at plan boundaries. If rows from a plan have already + * been exposed, callers must either replay the plan after recovery or persist + * their own logical rows-to-skip position alongside this buffer. + * Plans containing external data-file paths are rejected because version 1 + * recovery cannot revalidate those paths against a trusted manifest. + */ +struct paimon_result_bytes paimon_stream_plan_serialize(const struct paimon_stream_plan *plan); + +/** + * Return the next-snapshot cursor, or -1 before a startup position exists. + */ +int64_t paimon_stream_scan_checkpoint(const struct paimon_stream_scan *scan); + +/** + * Free a stream scan. It is valid to pass null. + */ +void paimon_stream_scan_free(struct paimon_stream_scan *scan); + +/** + * Fill stream options with forward-compatible defaults (`latest-full`, + * automatic delta/changelog selection). + */ +struct paimon_error *paimon_stream_scan_options_init(struct paimon_stream_scan_options *options); + +/** + * Poll once for a snapshot plan. This call never waits for a future snapshot. + * Calls using the same scan handle must not overlap on different threads. + */ +struct paimon_result_stream_poll paimon_stream_scan_poll(struct paimon_stream_scan *scan); + +/** + * Restore a next-snapshot cursor. Pass -1 to reapply the configured startup + * mode; non-negative values must name a valid Paimon snapshot position. + * This call must not overlap poll/checkpoint/free on the same handle. + */ +struct paimon_error *paimon_stream_scan_restore(struct paimon_stream_scan *scan, + int64_t next_snapshot_id); + +/** + * Abort standard commit messages. + */ +struct paimon_error *paimon_table_commit_abort(const struct paimon_table_commit *tc, + struct paimon_commit_messages *msgs); + +/** + * Abort files referenced by a durable prepared commit. + * + * Do not call this after an indeterminate commit response: retry + * `paimon_table_commit_commit_prepared` first so a successful commit is not + * followed by deletion of its files. The caller must also fence/serialize all + * commit and abort operations for the same `(table, commit_user)` across + * processes. If retained snapshot history cannot prove that abort is safe, + * this function fails closed and deletes nothing. + */ +struct paimon_error *paimon_table_commit_abort_prepared(const struct paimon_table_commit *tc, + const struct paimon_prepared_commit *prepared); + +/** + * Commit standard append messages. + */ +struct paimon_error *paimon_table_commit_commit(const struct paimon_table_commit *tc, + struct paimon_commit_messages *msgs); + +/** + * Commit a durable prepared commit using the retry-safe identifier path. + * + * This is the correct operation after restoring a prepared commit or after a + * previous commit returned an indeterminate transport/IO error. A successful + * earlier commit with the same `(commit_user, commit_identifier)` is filtered. + */ +struct paimon_error *paimon_table_commit_commit_prepared(const struct paimon_table_commit *tc, + const struct paimon_prepared_commit *prepared); + +/** + * Commit standard append messages with an identifier. + */ +struct paimon_error *paimon_table_commit_commit_with_identifier(const struct paimon_table_commit *tc, + struct paimon_commit_messages *msgs, + int64_t commit_identifier); + +/** + * Filter a committed identifier before committing standard append messages. + */ +struct paimon_error *paimon_table_commit_filter_and_commit_with_identifier(const struct paimon_table_commit *tc, + struct paimon_commit_messages *msgs, + int64_t commit_identifier); + +/** + * Free a standard TableCommit. + */ +void paimon_table_commit_free(struct paimon_table_commit *tc); + +/** + * Commit standard overwrite messages. + */ +struct paimon_error *paimon_table_commit_overwrite(const struct paimon_table_commit *tc, + struct paimon_commit_messages *msgs); + +/** + * Commit standard overwrite messages with an identifier. + */ +struct paimon_error *paimon_table_commit_overwrite_with_identifier(const struct paimon_table_commit *tc, + struct paimon_commit_messages *msgs, + int64_t commit_identifier); + +/** + * Truncate a table with a standard TableCommit. + */ +struct paimon_error *paimon_table_commit_truncate_table(const struct paimon_table_commit *tc); + +/** + * Truncate a table with a stable identifier. + */ +struct paimon_error *paimon_table_commit_truncate_table_with_identifier(const struct paimon_table_commit *tc, + int64_t commit_identifier); + +/** + * Free a paimon_table. + * + * # Safety + * Only call with a table returned from `paimon_catalog_get_table`, + * `paimon_table_from_schema_json`, or + * `paimon_table_from_schema_json_with_file_io`. + */ +void paimon_table_free(struct paimon_table *table); + +/** + * Create a table directly from a resolved Paimon table schema JSON. + * + * This constructor does not create a catalog or derive a warehouse. Storage + * options are used only to build FileIO; they are not merged into the supplied + * table schema. `branch` selects the branch-scoped managers while preserving + * the supplied schema; pass null to default to the `main` branch. + * + * # Safety + * All string pointers except `branch` must be valid null-terminated C strings. + * `branch` may be null to select the default `main` branch, or a valid + * null-terminated C string. `storage_options` must point to + * `storage_options_len` valid `paimon_option` values, or be null when + * `storage_options_len` is 0. + */ +struct paimon_result_get_table paimon_table_from_schema_json(const char *table_path, + const char *table_schema_json, + const char *database, + const char *table_name, + const char *branch, + const struct paimon_option *storage_options, + size_t storage_options_len); + +/** + * Create a table from a resolved schema and a caller-created FileIO. + * + * The FileIO is cloned into the table, so its handle may be freed immediately + * after this call. This additive API allows native embedders to share storage + * and an externally managed cache across tables. + * + * # Safety + * `file_io` must be returned by a Paimon FileIO constructor. All string + * pointers except `branch` must be valid null-terminated C strings. `branch` + * may be null to select `main`. + */ +struct paimon_result_get_table paimon_table_from_schema_json_with_file_io(const struct paimon_file_io *file_io, + const char *table_path, + const char *table_schema_json, + const char *database, + const char *table_name, + const char *branch); + +/** + * Create a reader using a table's FileIO. + * + * # Safety + * `table` is a valid handle returned by the Paimon C API. + */ +struct paimon_result_blob_reader paimon_table_new_blob_reader(const struct paimon_table *table); + +/** + * Create a one-shot fixed-bucket WriteBuilder for a postpone table. + * A bucket plan must be set before creating a writer. + * + * # Safety + * `table` must be a valid table pointer, or null (returns error). + */ +struct paimon_result_postpone_fixed_bucket_write_builder paimon_table_new_postpone_fixed_bucket_write_builder(const struct paimon_table *table); + +/** + * Create a fixed-bucket WriteBuilder with a stable commit identity. + * A bucket plan must be set before creating a writer. + * + * # Safety + * `table` must be a valid table pointer. `commit_user` must be a valid UTF-8 + * C string and a safe file-name segment. + */ +struct paimon_result_postpone_fixed_bucket_write_builder paimon_table_new_postpone_fixed_bucket_write_builder_with_commit_user(const struct paimon_table *table, + const char *commit_user); + +/** + * Create a new ReadBuilder from a Table. + * + * # Safety + * `table` must be a valid pointer from `paimon_catalog_get_table` or + * `paimon_table_from_schema_json`, or null (returns error). + */ +struct paimon_result_read_builder paimon_table_new_read_builder(const struct paimon_table *table); + +/** + * Create a ReadBuilder from a Table with scan options (e.g. time-travel + * selectors `scan.snapshot-id` / `scan.tag-name` / `scan.timestamp-millis` / + * `scan.watermark` / `scan.version`). At most one time-travel selector may be + * set. A selector that does not resolve to a snapshot is an error (never a + * silent read-of-latest). + * + * # Safety + * `table` must be a valid pointer. `options` must be a valid pointer to + * `options_len` `paimon_option` values, or null when `options_len` is 0. + */ +struct paimon_result_read_builder paimon_table_new_read_builder_with_options(const struct paimon_table *table, + const struct paimon_option *options, + size_t options_len); + +/** + * Create a new vector-search builder from a Table. + * + * # Safety + * `table` must be a valid pointer from `paimon_catalog_get_table` or + * `paimon_table_from_schema_json`, or null (returns error). + */ +struct paimon_result_vector_search_builder paimon_table_new_vector_search_builder(const struct paimon_table *table); + +/** + * Create a new WriteBuilder from a Table. + * + * The returned WriteBuilder holds a shared `commit_user` (UUID) that will be + * used by both `new_write()` and `new_commit()` for duplicate-commit detection. + * + * # Safety + * `table` must be a valid pointer from `paimon_catalog_get_table` or + * `paimon_table_from_schema_json`, or null (returns error). + */ +struct paimon_result_write_builder paimon_table_new_write_builder(const struct paimon_table *table); + +/** + * Create a WriteBuilder with a caller-provided stable commit identity. + * + * Writers whose messages are merged into one logical commit must use the + * same `commit_user`. + * + * # Safety + * `table` must be a valid table pointer. `commit_user` must be a valid UTF-8 + * C string and a safe file-name segment. + */ +struct paimon_result_write_builder paimon_table_new_write_builder_with_commit_user(const struct paimon_table *table, + const char *commit_user); + +/** + * Free a paimon_table_read. + * + * # Safety + * Only call with a read returned from `paimon_read_builder_new_read`. + */ +void paimon_table_read_free(struct paimon_table_read *read); + +/** + * Read table data as Arrow record batches via a streaming reader. + * + * Returns a `paimon_record_batch_reader` that yields one batch at a time + * via `paimon_record_batch_reader_next`. This avoids loading all batches + * into memory at once. + * + * `offset` and `length` select a contiguous sub-range of splits from the + * plan. The range is clamped to the available splits (out-of-range values + * are silently adjusted). + * + * # Safety + * `read` and `plan` must be valid pointers from previous paimon C calls, or null (returns error). + */ +struct paimon_result_record_batch_reader paimon_table_read_to_arrow(const struct paimon_table_read *read, + const struct paimon_plan *plan, + size_t offset, + size_t length); + +/** + * Free a paimon_table_scan. + * + * # Safety + * Only call with a scan returned from `paimon_read_builder_new_scan`. + */ +void paimon_table_scan_free(struct paimon_table_scan *scan); + +/** + * Execute a scan plan to get splits. + * + * # Safety + * `scan` must be a valid pointer from `paimon_read_builder_new_scan`, or null (returns error). + */ +struct paimon_result_plan paimon_table_scan_plan(const struct paimon_table_scan *scan); + +/** + * Free a standard TableWrite. + * + * # Safety + * Only call with a write returned from paimon_write_builder_new_write. + */ +void paimon_table_write_free(struct paimon_table_write *tw); + +/** + * Prepare standard commit messages. + * + * The returned handle remains owned by the caller. + */ +struct paimon_result_prepare_commit paimon_table_write_prepare_commit(struct paimon_table_write *tw); + +/** + * Write one Arrow record batch with a standard TableWrite. + * + * Ownership of array and schema is transferred once Arrow import starts. + */ +struct paimon_error *paimon_table_write_write_arrow_batch(struct paimon_table_write *tw, + void *array, + void *schema); + +/** + * Execute the vector search and return a streaming Arrow reader over the + * materialized rows (projected user columns plus `__paimon_search_score`). + * Works for both primary-key and data-evolution tables. Consume via + * `paimon_record_batch_reader_next` and free with `paimon_record_batch_reader_free`. + * + * # Safety + * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or + * null (returns an error result). + */ +struct paimon_result_record_batch_reader paimon_vector_search_builder_execute_read(struct paimon_vector_search_builder *b); + +/** + * Free a paimon_vector_search_builder. + * + * # Safety + * Only call with a builder returned from `paimon_table_new_vector_search_builder`. + */ +void paimon_vector_search_builder_free(struct paimon_vector_search_builder *b); + +/** + * Set an optional scalar residual filter for a vector-search builder. + * + * The predicate is consumed (ownership transferred to the builder). Pass null + * to clear any previously set filter. + * + * # Safety + * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or + * null (returns error). `predicate` must be a valid pointer from a + * `paimon_predicate_*` function, or null. + */ +struct paimon_error *paimon_vector_search_builder_with_filter(struct paimon_vector_search_builder *b, + struct paimon_predicate *predicate); + +/** + * Set the maximum number of results for a vector-search builder. + * + * # Safety + * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or + * null (returns error). + */ +struct paimon_error *paimon_vector_search_builder_with_limit(struct paimon_vector_search_builder *b, + size_t limit); + +/** + * Set scan/search options for a vector-search builder. + * + * # Safety + * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or + * null (returns error). `options` must be a valid pointer to `len` + * `paimon_option` values, or null when `len` is 0. + */ +struct paimon_error *paimon_vector_search_builder_with_options(struct paimon_vector_search_builder *b, + const struct paimon_option *options, + size_t len); + +/** + * Restrict the columns materialized by `paimon_vector_search_builder_execute_read` + * to `columns` (plus the always-appended `__paimon_search_score`). Without this + * call `execute_read` materializes every user table column. Only affects + * `execute_read`. + * + * `columns` is a null-terminated array of null-terminated C strings; output + * order follows the caller-specified order. An empty list is a valid zero-column + * projection (only the score column is materialized). Pass null to clear any + * previously set projection. + * + * Unlike `paimon_read_builder_with_projection`, this does not validate column + * names eagerly: the vector builder resolves the projection against the schema + * when the search runs, so an unknown column surfaces as an error from + * `paimon_vector_search_builder_execute_read`. + * + * # Safety + * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or + * null (returns error). `columns` must be a null-terminated array of + * null-terminated C strings, or null to clear the projection. + */ +struct paimon_error *paimon_vector_search_builder_with_projection(struct paimon_vector_search_builder *b, + const char *const *columns); + +/** + * Set the query vector for a vector-search builder. + * + * The `len` floats at `data` are copied into the builder; the caller retains + * ownership of `data`. An empty vector (`len == 0`) is rejected. + * + * # Safety + * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or + * null (returns error). `data` must point to `len` `f32` values when `len > 0`. + */ +struct paimon_error *paimon_vector_search_builder_with_query_vector(struct paimon_vector_search_builder *b, + const float *data, + size_t len); + +/** + * Set the target vector column for a vector-search builder. + * + * # Safety + * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or + * null (returns error). `column` must be a valid C string. + */ +struct paimon_error *paimon_vector_search_builder_with_vector_column(struct paimon_vector_search_builder *b, + const char *column); + +/** + * Free a paimon_write_builder. + * + * # Safety + * Only call with a write_builder returned from `paimon_table_new_write_builder`. + */ +void paimon_write_builder_free(struct paimon_write_builder *wb); + +/** + * Create a standard TableCommit from a standard WriteBuilder. + */ +struct paimon_result_table_commit paimon_write_builder_new_commit(const struct paimon_write_builder *wb); + +/** + * Create a standard TableWrite from a standard WriteBuilder. + * + * # Safety + * wb must be a valid standard builder, or null (returns error). + */ +struct paimon_result_table_write paimon_write_builder_new_write(const struct paimon_write_builder *wb); + +/** + * Enable overwrite mode for the WriteBuilder. + * + * # Safety + * `wb` must be a valid pointer from `paimon_table_new_write_builder`, or null (returns error). + */ +struct paimon_error *paimon_write_builder_with_overwrite(struct paimon_write_builder *wb); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif /* PAIMON_C_H */ diff --git a/bindings/c/src/error.rs b/bindings/c/src/error.rs index 7b0a88fc3..1613c9e58 100644 --- a/bindings/c/src/error.rs +++ b/bindings/c/src/error.rs @@ -19,15 +19,25 @@ use std::ffi::{c_char, CStr}; use crate::types::paimon_bytes; +pub const PAIMON_ERROR_UNEXPECTED: i32 = 0; +pub const PAIMON_ERROR_UNSUPPORTED: i32 = 1; +pub const PAIMON_ERROR_NOT_FOUND: i32 = 2; +pub const PAIMON_ERROR_ALREADY_EXISTS: i32 = 3; +pub const PAIMON_ERROR_INVALID_INPUT: i32 = 4; +pub const PAIMON_ERROR_IO: i32 = 5; +pub const PAIMON_ERROR_OUT_OF_RANGE: i32 = 6; + /// Error codes for paimon C API. #[repr(i32)] pub enum PaimonErrorCode { - Unexpected = 0, - Unsupported = 1, - NotFound = 2, - AlreadyExists = 3, - InvalidInput = 4, - IoError = 5, + Unexpected = PAIMON_ERROR_UNEXPECTED, + Unsupported = PAIMON_ERROR_UNSUPPORTED, + NotFound = PAIMON_ERROR_NOT_FOUND, + AlreadyExists = PAIMON_ERROR_ALREADY_EXISTS, + InvalidInput = PAIMON_ERROR_INVALID_INPUT, + IoError = PAIMON_ERROR_IO, + /// A requested streaming checkpoint or snapshot is no longer readable. + OutOfRange = PAIMON_ERROR_OUT_OF_RANGE, } /// C-compatible error type. @@ -53,9 +63,18 @@ impl paimon_error { paimon::Error::TableNotExist { .. } | paimon::Error::DatabaseNotExist { .. } | paimon::Error::ColumnNotExist { .. } => PaimonErrorCode::NotFound, + paimon::Error::SnapshotNotExist { .. } => PaimonErrorCode::OutOfRange, paimon::Error::TableAlreadyExist { .. } | paimon::Error::DatabaseAlreadyExist { .. } | paimon::Error::ColumnAlreadyExist { .. } => PaimonErrorCode::AlreadyExists, + paimon::Error::DataInvalid { message, .. } + if message.contains("snapshot") + && (message.contains("expired") + || message.contains("out of range") + || message.contains("too large")) => + { + PaimonErrorCode::OutOfRange + } paimon::Error::ConfigInvalid { .. } | paimon::Error::DataTypeInvalid { .. } | paimon::Error::DataInvalid { .. } diff --git a/bindings/c/src/lib.rs b/bindings/c/src/lib.rs index 0a5710ccc..297b9e355 100644 --- a/bindings/c/src/lib.rs +++ b/bindings/c/src/lib.rs @@ -25,11 +25,13 @@ mod error; mod file_io; mod identifier; mod result; +mod stream; mod table; #[cfg(test)] mod tests; mod types; mod vector_search; +mod version; mod write; use std::sync::OnceLock; diff --git a/bindings/c/src/result.rs b/bindings/c/src/result.rs index 94317572b..72e2cfe8c 100644 --- a/bindings/c/src/result.rs +++ b/bindings/c/src/result.rs @@ -146,6 +146,18 @@ pub struct paimon_result_prepare_commit { pub error: *mut paimon_error, } +#[repr(C)] +pub struct paimon_result_prepared_commit { + pub prepared: *mut paimon_prepared_commit, + pub error: *mut paimon_error, +} + +#[repr(C)] +pub struct paimon_result_bytes { + pub bytes: paimon_bytes, + pub error: *mut paimon_error, +} + #[repr(C)] pub struct paimon_result_postpone_fixed_bucket_write_builder { pub write_builder: *mut paimon_postpone_fixed_bucket_write_builder, diff --git a/bindings/c/src/stream.rs b/bindings/c/src/stream.rs new file mode 100644 index 000000000..6d0ab98dd --- /dev/null +++ b/bindings/c/src/stream.rs @@ -0,0 +1,1168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Stateful continuous-read C ABI. +//! +//! This is deliberately a pull API. It does not create a C++ callback thread, +//! so callers retain control of cancellation, backpressure and checkpoint +//! barriers. All handles in this module are single-thread-confined. + +use std::ffi::c_void; +use std::mem::size_of; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::ptr; + +use paimon::table::{ + ArrowRecordBatchStream, IncrementalPlan, IncrementalScanMode, IncrementalSplit, Plan, + StreamPlan, StreamScan, StreamScanFollowUpMode, StreamScanPoll, StreamScanStartupMode, + TableRead, +}; +use paimon::DataSplit; +use serde::{Deserialize, Serialize}; + +use crate::error::{check_non_null, paimon_error, PaimonErrorCode}; +use crate::result::{paimon_result_bytes, paimon_result_record_batch_reader}; +use crate::runtime; +use crate::types::{ + paimon_bytes, paimon_read_builder, paimon_record_batch_reader, paimon_table_read, + read_builder_fingerprint, ReadBuilderState, TableReadState, +}; + +pub const PAIMON_STREAM_STARTUP_LATEST_FULL: i32 = 0; +pub const PAIMON_STREAM_STARTUP_LATEST: i32 = 1; +pub const PAIMON_STREAM_STARTUP_FROM_SNAPSHOT: i32 = 2; +pub const PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL: i32 = 3; + +pub const PAIMON_STREAM_FOLLOW_UP_AUTO: i32 = 0; +pub const PAIMON_STREAM_FOLLOW_UP_DELTA: i32 = 1; +pub const PAIMON_STREAM_FOLLOW_UP_CHANGELOG: i32 = 2; + +pub const PAIMON_STREAM_POLL_DATA: i32 = 0; +pub const PAIMON_STREAM_POLL_WAITING: i32 = 1; +pub const PAIMON_STREAM_POLL_END: i32 = 2; + +pub const PAIMON_STREAM_READ_DATA: i32 = 0; +pub const PAIMON_STREAM_READ_AUDIT_LOG: i32 = 1; + +/// Extensible options for a continuous scan. +/// +/// Initialize this with `paimon_stream_scan_options_init`; future versions may +/// consume fields from `reserved` while preserving this prefix. +#[repr(C)] +pub struct paimon_stream_scan_options { + pub struct_size: u32, + pub startup_mode: i32, + pub follow_up_mode: i32, + pub snapshot_id: i64, + pub reserved: [u64; 4], +} + +#[repr(C)] +pub struct paimon_stream_scan { + pub inner: *mut c_void, +} + +#[repr(C)] +pub struct paimon_stream_plan { + pub inner: *mut c_void, +} + +#[repr(C)] +pub struct paimon_result_stream_scan { + pub scan: *mut paimon_stream_scan, + pub error: *mut paimon_error, +} + +#[repr(C)] +pub struct paimon_result_stream_poll { + pub status: i32, + pub plan: *mut paimon_stream_plan, + pub snapshot_id: i64, + pub next_snapshot_id: i64, + pub watermark: i64, + pub has_watermark: u8, + pub reserved: [u8; 7], + pub error: *mut paimon_error, +} + +const STREAM_PLAN_FORMAT: &str = "paimon-rust-stream-plan"; +// Version 3 also binds restored work to the table branch. Earlier versions are +// rejected because location + schema id alone cannot distinguish two branches. +const STREAM_PLAN_VERSION: u32 = 3; +const MAX_STREAM_PLAN_BYTES: usize = 64 * 1024 * 1024; +const MAX_STREAM_PLAN_SPLITS: usize = 100_000; +const MAX_STREAM_SPLIT_BYTES: usize = 16 * 1024 * 1024; +const MAX_STREAM_SPLIT_TOTAL_BYTES: usize = 64 * 1024 * 1024; +const MAX_STREAM_IDENTITY_BYTES: usize = 1024 * 1024; + +struct StreamScanState { + scan: StreamScan, + table_location: String, + table_branch: String, + schema_id: i64, + read_fingerprint: String, +} + +struct StreamPlanState { + plan: StreamPlan, + table_location: String, + table_branch: String, + schema_id: i64, + read_fingerprint: String, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StreamPlanEnvelope { + format: String, + version: u32, + table_location: String, + #[serde(default)] + table_branch: String, + schema_id: i64, + read_fingerprint: String, + kind: i32, + incremental_mode: i32, + snapshot_id: i64, + next_snapshot_id: i64, + watermark: Option, + #[serde(with = "bounded_splits")] + splits: Vec>, +} + +mod bounded_splits { + use std::fmt; + + use serde::de::{DeserializeSeed, Error, IgnoredAny, SeqAccess, Visitor}; + use serde::{Deserializer, Serialize, Serializer}; + + use super::{MAX_STREAM_PLAN_SPLITS, MAX_STREAM_SPLIT_BYTES, MAX_STREAM_SPLIT_TOTAL_BYTES}; + + pub fn serialize(splits: &[Vec], serializer: S) -> Result + where + S: Serializer, + { + splits.serialize(serializer) + } + + struct SplitBytesSeed; + + impl<'de> DeserializeSeed<'de> for SplitBytesSeed { + type Value = Vec; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_seq(SplitBytesVisitor) + } + } + + struct SplitBytesVisitor; + + impl<'de> Visitor<'de> for SplitBytesVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a bounded stream split byte array") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut bytes = Vec::with_capacity( + sequence + .size_hint() + .unwrap_or_default() + .min(MAX_STREAM_SPLIT_BYTES), + ); + while bytes.len() < MAX_STREAM_SPLIT_BYTES { + let Some(value) = sequence.next_element::()? else { + return Ok(bytes); + }; + bytes.push(value); + } + if sequence.next_element::()?.is_some() { + return Err(A::Error::custom(format!( + "stream plan split exceeds {MAX_STREAM_SPLIT_BYTES} bytes" + ))); + } + Ok(bytes) + } + } + + struct SplitsVisitor; + + impl<'de> Visitor<'de> for SplitsVisitor { + type Value = Vec>; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a bounded list of stream split byte arrays") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut splits = Vec::with_capacity( + sequence + .size_hint() + .unwrap_or_default() + .min(MAX_STREAM_PLAN_SPLITS), + ); + let mut total_bytes = 0usize; + while splits.len() < MAX_STREAM_PLAN_SPLITS { + let Some(split) = sequence.next_element_seed(SplitBytesSeed)? else { + return Ok(splits); + }; + total_bytes = total_bytes + .checked_add(split.len()) + .ok_or_else(|| A::Error::custom("stream plan split byte count overflows"))?; + if total_bytes > MAX_STREAM_SPLIT_TOTAL_BYTES { + return Err(A::Error::custom(format!( + "stream plan split bytes exceed {MAX_STREAM_SPLIT_TOTAL_BYTES}" + ))); + } + splits.push(split); + } + if sequence.next_element::()?.is_some() { + return Err(A::Error::custom(format!( + "stream plan contains more than {MAX_STREAM_PLAN_SPLITS} splits" + ))); + } + Ok(splits) + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_seq(SplitsVisitor) + } +} + +fn validate_stream_plan_envelope(envelope: &StreamPlanEnvelope) -> Result<(), *mut paimon_error> { + if envelope.table_location.is_empty() + || envelope.table_location.len() > MAX_STREAM_IDENTITY_BYTES + || envelope.table_branch.is_empty() + || envelope.table_branch.len() > MAX_STREAM_IDENTITY_BYTES + || envelope.schema_id < 0 + || envelope.read_fingerprint.is_empty() + || envelope.read_fingerprint.len() > MAX_STREAM_IDENTITY_BYTES + { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan contains an invalid table or read identity".to_string(), + )); + } + if envelope.splits.len() > MAX_STREAM_PLAN_SPLITS { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!( + "stream plan contains {} splits; maximum is {}", + envelope.splits.len(), + MAX_STREAM_PLAN_SPLITS + ), + )); + } + if envelope + .splits + .iter() + .any(|split| split.len() > MAX_STREAM_SPLIT_BYTES) + { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("stream plan split exceeds {MAX_STREAM_SPLIT_BYTES} bytes"), + )); + } + let total_split_bytes = envelope + .splits + .iter() + .try_fold(0usize, |total, split| total.checked_add(split.len())); + if total_split_bytes.is_none_or(|total| total > MAX_STREAM_SPLIT_TOTAL_BYTES) { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("stream plan split bytes exceed {MAX_STREAM_SPLIT_TOTAL_BYTES}"), + )); + } + let Some(expected_next_snapshot_id) = envelope.snapshot_id.checked_add(1) else { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan snapshot cursor overflows".to_string(), + )); + }; + let valid_next_snapshot_id = envelope.next_snapshot_id == expected_next_snapshot_id + || (envelope.kind == 0 && envelope.next_snapshot_id == envelope.snapshot_id); + if envelope.snapshot_id < 1 || !valid_next_snapshot_id { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan contains an invalid snapshot cursor".to_string(), + )); + } + Ok(()) +} + +fn validate_stream_plan_recovery_paths(state: &StreamPlanState) -> Result<(), *mut paimon_error> { + match &state.plan { + StreamPlan::Full { plan, .. } => { + for split in plan.splits() { + split + .validate_restored_containment(&state.table_location) + .map_err(paimon_error::from_paimon)?; + } + } + StreamPlan::Incremental { plan, .. } => { + for split in plan.splits() { + let IncrementalSplit::Data(split) = split else { + return Err(paimon_error::new( + PaimonErrorCode::Unsupported, + "DiffPair plans are not valid continuous stream plans".to_string(), + )); + }; + split + .validate_restored_containment(&state.table_location) + .map_err(paimon_error::from_paimon)?; + } + } + } + Ok(()) +} + +fn panic_error(operation: &str) -> *mut paimon_error { + paimon_error::new( + PaimonErrorCode::Unexpected, + format!("Rust panic while executing {operation}"), + ) +} + +fn invalid_mode(name: &str, value: i32) -> *mut paimon_error { + paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("invalid {name} value {value}"), + ) +} + +fn empty_bytes() -> paimon_bytes { + paimon_bytes { + data: ptr::null_mut(), + len: 0, + } +} + +fn empty_poll(status: i32, scan: Option<&StreamScan>) -> paimon_result_stream_poll { + paimon_result_stream_poll { + status, + plan: ptr::null_mut(), + snapshot_id: -1, + next_snapshot_id: scan.and_then(StreamScan::checkpoint).unwrap_or(-1), + watermark: scan.and_then(StreamScan::watermark).unwrap_or(0), + has_watermark: u8::from(scan.and_then(StreamScan::watermark).is_some()), + reserved: [0; 7], + error: ptr::null_mut(), + } +} + +fn error_poll(error: *mut paimon_error) -> paimon_result_stream_poll { + let mut result = empty_poll(PAIMON_STREAM_POLL_END, None); + result.error = error; + result +} + +fn configure_builder<'a>( + state: &'a ReadBuilderState, +) -> Result, *mut paimon_error> { + let mut builder = state.table.new_read_builder(); + builder.with_case_sensitive(state.case_sensitive); + if let Some(columns) = &state.projected_columns { + let columns: Vec<&str> = columns.iter().map(String::as_str).collect(); + builder + .with_projection(&columns) + .map_err(paimon_error::from_paimon)?; + } + if let Some(filter) = &state.filter { + builder.with_filter(filter.clone()); + } + Ok(builder) +} + +/// Fill stream options with forward-compatible defaults (`latest-full`, +/// automatic delta/changelog selection). +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_options_init( + options: *mut paimon_stream_scan_options, +) -> *mut paimon_error { + if let Err(error) = check_non_null(options, "options") { + return error; + } + ptr::write( + options, + paimon_stream_scan_options { + struct_size: size_of::() as u32, + startup_mode: PAIMON_STREAM_STARTUP_LATEST_FULL, + follow_up_mode: PAIMON_STREAM_FOLLOW_UP_AUTO, + snapshot_id: -1, + reserved: [0; 4], + }, + ); + ptr::null_mut() +} + +/// Create an owned stream scan from a read builder. +/// +/// The returned scan clones all required Rust state and remains valid after +/// the read builder and table handles are freed. A scan handle is +/// single-thread-confined: callers must serialize poll/checkpoint/restore/free. +#[no_mangle] +pub unsafe extern "C" fn paimon_read_builder_new_stream_scan( + read_builder: *const paimon_read_builder, + options: *const paimon_stream_scan_options, +) -> paimon_result_stream_scan { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(read_builder, "read_builder") { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error, + }; + } + if let Err(error) = check_non_null(options, "options") { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error, + }; + } + if (*options).struct_size < size_of::() as u32 { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::InvalidInput, + format!( + "stream options struct_size {} is smaller than required {}", + (*options).struct_size, + size_of::() + ), + ), + }; + } + if (*options).reserved.iter().any(|value| *value != 0) { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::Unsupported, + "stream options reserved fields must be zero for ABI version 1".to_string(), + ), + }; + } + let startup = match (*options).startup_mode { + PAIMON_STREAM_STARTUP_LATEST_FULL => StreamScanStartupMode::LatestFull, + PAIMON_STREAM_STARTUP_LATEST => StreamScanStartupMode::Latest, + PAIMON_STREAM_STARTUP_FROM_SNAPSHOT => { + StreamScanStartupMode::FromSnapshot((*options).snapshot_id) + } + PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL => { + StreamScanStartupMode::FromSnapshotFull((*options).snapshot_id) + } + value => { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error: invalid_mode("stream startup mode", value), + } + } + }; + let follow_up = match (*options).follow_up_mode { + PAIMON_STREAM_FOLLOW_UP_AUTO => StreamScanFollowUpMode::Auto, + PAIMON_STREAM_FOLLOW_UP_DELTA => StreamScanFollowUpMode::Delta, + PAIMON_STREAM_FOLLOW_UP_CHANGELOG => StreamScanFollowUpMode::Changelog, + value => { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error: invalid_mode("stream follow-up mode", value), + } + } + }; + let state = &*((*read_builder).inner as *const ReadBuilderState); + let table_location = state.table.location().to_string(); + let table_branch = state.table.branch().to_string(); + let schema_id = state.table.schema().id(); + let read_fingerprint = read_builder_fingerprint(state); + let builder = match configure_builder(state) { + Ok(builder) => builder, + Err(error) => { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error, + } + } + }; + match runtime().block_on(builder.new_stream_scan(startup, follow_up)) { + Ok(scan) => { + let inner = Box::into_raw(Box::new(StreamScanState { + scan, + table_location, + table_branch, + schema_id, + read_fingerprint, + })) as *mut c_void; + paimon_result_stream_scan { + scan: Box::into_raw(Box::new(paimon_stream_scan { inner })), + error: ptr::null_mut(), + } + } + Err(error) => paimon_result_stream_scan { + scan: ptr::null_mut(), + error: paimon_error::from_paimon(error), + }, + } + })); + outcome.unwrap_or_else(|_| paimon_result_stream_scan { + scan: ptr::null_mut(), + error: panic_error("paimon_read_builder_new_stream_scan"), + }) +} + +/// Poll once for a snapshot plan. This call never waits for a future snapshot. +/// Calls using the same scan handle must not overlap on different threads. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_poll( + scan: *mut paimon_stream_scan, +) -> paimon_result_stream_poll { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(scan, "scan") { + return error_poll(error); + } + let state = &mut *((*scan).inner as *mut StreamScanState); + match runtime().block_on(state.scan.poll_next()) { + Ok(StreamScanPoll::Data(plan)) => { + let snapshot_id = plan.snapshot_id(); + let next_snapshot_id = plan.next_snapshot_id(); + let watermark = plan.watermark(); + let inner = Box::into_raw(Box::new(StreamPlanState { + plan, + table_location: state.table_location.clone(), + table_branch: state.table_branch.clone(), + schema_id: state.schema_id, + read_fingerprint: state.read_fingerprint.clone(), + })) as *mut c_void; + paimon_result_stream_poll { + status: PAIMON_STREAM_POLL_DATA, + plan: Box::into_raw(Box::new(paimon_stream_plan { inner })), + snapshot_id, + next_snapshot_id, + watermark: watermark.unwrap_or(0), + has_watermark: u8::from(watermark.is_some()), + reserved: [0; 7], + error: ptr::null_mut(), + } + } + Ok(StreamScanPoll::Waiting) => { + empty_poll(PAIMON_STREAM_POLL_WAITING, Some(&state.scan)) + } + Ok(StreamScanPoll::End) => empty_poll(PAIMON_STREAM_POLL_END, Some(&state.scan)), + Err(error) => error_poll(paimon_error::from_paimon(error)), + } + })); + outcome.unwrap_or_else(|_| error_poll(panic_error("paimon_stream_scan_poll"))) +} + +/// Return the next-snapshot cursor, or -1 before a startup position exists. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_checkpoint(scan: *const paimon_stream_scan) -> i64 { + if scan.is_null() || (*scan).inner.is_null() { + return -1; + } + let state = &*((*scan).inner as *const StreamScanState); + state.scan.checkpoint().unwrap_or(-1) +} + +/// Restore a next-snapshot cursor. Pass -1 to reapply the configured startup +/// mode; non-negative values must name a valid Paimon snapshot position. +/// This call must not overlap poll/checkpoint/free on the same handle. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_restore( + scan: *mut paimon_stream_scan, + next_snapshot_id: i64, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(scan, "scan") { + return error; + } + if next_snapshot_id != -1 && next_snapshot_id < 1 { + return paimon_error::new( + PaimonErrorCode::InvalidInput, + "next_snapshot_id must be -1 or a positive snapshot id".to_string(), + ); + } + let state = &mut *((*scan).inner as *mut StreamScanState); + match state + .scan + .restore((next_snapshot_id >= 0).then_some(next_snapshot_id)) + { + Ok(()) => ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| panic_error("paimon_stream_scan_restore")) +} + +/// Free a stream scan. It is valid to pass null. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_free(scan: *mut paimon_stream_scan) { + if !scan.is_null() { + let wrapper = Box::from_raw(scan); + if !wrapper.inner.is_null() { + drop(Box::from_raw(wrapper.inner as *mut StreamScanState)); + } + } +} + +/// Return whether a stream plan is an initial full-snapshot plan. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_is_full(plan: *const paimon_stream_plan) -> u8 { + if plan.is_null() || (*plan).inner.is_null() { + return 0; + } + let state = &*((*plan).inner as *const StreamPlanState); + u8::from(state.plan.full_plan().is_some()) +} + +/// Return the number of work splits in a stream plan. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_num_splits(plan: *const paimon_stream_plan) -> usize { + if plan.is_null() || (*plan).inner.is_null() { + return 0; + } + let state = &*((*plan).inner as *const StreamPlanState); + match &state.plan { + StreamPlan::Full { plan, .. } => plan.splits().len(), + StreamPlan::Incremental { plan, .. } => plan.splits().len(), + } +} + +/// Serialize planned-but-not-yet-consumed work for an external checkpoint. +/// +/// The current format checkpoints at plan boundaries. If rows from a plan have already +/// been exposed, callers must either replay the plan after recovery or persist +/// their own logical rows-to-skip position alongside this buffer. +/// Plans containing external data-file paths are rejected because version 1 +/// recovery cannot revalidate those paths against a trusted manifest. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_serialize( + plan: *const paimon_stream_plan, +) -> paimon_result_bytes { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(plan, "plan") { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + let state = &*((*plan).inner as *const StreamPlanState); + if let Err(error) = validate_stream_plan_recovery_paths(state) { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + let envelope = match &state.plan { + StreamPlan::Full { + snapshot_id, + watermark, + next_snapshot_id, + plan, + } => { + let splits = match plan + .splits() + .iter() + .map(DataSplit::serialize_split_v1) + .collect::>>() + { + Ok(splits) => splits, + Err(error) => { + return paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::from_paimon(error), + } + } + }; + StreamPlanEnvelope { + format: STREAM_PLAN_FORMAT.to_string(), + version: STREAM_PLAN_VERSION, + table_location: state.table_location.clone(), + table_branch: state.table_branch.clone(), + schema_id: state.schema_id, + read_fingerprint: state.read_fingerprint.clone(), + kind: 0, + incremental_mode: -1, + snapshot_id: *snapshot_id, + next_snapshot_id: *next_snapshot_id, + watermark: *watermark, + splits, + } + } + StreamPlan::Incremental { + snapshot_id, + watermark, + next_snapshot_id, + plan, + } => { + let mut splits = Vec::with_capacity(plan.splits().len()); + for split in plan.splits() { + let IncrementalSplit::Data(split) = split else { + return paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::Unsupported, + "DiffPair plans are not valid continuous stream plans".to_string(), + ), + }; + }; + match split.serialize_split_v1() { + Ok(bytes) => splits.push(bytes), + Err(error) => { + return paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::from_paimon(error), + } + } + } + } + let incremental_mode = match plan.mode() { + IncrementalScanMode::Delta => 0, + IncrementalScanMode::Changelog => 1, + IncrementalScanMode::Auto | IncrementalScanMode::Diff => { + return paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::Unsupported, + "unresolved Auto and Diff plans are not valid continuous stream plans" + .to_string(), + ), + }; + } + }; + StreamPlanEnvelope { + format: STREAM_PLAN_FORMAT.to_string(), + version: STREAM_PLAN_VERSION, + table_location: state.table_location.clone(), + table_branch: state.table_branch.clone(), + schema_id: state.schema_id, + read_fingerprint: state.read_fingerprint.clone(), + kind: 1, + incremental_mode, + snapshot_id: *snapshot_id, + next_snapshot_id: *next_snapshot_id, + watermark: *watermark, + splits, + } + } + }; + if let Err(error) = validate_stream_plan_envelope(&envelope) { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + match serde_json::to_vec(&envelope) { + Ok(bytes) if bytes.len() <= MAX_STREAM_PLAN_BYTES => paimon_result_bytes { + bytes: paimon_bytes::new(bytes), + error: ptr::null_mut(), + }, + Ok(bytes) => paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::InvalidInput, + format!( + "serialized stream plan is {} bytes; maximum is {}", + bytes.len(), + MAX_STREAM_PLAN_BYTES + ), + ), + }, + Err(error) => paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::Unexpected, + format!("failed to serialize stream plan: {error}"), + ), + }, + } + })); + outcome.unwrap_or_else(|_| paimon_result_bytes { + bytes: empty_bytes(), + error: panic_error("paimon_stream_plan_serialize"), + }) +} + +/// Restore a stream plan serialized by `paimon_stream_plan_serialize`. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_deserialize( + data: *const u8, + len: usize, +) -> paimon_result_stream_poll { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if data.is_null() || len == 0 { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan buffer must not be null or empty".to_string(), + )); + } + if len > MAX_STREAM_PLAN_BYTES { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("stream plan buffer exceeds {MAX_STREAM_PLAN_BYTES} bytes"), + )); + } + let envelope: StreamPlanEnvelope = + match serde_json::from_slice(std::slice::from_raw_parts(data, len)) { + Ok(envelope) => envelope, + Err(error) => { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("invalid stream plan buffer: {error}"), + )) + } + }; + if envelope.format != STREAM_PLAN_FORMAT || envelope.version != STREAM_PLAN_VERSION { + return error_poll(paimon_error::new( + PaimonErrorCode::Unsupported, + format!( + "unsupported stream plan format '{}' version {}", + envelope.format, envelope.version + ), + )); + } + if let Err(error) = validate_stream_plan_envelope(&envelope) { + return error_poll(error); + } + let splits = match envelope + .splits + .iter() + .map(|split| DataSplit::deserialize_split_v1(split)) + .collect::>>() + { + Ok(splits) => splits, + Err(error) => return error_poll(paimon_error::from_paimon(error)), + }; + if splits + .iter() + .any(|split| split.snapshot_id() != envelope.snapshot_id) + { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan split snapshot does not match its envelope".to_string(), + )); + } + for split in &splits { + if let Err(error) = split.validate_restored_containment(&envelope.table_location) { + return error_poll(paimon_error::from_paimon(error)); + } + } + let plan = match (envelope.kind, envelope.incremental_mode) { + (0, -1) => StreamPlan::Full { + snapshot_id: envelope.snapshot_id, + watermark: envelope.watermark, + next_snapshot_id: envelope.next_snapshot_id, + plan: Plan::new(splits), + }, + (1, mode @ (0 | 1)) => { + let mode = if mode == 0 { + IncrementalScanMode::Delta + } else { + IncrementalScanMode::Changelog + }; + let splits = splits.into_iter().map(IncrementalSplit::Data).collect(); + let plan = match IncrementalPlan::try_new(mode, splits) { + Ok(plan) => plan, + Err(error) => return error_poll(paimon_error::from_paimon(error)), + }; + StreamPlan::Incremental { + snapshot_id: envelope.snapshot_id, + watermark: envelope.watermark, + next_snapshot_id: envelope.next_snapshot_id, + plan, + } + } + _ => { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan contains an invalid kind or mode".to_string(), + )) + } + }; + let snapshot_id = plan.snapshot_id(); + let next_snapshot_id = plan.next_snapshot_id(); + let watermark = plan.watermark(); + let inner = Box::into_raw(Box::new(StreamPlanState { + plan, + table_location: envelope.table_location, + table_branch: envelope.table_branch, + schema_id: envelope.schema_id, + read_fingerprint: envelope.read_fingerprint, + })) as *mut c_void; + paimon_result_stream_poll { + status: PAIMON_STREAM_POLL_DATA, + plan: Box::into_raw(Box::new(paimon_stream_plan { inner })), + snapshot_id, + next_snapshot_id, + watermark: watermark.unwrap_or(0), + has_watermark: u8::from(watermark.is_some()), + reserved: [0; 7], + error: ptr::null_mut(), + } + })); + outcome.unwrap_or_else(|_| error_poll(panic_error("paimon_stream_plan_deserialize"))) +} + +/// Read a contiguous split range from a stream plan. +/// +/// `read_mode=PAIMON_STREAM_READ_AUDIT_LOG` exposes a stable UTF-8 `rowkind` +/// column for incremental plans. Full startup plans currently support data +/// mode only; callers requiring one fixed audit schema should start at +/// `latest` or `from-snapshot`. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_read_to_arrow( + read: *const paimon_table_read, + plan: *const paimon_stream_plan, + offset: usize, + length: usize, + read_mode: i32, +) -> paimon_result_record_batch_reader { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(read, "read") { + return paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error, + }; + } + if let Err(error) = check_non_null(plan, "plan") { + return paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error, + }; + } + if read_mode != PAIMON_STREAM_READ_DATA && read_mode != PAIMON_STREAM_READ_AUDIT_LOG { + return paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error: invalid_mode("stream read mode", read_mode), + }; + } + let state = &*((*read).inner as *const TableReadState); + let plan_state = &*((*plan).inner as *const StreamPlanState); + if state.table_location != plan_state.table_location + || state.table_branch != plan_state.table_branch + || state.schema_id != plan_state.schema_id + || state.read_fingerprint != plan_state.read_fingerprint + { + return paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan was created for a different table, branch, schema, or read builder" + .to_string(), + ), + }; + } + let stream_plan = &plan_state.plan; + let table_read = TableRead::new( + &state.table, + state.read_type.clone(), + state.data_predicates.clone(), + ); + let stream_result: paimon::Result = match stream_plan { + StreamPlan::Full { plan, .. } => { + if read_mode == PAIMON_STREAM_READ_AUDIT_LOG { + Err(paimon::Error::Unsupported { + message: "Audit-log mode for a full stream startup plan is not implemented; use latest/from-snapshot startup or data mode".to_string(), + }) + } else { + let splits = plan.splits(); + let start = offset.min(splits.len()); + let end = offset.saturating_add(length).min(splits.len()); + table_read.to_arrow(&splits[start..end]) + } + } + StreamPlan::Incremental { plan, .. } => { + let splits = plan.splits(); + let start = offset.min(splits.len()); + let end = offset.saturating_add(length).min(splits.len()); + let selected = paimon::table::IncrementalPlan::try_new( + plan.mode(), + splits[start..end].to_vec(), + ); + match selected { + Ok(selected) if read_mode == PAIMON_STREAM_READ_AUDIT_LOG => { + table_read.to_audit_log_arrow(&selected) + } + Ok(selected) => table_read.to_incremental_arrow(&selected), + Err(error) => Err(error), + } + } + }; + match stream_result { + Ok(stream) => { + let inner = Box::into_raw(Box::new(stream)) as *mut c_void; + paimon_result_record_batch_reader { + reader: Box::into_raw(Box::new(paimon_record_batch_reader { inner })), + error: ptr::null_mut(), + } + } + Err(error) => paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error: paimon_error::from_paimon(error), + }, + } + })); + outcome.unwrap_or_else(|_| paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error: panic_error("paimon_stream_plan_read_to_arrow"), + }) +} + +/// Free a stream plan. It is valid to pass null. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_free(plan: *mut paimon_stream_plan) { + if !plan.is_null() { + let wrapper = Box::from_raw(plan); + if !wrapper.inner.is_null() { + drop(Box::from_raw(wrapper.inner as *mut StreamPlanState)); + } + } +} + +// C ABI signature guards. +const _: unsafe extern "C" fn( + *const paimon_read_builder, + *const paimon_stream_scan_options, +) -> paimon_result_stream_scan = paimon_read_builder_new_stream_scan; +const _: unsafe extern "C" fn(*mut paimon_stream_scan) -> paimon_result_stream_poll = + paimon_stream_scan_poll; +const _: unsafe extern "C" fn( + *const paimon_table_read, + *const paimon_stream_plan, + usize, + usize, + i32, +) -> paimon_result_record_batch_reader = paimon_stream_plan_read_to_arrow; +const _: unsafe extern "C" fn(*const paimon_stream_plan) -> paimon_result_bytes = + paimon_stream_plan_serialize; +const _: unsafe extern "C" fn(*const u8, usize) -> paimon_result_stream_poll = + paimon_stream_plan_deserialize; + +#[cfg(test)] +mod tests { + use paimon::spec::BinaryRow; + use paimon::table::DataSplitBuilder; + + use super::*; + + fn serialized_plan(bucket_path: &str, next_snapshot_id: i64) -> Vec { + let split = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(bucket_path.to_string()) + .with_total_buckets(1) + .with_data_files(Vec::new()) + .build() + .unwrap(); + serde_json::to_vec(&StreamPlanEnvelope { + format: STREAM_PLAN_FORMAT.to_string(), + version: STREAM_PLAN_VERSION, + table_location: "memory:/table".to_string(), + table_branch: "main".to_string(), + schema_id: 0, + read_fingerprint: "fingerprint".to_string(), + kind: 0, + incremental_mode: -1, + snapshot_id: 1, + next_snapshot_id, + watermark: None, + splits: vec![split.serialize_split_v1().unwrap()], + }) + .unwrap() + } + + fn full_plan_state(bucket_path: &str) -> StreamPlanState { + let split = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(bucket_path.to_string()) + .with_total_buckets(1) + .with_data_files(Vec::new()) + .build() + .unwrap(); + StreamPlanState { + plan: StreamPlan::Full { + snapshot_id: 1, + watermark: None, + next_snapshot_id: 2, + plan: Plan::new(vec![split]), + }, + table_location: "memory:/table".to_string(), + table_branch: "main".to_string(), + schema_id: 0, + read_fingerprint: "fingerprint".to_string(), + } + } + + #[test] + fn serialization_rejects_plan_which_cannot_be_restored() { + assert!( + validate_stream_plan_recovery_paths(&full_plan_state("memory:/table/bucket-0")).is_ok() + ); + let error = + validate_stream_plan_recovery_paths(&full_plan_state("memory:/table-evil/bucket-0")) + .unwrap_err(); + unsafe { crate::error::paimon_error_free(error) }; + } + + #[test] + fn restored_plan_rejects_bucket_path_outside_table() { + let bytes = serialized_plan("memory:/table-evil/bucket-0", 2); + let result = unsafe { paimon_stream_plan_deserialize(bytes.as_ptr(), bytes.len()) }; + assert!(result.plan.is_null()); + assert!(!result.error.is_null()); + unsafe { crate::error::paimon_error_free(result.error) }; + } + + #[test] + fn full_plan_allows_same_snapshot_follow_up_cursor() { + let bytes = serialized_plan("memory:/table/bucket-0", 1); + let result = unsafe { paimon_stream_plan_deserialize(bytes.as_ptr(), bytes.len()) }; + assert!(result.error.is_null()); + assert_eq!(result.next_snapshot_id, 1); + unsafe { paimon_stream_plan_free(result.plan) }; + } + + #[test] + fn older_plan_version_is_reported_as_unsupported() { + let bytes = serialized_plan("memory:/table/bucket-0", 2); + let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + value["version"] = serde_json::json!(2); + value.as_object_mut().unwrap().remove("table_branch"); + let bytes = serde_json::to_vec(&value).unwrap(); + let result = unsafe { paimon_stream_plan_deserialize(bytes.as_ptr(), bytes.len()) }; + assert!(result.plan.is_null()); + assert!(!result.error.is_null()); + assert_eq!( + unsafe { (*result.error).code }, + PaimonErrorCode::Unsupported as i32 + ); + unsafe { crate::error::paimon_error_free(result.error) }; + } +} diff --git a/bindings/c/src/table.rs b/bindings/c/src/table.rs index 0b18de703..08782fc15 100644 --- a/bindings/c/src/table.rs +++ b/bindings/c/src/table.rs @@ -675,6 +675,10 @@ pub unsafe extern "C" fn paimon_read_builder_new_read( table: state.table.clone(), read_type: table_read.read_type().to_vec(), data_predicates: table_read.data_predicates().to_vec(), + table_location: state.table.location().to_string(), + table_branch: state.table.branch().to_string(), + schema_id: state.table.schema().id(), + read_fingerprint: read_builder_fingerprint(state), }; paimon_result_new_read { read: box_table_read_state(read_state), diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs index 31b62de27..f2bcfa69a 100644 --- a/bindings/c/src/tests.rs +++ b/bindings/c/src/tests.rs @@ -48,6 +48,7 @@ use paimon::table::{SnapshotManager, Table}; use crate::blob_reader::*; use crate::error::*; use crate::file_io::*; +use crate::stream::*; use crate::table::*; use crate::types::*; use crate::vector_search::*; @@ -314,6 +315,42 @@ unsafe fn collect_rows(reader: *mut paimon_record_batch_reader) -> Vec<(i32, Str rows } +/// Collect the audit-log row-kind strings while exercising Arrow ownership. +unsafe fn collect_rowkinds(reader: *mut paimon_record_batch_reader) -> Vec { + let mut kinds = Vec::new(); + loop { + let result = paimon_record_batch_reader_next(reader); + assert!(result.error.is_null(), "reader_next should not error"); + if result.batch.array.is_null() { + break; + } + let ffi_array = ptr::read(result.batch.array as *const FFI_ArrowArray); + let ffi_schema = ptr::read(result.batch.schema as *const FFI_ArrowSchema); + let data = arrow_array::ffi::from_ffi(ffi_array, &ffi_schema).unwrap(); + ptr::write( + result.batch.array as *mut FFI_ArrowArray, + FFI_ArrowArray::empty(), + ); + ptr::write( + result.batch.schema as *mut FFI_ArrowSchema, + FFI_ArrowSchema::empty(), + ); + paimon_arrow_batch_free(result.batch); + + let batch = RecordBatch::from(StructArray::from(data)); + let rowkind = batch + .column_by_name("rowkind") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..batch.num_rows() { + kinds.push(rowkind.value(index).to_string()); + } + } + kinds +} + /// Full read via C FFI: read_builder -> scan -> plan -> read -> stream -> rows. /// Called OUTSIDE of any block_on — the C FFI functions use block_on internally. unsafe fn read_rows_ffi(table: *const paimon_table) -> Vec<(i32, String)> { @@ -348,6 +385,234 @@ unsafe fn read_rows_ffi(table: *const paimon_table) -> Vec<(i32, String)> { rows } +#[test] +fn test_stream_scan_tails_snapshots_and_restores_cursor() { + let path = "memory:/test_stream_scan_tails_snapshots"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io, + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + let handle = unsafe { wrap_table(table.clone()) }; + + unsafe { + let rb_result = paimon_table_new_read_builder(handle); + assert!(rb_result.error.is_null()); + let rb = rb_result.read_builder; + let read_result = paimon_read_builder_new_read(rb); + assert!(read_result.error.is_null()); + let read = read_result.read; + + let mut options = std::mem::MaybeUninit::::uninit(); + assert!(paimon_stream_scan_options_init(options.as_mut_ptr()).is_null()); + let mut options = options.assume_init(); + options.startup_mode = PAIMON_STREAM_STARTUP_LATEST; + options.follow_up_mode = PAIMON_STREAM_FOLLOW_UP_DELTA; + + let scan_result = paimon_read_builder_new_stream_scan(rb, &options); + assert!(scan_result.error.is_null()); + let scan = scan_result.scan; + assert_eq!(paimon_stream_scan_checkpoint(scan), 1); + + // Commit before the first poll. Eager initialization at scan creation + // must retain snapshot 1 instead of treating it as pre-existing data. + write_data_rust(&table, &[make_batch(vec![1], vec!["first"])]); + let first = paimon_stream_scan_poll(scan); + assert!(first.error.is_null()); + assert_eq!(first.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(first.snapshot_id, 1); + assert_eq!(first.next_snapshot_id, 2); + assert_eq!(paimon_stream_scan_checkpoint(scan), 2); + assert_eq!(paimon_stream_plan_is_full(first.plan), 0); + + let serialized = paimon_stream_plan_serialize(first.plan); + assert!(serialized.error.is_null()); + let checkpoint_bytes = + std::slice::from_raw_parts(serialized.bytes.data, serialized.bytes.len).to_vec(); + paimon_bytes_free(serialized.bytes); + let restored_plan = + paimon_stream_plan_deserialize(checkpoint_bytes.as_ptr(), checkpoint_bytes.len()); + assert!(restored_plan.error.is_null()); + assert_eq!(restored_plan.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(restored_plan.snapshot_id, 1); + assert_eq!(restored_plan.next_snapshot_id, 2); + let restored_reader = paimon_stream_plan_read_to_arrow( + read, + restored_plan.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(restored_reader.error.is_null()); + assert_eq!( + collect_rows(restored_reader.reader), + vec![(1, "first".into())] + ); + paimon_record_batch_reader_free(restored_reader.reader); + + let mismatched_builder = paimon_table_new_read_builder(handle); + assert!(mismatched_builder.error.is_null()); + assert!( + paimon_read_builder_with_case_sensitive(mismatched_builder.read_builder, false,) + .is_null() + ); + let mismatched_read = paimon_read_builder_new_read(mismatched_builder.read_builder); + assert!(mismatched_read.error.is_null()); + let mismatched_result = paimon_stream_plan_read_to_arrow( + mismatched_read.read, + restored_plan.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(mismatched_result.reader.is_null()); + assert!(!mismatched_result.error.is_null()); + assert_eq!( + (*mismatched_result.error).code, + PaimonErrorCode::InvalidInput as i32 + ); + paimon_error_free(mismatched_result.error); + paimon_table_read_free(mismatched_read.read); + paimon_read_builder_free(mismatched_builder.read_builder); + + let branch_table = Table::from_resolved_schema( + table.file_io().clone(), + Identifier::new("default", "test"), + path.to_string(), + table.schema().clone(), + "branch-review", + ) + .unwrap(); + let branch_handle = wrap_table(branch_table); + let branch_builder = paimon_table_new_read_builder(branch_handle); + assert!(branch_builder.error.is_null()); + let branch_read = paimon_read_builder_new_read(branch_builder.read_builder); + assert!(branch_read.error.is_null()); + let branch_result = paimon_stream_plan_read_to_arrow( + branch_read.read, + restored_plan.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(branch_result.reader.is_null()); + assert!(!branch_result.error.is_null()); + assert_eq!( + (*branch_result.error).code, + PaimonErrorCode::InvalidInput as i32 + ); + paimon_error_free(branch_result.error); + paimon_table_read_free(branch_read.read); + paimon_read_builder_free(branch_builder.read_builder); + unwrap_table(branch_handle); + + paimon_stream_plan_free(restored_plan.plan); + + let first_reader = paimon_stream_plan_read_to_arrow( + read, + first.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(first_reader.error.is_null()); + assert_eq!(collect_rows(first_reader.reader), vec![(1, "first".into())]); + paimon_record_batch_reader_free(first_reader.reader); + + let audit_reader = paimon_stream_plan_read_to_arrow( + read, + first.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_AUDIT_LOG, + ); + assert!(audit_reader.error.is_null()); + assert_eq!(collect_rowkinds(audit_reader.reader), vec!["+I"]); + paimon_record_batch_reader_free(audit_reader.reader); + paimon_stream_plan_free(first.plan); + + write_data_rust(&table, &[make_batch(vec![2], vec!["second"])]); + let second = paimon_stream_scan_poll(scan); + assert!(second.error.is_null()); + assert_eq!(second.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(second.snapshot_id, 2); + assert_eq!(second.next_snapshot_id, 3); + let second_reader = paimon_stream_plan_read_to_arrow( + read, + second.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(second_reader.error.is_null()); + assert_eq!( + collect_rows(second_reader.reader), + vec![(2, "second".into())] + ); + paimon_record_batch_reader_free(second_reader.reader); + paimon_stream_plan_free(second.plan); + + // Restoring nextSnapshotId=2 replays snapshot 2 rather than losing it. + assert!(paimon_stream_scan_restore(scan, 2).is_null()); + let replay = paimon_stream_scan_poll(scan); + assert!(replay.error.is_null()); + assert_eq!(replay.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(replay.snapshot_id, 2); + paimon_stream_plan_free(replay.plan); + + paimon_stream_scan_free(scan); + paimon_table_read_free(read); + paimon_read_builder_free(rb); + unwrap_table(handle); + } +} + +#[test] +fn test_stream_scan_latest_full_then_waits_for_follow_up() { + let path = "memory:/test_stream_scan_latest_full"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io, + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + write_data_rust(&table, &[make_batch(vec![7], vec!["existing"])]); + let handle = unsafe { wrap_table(table) }; + + unsafe { + let rb_result = paimon_table_new_read_builder(handle); + assert!(rb_result.error.is_null()); + let mut options = std::mem::MaybeUninit::::uninit(); + assert!(paimon_stream_scan_options_init(options.as_mut_ptr()).is_null()); + let options = options.assume_init(); + let scan_result = paimon_read_builder_new_stream_scan(rb_result.read_builder, &options); + assert!(scan_result.error.is_null()); + + let full = paimon_stream_scan_poll(scan_result.scan); + assert!(full.error.is_null()); + assert_eq!(full.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(full.snapshot_id, 1); + assert_eq!(paimon_stream_plan_is_full(full.plan), 1); + assert_eq!(paimon_stream_scan_checkpoint(scan_result.scan), 2); + paimon_stream_plan_free(full.plan); + + let waiting = paimon_stream_scan_poll(scan_result.scan); + assert!(waiting.error.is_null()); + assert_eq!(waiting.status, PAIMON_STREAM_POLL_WAITING); + + paimon_stream_scan_free(scan_result.scan); + paimon_read_builder_free(rb_result.read_builder); + unwrap_table(handle); + } +} + // ========================================================================= // Catalog-free table construction tests // ========================================================================= @@ -1803,6 +2068,298 @@ fn test_caller_supplied_commit_identity_is_shared_and_persisted() { assert_eq!(snapshot.id(), 1, "retry must not create another snapshot"); } +#[test] +fn test_stream_write_v1_reuses_writer_across_monotonic_checkpoints() { + let path = "memory:/test_stream_write_v1_reuses_writer"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io.clone(), + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + let handle = unsafe { wrap_table(table) }; + let commit_user = CString::new("stream-write-job-9").unwrap(); + + unsafe { + let wb_result = + paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()); + assert!(wb_result.error.is_null()); + let wb = wb_result.write_builder; + + let tw_result = paimon_write_builder_new_write(wb); + assert!(tw_result.error.is_null()); + let tw = tw_result.write; + + let commit_result = paimon_write_builder_new_commit(wb); + assert!(commit_result.error.is_null()); + let commit = commit_result.commit; + + // Checkpoint 100: retain the prepared messages until a successful + // filter-and-commit confirms an intentionally lost commit ACK. + let (array, schema) = export_batch_to_ffi(make_batch(vec![1], vec!["first"])); + let error = paimon_table_write_write_arrow_batch( + tw, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + + let prepared_100 = paimon_table_write_prepare_commit(tw); + assert!(prepared_100.error.is_null()); + let error = paimon_table_commit_commit_with_identifier(commit, prepared_100.messages, 100); + assert!(error.is_null()); + + let error = paimon_table_commit_filter_and_commit_with_identifier( + commit, + prepared_100.messages, + 100, + ); + assert!( + error.is_null(), + "a retry after a lost commit ACK must be idempotent" + ); + paimon_commit_messages_free(prepared_100.messages); + + // Checkpoint 101 deliberately reuses both the writer and committer. + // prepare_commit must drain only the data written since checkpoint 100. + let (array, schema) = export_batch_to_ffi(make_batch(vec![2], vec!["second"])); + let error = paimon_table_write_write_arrow_batch( + tw, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + + let prepared_101 = paimon_table_write_prepare_commit(tw); + assert!(prepared_101.error.is_null()); + let error = paimon_table_commit_commit_with_identifier(commit, prepared_101.messages, 101); + assert!(error.is_null()); + paimon_commit_messages_free(prepared_101.messages); + + assert_eq!( + read_rows_ffi(handle), + vec![(1, "first".into()), (2, "second".into())] + ); + + // A later prepared checkpoint can be abandoned without publishing a + // snapshot or making its rows visible. + let (array, schema) = export_batch_to_ffi(make_batch(vec![3], vec!["aborted"])); + let error = paimon_table_write_write_arrow_batch( + tw, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + + let abandoned = paimon_table_write_prepare_commit(tw); + assert!(abandoned.error.is_null()); + let abandoned_prepared = paimon_commit_messages_prepare(abandoned.messages, 102); + assert!(abandoned_prepared.error.is_null()); + paimon_commit_messages_free(abandoned.messages); + let error = paimon_table_commit_abort_prepared(commit, abandoned_prepared.prepared); + assert!(error.is_null()); + paimon_prepared_commit_free(abandoned_prepared.prepared); + + assert_eq!( + read_rows_ffi(handle), + vec![(1, "first".into()), (2, "second".into())] + ); + + paimon_table_commit_free(commit); + paimon_table_write_free(tw); + paimon_write_builder_free(wb); + unwrap_table(handle); + } + + let snapshots = crate::runtime().block_on(async { + let manager = SnapshotManager::new(file_io, path.to_string()); + ( + manager.get_snapshot(1).await.unwrap(), + manager.get_snapshot(2).await.unwrap(), + manager.get_latest_snapshot_id().await.unwrap(), + ) + }); + assert_eq!(snapshots.0.commit_user(), "stream-write-job-9"); + assert_eq!(snapshots.0.commit_identifier(), 100); + assert_eq!(snapshots.1.commit_user(), "stream-write-job-9"); + assert_eq!(snapshots.1.commit_identifier(), 101); + assert_eq!( + snapshots.2, + Some(2), + "the retry and abort must not publish snapshots" + ); +} + +#[test] +fn test_prepared_commit_roundtrip_and_lost_ack_retry() { + let path = "memory:/test_prepared_commit_roundtrip"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io.clone(), + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + let handle = unsafe { wrap_table(table) }; + let commit_user = CString::new("durable-stream-job-5").unwrap(); + + unsafe { + let writer_builder = + paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()); + assert!(writer_builder.error.is_null()); + let writer_builder = writer_builder.write_builder; + + let writer = paimon_write_builder_new_write(writer_builder); + assert!(writer.error.is_null()); + let writer = writer.write; + + let (array, schema) = export_batch_to_ffi(make_batch(vec![5], vec!["durable"])); + let error = paimon_table_write_write_arrow_batch( + writer, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + + let messages = paimon_table_write_prepare_commit(writer); + assert!(messages.error.is_null()); + let reserved = paimon_commit_messages_prepare(messages.messages, i64::MAX); + assert!(reserved.prepared.is_null()); + assert!(!reserved.error.is_null()); + assert_eq!((*reserved.error).code, PaimonErrorCode::InvalidInput as i32); + paimon_error_free(reserved.error); + let prepared = paimon_commit_messages_prepare(messages.messages, 500); + assert!(prepared.error.is_null()); + assert_eq!(paimon_prepared_commit_identifier(prepared.prepared), 500); + + let serialized = paimon_prepared_commit_serialize(prepared.prepared); + assert!(serialized.error.is_null()); + assert!(!serialized.bytes.data.is_null()); + assert!(serialized.bytes.len > 0); + + let mut unsafe_checkpoint: serde_json::Value = serde_json::from_slice( + std::slice::from_raw_parts(serialized.bytes.data, serialized.bytes.len), + ) + .unwrap(); + unsafe_checkpoint["messages"][0]["new_files"][0]["_EXTERNAL_PATH"] = + serde_json::json!("file:/tmp/not-owned-by-the-prepared-commit"); + let unsafe_checkpoint = serde_json::to_vec(&unsafe_checkpoint).unwrap(); + let rejected = + paimon_prepared_commit_deserialize(unsafe_checkpoint.as_ptr(), unsafe_checkpoint.len()); + assert!(rejected.prepared.is_null()); + assert!(!rejected.error.is_null()); + assert_eq!((*rejected.error).code, PaimonErrorCode::InvalidInput as i32); + paimon_error_free(rejected.error); + + let mut duplicated_checkpoint: serde_json::Value = serde_json::from_slice( + std::slice::from_raw_parts(serialized.bytes.data, serialized.bytes.len), + ) + .unwrap(); + let duplicate_message = duplicated_checkpoint["messages"][0].clone(); + duplicated_checkpoint["messages"] + .as_array_mut() + .unwrap() + .push(duplicate_message); + let duplicated_checkpoint = serde_json::to_vec(&duplicated_checkpoint).unwrap(); + + let mut conflicting_checkpoint: serde_json::Value = serde_json::from_slice( + std::slice::from_raw_parts(serialized.bytes.data, serialized.bytes.len), + ) + .unwrap(); + let mut conflicting_message = conflicting_checkpoint["messages"][0].clone(); + conflicting_message["new_files"][0]["_FILE_SIZE"] = serde_json::json!(123456789); + conflicting_checkpoint["messages"] + .as_array_mut() + .unwrap() + .push(conflicting_message); + let conflicting_checkpoint = serde_json::to_vec(&conflicting_checkpoint).unwrap(); + let rejected = paimon_prepared_commit_deserialize( + conflicting_checkpoint.as_ptr(), + conflicting_checkpoint.len(), + ); + assert!(rejected.prepared.is_null()); + assert!(!rejected.error.is_null()); + assert!(error_message(rejected.error).contains("same file identity")); + paimon_error_free(rejected.error); + + // The serialized bytes, rather than either in-process source handle, + // are the durable checkpoint boundary. + paimon_commit_messages_free(messages.messages); + paimon_prepared_commit_free(prepared.prepared); + + let restored = paimon_prepared_commit_deserialize( + duplicated_checkpoint.as_ptr(), + duplicated_checkpoint.len(), + ); + assert!(restored.error.is_null()); + assert_eq!(paimon_prepared_commit_identifier(restored.prepared), 500); + + let first_committer_builder = + paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()); + assert!(first_committer_builder.error.is_null()); + let first_committer_builder = first_committer_builder.write_builder; + let first_committer = paimon_write_builder_new_commit(first_committer_builder); + assert!(first_committer.error.is_null()); + let error = paimon_table_commit_commit_prepared(first_committer.commit, restored.prepared); + assert!(error.is_null()); + + // A stale abort request after a successful commit (including a lost + // acknowledgement recovered by identifier) must not delete files now + // referenced by the committed snapshot. + let error = paimon_table_commit_abort_prepared(first_committer.commit, restored.prepared); + assert!(error.is_null()); + assert_eq!(read_rows_ffi(handle), vec![(5, "durable".into())]); + + // Treat the successful return above as a lost ACK. Discard all + // in-memory commit state, recover from the same durable bytes, and + // retry through the identifier-filtering commit path. + paimon_prepared_commit_free(restored.prepared); + paimon_table_commit_free(first_committer.commit); + paimon_write_builder_free(first_committer_builder); + + let retry = paimon_prepared_commit_deserialize( + serialized.bytes.data.cast_const(), + serialized.bytes.len, + ); + assert!(retry.error.is_null()); + let retry_committer_builder = + paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()); + assert!(retry_committer_builder.error.is_null()); + let retry_committer_builder = retry_committer_builder.write_builder; + let retry_committer = paimon_write_builder_new_commit(retry_committer_builder); + assert!(retry_committer.error.is_null()); + let error = paimon_table_commit_commit_prepared(retry_committer.commit, retry.prepared); + assert!( + error.is_null(), + "recovered commit_prepared must filter a previously committed identifier" + ); + + paimon_prepared_commit_free(retry.prepared); + paimon_bytes_free(serialized.bytes); + paimon_table_commit_free(retry_committer.commit); + paimon_write_builder_free(retry_committer_builder); + + assert_eq!(read_rows_ffi(handle), vec![(5, "durable".into())]); + + paimon_table_write_free(writer); + paimon_write_builder_free(writer_builder); + unwrap_table(handle); + } + + let snapshot = crate::runtime() + .block_on(SnapshotManager::new(file_io, path.to_string()).get_latest_snapshot()) + .unwrap() + .unwrap(); + assert_eq!(snapshot.id(), 1, "the lost-ACK retry must be a no-op"); + assert_eq!(snapshot.commit_user(), "durable-stream-job-5"); + assert_eq!(snapshot.commit_identifier(), 500); +} + #[test] fn test_commit_messages_merge_preserves_all_writer_files() { let path = "memory:/test_commit_messages_merge"; @@ -1840,6 +2397,11 @@ fn test_commit_messages_merge_preserves_all_writer_files() { let messages2 = paimon_table_write_prepare_commit(tw2).messages; let err = paimon_commit_messages_merge(messages1, messages2); assert!(err.is_null()); + let err = paimon_commit_messages_merge(messages1, messages2); + assert!( + err.is_null(), + "re-merging the same fragment must be a no-op" + ); let commit = paimon_write_builder_new_commit(wb1).commit; let err = paimon_table_commit_commit_with_identifier(commit, messages1, 7); @@ -1860,6 +2422,81 @@ fn test_commit_messages_merge_preserves_all_writer_files() { } } +#[test] +fn test_prepared_commit_merge_preserves_parallel_writer_files() { + let path = "memory:/test_prepared_commit_merge"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io, + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + let handle = unsafe { wrap_table(table) }; + let commit_user = CString::new("durable-distributed-job-700").unwrap(); + + unsafe { + let wb1 = paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()) + .write_builder; + let wb2 = paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()) + .write_builder; + let tw1 = paimon_write_builder_new_write(wb1).write; + let tw2 = paimon_write_builder_new_write(wb2).write; + + for (writer, ids, names) in [ + (tw1, vec![10], vec!["left"]), + (tw2, vec![20], vec!["right"]), + ] { + let (array, schema) = export_batch_to_ffi(make_batch(ids, names)); + let error = paimon_table_write_write_arrow_batch( + writer, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + } + + let messages1 = paimon_table_write_prepare_commit(tw1); + assert!(messages1.error.is_null()); + let messages2 = paimon_table_write_prepare_commit(tw2); + assert!(messages2.error.is_null()); + let prepared1 = paimon_commit_messages_prepare(messages1.messages, 700); + assert!(prepared1.error.is_null()); + let prepared2 = paimon_commit_messages_prepare(messages2.messages, 700); + assert!(prepared2.error.is_null()); + paimon_commit_messages_free(messages2.messages); + paimon_commit_messages_free(messages1.messages); + + let error = paimon_prepared_commit_merge(prepared1.prepared, prepared2.prepared); + assert!(error.is_null()); + let error = paimon_prepared_commit_merge(prepared1.prepared, prepared2.prepared); + assert!( + error.is_null(), + "re-merging the same durable fragment must be a no-op" + ); + let commit = paimon_write_builder_new_commit(wb1); + assert!(commit.error.is_null()); + let error = paimon_table_commit_commit_prepared(commit.commit, prepared1.prepared); + assert!(error.is_null()); + + assert_eq!( + read_rows_ffi(handle), + vec![(10, "left".into()), (20, "right".into())] + ); + + paimon_table_commit_free(commit.commit); + paimon_prepared_commit_free(prepared2.prepared); + paimon_prepared_commit_free(prepared1.prepared); + paimon_table_write_free(tw2); + paimon_table_write_free(tw1); + paimon_write_builder_free(wb2); + paimon_write_builder_free(wb1); + unwrap_table(handle); + } +} + #[test] fn test_postpone_bucket_plan_arrow_ownership_on_errors() { let path = "memory:/test_postpone_bucket_plan_arrow_ownership"; @@ -2125,6 +2762,17 @@ fn test_write_multiple_batches() { let tc_result = paimon_write_builder_new_commit(wb); let tc = tc_result.commit; + for invalid in [-1, i64::MAX] { + let err = paimon_table_commit_commit_with_identifier(tc, pc_result.messages, invalid); + assert!(!err.is_null()); + assert_eq!((*err).code, PaimonErrorCode::InvalidInput as i32); + paimon_error_free(err); + let err = paimon_table_commit_truncate_table_with_identifier(tc, invalid); + assert!(!err.is_null()); + assert_eq!((*err).code, PaimonErrorCode::InvalidInput as i32); + paimon_error_free(err); + } + let err = paimon_table_commit_commit(tc, pc_result.messages); assert!(err.is_null()); paimon_commit_messages_free(pc_result.messages); diff --git a/bindings/c/src/types.rs b/bindings/c/src/types.rs index 4e6d2709f..5bc4d5b66 100644 --- a/bindings/c/src/types.rs +++ b/bindings/c/src/types.rs @@ -24,6 +24,7 @@ use paimon::table::{ CommitMessage, PostponeBucketPlan, PostponeFixedBucketTableCommit, PostponeFixedBucketTableWrite, Table, TableCommit, TableWrite, }; +use sha2::{Digest, Sha256}; /// C-compatible key-value pair for options. #[repr(C)] @@ -201,6 +202,68 @@ pub(crate) struct ReadBuilderState { pub case_sensitive: bool, } +fn digest_read_builder_canonical(canonical: &str) -> String { + let digest = Sha256::digest(canonical.as_bytes()); + format!("paimon-c-read-builder-sha256-v1:{digest:x}") +} + +/// Build the stable identity component shared by a stream plan and the +/// `TableRead` which consumes it. +/// +/// The canonical input uses length-prefixed components and the persisted value +/// is a SHA-256 digest, so predicate literals are not exposed in a checkpoint. +/// Predicate `Debug` formatting is versioned by the fingerprint prefix and +/// must be bumped if it changes incompatibly. +pub(crate) fn read_builder_fingerprint(state: &ReadBuilderState) -> String { + fn push_component(target: &mut String, value: &str) { + target.push_str(&value.len().to_string()); + target.push(':'); + target.push_str(value); + target.push(';'); + } + + let mut canonical = String::from("paimon-c-read-builder-canonical-v1;"); + canonical.push_str(if state.case_sensitive { + "case=1;" + } else { + "case=0;" + }); + match &state.projected_columns { + None => canonical.push_str("projection=none;"), + Some(columns) => { + canonical.push_str("projection=some;"); + canonical.push_str(&columns.len().to_string()); + canonical.push(';'); + for column in columns { + push_component(&mut canonical, column); + } + } + } + match &state.filter { + None => canonical.push_str("filter=none;"), + Some(filter) => { + canonical.push_str("filter=some;"); + push_component(&mut canonical, &format!("{filter:?}")); + } + } + digest_read_builder_canonical(&canonical) +} + +#[cfg(test)] +mod fingerprint_tests { + use super::digest_read_builder_canonical; + + #[test] + fn digest_does_not_expose_predicate_literals() { + let canonical = "filter=some;24:secret-customer-id=42;"; + let digest = digest_read_builder_canonical(canonical); + assert!(digest.starts_with("paimon-c-read-builder-sha256-v1:")); + assert!(!digest.contains("secret-customer-id")); + assert_eq!(digest.len(), "paimon-c-read-builder-sha256-v1:".len() + 64); + assert_eq!(digest, digest_read_builder_canonical(canonical)); + } +} + /// Internal state for TableScan that stores table and filter. pub(crate) struct TableScanState { pub table: Table, @@ -222,6 +285,10 @@ pub(crate) struct TableReadState { pub table: Table, pub read_type: Vec, pub data_predicates: Vec, + pub table_location: String, + pub table_branch: String, + pub schema_id: i64, + pub read_fingerprint: String, } #[repr(C)] @@ -368,6 +435,15 @@ pub(crate) struct CommitMessagesState { pub commit_user: String, } +/// Durable, versioned representation of one standard streaming checkpoint. +/// +/// Unlike `paimon_commit_messages`, this state also carries the monotonically +/// increasing commit identifier and can be serialized across process restarts. +pub(crate) struct PreparedCommitState { + pub commit_identifier: i64, + pub messages: CommitMessagesState, +} + pub(crate) struct PostponeFixedBucketCommitMessagesState { pub messages: Vec, pub overwrite: bool, @@ -396,6 +472,12 @@ pub struct paimon_commit_messages { pub inner: *mut c_void, } +/// Opaque durable prepared-commit handle for a standard table write. +#[repr(C)] +pub struct paimon_prepared_commit { + pub inner: *mut c_void, +} + #[repr(C)] pub struct paimon_postpone_fixed_bucket_write_builder { pub inner: *mut c_void, diff --git a/bindings/c/src/version.rs b/bindings/c/src/version.rs new file mode 100644 index 000000000..c3f277ab0 --- /dev/null +++ b/bindings/c/src/version.rs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::types::paimon_bytes; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::ptr; + +/// ABI version for the native C boundary. +/// +/// Version 1 is additive: callers must still feature-detect newer symbols when +/// loading the shared library dynamically. +#[no_mangle] +pub extern "C" fn paimon_abi_version() -> u32 { + 1 +} + +/// Return the paimon-rust package version as an owned UTF-8 byte buffer. +/// +/// The returned bytes are not NUL terminated and must be released with +/// `paimon_bytes_free`. +#[no_mangle] +pub extern "C" fn paimon_library_version() -> paimon_bytes { + catch_unwind(AssertUnwindSafe(|| { + paimon_bytes::new(env!("CARGO_PKG_VERSION").as_bytes().to_vec()) + })) + .unwrap_or(paimon_bytes { + data: ptr::null_mut(), + len: 0, + }) +} + +const _: extern "C" fn() -> u32 = paimon_abi_version; +const _: extern "C" fn() -> paimon_bytes = paimon_library_version; diff --git a/bindings/c/src/write.rs b/bindings/c/src/write.rs index c6e9afa7d..474156942 100644 --- a/bindings/c/src/write.rs +++ b/bindings/c/src/write.rs @@ -15,22 +15,28 @@ // specific language governing permissions and limitations // under the License. +use std::collections::{HashMap, HashSet}; use std::ffi::{c_char, c_void}; +use std::fmt; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::ptr; use std::sync::Arc; use arrow_array::ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}; use arrow_array::{Array, RecordBatch, RecordBatchOptions, StructArray}; use arrow_schema::{DataType as ArrowDataType, Schema as ArrowSchema}; -use paimon::table::{PostponeBucketPlan, Table}; +use paimon::table::{CommitMessage, PostponeBucketPlan, Table}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; use crate::error::{check_non_null, paimon_error, validate_cstr, PaimonErrorCode}; use crate::result::{ - paimon_result_postpone_fixed_bucket_prepare_commit, + paimon_result_bytes, paimon_result_postpone_fixed_bucket_prepare_commit, paimon_result_postpone_fixed_bucket_table_commit, paimon_result_postpone_fixed_bucket_table_write, paimon_result_postpone_fixed_bucket_write_builder, paimon_result_prepare_commit, - paimon_result_table_commit, paimon_result_table_write, paimon_result_write_builder, + paimon_result_prepared_commit, paimon_result_table_commit, paimon_result_table_write, + paimon_result_write_builder, }; use crate::runtime; use crate::types::*; @@ -817,6 +823,490 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_commit_messages_free( } } +const PREPARED_COMMIT_FORMAT: &str = "paimon-rust-prepared-commit"; +// Version 2 adds strict resource and path validation. Version 1 is rejected: +// accepting its unconstrained internal CommitMessage representation would +// reintroduce unsafe file references after recovery. +const PREPARED_COMMIT_VERSION: u32 = 2; +const MAX_PREPARED_COMMIT_BYTES: usize = 64 * 1024 * 1024; +const MAX_PREPARED_MESSAGES: usize = 100_000; +const MAX_PREPARED_MESSAGE_BYTES: usize = 16 * 1024 * 1024; +const MAX_FILES_PER_MESSAGE: usize = 100_000; +const MAX_TOTAL_FILE_REFERENCES: usize = 1_000_000; +const MAX_EXTRA_FILES_PER_DATA_FILE: usize = 10_000; +const MAX_PARTITION_BYTES: usize = 16 * 1024 * 1024; +const MAX_IDENTITY_BYTES: usize = 1024 * 1024; +const MAX_FILE_NAME_BYTES: usize = 4 * 1024; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PreparedCommitEnvelope { + format: String, + version: u32, + commit_identifier: i64, + table_location: String, + commit_user: String, + overwrite: bool, + messages: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawPreparedCommitEnvelope<'a> { + format: String, + version: u32, + commit_identifier: i64, + table_location: String, + commit_user: String, + overwrite: bool, + #[serde(borrow, deserialize_with = "deserialize_bounded_raw_messages")] + messages: Vec<&'a RawValue>, +} + +fn deserialize_bounded_raw_messages<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + use serde::de::{Error, IgnoredAny, SeqAccess, Visitor}; + + struct RawMessagesVisitor; + + impl<'de> Visitor<'de> for RawMessagesVisitor { + type Value = Vec<&'de RawValue>; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a bounded list of prepared commit messages") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut messages = Vec::with_capacity( + sequence + .size_hint() + .unwrap_or_default() + .min(MAX_PREPARED_MESSAGES), + ); + while messages.len() < MAX_PREPARED_MESSAGES { + let Some(message) = sequence.next_element::<&'de RawValue>()? else { + return Ok(messages); + }; + messages.push(message); + } + if sequence.next_element::()?.is_some() { + return Err(A::Error::custom(format!( + "prepared commit contains more than {MAX_PREPARED_MESSAGES} messages" + ))); + } + Ok(messages) + } + } + + deserializer.deserialize_seq(RawMessagesVisitor) +} + +fn prepared_panic_error(operation: &str) -> *mut paimon_error { + paimon_error::new( + PaimonErrorCode::Unexpected, + format!("Rust panic while executing {operation}"), + ) +} + +fn validate_file_component(kind: &str, name: &str) -> Result<(), *mut paimon_error> { + if name.is_empty() + || name.len() > MAX_FILE_NAME_BYTES + || name == "." + || name == ".." + || name.contains('/') + || name.contains('\\') + || name.contains('\0') + { + return Err(invalid_input(format!( + "prepared commit contains unsafe {kind} '{name}'" + ))); + } + Ok(()) +} + +fn validate_data_file(file: &paimon::spec::DataFileMeta) -> Result { + if file.external_path.is_some() { + return Err(invalid_input( + "prepared commits with external data-file paths are not supported", + )); + } + validate_file_component("data file name", &file.file_name)?; + if file.extra_files.len() > MAX_EXTRA_FILES_PER_DATA_FILE { + return Err(invalid_input(format!( + "data file contains {} extra files; maximum is {}", + file.extra_files.len(), + MAX_EXTRA_FILES_PER_DATA_FILE + ))); + } + for extra in &file.extra_files { + validate_file_component("extra file name", extra)?; + } + Ok(1 + file.extra_files.len()) +} + +fn validate_index_file(file: &paimon::spec::IndexFileMeta) -> Result { + validate_file_component("index file name", &file.file_name)?; + if let Some(ranges) = &file.deletion_vectors_ranges { + for data_file_name in ranges.keys() { + validate_file_component("deletion-vector data file name", data_file_name)?; + } + } + Ok(1) +} + +fn validate_prepared_commit_envelope( + envelope: &PreparedCommitEnvelope, +) -> Result<(), *mut paimon_error> { + if envelope.commit_identifier < 0 + || envelope.commit_identifier == i64::MAX + || envelope.table_location.is_empty() + || envelope.table_location.len() > MAX_IDENTITY_BYTES + || envelope.commit_user.is_empty() + || envelope.commit_user.len() > MAX_IDENTITY_BYTES + { + return Err(invalid_input( + "prepared commit contains an invalid identity", + )); + } + if envelope.messages.len() > MAX_PREPARED_MESSAGES { + return Err(invalid_input(format!( + "prepared commit contains {} messages; maximum is {}", + envelope.messages.len(), + MAX_PREPARED_MESSAGES + ))); + } + + let mut total_file_references = 0usize; + for message in &envelope.messages { + if message.partition.len() > MAX_PARTITION_BYTES { + return Err(invalid_input(format!( + "prepared commit partition exceeds {MAX_PARTITION_BYTES} bytes" + ))); + } + let message_file_count = message + .new_files + .len() + .checked_add(message.new_changelog_files.len()) + .and_then(|count| count.checked_add(message.deleted_files.len())) + .and_then(|count| count.checked_add(message.new_index_files.len())) + .and_then(|count| count.checked_add(message.deleted_index_files.len())) + .ok_or_else(|| invalid_input("prepared commit file count overflows"))?; + if message_file_count > MAX_FILES_PER_MESSAGE { + return Err(invalid_input(format!( + "prepared commit message contains {message_file_count} files; maximum is {MAX_FILES_PER_MESSAGE}" + ))); + } + for file in message + .new_files + .iter() + .chain(message.new_changelog_files.iter()) + .chain(message.deleted_files.iter()) + { + total_file_references = total_file_references + .checked_add(validate_data_file(file)?) + .ok_or_else(|| invalid_input("prepared commit file count overflows"))?; + } + for file in message + .new_index_files + .iter() + .chain(message.deleted_index_files.iter()) + { + total_file_references = total_file_references + .checked_add(validate_index_file(file)?) + .ok_or_else(|| invalid_input("prepared commit file count overflows"))?; + } + if total_file_references > MAX_TOTAL_FILE_REFERENCES { + return Err(invalid_input(format!( + "prepared commit contains more than {MAX_TOTAL_FILE_REFERENCES} file references" + ))); + } + } + Ok(()) +} + +fn empty_bytes() -> paimon_bytes { + paimon_bytes { + data: ptr::null_mut(), + len: 0, + } +} + +/// Bind standard commit messages to a monotonically increasing streaming +/// commit identifier. The returned prepared commit owns a clone of the +/// messages, so the source handle remains valid. Valid identifiers are in +/// `[0, INT64_MAX)`; `INT64_MAX` is reserved for unidentified batch commits. +#[no_mangle] +pub unsafe extern "C" fn paimon_commit_messages_prepare( + msgs: *const paimon_commit_messages, + commit_identifier: i64, +) -> paimon_result_prepared_commit { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(msgs, "msgs") { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error, + }; + } + if commit_identifier < 0 || commit_identifier == i64::MAX { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ), + }; + } + let source = &*((*msgs).inner as *const CommitMessagesState); + let mut messages = Vec::new(); + if let Err(error) = merge_messages_idempotently(&mut messages, &source.messages) { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error, + }; + } + let state = PreparedCommitState { + commit_identifier, + messages: CommitMessagesState { + messages, + overwrite: source.overwrite, + table_location: source.table_location.clone(), + commit_user: source.commit_user.clone(), + }, + }; + let inner = Box::into_raw(Box::new(state)) as *mut c_void; + paimon_result_prepared_commit { + prepared: Box::into_raw(Box::new(paimon_prepared_commit { inner })), + error: ptr::null_mut(), + } + })); + outcome.unwrap_or_else(|_| paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: prepared_panic_error("paimon_commit_messages_prepare"), + }) +} + +/// Serialize a prepared commit into a process-independent, versioned buffer. +/// The bytes must be released with `paimon_bytes_free`. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_serialize( + prepared: *const paimon_prepared_commit, +) -> paimon_result_bytes { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(prepared, "prepared") { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + let state = &*((*prepared).inner as *const PreparedCommitState); + let envelope = PreparedCommitEnvelope { + format: PREPARED_COMMIT_FORMAT.to_string(), + version: PREPARED_COMMIT_VERSION, + commit_identifier: state.commit_identifier, + table_location: state.messages.table_location.clone(), + commit_user: state.messages.commit_user.clone(), + overwrite: state.messages.overwrite, + messages: state.messages.messages.clone(), + }; + if let Err(error) = validate_prepared_commit_envelope(&envelope) { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + match serde_json::to_vec(&envelope) { + Ok(bytes) if bytes.len() <= MAX_PREPARED_COMMIT_BYTES => paimon_result_bytes { + bytes: paimon_bytes::new(bytes), + error: ptr::null_mut(), + }, + Ok(bytes) => paimon_result_bytes { + bytes: empty_bytes(), + error: invalid_input(format!( + "serialized prepared commit is {} bytes; maximum is {}", + bytes.len(), + MAX_PREPARED_COMMIT_BYTES + )), + }, + Err(error) => paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::Unexpected, + format!("failed to serialize prepared commit: {error}"), + ), + }, + } + })); + outcome.unwrap_or_else(|_| paimon_result_bytes { + bytes: empty_bytes(), + error: prepared_panic_error("paimon_prepared_commit_serialize"), + }) +} + +/// Restore a prepared commit serialized by `paimon_prepared_commit_serialize`. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_deserialize( + data: *const u8, + len: usize, +) -> paimon_result_prepared_commit { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if data.is_null() || len == 0 { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input("prepared commit buffer must not be null or empty"), + }; + } + if len > MAX_PREPARED_COMMIT_BYTES { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!( + "prepared commit buffer exceeds {MAX_PREPARED_COMMIT_BYTES} bytes" + )), + }; + } + let bytes = std::slice::from_raw_parts(data, len); + let raw: RawPreparedCommitEnvelope<'_> = match serde_json::from_slice(bytes) { + Ok(envelope) => envelope, + Err(error) => { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!("invalid prepared commit buffer: {error}")), + }; + } + }; + if raw.format != PREPARED_COMMIT_FORMAT || raw.version != PREPARED_COMMIT_VERSION { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::Unsupported, + format!( + "unsupported prepared commit format '{}' version {}", + raw.format, raw.version + ), + ), + }; + } + if raw.commit_identifier < 0 + || raw.commit_identifier == i64::MAX + || raw.table_location.is_empty() + || raw.table_location.len() > MAX_IDENTITY_BYTES + || raw.commit_user.is_empty() + || raw.commit_user.len() > MAX_IDENTITY_BYTES + { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input("prepared commit contains an invalid identity"), + }; + } + if raw.messages.len() > MAX_PREPARED_MESSAGES { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!( + "prepared commit contains {} messages; maximum is {}", + raw.messages.len(), + MAX_PREPARED_MESSAGES + )), + }; + } + let mut messages = Vec::with_capacity(raw.messages.len()); + for (index, raw_message) in raw.messages.into_iter().enumerate() { + if raw_message.get().len() > MAX_PREPARED_MESSAGE_BYTES { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!( + "prepared commit message {index} exceeds {MAX_PREPARED_MESSAGE_BYTES} bytes" + )), + }; + } + match serde_json::from_str::(raw_message.get()) { + Ok(message) => messages.push(message), + Err(error) => { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!( + "invalid prepared commit message {index}: {error}" + )), + }; + } + } + } + let mut envelope = PreparedCommitEnvelope { + format: raw.format, + version: raw.version, + commit_identifier: raw.commit_identifier, + table_location: raw.table_location, + commit_user: raw.commit_user, + overwrite: raw.overwrite, + messages, + }; + if let Err(error) = validate_prepared_commit_envelope(&envelope) { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error, + }; + } + let mut normalized_messages = Vec::new(); + if let Err(error) = + merge_messages_idempotently(&mut normalized_messages, &envelope.messages) + { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error, + }; + } + envelope.messages = normalized_messages; + let state = PreparedCommitState { + commit_identifier: envelope.commit_identifier, + messages: CommitMessagesState { + messages: envelope.messages, + overwrite: envelope.overwrite, + table_location: envelope.table_location, + commit_user: envelope.commit_user, + }, + }; + let inner = Box::into_raw(Box::new(state)) as *mut c_void; + paimon_result_prepared_commit { + prepared: Box::into_raw(Box::new(paimon_prepared_commit { inner })), + error: ptr::null_mut(), + } + })); + outcome.unwrap_or_else(|_| paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: prepared_panic_error("paimon_prepared_commit_deserialize"), + }) +} + +/// Return the commit identifier carried by a prepared commit, or -1 for null. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_identifier( + prepared: *const paimon_prepared_commit, +) -> i64 { + catch_unwind(AssertUnwindSafe(|| { + if prepared.is_null() || (*prepared).inner.is_null() { + return -1; + } + let state = &*((*prepared).inner as *const PreparedCommitState); + state.commit_identifier + })) + .unwrap_or(-1) +} + +/// Free a prepared commit. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_free(prepared: *mut paimon_prepared_commit) { + let _ = catch_unwind(AssertUnwindSafe(|| { + if !prepared.is_null() { + let wrapper = Box::from_raw(prepared); + if !wrapper.inner.is_null() { + drop(Box::from_raw(wrapper.inner as *mut PreparedCommitState)); + } + } + })); +} + fn validate_message_context( target_table: &str, target_user: &str, @@ -838,6 +1328,90 @@ fn validate_message_context( Ok(()) } +type CommitMessageGroupKey = (Vec, i32); +type CommitFileKey = (u8, String); + +fn commit_message_file_keys(message: &CommitMessage) -> Vec { + let mut keys = Vec::new(); + let mut add_data_files = |category: u8, files: &[paimon::spec::DataFileMeta]| { + for file in files { + keys.push((category, file.file_name.clone())); + for extra in &file.extra_files { + keys.push((category + 1, extra.clone())); + } + } + }; + add_data_files(0, &message.new_files); + add_data_files(2, &message.new_changelog_files); + add_data_files(4, &message.deleted_files); + for (category, files) in [ + (6u8, &message.new_index_files), + (7u8, &message.deleted_index_files), + ] { + for file in files { + keys.push((category, file.file_name.clone())); + } + } + keys +} + +fn merge_messages_idempotently( + target: &mut Vec, + source: &[CommitMessage], +) -> Result<(), *mut paimon_error> { + let capacity = target + .len() + .checked_add(source.len()) + .unwrap_or(MAX_PREPARED_MESSAGES) + .min(MAX_PREPARED_MESSAGES); + let mut merged = Vec::with_capacity(capacity); + let mut key_owners: HashMap> = + HashMap::new(); + + for message in target.iter().chain(source) { + let message_keys = commit_message_file_keys(message); + // Empty writer fragments do not publish any metadata and can be + // removed without changing commit semantics. + if message_keys.is_empty() { + continue; + } + let unique_keys = message_keys.iter().cloned().collect::>(); + if unique_keys.len() != message_keys.len() { + return Err(invalid_input( + "commit message contains a duplicate file identity", + )); + } + + // Hash/copy a partition only once per message. Putting it in every + // file key makes merge CPU and memory proportional to + // partition_bytes * file_count. + let group = (message.partition.clone(), message.bucket); + let group_owners = key_owners.entry(group).or_default(); + let owners = unique_keys + .iter() + .filter_map(|key| group_owners.get(key).copied()) + .collect::>(); + if !owners.is_empty() { + if owners.iter().any(|index| merged[*index] == *message) { + continue; + } + return Err(invalid_input( + "commit message merge found the same file identity with different fragment metadata", + )); + } + if merged.len() >= MAX_PREPARED_MESSAGES { + return Err(invalid_input(format!( + "merged commit contains more than {MAX_PREPARED_MESSAGES} messages" + ))); + } + let owner = merged.len(); + group_owners.extend(unique_keys.into_iter().map(|key| (key, owner))); + merged.push(message.clone()); + } + *target = merged; + Ok(()) +} + /// Merge standard commit messages for one logical commit. #[no_mangle] pub unsafe extern "C" fn paimon_commit_messages_merge( @@ -865,8 +1439,51 @@ pub unsafe extern "C" fn paimon_commit_messages_merge( ) { return error; } - target.messages.extend(source.messages.clone()); - ptr::null_mut() + match merge_messages_idempotently(&mut target.messages, &source.messages) { + Ok(()) => ptr::null_mut(), + Err(error) => error, + } +} + +/// Merge two durable prepared commits produced by parallel writers for the +/// same table, commit user, mode and identifier. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_merge( + target: *mut paimon_prepared_commit, + source: *const paimon_prepared_commit, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(target, "target") { + return error; + } + if let Err(error) = check_non_null(source, "source") { + return error; + } + if ptr::eq(target, source.cast_mut()) { + return invalid_input("target and source prepared commits must be distinct handles"); + } + let target = &mut *((*target).inner as *mut PreparedCommitState); + let source = &*((*source).inner as *const PreparedCommitState); + if target.commit_identifier != source.commit_identifier { + return invalid_input("prepared commits must have the same commit_identifier"); + } + if let Err(error) = validate_message_context( + &target.messages.table_location, + &target.messages.commit_user, + target.messages.overwrite, + &source.messages.table_location, + &source.messages.commit_user, + source.messages.overwrite, + ) { + return error; + } + match merge_messages_idempotently(&mut target.messages.messages, &source.messages.messages) + { + Ok(()) => ptr::null_mut(), + Err(error) => error, + } + })); + outcome.unwrap_or_else(|_| prepared_panic_error("paimon_prepared_commit_merge")) } /// Merge postpone fixed-bucket messages for one logical commit. @@ -896,8 +1513,10 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_commit_messages_merge( ) { return error; } - target.messages.extend(source.messages.clone()); - ptr::null_mut() + match merge_messages_idempotently(&mut target.messages, &source.messages) { + Ok(()) => ptr::null_mut(), + Err(error) => error, + } } // ======================= Commit operations =============================== @@ -929,11 +1548,107 @@ fn validate_commit_context( Ok(()) } +/// Commit a durable prepared commit using the retry-safe identifier path. +/// +/// This is the correct operation after restoring a prepared commit or after a +/// previous commit returned an indeterminate transport/IO error. A successful +/// earlier commit with the same `(commit_user, commit_identifier)` is filtered. +#[no_mangle] +pub unsafe extern "C" fn paimon_table_commit_commit_prepared( + tc: *const paimon_table_commit, + prepared: *const paimon_prepared_commit, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(tc, "tc") { + return error; + } + if let Err(error) = check_non_null(prepared, "prepared") { + return error; + } + let table_commit = &*((*tc).inner as *const TableCommitState); + let prepared = &*((*prepared).inner as *const PreparedCommitState); + let messages = &prepared.messages; + if let Err(error) = validate_commit_context( + &table_commit.table_location, + &table_commit.commit_user, + table_commit.overwrite, + &messages.table_location, + &messages.commit_user, + messages.overwrite, + ) { + return error; + } + let result = if messages.overwrite { + runtime().block_on(table_commit.commit.overwrite_with_identifier( + messages.messages.clone(), + None, + prepared.commit_identifier, + )) + } else { + runtime().block_on(table_commit.commit.filter_and_commit_with_identifier( + messages.messages.clone(), + prepared.commit_identifier, + )) + }; + match result { + Ok(()) => ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| prepared_panic_error("paimon_table_commit_commit_prepared")) +} + +/// Abort files referenced by a durable prepared commit. +/// +/// Do not call this after an indeterminate commit response: retry +/// `paimon_table_commit_commit_prepared` first so a successful commit is not +/// followed by deletion of its files. The caller must also fence/serialize all +/// commit and abort operations for the same `(table, commit_user)` across +/// processes. If retained snapshot history cannot prove that abort is safe, +/// this function fails closed and deletes nothing. +#[no_mangle] +pub unsafe extern "C" fn paimon_table_commit_abort_prepared( + tc: *const paimon_table_commit, + prepared: *const paimon_prepared_commit, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(tc, "tc") { + return error; + } + if let Err(error) = check_non_null(prepared, "prepared") { + return error; + } + let table_commit = &*((*tc).inner as *const TableCommitState); + let prepared = &*((*prepared).inner as *const PreparedCommitState); + let messages = &prepared.messages; + if let Err(error) = validate_commit_context( + &table_commit.table_location, + &table_commit.commit_user, + table_commit.overwrite, + &messages.table_location, + &messages.commit_user, + messages.overwrite, + ) { + return error; + } + match runtime().block_on( + table_commit + .commit + .abort_if_uncommitted(&messages.messages, prepared.commit_identifier), + ) { + Ok(()) => ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| prepared_panic_error("paimon_table_commit_abort_prepared")) +} + unsafe fn standard_commit_with_identifier_impl( tc: *const paimon_table_commit, msgs: *mut paimon_commit_messages, commit_identifier: i64, filter_committed: bool, + batch_commit: bool, ) -> *mut paimon_error { if let Err(error) = check_non_null(tc, "tc") { return error; @@ -941,6 +1656,11 @@ unsafe fn standard_commit_with_identifier_impl( if let Err(error) = check_non_null(msgs, "msgs") { return error; } + if commit_identifier < 0 || (!batch_commit && commit_identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const TableCommitState); let messages = &*((*msgs).inner as *const CommitMessagesState); if let Err(error) = validate_commit_context( @@ -959,7 +1679,9 @@ unsafe fn standard_commit_with_identifier_impl( ); } let messages = messages.messages.clone(); - let result = if filter_committed { + let result = if batch_commit { + runtime().block_on(table_commit.commit.commit(messages)) + } else if filter_committed { runtime().block_on( table_commit .commit @@ -984,7 +1706,7 @@ pub unsafe extern "C" fn paimon_table_commit_commit( tc: *const paimon_table_commit, msgs: *mut paimon_commit_messages, ) -> *mut paimon_error { - paimon_table_commit_commit_with_identifier(tc, msgs, i64::MAX) + standard_commit_with_identifier_impl(tc, msgs, i64::MAX, false, true) } /// Commit standard append messages with an identifier. @@ -994,7 +1716,7 @@ pub unsafe extern "C" fn paimon_table_commit_commit_with_identifier( msgs: *mut paimon_commit_messages, commit_identifier: i64, ) -> *mut paimon_error { - standard_commit_with_identifier_impl(tc, msgs, commit_identifier, false) + standard_commit_with_identifier_impl(tc, msgs, commit_identifier, false, false) } /// Filter a committed identifier before committing standard append messages. @@ -1004,7 +1726,7 @@ pub unsafe extern "C" fn paimon_table_commit_filter_and_commit_with_identifier( msgs: *mut paimon_commit_messages, commit_identifier: i64, ) -> *mut paimon_error { - standard_commit_with_identifier_impl(tc, msgs, commit_identifier, true) + standard_commit_with_identifier_impl(tc, msgs, commit_identifier, true, false) } /// Commit standard overwrite messages. @@ -1037,6 +1759,11 @@ unsafe fn standard_overwrite_impl( if let Err(error) = check_non_null(msgs, "msgs") { return error; } + if commit_identifier.is_some_and(|identifier| identifier < 0 || identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const TableCommitState); let messages = &*((*msgs).inner as *const CommitMessagesState); if let Err(error) = validate_commit_context( @@ -1093,6 +1820,11 @@ unsafe fn paimon_table_commit_truncate_table_impl( if let Err(error) = check_non_null(tc, "tc") { return error; } + if commit_identifier.is_some_and(|identifier| identifier < 0 || identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const TableCommitState); let result = match commit_identifier { Some(commit_identifier) => runtime().block_on( @@ -1143,6 +1875,7 @@ unsafe fn fixed_commit_with_identifier_impl( msgs: *mut paimon_postpone_fixed_bucket_commit_messages, commit_identifier: i64, filter_committed: bool, + batch_commit: bool, ) -> *mut paimon_error { if let Err(error) = check_non_null(tc, "tc") { return error; @@ -1150,6 +1883,11 @@ unsafe fn fixed_commit_with_identifier_impl( if let Err(error) = check_non_null(msgs, "msgs") { return error; } + if commit_identifier < 0 || (!batch_commit && commit_identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const PostponeFixedBucketTableCommitState); let messages = &*((*msgs).inner as *const PostponeFixedBucketCommitMessagesState); if let Err(error) = validate_commit_context( @@ -1163,7 +1901,9 @@ unsafe fn fixed_commit_with_identifier_impl( return error; } let messages = messages.messages.clone(); - let result = if filter_committed { + let result = if batch_commit { + runtime().block_on(table_commit.commit.commit(messages)) + } else if filter_committed { runtime().block_on( table_commit .commit @@ -1188,7 +1928,7 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_table_commit_commit( tc: *const paimon_postpone_fixed_bucket_table_commit, msgs: *mut paimon_postpone_fixed_bucket_commit_messages, ) -> *mut paimon_error { - paimon_postpone_fixed_bucket_table_commit_commit_with_identifier(tc, msgs, i64::MAX) + fixed_commit_with_identifier_impl(tc, msgs, i64::MAX, false, true) } /// Commit postpone fixed-bucket messages with an identifier. @@ -1198,7 +1938,7 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_table_commit_commit_with_i msgs: *mut paimon_postpone_fixed_bucket_commit_messages, commit_identifier: i64, ) -> *mut paimon_error { - fixed_commit_with_identifier_impl(tc, msgs, commit_identifier, false) + fixed_commit_with_identifier_impl(tc, msgs, commit_identifier, false, false) } /// Filter a committed identifier before committing fixed-bucket messages. @@ -1208,7 +1948,7 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_table_commit_filter_and_co msgs: *mut paimon_postpone_fixed_bucket_commit_messages, commit_identifier: i64, ) -> *mut paimon_error { - fixed_commit_with_identifier_impl(tc, msgs, commit_identifier, true) + fixed_commit_with_identifier_impl(tc, msgs, commit_identifier, true, false) } /// Truncate a table with a postpone fixed-bucket TableCommit. @@ -1235,6 +1975,11 @@ unsafe fn fixed_truncate_table_impl( if let Err(error) = check_non_null(tc, "tc") { return error; } + if commit_identifier.is_some_and(|identifier| identifier < 0 || identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const PostponeFixedBucketTableCommitState); let result = match commit_identifier { Some(commit_identifier) => runtime().block_on( @@ -1322,6 +2067,16 @@ const _: unsafe extern "C" fn( *mut paimon_commit_messages, *const paimon_commit_messages, ) -> *mut paimon_error = paimon_commit_messages_merge; +const _: unsafe extern "C" fn(*const paimon_commit_messages, i64) -> paimon_result_prepared_commit = + paimon_commit_messages_prepare; +const _: unsafe extern "C" fn(*const paimon_prepared_commit) -> paimon_result_bytes = + paimon_prepared_commit_serialize; +const _: unsafe extern "C" fn(*const u8, usize) -> paimon_result_prepared_commit = + paimon_prepared_commit_deserialize; +const _: unsafe extern "C" fn( + *mut paimon_prepared_commit, + *const paimon_prepared_commit, +) -> *mut paimon_error = paimon_prepared_commit_merge; const _: unsafe extern "C" fn( *mut paimon_postpone_fixed_bucket_commit_messages, *const paimon_postpone_fixed_bucket_commit_messages, @@ -1382,7 +2137,39 @@ const _: unsafe extern "C" fn( *const paimon_table_commit, *mut paimon_commit_messages, ) -> *mut paimon_error = paimon_table_commit_abort; +const _: unsafe extern "C" fn( + *const paimon_table_commit, + *const paimon_prepared_commit, +) -> *mut paimon_error = paimon_table_commit_commit_prepared; +const _: unsafe extern "C" fn( + *const paimon_table_commit, + *const paimon_prepared_commit, +) -> *mut paimon_error = paimon_table_commit_abort_prepared; const _: unsafe extern "C" fn( *const paimon_postpone_fixed_bucket_table_commit, *mut paimon_postpone_fixed_bucket_commit_messages, ) -> *mut paimon_error = paimon_postpone_fixed_bucket_table_commit_abort; + +#[cfg(test)] +mod raw_message_limit_tests { + use super::{RawPreparedCommitEnvelope, MAX_PREPARED_MESSAGES}; + + #[test] + fn raw_message_count_is_rejected_during_deserialization() { + let messages = (0..=MAX_PREPARED_MESSAGES) + .map(|_| "{}") + .collect::>() + .join(","); + let json = format!( + r#"{{"format":"paimon-rust-prepared-commit","version":2,"commit_identifier":1,"table_location":"memory:/table","commit_user":"job","overwrite":false,"messages":[{messages}]}}"# + ); + let error = match serde_json::from_str::>(&json) { + Ok(_) => panic!("oversized raw message list must be rejected"), + Err(error) => error, + }; + assert!(error.to_string().contains("more than")); + assert!(error + .to_string() + .contains(&MAX_PREPARED_MESSAGES.to_string())); + } +} diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt new file mode 100644 index 000000000..b4339360a --- /dev/null +++ b/bindings/cpp/CMakeLists.txt @@ -0,0 +1,341 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 3.15) +project(PaimonCpp VERSION 0.1.0 LANGUAGES C CXX) + +include(CMakePackageConfigHelpers) +include(GNUInstallDirs) + +option(PAIMON_CPP_BUILD_EXAMPLES "Build the C++ facade examples" OFF) +option(PAIMON_CPP_BUILD_TESTS "Build the header compile smoke test" OFF) +set(PAIMON_C_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../c/include" CACHE PATH + "Directory containing the cbindgen-generated paimon.h") +set(PAIMON_C_LIBRARY "" CACHE FILEPATH "Path to libpaimon_c") + +if(PAIMON_C_LIBRARY AND NOT EXISTS "${PAIMON_C_LIBRARY}") + message(FATAL_ERROR "PAIMON_C_LIBRARY does not exist: ${PAIMON_C_LIBRARY}") +endif() + +set(PAIMON_C_INSTALL_FILENAME + "${CMAKE_SHARED_LIBRARY_PREFIX}paimon_c${CMAKE_SHARED_LIBRARY_SUFFIX}") +if(PAIMON_C_LIBRARY) + get_filename_component( + PAIMON_C_INSTALL_FILENAME "${PAIMON_C_LIBRARY}" NAME) +endif() + +# Keep the C ABI dependency as a target in both the build and install trees. +# PaimonCppConfig.cmake recreates this imported target for installed consumers. +if(NOT TARGET Paimon::c) + if(PAIMON_C_LIBRARY) + add_library(Paimon::c SHARED IMPORTED GLOBAL) + set_target_properties( + Paimon::c + PROPERTIES + IMPORTED_LOCATION "${PAIMON_C_LIBRARY}" + IMPORTED_NO_SONAME TRUE + INTERFACE_INCLUDE_DIRECTORIES "${PAIMON_C_INCLUDE_DIR}") + else() + add_library(Paimon::c INTERFACE IMPORTED GLOBAL) + set_target_properties( + Paimon::c + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${PAIMON_C_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES paimon_c) + endif() +endif() + +add_library(paimon_cpp INTERFACE) +add_library(Paimon::cpp ALIAS paimon_cpp) +set_target_properties(paimon_cpp PROPERTIES EXPORT_NAME cpp) +target_compile_features(paimon_cpp INTERFACE cxx_std_17) +target_include_directories( + paimon_cpp + INTERFACE + "$" + "$") + +if(PAIMON_C_INCLUDE_DIR) + target_include_directories( + paimon_cpp INTERFACE "$") +endif() + +target_link_libraries(paimon_cpp INTERFACE Paimon::c) + +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/PaimonNoRuntimePlugin.cmake") + +if(PAIMON_CPP_BUILD_EXAMPLES) + if(NOT PAIMON_C_INCLUDE_DIR) + message(FATAL_ERROR "PAIMON_C_INCLUDE_DIR is required for examples") + endif() + add_executable(paimon_cpp_batch_read examples/batch_read.cpp) + target_link_libraries(paimon_cpp_batch_read PRIVATE Paimon::cpp) + add_executable(paimon_cpp_streaming_write examples/streaming_write.cpp) + target_link_libraries(paimon_cpp_streaming_write PRIVATE Paimon::cpp) + add_executable(paimon_cpp_stream_read examples/stream_read.cpp) + target_link_libraries(paimon_cpp_stream_read PRIVATE Paimon::cpp) +endif() + +if(PAIMON_CPP_BUILD_TESTS) + enable_testing() + add_library(paimon_cpp_header_smoke OBJECT tests/header_smoke.cpp) + target_compile_definitions( + paimon_cpp_header_smoke + PRIVATE PAIMON_C_HEADER="paimon_test_stub.h") + target_include_directories( + paimon_cpp_header_smoke PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/tests") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + target_compile_options( + paimon_cpp_header_smoke + PRIVATE + -fno-exceptions + -fno-rtti + -fvisibility=hidden + -fvisibility-inlines-hidden) + endif() + target_link_libraries(paimon_cpp_header_smoke PRIVATE Paimon::cpp) + add_test( + NAME paimon_cpp_header_compile_smoke + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} + --target paimon_cpp_header_smoke) + + if(PAIMON_C_LIBRARY AND UNIX AND NOT APPLE) + # Compile as C++, but deliberately invoke the C linker driver. This proves + # the facade itself needs no libstdc++/libc++ symbols while resolving every + # wrapped function against the real libpaimon_c artifact. + set(paimon_cpp_smoke_object + "${CMAKE_CURRENT_BINARY_DIR}/paimon_cpp_real_link_smoke.o") + set(paimon_cpp_smoke_library + "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_cpp_real_link_smoke${CMAKE_SHARED_LIBRARY_SUFFIX}") + get_filename_component( + PAIMON_C_LIBRARY_DIR "${PAIMON_C_LIBRARY}" DIRECTORY) + add_custom_command( + OUTPUT "${paimon_cpp_smoke_object}" + COMMAND + "${CMAKE_CXX_COMPILER}" -std=c++17 -fPIC -fno-exceptions -fno-rtti + -fvisibility=hidden -fvisibility-inlines-hidden + "-I${CMAKE_CURRENT_SOURCE_DIR}/include" + "-I${PAIMON_C_INCLUDE_DIR}" + -c "${CMAKE_CURRENT_SOURCE_DIR}/tests/header_smoke.cpp" + -o "${paimon_cpp_smoke_object}" + DEPENDS + tests/header_smoke.cpp + include/paimon/paimon.hpp + "${PAIMON_C_INCLUDE_DIR}/paimon.h" + VERBATIM) + add_custom_command( + OUTPUT "${paimon_cpp_smoke_library}" + COMMAND + "${CMAKE_C_COMPILER}" -shared + "${paimon_cpp_smoke_object}" + "-L${PAIMON_C_LIBRARY_DIR}" -lpaimon_c -Wl,-z,defs + -o "${paimon_cpp_smoke_library}" + DEPENDS "${paimon_cpp_smoke_object}" "${PAIMON_C_LIBRARY}" + VERBATIM) + add_custom_target( + paimon_cpp_real_link_smoke ALL DEPENDS "${paimon_cpp_smoke_library}") + + # The helper always compiles with the configured CXX compiler and links the + # resulting object with the configured C compiler. This remains correct when + # those drivers are different versions, as on an older deployment host. + paimon_add_no_runtime_plugin( + paimon_cpp_no_runtime_plugin + SOURCES tests/no_cpp_runtime_plugin.cpp + INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/tests/helper_config" + COMPILE_DEFINITIONS PAIMON_CPP_HELPER_COMPILE_DEFINITION=73) + + configure_file( + tests/relink_probe.cpp + "${CMAKE_CURRENT_BINARY_DIR}/relink_probe.cpp" + COPYONLY) + paimon_add_no_runtime_plugin( + paimon_cpp_relink_probe + SOURCES "${CMAKE_CURRENT_BINARY_DIR}/relink_probe.cpp") + + add_executable(paimon_cpp_dlopen_smoke tests/dlopen_smoke.c) + target_link_libraries(paimon_cpp_dlopen_smoke PRIVATE ${CMAKE_DL_LIBS}) + + add_test( + NAME paimon_c_no_cpp_runtime + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + "${PAIMON_C_LIBRARY}") + add_test( + NAME paimon_cpp_facade_no_cpp_runtime + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + "${paimon_cpp_smoke_library}") + add_test( + NAME paimon_cpp_plugin_no_cpp_runtime + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + "$") + + add_library( + paimon_elf_fixture_undefined_operator_new SHARED + tests/elf_fixtures/undefined_operator_new.c) + add_test( + NAME paimon_elf_guard_rejects_operator_new + COMMAND + "${CMAKE_COMMAND}" + "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + "-DLIBRARY=$" + "-DEXPECTED=C++ mangled" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") + + add_library( + paimon_elf_fixture_undefined_host_hook SHARED + tests/elf_fixtures/undefined_host_hook.c) + add_test( + NAME paimon_elf_guard_rejects_host_hook + COMMAND + "${CMAKE_COMMAND}" + "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + "-DLIBRARY=$" + "-DEXPECTED=unversioned undefined symbol" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") + + add_library( + paimon_elf_fixture_executable_stack SHARED + tests/elf_fixtures/executable_stack.c) + target_link_options( + paimon_elf_fixture_executable_stack PRIVATE -Wl,-z,execstack) + add_test( + NAME paimon_elf_guard_rejects_executable_stack + COMMAND + "${CMAKE_COMMAND}" + "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + "-DLIBRARY=$" + "-DEXPECTED=executable GNU_STACK" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") + + add_executable( + paimon_elf_fixture_pie + tests/elf_fixtures/pie_executable.c) + target_compile_options(paimon_elf_fixture_pie PRIVATE -fPIE) + target_link_options(paimon_elf_fixture_pie PRIVATE -pie) + add_test( + NAME paimon_elf_guard_rejects_pie_executable + COMMAND + "${CMAKE_COMMAND}" + "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + "-DLIBRARY=$" + "-DEXPECTED=PIE executable" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") + + add_test( + NAME paimon_cpp_plugin_incremental_relink + COMMAND + "${CMAKE_COMMAND}" + "-DBUILD_DIR=${CMAKE_BINARY_DIR}" + "-DSOURCE=${CMAKE_CURRENT_BINARY_DIR}/relink_probe.cpp" + "-DLIBRARY=$" + "-DTARGET=paimon_cpp_relink_probe" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/check_incremental_relink.cmake") + + add_test( + NAME paimon_cpp_plugin_isolated_load + COMMAND + "${CMAKE_COMMAND}" + "-DLOADER=$" + "-DPLUGIN=$" + "-DPAIMON_C_LIBRARY=${PAIMON_C_LIBRARY}" + "-DTEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/isolated-load-test" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_isolated_load.cmake") + + if(CMAKE_C_COMPILER_ID MATCHES "Clang|GNU") + add_library( + paimon_elf_fixture_libgcc SHARED + tests/elf_fixtures/needs_libgcc.c) + target_link_options( + paimon_elf_fixture_libgcc PRIVATE -Wl,--no-as-needed) + target_link_libraries(paimon_elf_fixture_libgcc PRIVATE gcc_s) + add_test( + NAME paimon_elf_guard_rejects_libgcc + COMMAND + "${CMAKE_COMMAND}" + "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + "-DLIBRARY=$" + "-DEXPECTED=libgcc_s" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") + endif() + + set(paimon_cpp_install_test_root + "${CMAKE_CURRENT_BINARY_DIR}/install-tree-consumer-test") + add_test( + NAME paimon_cpp_install_tree_consumer + COMMAND + "${CMAKE_COMMAND}" + "-DMAIN_BUILD_DIR=${CMAKE_BINARY_DIR}" + "-DCONSUMER_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/tests/install_tree_consumer" + "-DTEST_ROOT=${paimon_cpp_install_test_root}" + "-DC_COMPILER=${CMAKE_C_COMPILER}" + "-DCXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DPLUGIN_FILENAME=${CMAKE_SHARED_LIBRARY_PREFIX}paimon_install_tree_consumer${CMAKE_SHARED_LIBRARY_SUFFIX}" + "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_install_tree_consumer.cmake") + add_test( + NAME paimon_cpp_install_tree_guard_rejects_runtime + COMMAND + "${CMAKE_COMMAND}" + "-DMAIN_BUILD_DIR=${CMAKE_BINARY_DIR}" + "-DCONSUMER_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/tests/install_tree_consumer" + "-DTEST_ROOT=${paimon_cpp_install_test_root}-forbidden" + "-DC_COMPILER=${CMAKE_C_COMPILER}" + "-DCXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DPLUGIN_FILENAME=${CMAKE_SHARED_LIBRARY_PREFIX}paimon_install_tree_consumer${CMAKE_SHARED_LIBRARY_SUFFIX}" + "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + -DEXPECT_BUILD_FAILURE=ON + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_install_tree_consumer.cmake") + endif() +endif() + +install(DIRECTORY include/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +if(EXISTS "${PAIMON_C_INCLUDE_DIR}/paimon.h") + install( + FILES "${PAIMON_C_INCLUDE_DIR}/paimon.h" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +endif() +if(PAIMON_C_LIBRARY) + install( + FILES "${PAIMON_C_LIBRARY}" + DESTINATION "${CMAKE_INSTALL_LIBDIR}") +endif() +install( + TARGETS paimon_cpp + EXPORT PaimonCppTargets + INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +install( + EXPORT PaimonCppTargets + FILE PaimonCppTargets.cmake + NAMESPACE Paimon:: + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp") + +configure_package_config_file( + cmake/PaimonCppConfig.cmake.in + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfig.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp") +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfigVersion.cmake" + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion) +install( + FILES + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfigVersion.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/PaimonNoRuntimePlugin.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp") +install( + PROGRAMS "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp") diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md new file mode 100644 index 000000000..795831a31 --- /dev/null +++ b/bindings/cpp/README.md @@ -0,0 +1,146 @@ + + +# Paimon C++ facade + +This directory provides a header-only C++17 RAII facade over the stable Paimon +C ABI. It deliberately builds no C++ shared library: the only Paimon binary is +`libpaimon_c`, produced by Rust, and every symbol called by the facade has C +linkage. The facade does not depend on Arrow C++; Arrow batches cross the API as +raw Arrow C Data `array` and `schema` pointers. + +All native handles are move-only. Their destructors are `noexcept` and only +release resources. In particular, destroying `PreparedMessages` or +`PreparedCommit` never commits or aborts them. A streaming writer binds messages +to a checkpoint with `PreparedMessages::prepare`, persists the bytes from +`PreparedCommit::serialize`, then calls `TableCommit::commit_prepared`. After an +uncertain result or process restart, deserialize the same bytes and retry with +the same stable `commit_user`; the identifier path filters duplicate commits. +Exactly-once filtering is recorded in retained snapshot metadata. Keep snapshot +history for at least the maximum writer-recovery horizon, never retry a +checkpoint older than that horizon, and give each fresh job a new globally +unique `commit_user`. Checkpoint identifiers must be in `[0, INT64_MAX)`; +`INT64_MAX` is reserved for unidentified batch commits. +Checkpoint blobs are trusted state, not a security token: store them behind +normal integrity/access controls. Before `abort_prepared`, fence every commit +and abort for the same `(table, commit_user)` across processes. If snapshot +history is too old to prove safety, abort fails closed and leaves cleanup to an +orphan-file policy. +Filesystem-catalog commits require a backend with atomic publish-if-absent +(conditional rename/copy/write). Unsupported backends fail closed; use REST +commit or an external lock instead of relying on a racy existence check. + +Continuous reading is a pull API. `StreamScan::poll` immediately returns data, +waiting, or end; it never starts a callback thread and never waits for a future +snapshot. A data result owns a `StreamPlan`, which can be read in data or audit +log mode using the same Arrow C Data `RecordBatchReader` as bounded reads. +Each `StreamScan` is single-thread-confined; serialize poll, checkpoint, +restore, and destruction. Decoupled changelog fallback and consumer-retention +registration are not implemented yet, so snapshot retention must cover the +maximum expected reader lag. +Persisted stream plans currently reject external data-file paths. The failure +is reported by `StreamPlan::serialize` before a checkpoint can be acknowledged, +instead of producing a checkpoint that cannot be restored. + +## Build + +Generate the C header and build the Rust library first: + +```bash +cargo build --release -p paimon-c +cbindgen --config bindings/c/cbindgen.toml bindings/c \ + --output bindings/c/include/paimon.h +``` + +For a shared plugin that must load without `libstdc++`, `libc++`, or +`libgcc_s`, compile the C++ source without exceptions/RTTI and use the C linker +driver for the final link: + +```bash +c++ -std=c++17 -fPIC -fno-exceptions -fno-rtti \ + -Ibindings/cpp/include -Ibindings/c/include \ + -c plugin.cpp -o plugin.o +cc -shared plugin.o -Ltarget/release -lpaimon_c -Wl,-z,defs \ + -o libplugin.so +bindings/cpp/scripts/verify_linux_elf.sh libplugin.so +``` + +The plugin must expose each public entry point with +`PAIMON_CPP_PLUGIN_EXPORT` and stay within the facade's allocation-free, +no-exceptions subset. Linking the final `.so` with a C++ driver can add a C++ +runtime even when the source does not call that runtime directly. + +Or install its CMake interface target: + +```bash +cmake -S bindings/cpp -B target/cpp-build \ + -DPAIMON_C_LIBRARY="$PWD/target/release/libpaimon_c.so" \ + -DPAIMON_CPP_BUILD_EXAMPLES=ON +cmake --build target/cpp-build +cmake --install target/cpp-build --prefix /your/prefix +``` + +When `PAIMON_C_LIBRARY` is set, installation copies `libpaimon_c` into the +prefix and the package exports imported target `Paimon::c`. Installed consumers +can build a verified no-runtime plugin with the provided helper: + +```cmake +cmake_minimum_required(VERSION 3.15) +project(MyPaimonPlugin LANGUAGES C CXX) +find_package(PaimonCpp CONFIG REQUIRED) +paimon_add_no_runtime_plugin( + my_paimon_plugin + SOURCES plugin.cpp + INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/include" + COMPILE_DEFINITIONS MY_PLUGIN_ABI=1) +``` + +Configure C++ compilation through the helper's `SOURCES`, +`INCLUDE_DIRECTORIES`, `COMPILE_DEFINITIONS`, `COMPILE_OPTIONS`, and +`LINK_LIBRARIES` arguments. Do not add C++ sources to the returned C-link +target with `target_sources`; doing so bypasses the split compile/link model. +The helper hides all non-exported C++ symbols, links with the C driver, embeds +only `$ORIGIN` as its runtime search path, and runs the installed ELF guard +after every successful link. + +`Paimon::cpp` remains the header-only facade target for consumers that manage +their own final link. `PaimonCpp_C_LIBRARY` may be set before `find_package` to +select an externally installed `libpaimon_c`. + +## Linux runtime guard + +Run the ELF guard on every release artifact: + +```bash +bindings/cpp/scripts/verify_linux_elf.sh target/release/libpaimon_c.so +``` + +It prints the build host's `ldd --version` and applies a `DT_NEEDED` allowlist +containing glibc components and `libpaimon_c`. It rejects C++ runtimes, +`libgcc_s`, `libunwind`, `libatomic`, `GLIBCXX`/`CXXABI`/`GCC` symbol versions, +undefined or exported C++ mangled symbols, unversioned host hooks, operator +new/delete, RTTI/dynamic-cast support, absolute runtime paths, and +private/non-baseline glibc ABI versions. Glibc's C-level `__cxa_atexit`, +`__cxa_finalize`, and `__cxa_thread_atexit_impl` remain allowed. The highest +referenced numeric `GLIBC_*` symbol version must not exceed 2.17. + +`Scan::plan()` remains a bounded scan. Use `StreamScanOptions` and +`ReadBuilder::new_stream_scan` for a stateful continuous scan. Persist +`StreamScan::checkpoint()` only after every split in the returned plan has been +durably accounted for by the surrounding checkpoint barrier. diff --git a/bindings/cpp/cmake/PaimonCppConfig.cmake.in b/bindings/cpp/cmake/PaimonCppConfig.cmake.in new file mode 100644 index 000000000..f37f23993 --- /dev/null +++ b/bindings/cpp/cmake/PaimonCppConfig.cmake.in @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +@PACKAGE_INIT@ + +if(NOT TARGET Paimon::c) + set(_paimon_c_bundled + "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_LIBDIR@/@PAIMON_C_INSTALL_FILENAME@") + if(DEFINED PaimonCpp_C_LIBRARY) + set(_paimon_c_library "${PaimonCpp_C_LIBRARY}") + elseif(EXISTS "${_paimon_c_bundled}") + set(_paimon_c_library "${_paimon_c_bundled}") + else() + unset(_PaimonCpp_DISCOVERED_C_LIBRARY CACHE) + find_library( + _PaimonCpp_DISCOVERED_C_LIBRARY + NAMES paimon_c + HINTS "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_LIBDIR@") + set(_paimon_c_library "${_PaimonCpp_DISCOVERED_C_LIBRARY}") + unset(_PaimonCpp_DISCOVERED_C_LIBRARY CACHE) + endif() + + if(NOT _paimon_c_library OR NOT EXISTS "${_paimon_c_library}") + set(PaimonCpp_FOUND FALSE) + set(PaimonCpp_NOT_FOUND_MESSAGE + "libpaimon_c was not found; install it under the package prefix or set PaimonCpp_C_LIBRARY") + return() + endif() + + add_library(Paimon::c SHARED IMPORTED) + set_target_properties( + Paimon::c + PROPERTIES + IMPORTED_LOCATION "${_paimon_c_library}" + IMPORTED_NO_SONAME TRUE + INTERFACE_INCLUDE_DIRECTORIES "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/PaimonCppTargets.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/PaimonNoRuntimePlugin.cmake") +check_required_components(PaimonCpp) diff --git a/bindings/cpp/cmake/PaimonNoRuntimePlugin.cmake b/bindings/cpp/cmake/PaimonNoRuntimePlugin.cmake new file mode 100644 index 000000000..977f4121e --- /dev/null +++ b/bindings/cpp/cmake/PaimonNoRuntimePlugin.cmake @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +include_guard(GLOBAL) + +# Capture this while the module is included. CMAKE_CURRENT_LIST_DIR inside a +# function can otherwise refer to the consumer's calling list file. +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/verify_linux_elf.sh") + set(_PAIMON_NO_RUNTIME_ELF_VERIFIER + "${CMAKE_CURRENT_LIST_DIR}/verify_linux_elf.sh") +else() + set(_PAIMON_NO_RUNTIME_ELF_VERIFIER + "${CMAKE_CURRENT_LIST_DIR}/../scripts/verify_linux_elf.sh") +endif() + +# Build a C-linkage plugin from C++17 sources without linking a C++ runtime. +# The source must itself stay within the no-exceptions/no-RTTI subset used by +# the Paimon facade. The final target is linked by the configured C driver. +function(paimon_add_no_runtime_plugin target) + if(NOT TARGET Paimon::cpp OR NOT TARGET Paimon::c) + message(FATAL_ERROR + "paimon_add_no_runtime_plugin requires Paimon::cpp and Paimon::c") + endif() + set(multi_value_args + SOURCES INCLUDE_DIRECTORIES COMPILE_DEFINITIONS COMPILE_OPTIONS + LINK_LIBRARIES) + cmake_parse_arguments(PAIMON_PLUGIN "" "" "${multi_value_args}" ${ARGN}) + if(PAIMON_PLUGIN_KEYWORDS_MISSING_VALUES) + message(FATAL_ERROR + "missing values for: ${PAIMON_PLUGIN_KEYWORDS_MISSING_VALUES}") + endif() + if(PAIMON_PLUGIN_SOURCES) + if(PAIMON_PLUGIN_UNPARSED_ARGUMENTS) + message(FATAL_ERROR + "unexpected no-runtime plugin arguments: ${PAIMON_PLUGIN_UNPARSED_ARGUMENTS}") + endif() + set(plugin_sources ${PAIMON_PLUGIN_SOURCES}) + else() + # Preserve the original positional-source form. + set(plugin_sources ${PAIMON_PLUGIN_UNPARSED_ARGUMENTS}) + endif() + if(NOT plugin_sources) + message(FATAL_ERROR + "paimon_add_no_runtime_plugin(${target}) requires source files") + endif() + if(TARGET "${target}" OR TARGET "${target}__paimon_cpp_objects") + message(FATAL_ERROR "target already exists: ${target}") + endif() + if(NOT UNIX OR APPLE OR + NOT CMAKE_C_COMPILER_ID MATCHES "Clang|GNU" OR + NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + message(FATAL_ERROR + "no-runtime plugins currently require GNU/Clang C and C++ compilers on Linux") + endif() + + if(NOT EXISTS "${_PAIMON_NO_RUNTIME_ELF_VERIFIER}") + message(FATAL_ERROR + "Paimon ELF verifier is missing: ${_PAIMON_NO_RUNTIME_ELF_VERIFIER}") + endif() + + add_library("${target}__paimon_cpp_objects" OBJECT ${plugin_sources}) + set_target_properties( + "${target}__paimon_cpp_objects" + PROPERTIES + POSITION_INDEPENDENT_CODE ON + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF) + target_compile_options( + "${target}__paimon_cpp_objects" + PRIVATE + -fno-exceptions + -fno-rtti + -fvisibility=hidden + -fvisibility-inlines-hidden + ${PAIMON_PLUGIN_COMPILE_OPTIONS}) + if(PAIMON_PLUGIN_INCLUDE_DIRECTORIES) + target_include_directories( + "${target}__paimon_cpp_objects" + PRIVATE ${PAIMON_PLUGIN_INCLUDE_DIRECTORIES}) + endif() + if(PAIMON_PLUGIN_COMPILE_DEFINITIONS) + target_compile_definitions( + "${target}__paimon_cpp_objects" + PRIVATE ${PAIMON_PLUGIN_COMPILE_DEFINITIONS}) + endif() + target_link_libraries( + "${target}__paimon_cpp_objects" + PRIVATE Paimon::cpp ${PAIMON_PLUGIN_LINK_LIBRARIES}) + + # Hide the C++ object language from the final C target. CMake otherwise adds + # its configured implicit C++ libraries even when LINKER_LANGUAGE is C. + set(archive_target "${target}__paimon_cpp_archive") + add_library( + "${archive_target}" STATIC + $) + set_target_properties("${archive_target}" PROPERTIES LINKER_LANGUAGE CXX) + + set(link_stub "${CMAKE_CURRENT_BINARY_DIR}/${target}__paimon_link_stub.c") + file(GENERATE OUTPUT "${link_stub}" + CONTENT "/* Generated C link anchor for a Paimon no-runtime plugin. */\n") + add_library("${target}" SHARED "${link_stub}") + set_target_properties( + "${target}" + PROPERTIES + LINKER_LANGUAGE C + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH "\$ORIGIN") + add_dependencies("${target}" "${archive_target}") + set_property( + TARGET "${target}" APPEND PROPERTY + LINK_DEPENDS "$") + target_link_options( + "${target}" + PRIVATE + "-Wl,--whole-archive,$,--no-whole-archive" + -Wl,-z,defs) + target_link_libraries( + "${target}" PRIVATE Paimon::c ${PAIMON_PLUGIN_LINK_LIBRARIES}) + add_custom_command( + TARGET "${target}" + POST_BUILD + COMMAND "${_PAIMON_NO_RUNTIME_ELF_VERIFIER}" "$" + COMMENT "Verifying that ${target} has a C-only dynamic ABI" + VERBATIM) +endfunction() diff --git a/bindings/cpp/examples/batch_read.cpp b/bindings/cpp/examples/batch_read.cpp new file mode 100644 index 000000000..d8654179d --- /dev/null +++ b/bindings/cpp/examples/batch_read.cpp @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +namespace { + +void print_error(const paimon::Error& error) { + const auto message = error.message(); + std::fprintf(stderr, "Paimon error %d: %.*s\n", + static_cast(error.code()), + static_cast(message.size()), message.data()); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 4) { + std::fprintf(stderr, "usage: %s WAREHOUSE DATABASE TABLE\n", argv[0]); + return EXIT_FAILURE; + } + + const paimon::Option options[] = {{"warehouse", argv[1]}}; + auto catalog_result = paimon::Catalog::create(options); + if (!catalog_result) { + print_error(catalog_result.error()); + return EXIT_FAILURE; + } + auto catalog = std::move(catalog_result).value(); + + auto identifier_result = paimon::Identifier::create(argv[2], argv[3]); + if (!identifier_result) { + print_error(identifier_result.error()); + return EXIT_FAILURE; + } + auto identifier = std::move(identifier_result).value(); + + auto table_result = catalog.get_table(identifier); + if (!table_result) { + print_error(table_result.error()); + return EXIT_FAILURE; + } + auto table = std::move(table_result).value(); + + auto builder_result = table.new_read_builder(); + if (!builder_result) { + print_error(builder_result.error()); + return EXIT_FAILURE; + } + auto builder = std::move(builder_result).value(); + + auto scan_result = builder.new_scan(); + auto read_result = builder.new_read(); + if (!scan_result || !read_result) { + print_error(!scan_result ? scan_result.error() : read_result.error()); + return EXIT_FAILURE; + } + auto scan = std::move(scan_result).value(); + auto read = std::move(read_result).value(); + + auto plan_result = scan.plan(); + if (!plan_result) { + print_error(plan_result.error()); + return EXIT_FAILURE; + } + auto plan = std::move(plan_result).value(); + + auto reader_result = read.to_arrow(plan); + if (!reader_result) { + print_error(reader_result.error()); + return EXIT_FAILURE; + } + auto reader = std::move(reader_result).value(); + + std::size_t batch_count = 0; + for (;;) { + auto next = reader.next(); + if (!next) { + print_error(next.error()); + return EXIT_FAILURE; + } + auto batch = std::move(next).value(); + if (!batch) { + break; + } + // Import batch.array()/batch.schema() with any Arrow C Data consumer here. + // ArrowBatch releases both native containers when it leaves this scope. + ++batch_count; + } + + std::printf("splits=%zu batches=%zu\n", plan.num_splits(), batch_count); + return EXIT_SUCCESS; +} diff --git a/bindings/cpp/examples/stream_read.cpp b/bindings/cpp/examples/stream_read.cpp new file mode 100644 index 000000000..ffdae6a02 --- /dev/null +++ b/bindings/cpp/examples/stream_read.cpp @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +namespace { + +void print_error(const paimon::Error& error) { + const auto message = error.message(); + std::fprintf(stderr, "Paimon error %d: %.*s\n", + static_cast(error.code()), + static_cast(message.size()), message.data()); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 4) { + std::fprintf(stderr, "usage: %s WAREHOUSE DATABASE TABLE\n", argv[0]); + return EXIT_FAILURE; + } + + const paimon::Option catalog_options[] = {{"warehouse", argv[1]}}; + auto catalog_result = paimon::Catalog::create(catalog_options); + auto identifier_result = paimon::Identifier::create(argv[2], argv[3]); + if (!catalog_result || !identifier_result) { + print_error(!catalog_result ? catalog_result.error() + : identifier_result.error()); + return EXIT_FAILURE; + } + auto catalog = std::move(catalog_result).value(); + auto identifier = std::move(identifier_result).value(); + + auto table_result = catalog.get_table(identifier); + if (!table_result) { + print_error(table_result.error()); + return EXIT_FAILURE; + } + auto table = std::move(table_result).value(); + + auto builder_result = table.new_read_builder(); + auto options_result = paimon::StreamScanOptions::defaults(); + if (!builder_result || !options_result) { + print_error(!builder_result ? builder_result.error() + : options_result.error()); + return EXIT_FAILURE; + } + auto builder = std::move(builder_result).value(); + auto options = std::move(options_result).value(); + options.with_startup(paimon::StreamStartupMode::latest_full) + .with_follow_up(paimon::StreamFollowUpMode::automatic); + + auto read_result = builder.new_read(); + auto scan_result = builder.new_stream_scan(options); + if (!read_result || !scan_result) { + print_error(!read_result ? read_result.error() : scan_result.error()); + return EXIT_FAILURE; + } + auto read = std::move(read_result).value(); + auto scan = std::move(scan_result).value(); + + // poll() is a pull operation and never waits. A scheduler should call it + // again later after Waiting; this standalone example exits instead. + auto poll_result = scan.poll(); + if (!poll_result) { + print_error(poll_result.error()); + return EXIT_FAILURE; + } + auto poll = std::move(poll_result).value(); + if (poll.waiting()) { + std::printf("waiting next_snapshot_id=%lld\n", + static_cast(poll.next_snapshot_id())); + return EXIT_SUCCESS; + } + if (poll.end()) { + std::puts("end"); + return EXIT_SUCCESS; + } + + auto pending_plan_result = poll.plan().serialize(); + if (!pending_plan_result) { + print_error(pending_plan_result.error()); + return EXIT_FAILURE; + } + auto pending_plan = std::move(pending_plan_result).value(); + // Persist pending_plan together with the cursor before exposing rows. On + // recovery, StreamPlan::deserialize recreates this PollResult for replay. + std::printf("pending-plan-bytes=%zu\n", pending_plan.size()); + + auto reader_result = poll.plan().read_to_arrow( + read, paimon::StreamReadMode::data); + if (!reader_result) { + print_error(reader_result.error()); + return EXIT_FAILURE; + } + auto reader = std::move(reader_result).value(); + std::size_t batches = 0; + for (;;) { + auto next = reader.next(); + if (!next) { + print_error(next.error()); + return EXIT_FAILURE; + } + auto batch = std::move(next).value(); + if (!batch) { + break; + } + // Import through any Arrow C Data consumer before batch is destroyed. + ++batches; + } + + // Persist this cursor only after the plan's split progress is durably part of + // the surrounding checkpoint barrier. + std::printf("snapshot=%lld splits=%zu batches=%zu checkpoint=%lld\n", + static_cast(poll.snapshot_id()), + poll.plan().num_splits(), batches, + static_cast(scan.checkpoint())); + return EXIT_SUCCESS; +} diff --git a/bindings/cpp/examples/streaming_write.cpp b/bindings/cpp/examples/streaming_write.cpp new file mode 100644 index 000000000..5e8b2fe2b --- /dev/null +++ b/bindings/cpp/examples/streaming_write.cpp @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +// Data and checkpoint barriers are separate events so an idle stream can still +// checkpoint. Paimon consumes Arrow contents in place; the producer continues +// to own the ArrowArray and ArrowSchema container memory. +enum class StreamEventKind : std::uint8_t { batch, checkpoint, end }; + +struct StreamEvent { + StreamEventKind kind; + void* array; + void* schema; +}; + +using Producer = paimon::Status (*)(void* context, StreamEvent* output); +using Persist = paimon::Status (*)(void* context, std::int64_t checkpoint_id, + const std::uint8_t* data, std::size_t size); + +paimon::Status recover_checkpoint(const paimon::TableCommit& committer, + const std::uint8_t* data, + std::size_t size) { + // A zero-length checkpoint records source progress but has no Paimon data to + // commit. The surrounding engine owns that source-state representation. + if (size == 0) { + return paimon::Status::success(); + } + auto restored_result = paimon::PreparedCommit::deserialize(data, size); + if (!restored_result) { + return paimon::Status::failure(std::move(restored_result).error()); + } + auto restored = std::move(restored_result).value(); + return committer.commit_prepared(restored); +} + +paimon::Status complete_checkpoint(paimon::TableWrite& writer, + const paimon::TableCommit& committer, + Persist persist, void* context, + std::int64_t checkpoint_id) { + auto prepared_result = writer.prepare_commit(); + if (!prepared_result) { + return paimon::Status::failure(std::move(prepared_result).error()); + } + auto prepared = std::move(prepared_result).value(); + + auto durable_result = prepared.prepare(checkpoint_id); + if (!durable_result) { + return paimon::Status::failure(std::move(durable_result).error()); + } + auto durable = std::move(durable_result).value(); + auto bytes_result = durable.serialize(); + if (!bytes_result) { + return paimon::Status::failure(std::move(bytes_result).error()); + } + auto bytes = std::move(bytes_result).value(); + + // persist must not report success until the checkpoint blob and the engine's + // source state are durable in the same checkpoint protocol. + auto persist_status = + persist(context, checkpoint_id, bytes.data(), bytes.size()); + if (!persist_status) { + return persist_status; + } + + // commit_prepared is retry-safe. After a crash, deserialize the persisted + // blob and call this again with the same stable commit_user. + return committer.commit_prepared(durable); +} + +paimon::Status run_stream(paimon::TableWrite& writer, + const paimon::TableCommit& committer, + Producer producer, void* context, + Persist persist, + std::int64_t first_checkpoint_id) { + auto checkpoint_id = first_checkpoint_id; + bool dirty = false; + for (;;) { + StreamEvent event{StreamEventKind::end, nullptr, nullptr}; + auto producer_status = producer(context, &event); + if (!producer_status) { + return producer_status; + } + + switch (event.kind) { + case StreamEventKind::batch: { + auto write_status = writer.write_arrow(event.array, event.schema); + if (!write_status) { + return write_status; + } + dirty = true; + break; + } + case StreamEventKind::checkpoint: { + auto checkpoint_status = + dirty ? complete_checkpoint(writer, committer, persist, context, + checkpoint_id) + : persist(context, checkpoint_id, nullptr, 0); + if (!checkpoint_status) { + return checkpoint_status; + } + dirty = false; + ++checkpoint_id; + break; + } + case StreamEventKind::end: + // Never discard a tail batch merely because the producer ended before + // emitting its next periodic checkpoint barrier. + return dirty ? complete_checkpoint(writer, committer, persist, context, + checkpoint_id) + : paimon::Status::success(); + } + } +} + +namespace { + +void print_error(const paimon::Error& error) { + const auto message = error.message(); + std::fprintf(stderr, "Paimon error %d: %.*s\n", + static_cast(error.code()), + static_cast(message.size()), message.data()); +} + +paimon::Status no_input(void*, StreamEvent* event) { + *event = {StreamEventKind::end, nullptr, nullptr}; + return paimon::Status::success(); +} + +paimon::Status no_op_persist(void*, std::int64_t, const std::uint8_t*, + std::size_t) { + // Replace this with fsync/rename or the surrounding engine's durable state. + return paimon::Status::success(); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 5) { + std::fprintf(stderr, + "usage: %s WAREHOUSE DATABASE TABLE STABLE_COMMIT_USER\n", + argv[0]); + return EXIT_FAILURE; + } + + const paimon::Option options[] = {{"warehouse", argv[1]}}; + auto catalog_result = paimon::Catalog::create(options); + auto identifier_result = paimon::Identifier::create(argv[2], argv[3]); + if (!catalog_result || !identifier_result) { + print_error(!catalog_result ? catalog_result.error() + : identifier_result.error()); + return EXIT_FAILURE; + } + auto catalog = std::move(catalog_result).value(); + auto identifier = std::move(identifier_result).value(); + + auto table_result = catalog.get_table(identifier); + if (!table_result) { + print_error(table_result.error()); + return EXIT_FAILURE; + } + auto table = std::move(table_result).value(); + + auto builder_result = table.new_write_builder(argv[4]); + if (!builder_result) { + print_error(builder_result.error()); + return EXIT_FAILURE; + } + auto builder = std::move(builder_result).value(); + + auto writer_result = builder.new_write(); + auto committer_result = builder.new_commit(); + if (!writer_result || !committer_result) { + print_error(!writer_result ? writer_result.error() + : committer_result.error()); + return EXIT_FAILURE; + } + auto writer = std::move(writer_result).value(); + auto committer = std::move(committer_result).value(); + + // Replace no_input and no_op_persist with the application's Arrow producer + // and durable checkpoint store. + auto status = run_stream(writer, committer, no_input, nullptr, + no_op_persist, 1); + if (!status) { + print_error(status.error()); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/bindings/cpp/include/paimon/paimon.hpp b/bindings/cpp/include/paimon/paimon.hpp new file mode 100644 index 000000000..fc456dd01 --- /dev/null +++ b/bindings/cpp/include/paimon/paimon.hpp @@ -0,0 +1,1489 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef PAIMON_CPP_PAIMON_HPP +#define PAIMON_CPP_PAIMON_HPP + +#include +#include +#include +#include +#include +#include +#include + +// Tests and embedders may override this with a quoted header name. Normal +// consumers use the cbindgen-generated paimon.h shipped with libpaimon_c. +#ifndef PAIMON_C_HEADER +#define PAIMON_C_HEADER +#endif + +// The repository's plain `cbindgen --lang c` output does not add a C++ +// compatibility guard. Force C linkage here; nesting is harmless when a +// packaged paimon.h already supplies its own extern "C" block. +extern "C" { +#include PAIMON_C_HEADER +} + +// Mark the deliberately small C ABI exported by a no-runtime plugin. The +// CMake helper hides every other C++ symbol so inline facade implementation +// details cannot leak into the plugin's dynamic ABI. +#if defined(_WIN32) +#define PAIMON_CPP_PLUGIN_EXPORT extern "C" __declspec(dllexport) +#elif defined(__GNUC__) || defined(__clang__) +#define PAIMON_CPP_PLUGIN_EXPORT \ + extern "C" __attribute__((visibility("default"))) +#else +#define PAIMON_CPP_PLUGIN_EXPORT extern "C" +#endif + +namespace paimon { + +struct adopt_handle_t { + explicit constexpr adopt_handle_t() noexcept = default; +}; + +inline constexpr adopt_handle_t adopt_handle{}; + +enum class ErrorCode : std::int32_t { + unexpected = PAIMON_ERROR_UNEXPECTED, + unsupported = PAIMON_ERROR_UNSUPPORTED, + not_found = PAIMON_ERROR_NOT_FOUND, + already_exists = PAIMON_ERROR_ALREADY_EXISTS, + invalid_input = PAIMON_ERROR_INVALID_INPUT, + io_error = PAIMON_ERROR_IO, + out_of_range = PAIMON_ERROR_OUT_OF_RANGE, +}; + +// Owns one paimon_error. Error is also used as the storage for Status: a null +// native handle means success. The message view remains valid until this Error +// is moved, reset, or destroyed. +class Error final { + public: + constexpr Error() noexcept = default; + + explicit Error(adopt_handle_t, ::paimon_error* error) noexcept + : error_(error) {} + + Error(const Error&) = delete; + Error& operator=(const Error&) = delete; + + Error(Error&& other) noexcept : error_(other.release()) {} + + Error& operator=(Error&& other) noexcept { + if (this != &other) { + reset(other.release()); + } + return *this; + } + + ~Error() noexcept { reset(); } + + [[nodiscard]] bool ok() const noexcept { return error_ == nullptr; } + [[nodiscard]] explicit operator bool() const noexcept { return !ok(); } + + [[nodiscard]] ErrorCode code() const noexcept { + return error_ == nullptr + ? ErrorCode::unexpected + : static_cast(error_->code); + } + + [[nodiscard]] std::string_view message() const noexcept { + if (error_ == nullptr || error_->message.data == nullptr) { + return {}; + } + return {reinterpret_cast(error_->message.data), + error_->message.len}; + } + + [[nodiscard]] ::paimon_error* native_handle() const noexcept { + return error_; + } + + [[nodiscard]] ::paimon_error* release() noexcept { + auto* result = error_; + error_ = nullptr; + return result; + } + + void reset(::paimon_error* error = nullptr) noexcept { + if (error_ == error) { + return; + } + if (error_ != nullptr) { + ::paimon_error_free(error_); + } + error_ = error; + } + + private: + ::paimon_error* error_ = nullptr; +}; + +// Owns a byte buffer allocated by libpaimon_c. This is used by the version and +// durable prepared-commit APIs and never allocates through a C++ runtime. +class Bytes final { + public: + constexpr Bytes() noexcept : bytes_{nullptr, 0} {} + explicit constexpr Bytes(adopt_handle_t, ::paimon_bytes bytes) noexcept + : bytes_(bytes) {} + + Bytes(const Bytes&) = delete; + Bytes& operator=(const Bytes&) = delete; + + Bytes(Bytes&& other) noexcept : bytes_(other.release()) {} + + Bytes& operator=(Bytes&& other) noexcept { + if (this != &other) { + reset(); + bytes_ = other.release(); + } + return *this; + } + + ~Bytes() noexcept { reset(); } + + [[nodiscard]] const std::uint8_t* data() const noexcept { + return bytes_.data; + } + + [[nodiscard]] std::size_t size() const noexcept { return bytes_.len; } + [[nodiscard]] bool empty() const noexcept { return bytes_.len == 0; } + + [[nodiscard]] std::string_view string_view() const noexcept { + if (bytes_.data == nullptr) { + return {}; + } + return {reinterpret_cast(bytes_.data), bytes_.len}; + } + + [[nodiscard]] ::paimon_bytes native_handle() const noexcept { return bytes_; } + + [[nodiscard]] ::paimon_bytes release() noexcept { + const auto result = bytes_; + bytes_ = {nullptr, 0}; + return result; + } + + void reset() noexcept { + if (bytes_.data != nullptr) { + ::paimon_bytes_free(bytes_); + bytes_ = {nullptr, 0}; + } + } + + private: + ::paimon_bytes bytes_; +}; + +// A small C++17 expected-like result. It deliberately does not throw and does +// not allocate. Accessing the wrong alternative is a programming error. +template +class [[nodiscard]] Result final { + public: + Result(const Result&) = delete; + Result& operator=(const Result&) = delete; + + Result(Result&& other) noexcept( + std::is_nothrow_move_constructible::value) + : has_value_(other.has_value_) { + if (has_value_) { + new (&storage_.value) T(std::move(other.storage_.value)); + } else { + new (&storage_.error) Error(std::move(other.storage_.error)); + } + } + + Result& operator=(Result&& other) noexcept( + std::is_nothrow_move_constructible::value) { + if (this != &other) { + destroy(); + has_value_ = other.has_value_; + if (has_value_) { + new (&storage_.value) T(std::move(other.storage_.value)); + } else { + new (&storage_.error) Error(std::move(other.storage_.error)); + } + } + return *this; + } + + ~Result() noexcept { destroy(); } + + static Result success(T value) noexcept( + std::is_nothrow_move_constructible::value) { + return Result(value_tag{}, std::move(value)); + } + + static Result failure(Error error) noexcept { + return Result(error_tag{}, std::move(error)); + } + + [[nodiscard]] bool ok() const noexcept { return has_value_; } + [[nodiscard]] explicit operator bool() const noexcept { return ok(); } + + [[nodiscard]] T& value() & noexcept { + assert(has_value_); + return storage_.value; + } + + [[nodiscard]] const T& value() const& noexcept { + assert(has_value_); + return storage_.value; + } + + [[nodiscard]] T&& value() && noexcept { + assert(has_value_); + return std::move(storage_.value); + } + + [[nodiscard]] Error& error() & noexcept { + assert(!has_value_); + return storage_.error; + } + + [[nodiscard]] const Error& error() const& noexcept { + assert(!has_value_); + return storage_.error; + } + + [[nodiscard]] Error&& error() && noexcept { + assert(!has_value_); + return std::move(storage_.error); + } + + private: + struct value_tag {}; + struct error_tag {}; + + union Storage { + T value; + Error error; + + Storage() noexcept {} + ~Storage() noexcept {} + } storage_; + + explicit Result(value_tag, T&& value) noexcept( + std::is_nothrow_move_constructible::value) + : has_value_(true) { + new (&storage_.value) T(std::move(value)); + } + + explicit Result(error_tag, Error&& error) noexcept : has_value_(false) { + new (&storage_.error) Error(std::move(error)); + } + + void destroy() noexcept { + if (has_value_) { + storage_.value.~T(); + } else { + storage_.error.~Error(); + } + } + + bool has_value_; +}; + +template <> +class [[nodiscard]] Result final { + public: + Result(const Result&) = delete; + Result& operator=(const Result&) = delete; + Result(Result&&) noexcept = default; + Result& operator=(Result&&) noexcept = default; + ~Result() noexcept = default; + + static Result success() noexcept { return Result(Error{}); } + static Result failure(Error error) noexcept { + return Result(std::move(error)); + } + + [[nodiscard]] bool ok() const noexcept { return error_.ok(); } + [[nodiscard]] explicit operator bool() const noexcept { return ok(); } + + [[nodiscard]] Error& error() & noexcept { + assert(!ok()); + return error_; + } + + [[nodiscard]] const Error& error() const& noexcept { + assert(!ok()); + return error_; + } + + [[nodiscard]] Error&& error() && noexcept { + assert(!ok()); + return std::move(error_); + } + + private: + explicit Result(Error error) noexcept : error_(std::move(error)) {} + Error error_; +}; + +using Status = Result; +using Option = ::paimon_option; + +[[nodiscard]] inline std::uint32_t abi_version() noexcept { + return ::paimon_abi_version(); +} + +[[nodiscard]] inline Bytes library_version() noexcept { + return Bytes(adopt_handle, ::paimon_library_version()); +} + +namespace detail { + +inline Status status_from(::paimon_error* error) noexcept { + if (error == nullptr) { + return Status::success(); + } + return Status::failure(Error(adopt_handle, error)); +} + +template +class UniqueHandle final { + public: + constexpr UniqueHandle() noexcept = default; + explicit UniqueHandle(adopt_handle_t, Raw* raw) noexcept : raw_(raw) {} + + UniqueHandle(const UniqueHandle&) = delete; + UniqueHandle& operator=(const UniqueHandle&) = delete; + + UniqueHandle(UniqueHandle&& other) noexcept : raw_(other.release()) {} + + UniqueHandle& operator=(UniqueHandle&& other) noexcept { + if (this != &other) { + reset(other.release()); + } + return *this; + } + + ~UniqueHandle() noexcept { reset(); } + + [[nodiscard]] Raw* get() const noexcept { return raw_; } + [[nodiscard]] explicit operator bool() const noexcept { + return raw_ != nullptr; + } + + [[nodiscard]] Raw* release() noexcept { + Raw* result = raw_; + raw_ = nullptr; + return result; + } + + void reset(Raw* raw = nullptr) noexcept { + if (raw_ != nullptr) { + Free(raw_); + } + raw_ = raw; + } + + private: + Raw* raw_ = nullptr; +}; + +} // namespace detail + +class Identifier; +class Table; +class ReadBuilder; +class Scan; +class Plan; +class TableRead; +class RecordBatchReader; +class WriteBuilder; +class TableWrite; +class PreparedMessages; +class TableCommit; +class PreparedCommit; +class StreamScan; +class StreamPlan; +class PollResult; + +enum class StreamStartupMode : std::int32_t { + latest_full = PAIMON_STREAM_STARTUP_LATEST_FULL, + latest = PAIMON_STREAM_STARTUP_LATEST, + from_snapshot = PAIMON_STREAM_STARTUP_FROM_SNAPSHOT, + from_snapshot_full = PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL, +}; + +enum class StreamFollowUpMode : std::int32_t { + automatic = PAIMON_STREAM_FOLLOW_UP_AUTO, + delta = PAIMON_STREAM_FOLLOW_UP_DELTA, + changelog = PAIMON_STREAM_FOLLOW_UP_CHANGELOG, +}; + +enum class StreamPollStatus : std::int32_t { + data = PAIMON_STREAM_POLL_DATA, + waiting = PAIMON_STREAM_POLL_WAITING, + end = PAIMON_STREAM_POLL_END, +}; + +enum class StreamReadMode : std::int32_t { + data = PAIMON_STREAM_READ_DATA, + audit_log = PAIMON_STREAM_READ_AUDIT_LOG, +}; + +class StreamScanOptions final { + public: + StreamScanOptions(const StreamScanOptions&) noexcept = default; + StreamScanOptions& operator=(const StreamScanOptions&) noexcept = default; + StreamScanOptions(StreamScanOptions&&) noexcept = default; + StreamScanOptions& operator=(StreamScanOptions&&) noexcept = default; + ~StreamScanOptions() noexcept = default; + + [[nodiscard]] static Result defaults() noexcept; + + StreamScanOptions& with_startup(StreamStartupMode mode, + std::int64_t snapshot_id = -1) noexcept { + options_.startup_mode = static_cast(mode); + options_.snapshot_id = snapshot_id; + return *this; + } + + StreamScanOptions& with_follow_up(StreamFollowUpMode mode) noexcept { + options_.follow_up_mode = static_cast(mode); + return *this; + } + + [[nodiscard]] const ::paimon_stream_scan_options* native_handle() + const noexcept { + return &options_; + } + + [[nodiscard]] ::paimon_stream_scan_options* native_handle() noexcept { + return &options_; + } + + private: + explicit StreamScanOptions(::paimon_stream_scan_options options) noexcept + : options_(options) {} + + ::paimon_stream_scan_options options_{}; +}; + +class Catalog final { + public: + Catalog() noexcept = default; + explicit Catalog(adopt_handle_t tag, ::paimon_catalog* raw) noexcept + : handle_(tag, raw) {} + + Catalog(const Catalog&) = delete; + Catalog& operator=(const Catalog&) = delete; + Catalog(Catalog&&) noexcept = default; + Catalog& operator=(Catalog&&) noexcept = default; + ~Catalog() noexcept = default; + + static Result create(const Option* options = nullptr, + std::size_t options_len = 0) noexcept; + + template + static Result create(const Option (&options)[N]) noexcept { + return create(options, N); + } + + [[nodiscard]] Result get_table( + const Identifier& identifier) const noexcept; + + [[nodiscard]] ::paimon_catalog* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_catalog, ::paimon_catalog_free> handle_; +}; + +class Identifier final { + public: + Identifier() noexcept = default; + explicit Identifier(adopt_handle_t tag, ::paimon_identifier* raw) noexcept + : handle_(tag, raw) {} + + Identifier(const Identifier&) = delete; + Identifier& operator=(const Identifier&) = delete; + Identifier(Identifier&&) noexcept = default; + Identifier& operator=(Identifier&&) noexcept = default; + ~Identifier() noexcept = default; + + static Result create(const char* database, + const char* object) noexcept; + + [[nodiscard]] ::paimon_identifier* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_identifier, ::paimon_identifier_free> handle_; +}; + +class Table final { + public: + Table() noexcept = default; + explicit Table(adopt_handle_t tag, ::paimon_table* raw) noexcept + : handle_(tag, raw) {} + + Table(const Table&) = delete; + Table& operator=(const Table&) = delete; + Table(Table&&) noexcept = default; + Table& operator=(Table&&) noexcept = default; + ~Table() noexcept = default; + + static Result
from_schema_json( + const char* table_path, const char* table_schema_json, + const char* database, const char* table_name, const char* branch = nullptr, + const Option* storage_options = nullptr, + std::size_t storage_options_len = 0) noexcept; + + [[nodiscard]] Result new_read_builder() const noexcept; + [[nodiscard]] Result new_read_builder( + const Option* options, std::size_t options_len) const noexcept; + + template + [[nodiscard]] Result new_read_builder( + const Option (&options)[N]) const noexcept; + + [[nodiscard]] Result new_write_builder() const noexcept; + [[nodiscard]] Result new_write_builder( + const char* stable_commit_user) const noexcept; + + [[nodiscard]] ::paimon_table* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table, ::paimon_table_free> handle_; +}; + +class ReadBuilder final { + public: + ReadBuilder() noexcept = default; + explicit ReadBuilder(adopt_handle_t tag, ::paimon_read_builder* raw) noexcept + : handle_(tag, raw) {} + + ReadBuilder(const ReadBuilder&) = delete; + ReadBuilder& operator=(const ReadBuilder&) = delete; + ReadBuilder(ReadBuilder&&) noexcept = default; + ReadBuilder& operator=(ReadBuilder&&) noexcept = default; + ~ReadBuilder() noexcept = default; + + // columns must be a null-terminated array. Passing nullptr clears projection. + [[nodiscard]] Status with_projection( + const char* const* columns) noexcept { + return detail::status_from( + ::paimon_read_builder_with_projection(handle_.get(), columns)); + } + + [[nodiscard]] Status with_case_sensitive(bool case_sensitive) noexcept { + return detail::status_from(::paimon_read_builder_with_case_sensitive( + handle_.get(), case_sensitive)); + } + + [[nodiscard]] Result new_scan() const noexcept; + [[nodiscard]] Result new_read() const noexcept; + [[nodiscard]] Result new_stream_scan( + const StreamScanOptions& options) const noexcept; + + [[nodiscard]] ::paimon_read_builder* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_read_builder, ::paimon_read_builder_free> + handle_; +}; + +class Scan final { + public: + Scan() noexcept = default; + explicit Scan(adopt_handle_t tag, ::paimon_table_scan* raw) noexcept + : handle_(tag, raw) {} + + Scan(const Scan&) = delete; + Scan& operator=(const Scan&) = delete; + Scan(Scan&&) noexcept = default; + Scan& operator=(Scan&&) noexcept = default; + ~Scan() noexcept = default; + + [[nodiscard]] Result plan() const noexcept; + + [[nodiscard]] ::paimon_table_scan* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table_scan, ::paimon_table_scan_free> handle_; +}; + +class Plan final { + public: + Plan() noexcept = default; + explicit Plan(adopt_handle_t tag, ::paimon_plan* raw) noexcept + : handle_(tag, raw) {} + + Plan(const Plan&) = delete; + Plan& operator=(const Plan&) = delete; + Plan(Plan&&) noexcept = default; + Plan& operator=(Plan&&) noexcept = default; + ~Plan() noexcept = default; + + static Result from_split_bytes(const std::uint8_t* data, + std::size_t size) noexcept; + + [[nodiscard]] std::size_t num_splits() const noexcept { + return ::paimon_plan_num_splits(handle_.get()); + } + + [[nodiscard]] ::paimon_plan* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_plan, ::paimon_plan_free> handle_; +}; + +// Owns the two heap-allocated Arrow C Data container structs returned by +// libpaimon_c. This type intentionally has no dependency on Arrow C++. +class ArrowBatch final { + public: + constexpr ArrowBatch() noexcept : batch_{nullptr, nullptr} {} + explicit constexpr ArrowBatch(adopt_handle_t, + ::paimon_arrow_batch batch) noexcept + : batch_(batch) {} + + ArrowBatch(const ArrowBatch&) = delete; + ArrowBatch& operator=(const ArrowBatch&) = delete; + + ArrowBatch(ArrowBatch&& other) noexcept : batch_(other.release()) {} + + ArrowBatch& operator=(ArrowBatch&& other) noexcept { + if (this != &other) { + reset(); + batch_ = other.release(); + } + return *this; + } + + ~ArrowBatch() noexcept { reset(); } + + [[nodiscard]] bool empty() const noexcept { + return batch_.array == nullptr && batch_.schema == nullptr; + } + + [[nodiscard]] explicit operator bool() const noexcept { return !empty(); } + [[nodiscard]] void* array() const noexcept { return batch_.array; } + [[nodiscard]] void* schema() const noexcept { return batch_.schema; } + + [[nodiscard]] ::paimon_arrow_batch native_handle() const noexcept { + return batch_; + } + + // The caller becomes responsible for paimon_arrow_batch_free(raw). + [[nodiscard]] ::paimon_arrow_batch release() noexcept { + const auto result = batch_; + batch_ = {nullptr, nullptr}; + return result; + } + + void reset() noexcept { + if (!empty()) { + ::paimon_arrow_batch_free(batch_); + batch_ = {nullptr, nullptr}; + } + } + + private: + ::paimon_arrow_batch batch_; +}; + +class RecordBatchReader final { + public: + RecordBatchReader() noexcept = default; + explicit RecordBatchReader(adopt_handle_t tag, + ::paimon_record_batch_reader* raw) noexcept + : handle_(tag, raw) {} + + RecordBatchReader(const RecordBatchReader&) = delete; + RecordBatchReader& operator=(const RecordBatchReader&) = delete; + RecordBatchReader(RecordBatchReader&&) noexcept = default; + RecordBatchReader& operator=(RecordBatchReader&&) noexcept = default; + ~RecordBatchReader() noexcept = default; + + // A successful empty ArrowBatch is end-of-stream. + [[nodiscard]] Result next() noexcept; + + [[nodiscard]] ::paimon_record_batch_reader* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_record_batch_reader, + ::paimon_record_batch_reader_free> + handle_; +}; + +class TableRead final { + public: + TableRead() noexcept = default; + explicit TableRead(adopt_handle_t tag, ::paimon_table_read* raw) noexcept + : handle_(tag, raw) {} + + TableRead(const TableRead&) = delete; + TableRead& operator=(const TableRead&) = delete; + TableRead(TableRead&&) noexcept = default; + TableRead& operator=(TableRead&&) noexcept = default; + ~TableRead() noexcept = default; + + [[nodiscard]] Result to_arrow( + const Plan& plan, std::size_t offset = 0, + std::size_t length = static_cast(-1)) const noexcept; + + [[nodiscard]] ::paimon_table_read* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table_read, ::paimon_table_read_free> handle_; +}; + +class StreamPlan final { + public: + StreamPlan() noexcept = default; + explicit StreamPlan(adopt_handle_t tag, ::paimon_stream_plan* raw) noexcept + : handle_(tag, raw) {} + + StreamPlan(const StreamPlan&) = delete; + StreamPlan& operator=(const StreamPlan&) = delete; + StreamPlan(StreamPlan&&) noexcept = default; + StreamPlan& operator=(StreamPlan&&) noexcept = default; + ~StreamPlan() noexcept = default; + + [[nodiscard]] bool is_full() const noexcept { + return ::paimon_stream_plan_is_full(handle_.get()) != 0; + } + + [[nodiscard]] std::size_t num_splits() const noexcept { + return ::paimon_stream_plan_num_splits(handle_.get()); + } + + [[nodiscard]] Result serialize() const noexcept; + + [[nodiscard]] static Result deserialize( + const std::uint8_t* data, std::size_t size) noexcept; + + [[nodiscard]] Result read_to_arrow( + const TableRead& read, StreamReadMode mode = StreamReadMode::data, + std::size_t offset = 0, + std::size_t length = static_cast(-1)) const noexcept; + + [[nodiscard]] ::paimon_stream_plan* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_stream_plan, ::paimon_stream_plan_free> handle_; +}; + +class PollResult final { + public: + PollResult(StreamPollStatus status, StreamPlan plan, + std::int64_t snapshot_id, std::int64_t next_snapshot_id, + std::int64_t watermark, bool has_watermark) noexcept + : status_(status), + plan_(std::move(plan)), + snapshot_id_(snapshot_id), + next_snapshot_id_(next_snapshot_id), + watermark_(watermark), + has_watermark_(has_watermark) {} + + PollResult(const PollResult&) = delete; + PollResult& operator=(const PollResult&) = delete; + PollResult(PollResult&&) noexcept = default; + PollResult& operator=(PollResult&&) noexcept = default; + ~PollResult() noexcept = default; + + [[nodiscard]] StreamPollStatus status() const noexcept { return status_; } + [[nodiscard]] bool has_data() const noexcept { + return status_ == StreamPollStatus::data; + } + [[nodiscard]] bool waiting() const noexcept { + return status_ == StreamPollStatus::waiting; + } + [[nodiscard]] bool end() const noexcept { + return status_ == StreamPollStatus::end; + } + + [[nodiscard]] StreamPlan& plan() & noexcept { + assert(has_data()); + return plan_; + } + + [[nodiscard]] const StreamPlan& plan() const& noexcept { + assert(has_data()); + return plan_; + } + + [[nodiscard]] StreamPlan&& plan() && noexcept { + assert(has_data()); + return std::move(plan_); + } + + [[nodiscard]] std::int64_t snapshot_id() const noexcept { + return snapshot_id_; + } + + [[nodiscard]] std::int64_t next_snapshot_id() const noexcept { + return next_snapshot_id_; + } + + [[nodiscard]] bool has_watermark() const noexcept { return has_watermark_; } + + [[nodiscard]] std::int64_t watermark() const noexcept { + assert(has_watermark_); + return watermark_; + } + + private: + StreamPollStatus status_; + StreamPlan plan_; + std::int64_t snapshot_id_; + std::int64_t next_snapshot_id_; + std::int64_t watermark_; + bool has_watermark_; +}; + +class StreamScan final { + public: + StreamScan() noexcept = default; + explicit StreamScan(adopt_handle_t tag, ::paimon_stream_scan* raw) noexcept + : handle_(tag, raw) {} + + StreamScan(const StreamScan&) = delete; + StreamScan& operator=(const StreamScan&) = delete; + StreamScan(StreamScan&&) noexcept = default; + StreamScan& operator=(StreamScan&&) noexcept = default; + ~StreamScan() noexcept = default; + + // poll() never waits for a future snapshot. Waiting is a normal result, not + // an error, so the caller controls scheduling, cancellation and backpressure. + // One StreamScan is single-thread-confined; poll/checkpoint/restore/free must + // be externally serialized. + [[nodiscard]] Result poll() noexcept; + + // This cursor is safe to persist only after every split in the returned plan + // has been durably accounted for by the caller's checkpoint barrier. + [[nodiscard]] std::int64_t checkpoint() const noexcept { + return ::paimon_stream_scan_checkpoint(handle_.get()); + } + + [[nodiscard]] Status restore(std::int64_t next_snapshot_id) noexcept { + return detail::status_from( + ::paimon_stream_scan_restore(handle_.get(), next_snapshot_id)); + } + + [[nodiscard]] ::paimon_stream_scan* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_stream_scan, ::paimon_stream_scan_free> handle_; +}; + +class PreparedMessages final { + public: + PreparedMessages() noexcept = default; + explicit PreparedMessages(adopt_handle_t tag, + ::paimon_commit_messages* raw) noexcept + : handle_(tag, raw) {} + + PreparedMessages(const PreparedMessages&) = delete; + PreparedMessages& operator=(const PreparedMessages&) = delete; + PreparedMessages(PreparedMessages&&) noexcept = default; + PreparedMessages& operator=(PreparedMessages&&) noexcept = default; + + // Destruction only frees the messages. It never commits or aborts files. + ~PreparedMessages() noexcept = default; + + [[nodiscard]] Status merge(const PreparedMessages& source) noexcept { + return detail::status_from(::paimon_commit_messages_merge( + handle_.get(), source.handle_.get())); + } + + [[nodiscard]] Result prepare( + std::int64_t checkpoint_id) const noexcept; + + [[nodiscard]] ::paimon_commit_messages* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_commit_messages, + ::paimon_commit_messages_free> + handle_; +}; + +class PreparedCommit final { + public: + PreparedCommit() noexcept = default; + explicit PreparedCommit(adopt_handle_t tag, + ::paimon_prepared_commit* raw) noexcept + : handle_(tag, raw) {} + + PreparedCommit(const PreparedCommit&) = delete; + PreparedCommit& operator=(const PreparedCommit&) = delete; + PreparedCommit(PreparedCommit&&) noexcept = default; + PreparedCommit& operator=(PreparedCommit&&) noexcept = default; + + // Destruction only releases the durable in-memory envelope. It never commits + // or aborts the referenced data files. + ~PreparedCommit() noexcept = default; + + [[nodiscard]] static Result deserialize( + const std::uint8_t* data, std::size_t size) noexcept; + + [[nodiscard]] std::int64_t identifier() const noexcept { + return ::paimon_prepared_commit_identifier(handle_.get()); + } + + [[nodiscard]] Result serialize() const noexcept; + + [[nodiscard]] Status merge(const PreparedCommit& source) noexcept { + return detail::status_from(::paimon_prepared_commit_merge( + handle_.get(), source.handle_.get())); + } + + [[nodiscard]] ::paimon_prepared_commit* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_prepared_commit, + ::paimon_prepared_commit_free> + handle_; +}; + +class TableWrite final { + public: + TableWrite() noexcept = default; + explicit TableWrite(adopt_handle_t tag, ::paimon_table_write* raw) noexcept + : handle_(tag, raw) {} + + TableWrite(const TableWrite&) = delete; + TableWrite& operator=(const TableWrite&) = delete; + TableWrite(TableWrite&&) noexcept = default; + TableWrite& operator=(TableWrite&&) noexcept = default; + ~TableWrite() noexcept = default; + + // The Arrow C Data contents are consumed in place once import begins. The + // caller continues to own the ArrowArray/ArrowSchema container memory. + [[nodiscard]] Status write_arrow(void* array, void* schema) noexcept { + return detail::status_from(::paimon_table_write_write_arrow_batch( + handle_.get(), array, schema)); + } + + // Convenient bridge for a Rust-allocated batch. Its heap container structs + // remain owned by batch and are released before this call returns. + [[nodiscard]] Status write_arrow(ArrowBatch&& batch) noexcept { + auto status = write_arrow(batch.array(), batch.schema()); + batch.reset(); + return status; + } + + [[nodiscard]] Result prepare_commit() noexcept; + + [[nodiscard]] ::paimon_table_write* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table_write, ::paimon_table_write_free> handle_; +}; + +class TableCommit final { + public: + TableCommit() noexcept = default; + explicit TableCommit(adopt_handle_t tag, ::paimon_table_commit* raw) noexcept + : handle_(tag, raw) {} + + TableCommit(const TableCommit&) = delete; + TableCommit& operator=(const TableCommit&) = delete; + TableCommit(TableCommit&&) noexcept = default; + TableCommit& operator=(TableCommit&&) noexcept = default; + ~TableCommit() noexcept = default; + + // Commit calls never consume messages. Keep them until the outcome is known; + // retry an uncertain outcome with filter_and_commit(checkpoint_id). + [[nodiscard]] Status commit(PreparedMessages& messages) const noexcept { + return detail::status_from(::paimon_table_commit_commit( + handle_.get(), messages.native_handle())); + } + + [[nodiscard]] Status commit(PreparedMessages& messages, + std::int64_t checkpoint_id) const noexcept { + return detail::status_from(::paimon_table_commit_commit_with_identifier( + handle_.get(), messages.native_handle(), checkpoint_id)); + } + + [[nodiscard]] Status filter_and_commit( + PreparedMessages& messages, std::int64_t checkpoint_id) const noexcept { + return detail::status_from( + ::paimon_table_commit_filter_and_commit_with_identifier( + handle_.get(), messages.native_handle(), checkpoint_id)); + } + + // Retry-safe commit for a serialized/restored checkpoint. The PreparedCommit + // remains owned by the caller and can be retried after an uncertain result. + [[nodiscard]] Status commit_prepared( + const PreparedCommit& prepared) const noexcept { + return detail::status_from(::paimon_table_commit_commit_prepared( + handle_.get(), prepared.native_handle())); + } + + [[nodiscard]] Status overwrite(PreparedMessages& messages) const noexcept { + return detail::status_from(::paimon_table_commit_overwrite( + handle_.get(), messages.native_handle())); + } + + [[nodiscard]] Status overwrite(PreparedMessages& messages, + std::int64_t checkpoint_id) const noexcept { + return detail::status_from( + ::paimon_table_commit_overwrite_with_identifier( + handle_.get(), messages.native_handle(), checkpoint_id)); + } + + [[nodiscard]] Status truncate_table() const noexcept { + return detail::status_from( + ::paimon_table_commit_truncate_table(handle_.get())); + } + + [[nodiscard]] Status truncate_table( + std::int64_t checkpoint_id) const noexcept { + return detail::status_from( + ::paimon_table_commit_truncate_table_with_identifier( + handle_.get(), checkpoint_id)); + } + + // Abort is always explicit. PreparedMessages destruction does not call it. + [[nodiscard]] Status abort(PreparedMessages& messages) const noexcept { + return detail::status_from(::paimon_table_commit_abort( + handle_.get(), messages.native_handle())); + } + + // Fence all commit/abort calls for the same table and commit_user across + // processes. Truncated snapshot history is reported as an error and no file + // is removed. + [[nodiscard]] Status abort_prepared( + const PreparedCommit& prepared) const noexcept { + return detail::status_from(::paimon_table_commit_abort_prepared( + handle_.get(), prepared.native_handle())); + } + + [[nodiscard]] ::paimon_table_commit* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table_commit, ::paimon_table_commit_free> + handle_; +}; + +class WriteBuilder final { + public: + WriteBuilder() noexcept = default; + explicit WriteBuilder(adopt_handle_t tag, ::paimon_write_builder* raw) noexcept + : handle_(tag, raw) {} + + WriteBuilder(const WriteBuilder&) = delete; + WriteBuilder& operator=(const WriteBuilder&) = delete; + WriteBuilder(WriteBuilder&&) noexcept = default; + WriteBuilder& operator=(WriteBuilder&&) noexcept = default; + ~WriteBuilder() noexcept = default; + + [[nodiscard]] Status with_overwrite() noexcept { + return detail::status_from( + ::paimon_write_builder_with_overwrite(handle_.get())); + } + + [[nodiscard]] Result new_write() const noexcept; + [[nodiscard]] Result new_commit() const noexcept; + + [[nodiscard]] ::paimon_write_builder* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_write_builder, ::paimon_write_builder_free> + handle_; +}; + +inline Result Catalog::create(const Option* options, + std::size_t options_len) noexcept { + const auto result = ::paimon_catalog_create(options, options_len); + if (result.error != nullptr) { + if (result.catalog != nullptr) { + ::paimon_catalog_free(result.catalog); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + Catalog(adopt_handle, result.catalog)); +} + +inline Result StreamScanOptions::defaults() noexcept { + ::paimon_stream_scan_options options{}; + auto* error = ::paimon_stream_scan_options_init(&options); + if (error != nullptr) { + return Result::failure(Error(adopt_handle, error)); + } + return Result::success(StreamScanOptions(options)); +} + +inline Result Identifier::create(const char* database, + const char* object) noexcept { + const auto result = ::paimon_identifier_new(database, object); + if (result.error != nullptr) { + if (result.identifier != nullptr) { + ::paimon_identifier_free(result.identifier); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + Identifier(adopt_handle, result.identifier)); +} + +inline Result
Catalog::get_table( + const Identifier& identifier) const noexcept { + const auto result = + ::paimon_catalog_get_table(handle_.get(), identifier.native_handle()); + if (result.error != nullptr) { + if (result.table != nullptr) { + ::paimon_table_free(result.table); + } + return Result
::failure(Error(adopt_handle, result.error)); + } + return Result
::success(Table(adopt_handle, result.table)); +} + +inline Result
Table::from_schema_json( + const char* table_path, const char* table_schema_json, const char* database, + const char* table_name, const char* branch, const Option* storage_options, + std::size_t storage_options_len) noexcept { + const auto result = ::paimon_table_from_schema_json( + table_path, table_schema_json, database, table_name, branch, + storage_options, storage_options_len); + if (result.error != nullptr) { + if (result.table != nullptr) { + ::paimon_table_free(result.table); + } + return Result
::failure(Error(adopt_handle, result.error)); + } + return Result
::success(Table(adopt_handle, result.table)); +} + +inline Result Table::new_read_builder() const noexcept { + const auto result = ::paimon_table_new_read_builder(handle_.get()); + if (result.error != nullptr) { + if (result.read_builder != nullptr) { + ::paimon_read_builder_free(result.read_builder); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + ReadBuilder(adopt_handle, result.read_builder)); +} + +inline Result Table::new_read_builder( + const Option* options, std::size_t options_len) const noexcept { + const auto result = ::paimon_table_new_read_builder_with_options( + handle_.get(), options, options_len); + if (result.error != nullptr) { + if (result.read_builder != nullptr) { + ::paimon_read_builder_free(result.read_builder); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + ReadBuilder(adopt_handle, result.read_builder)); +} + +template +inline Result Table::new_read_builder( + const Option (&options)[N]) const noexcept { + return new_read_builder(options, N); +} + +inline Result ReadBuilder::new_scan() const noexcept { + const auto result = ::paimon_read_builder_new_scan(handle_.get()); + if (result.error != nullptr) { + if (result.scan != nullptr) { + ::paimon_table_scan_free(result.scan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Scan(adopt_handle, result.scan)); +} + +inline Result ReadBuilder::new_read() const noexcept { + const auto result = ::paimon_read_builder_new_read(handle_.get()); + if (result.error != nullptr) { + if (result.read != nullptr) { + ::paimon_table_read_free(result.read); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(TableRead(adopt_handle, result.read)); +} + +inline Result ReadBuilder::new_stream_scan( + const StreamScanOptions& options) const noexcept { + const auto result = ::paimon_read_builder_new_stream_scan( + handle_.get(), options.native_handle()); + if (result.error != nullptr) { + if (result.scan != nullptr) { + ::paimon_stream_scan_free(result.scan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(StreamScan(adopt_handle, result.scan)); +} + +inline Result Scan::plan() const noexcept { + const auto result = ::paimon_table_scan_plan(handle_.get()); + if (result.error != nullptr) { + if (result.plan != nullptr) { + ::paimon_plan_free(result.plan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Plan(adopt_handle, result.plan)); +} + +inline Result Plan::from_split_bytes(const std::uint8_t* data, + std::size_t size) noexcept { + const auto result = ::paimon_plan_from_split_bytes(data, size); + if (result.error != nullptr) { + if (result.plan != nullptr) { + ::paimon_plan_free(result.plan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Plan(adopt_handle, result.plan)); +} + +inline Result TableRead::to_arrow( + const Plan& plan, std::size_t offset, std::size_t length) const noexcept { + const auto result = ::paimon_table_read_to_arrow( + handle_.get(), plan.native_handle(), offset, length); + if (result.error != nullptr) { + if (result.reader != nullptr) { + ::paimon_record_batch_reader_free(result.reader); + } + return Result::failure( + Error(adopt_handle, result.error)); + } + return Result::success( + RecordBatchReader(adopt_handle, result.reader)); +} + +inline Result RecordBatchReader::next() noexcept { + auto result = ::paimon_record_batch_reader_next(handle_.get()); + if (result.error != nullptr) { + if (result.batch.array != nullptr || result.batch.schema != nullptr) { + ::paimon_arrow_batch_free(result.batch); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + ArrowBatch(adopt_handle, result.batch)); +} + +inline Result StreamPlan::read_to_arrow( + const TableRead& read, StreamReadMode mode, std::size_t offset, + std::size_t length) const noexcept { + const auto result = ::paimon_stream_plan_read_to_arrow( + read.native_handle(), handle_.get(), offset, length, + static_cast(mode)); + if (result.error != nullptr) { + if (result.reader != nullptr) { + ::paimon_record_batch_reader_free(result.reader); + } + return Result::failure( + Error(adopt_handle, result.error)); + } + return Result::success( + RecordBatchReader(adopt_handle, result.reader)); +} + +inline Result StreamPlan::serialize() const noexcept { + auto result = ::paimon_stream_plan_serialize(handle_.get()); + if (result.error != nullptr) { + if (result.bytes.data != nullptr) { + ::paimon_bytes_free(result.bytes); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Bytes(adopt_handle, result.bytes)); +} + +inline Result StreamPlan::deserialize( + const std::uint8_t* data, std::size_t size) noexcept { + auto result = ::paimon_stream_plan_deserialize(data, size); + if (result.error != nullptr) { + if (result.plan != nullptr) { + ::paimon_stream_plan_free(result.plan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(PollResult( + static_cast(result.status), + StreamPlan(adopt_handle, result.plan), result.snapshot_id, + result.next_snapshot_id, result.watermark, result.has_watermark != 0)); +} + +inline Result StreamScan::poll() noexcept { + auto result = ::paimon_stream_scan_poll(handle_.get()); + if (result.error != nullptr) { + if (result.plan != nullptr) { + ::paimon_stream_plan_free(result.plan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(PollResult( + static_cast(result.status), + StreamPlan(adopt_handle, result.plan), result.snapshot_id, + result.next_snapshot_id, result.watermark, result.has_watermark != 0)); +} + +inline Result Table::new_write_builder() const noexcept { + const auto result = ::paimon_table_new_write_builder(handle_.get()); + if (result.error != nullptr) { + if (result.write_builder != nullptr) { + ::paimon_write_builder_free(result.write_builder); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + WriteBuilder(adopt_handle, result.write_builder)); +} + +inline Result Table::new_write_builder( + const char* stable_commit_user) const noexcept { + const auto result = ::paimon_table_new_write_builder_with_commit_user( + handle_.get(), stable_commit_user); + if (result.error != nullptr) { + if (result.write_builder != nullptr) { + ::paimon_write_builder_free(result.write_builder); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + WriteBuilder(adopt_handle, result.write_builder)); +} + +inline Result WriteBuilder::new_write() const noexcept { + const auto result = ::paimon_write_builder_new_write(handle_.get()); + if (result.error != nullptr) { + if (result.write != nullptr) { + ::paimon_table_write_free(result.write); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(TableWrite(adopt_handle, result.write)); +} + +inline Result WriteBuilder::new_commit() const noexcept { + const auto result = ::paimon_write_builder_new_commit(handle_.get()); + if (result.error != nullptr) { + if (result.commit != nullptr) { + ::paimon_table_commit_free(result.commit); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + TableCommit(adopt_handle, result.commit)); +} + +inline Result TableWrite::prepare_commit() noexcept { + const auto result = ::paimon_table_write_prepare_commit(handle_.get()); + if (result.error != nullptr) { + if (result.messages != nullptr) { + ::paimon_commit_messages_free(result.messages); + } + return Result::failure( + Error(adopt_handle, result.error)); + } + return Result::success( + PreparedMessages(adopt_handle, result.messages)); +} + +inline Result PreparedMessages::prepare( + std::int64_t checkpoint_id) const noexcept { + const auto result = + ::paimon_commit_messages_prepare(handle_.get(), checkpoint_id); + if (result.error != nullptr) { + if (result.prepared != nullptr) { + ::paimon_prepared_commit_free(result.prepared); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + PreparedCommit(adopt_handle, result.prepared)); +} + +inline Result PreparedCommit::serialize() const noexcept { + auto result = ::paimon_prepared_commit_serialize(handle_.get()); + if (result.error != nullptr) { + if (result.bytes.data != nullptr) { + ::paimon_bytes_free(result.bytes); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Bytes(adopt_handle, result.bytes)); +} + +inline Result PreparedCommit::deserialize( + const std::uint8_t* data, std::size_t size) noexcept { + const auto result = ::paimon_prepared_commit_deserialize(data, size); + if (result.error != nullptr) { + if (result.prepared != nullptr) { + ::paimon_prepared_commit_free(result.prepared); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + PreparedCommit(adopt_handle, result.prepared)); +} + +static_assert(!std::is_copy_constructible::value, + "native handles must stay move-only"); +static_assert(std::is_nothrow_destructible::value, + "native handle destructors must be noexcept"); +static_assert(!std::is_copy_constructible::value, + "prepared messages must stay move-only"); +static_assert(std::is_nothrow_destructible::value, + "prepared-message destruction must be noexcept"); +static_assert(!std::is_copy_constructible::value, + "stream scans must stay move-only"); +static_assert(std::is_nothrow_destructible::value, + "stream plan destruction must be noexcept"); +static_assert(!std::is_copy_constructible::value, + "durable prepared commits must stay move-only"); + +} // namespace paimon + +#endif // PAIMON_CPP_PAIMON_HPP diff --git a/bindings/cpp/scripts/verify_linux_elf.sh b/bindings/cpp/scripts/verify_linux_elf.sh new file mode 100755 index 000000000..89a144271 --- /dev/null +++ b/bindings/cpp/scripts/verify_linux_elf.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env sh +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +if [ "$#" -ne 1 ]; then + echo "usage: $0 /path/to/libpaimon_c.so" >&2 + exit 2 +fi + +library=$1 +if [ ! -f "$library" ]; then + echo "not a file: $library" >&2 + exit 2 +fi + +if command -v readelf >/dev/null 2>&1; then + readelf_cmd=readelf +elif command -v llvm-readelf >/dev/null 2>&1; then + readelf_cmd=llvm-readelf +else + echo "readelf or llvm-readelf is required" >&2 + exit 2 +fi + +elf_header=$($readelf_cmd -h "$library") +if ! printf '%s\n' "$elf_header" | grep -q 'ELF'; then + echo "not an ELF shared object: $library" >&2 + exit 1 +fi +if ! printf '%s\n' "$elf_header" | grep -Eq 'Type:[[:space:]]+DYN'; then + echo "ELF artifact is not a shared object: $library" >&2 + exit 1 +fi + +program_headers=$($readelf_cmd -W -l "$library") +if printf '%s\n' "$program_headers" | grep -q 'INTERP'; then + echo "ELF artifact is a PIE executable, not a shared object" >&2 + printf '%s\n' "$program_headers" | grep 'INTERP' >&2 + exit 1 +fi +if printf '%s\n' "$program_headers" | grep -Eq \ + 'GNU_STACK.*W.*E|GNU_STACK.*E.*W'; then + echo "forbidden executable GNU_STACK segment" >&2 + printf '%s\n' "$program_headers" | grep 'GNU_STACK' >&2 + exit 1 +fi +if printf '%s\n' "$program_headers" | grep -Eq \ + 'LOAD.*W.*E|LOAD.*E.*W'; then + echo "forbidden writable and executable LOAD segment" >&2 + printf '%s\n' "$program_headers" | grep 'LOAD' >&2 + exit 1 +fi + +dynamic_section=$($readelf_cmd -d "$library") +if printf '%s\n' "$dynamic_section" | grep -q 'TEXTREL'; then + echo "forbidden text relocation" >&2 + printf '%s\n' "$dynamic_section" | grep 'TEXTREL' >&2 + exit 1 +fi + +if command -v ldd >/dev/null 2>&1; then + ldd --version 2>&1 | sed -n '1,2p' +fi + +needed=$(printf '%s\n' "$dynamic_section" | grep 'NEEDED' || true) +printf '%s\n' "$needed" + +runtime_paths=$(printf '%s\n' "$dynamic_section" | + sed -n 's/.*(RPATH).*Library rpath: \[\([^]]*\)\].*/\1/p; + s/.*(RUNPATH).*Library runpath: \[\([^]]*\)\].*/\1/p') +old_ifs=$IFS +IFS=: +for runtime_path in $runtime_paths; do + case "$runtime_path" in + '$ORIGIN'|'${ORIGIN}') + ;; + *) + echo "forbidden runtime search path: $runtime_path" >&2 + exit 1 + ;; + esac +done +IFS=$old_ifs + +needed_names=$(printf '%s\n' "$needed" | + sed -n 's/.*Shared library: \[\([^]]*\)\].*/\1/p') +for dependency in $needed_names; do + case "$dependency" in + libstdc++*|libc++*|libsupc++*|libgcc_s*|libunwind*|libatomic*) + echo "forbidden non-C runtime dependency in DT_NEEDED: $dependency" >&2 + exit 1 + ;; + libc.so.*|libm.so.*|libpthread.so.*|libdl.so.*|librt.so.*|libutil.so.*|libresolv.so.*|libanl.so.*|libBrokenLocale.so.*|libcrypt.so.*|libnss_*.so.*|ld-linux*.so.*|ld64.so.*|ld.so.*|libpaimon_c.so*) + ;; + *) + echo "dependency is outside the glibc/libpaimon_c allowlist: $dependency" >&2 + exit 1 + ;; + esac +done + +undefined=$($readelf_cmd --dyn-syms --wide "$library" | grep ' UND ' || true) +unexpected_unversioned=$(printf '%s\n' "$undefined" | awk ' + { + bind = $5 + name = $8 + base = name + sub(/@.*/, "", base) + if (base == "" || name ~ /@GLIBC_[0-9]/ || base ~ /^paimon_/ || + base ~ /^_Z/) { + next + } + if (bind == "WEAK" && + (base == "_ITM_deregisterTMCloneTable" || + base == "_ITM_registerTMCloneTable" || + base == "__gmon_start__" || + base == "_Jv_RegisterClasses" || + base == "ZSTD_trace_compress_begin" || + base == "ZSTD_trace_compress_end" || + base == "ZSTD_trace_decompress_begin" || + base == "ZSTD_trace_decompress_end" || + base == "OPENSSL_memory_alloc" || + base == "OPENSSL_memory_free" || + base == "OPENSSL_memory_get_size" || + base == "OPENSSL_memory_realloc" || + base == "sdallocx" || + base == "gettid" || + base == "statx" || + base == "getrandom" || + base == "copy_file_range" || + base == "__cxa_thread_atexit_impl")) { + next + } + print + }') +if [ -n "$unexpected_unversioned" ]; then + echo "forbidden unversioned undefined symbol; only paimon_* and narrow weak CRT hooks are allowed" >&2 + printf '%s\n' "$unexpected_unversioned" >&2 + exit 1 +fi + +defined=$($readelf_cmd --dyn-syms --wide "$library" | + awk '$7 != "UND" && $8 != "" { print }') +if printf '%s\n' "$defined" | grep -Eq \ + '[[:space:]]_Z[A-Za-z0-9_$.@]*'; then + echo "forbidden exported C++ mangled symbol" >&2 + printf '%s\n' "$defined" | + grep -E '[[:space:]]_Z[A-Za-z0-9_$.@]*' >&2 + exit 1 +fi + +version_info=$($readelf_cmd --version-info --wide "$library" || true) +symbol_versions=$(printf '%s\n%s\n' "$undefined" "$version_info") +if printf '%s\n' "$symbol_versions" | grep -Eq 'GLIBCXX_|CXXABI_|GCC_[0-9]'; then + echo "forbidden C++/compiler runtime symbol version" >&2 + printf '%s\n' "$symbol_versions" | + grep -E 'GLIBCXX_|CXXABI_|GCC_[0-9]' >&2 + exit 1 +fi + +if printf '%s\n' "$undefined" | grep -Eq \ + '[[:space:]]_Z[A-Za-z0-9_$.@]*'; then + echo "forbidden C++ mangled undefined symbol" >&2 + printf '%s\n' "$undefined" | + grep -E '[[:space:]]_Z[A-Za-z0-9_$.@]*' >&2 + exit 1 +fi + +cxa_symbols=$(printf '%s\n' "$undefined" | + grep '__cxa_' | + grep -Ev '__cxa_(atexit|finalize|thread_atexit_impl)(@|$)' || true) +if [ -n "$cxa_symbols" ] || + printf '%s\n' "$undefined" | grep -Eq '__gxx_personality_v0|__dynamic_cast'; then + echo "forbidden C++ ABI undefined symbol" >&2 + printf '%s\n' "$cxa_symbols" >&2 + printf '%s\n' "$undefined" | + grep -E '__gxx_personality_v0|__dynamic_cast' >&2 || true + exit 1 +fi + +if printf '%s\n' "$symbol_versions" | grep -Eq 'GLIBC_(PRIVATE|ABI_)'; then + echo "private or non-baseline glibc ABI requirement" >&2 + printf '%s\n' "$symbol_versions" | grep -E 'GLIBC_(PRIVATE|ABI_)' >&2 + exit 1 +fi + +max_glibc=$(printf '%s\n' "$symbol_versions" | + grep -Eo 'GLIBC_[0-9][0-9.]*' | + sed 's/^GLIBC_//' | + sort -V | + tail -n 1 || true) +if [ -n "$max_glibc" ]; then + newest=$(printf '%s\n' 2.17 "$max_glibc" | sort -V | tail -n 1) + if [ "$newest" != "2.17" ]; then + echo "GLIBC symbol version $max_glibc exceeds supported baseline 2.17" >&2 + printf '%s\n' "$symbol_versions" | grep "GLIBC_$max_glibc" >&2 + exit 1 + fi +fi + +if ! command -v c++filt >/dev/null 2>&1; then + echo "c++filt is required to inspect demangled undefined symbols" >&2 + exit 2 +fi + +demangled=$(printf '%s\n' "$undefined" | c++filt) +if printf '%s\n' "$demangled" | grep -Eq \ + 'std::|__gnu_cxx::|typeinfo for|vtable for|operator (new|delete)(\[\])?\(|__dynamic_cast'; then + echo "forbidden demangled C++ undefined symbol" >&2 + printf '%s\n' "$demangled" | grep -E \ + 'std::|__gnu_cxx::|typeinfo for|vtable for|operator (new|delete)(\[\])?\(|__dynamic_cast' >&2 + exit 1 +fi + +echo "ELF C++ runtime check passed: $library" diff --git a/bindings/cpp/tests/check_incremental_relink.cmake b/bindings/cpp/tests/check_incremental_relink.cmake new file mode 100644 index 000000000..b18e4277d --- /dev/null +++ b/bindings/cpp/tests/check_incremental_relink.cmake @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +foreach(required IN ITEMS BUILD_DIR SOURCE LIBRARY TARGET) + if(NOT DEFINED ${required}) + message(FATAL_ERROR "missing -D${required}=...") + endif() +endforeach() + +file(SHA256 "${LIBRARY}" before_hash) +file(READ "${SOURCE}" source_text) +if(source_text MATCHES "\\+ 1001") + string(REPLACE "+ 1001" "+ 1002" source_text "${source_text}") +elseif(source_text MATCHES "\\+ 1002") + string(REPLACE "+ 1002" "+ 1001" source_text "${source_text}") +else() + message(FATAL_ERROR "relink probe source does not contain its toggle") +endif() +file(WRITE "${SOURCE}" "${source_text}") + +execute_process( + COMMAND "${CMAKE_COMMAND}" --build "${BUILD_DIR}" --target "${TARGET}" + RESULT_VARIABLE build_result + OUTPUT_VARIABLE build_stdout + ERROR_VARIABLE build_stderr) +if(NOT build_result EQUAL 0) + message(FATAL_ERROR + "incremental plugin rebuild failed:\n${build_stdout}\n${build_stderr}") +endif() + +file(SHA256 "${LIBRARY}" after_hash) +if(before_hash STREQUAL after_hash) + message(FATAL_ERROR + "plugin did not relink after its C++ object archive changed") +endif() diff --git a/bindings/cpp/tests/dlopen_smoke.c b/bindings/cpp/tests/dlopen_smoke.c new file mode 100644 index 000000000..4b4a18d68 --- /dev/null +++ b/bindings/cpp/tests/dlopen_smoke.c @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +typedef uint32_t (*abi_version_fn)(void); +typedef int32_t (*self_reset_fn)(void); + +static void *required_symbol(void *library, const char *name) { + void *symbol; + dlerror(); + symbol = dlsym(library, name); + if (symbol == NULL || dlerror() != NULL) { + fprintf(stderr, "missing plugin symbol: %s\n", name); + return NULL; + } + return symbol; +} + +int main(int argc, char **argv) { + void *library; + void *symbol; + abi_version_fn abi_version; + self_reset_fn self_reset; + if (argc != 2) { + return 2; + } + library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (library == NULL) { + fprintf(stderr, "dlopen failed: %s\n", dlerror()); + return 1; + } + symbol = required_symbol(library, "paimon_cpp_plugin_abi_version"); + if (symbol == NULL) { + return 1; + } + memcpy(&abi_version, &symbol, sizeof(abi_version)); + symbol = required_symbol(library, "paimon_cpp_plugin_error_self_reset"); + if (symbol == NULL) { + return 1; + } + memcpy(&self_reset, &symbol, sizeof(self_reset)); + if (abi_version() != 1 || self_reset() != 0) { + return 1; + } + return dlclose(library) == 0 ? 0 : 1; +} diff --git a/bindings/cpp/tests/elf_fixtures/executable_stack.c b/bindings/cpp/tests/elf_fixtures/executable_stack.c new file mode 100644 index 000000000..3528d690b --- /dev/null +++ b/bindings/cpp/tests/elf_fixtures/executable_stack.c @@ -0,0 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +int paimon_elf_fixture_executable_stack(void) { + return 0; +} diff --git a/bindings/cpp/tests/elf_fixtures/needs_libgcc.c b/bindings/cpp/tests/elf_fixtures/needs_libgcc.c new file mode 100644 index 000000000..4c460b24a --- /dev/null +++ b/bindings/cpp/tests/elf_fixtures/needs_libgcc.c @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +extern void _Unwind_Resume(void *exception_object); + +void paimon_fixture_force_libgcc(void *exception_object) { + _Unwind_Resume(exception_object); +} diff --git a/bindings/cpp/tests/elf_fixtures/pie_executable.c b/bindings/cpp/tests/elf_fixtures/pie_executable.c new file mode 100644 index 000000000..3a621d61f --- /dev/null +++ b/bindings/cpp/tests/elf_fixtures/pie_executable.c @@ -0,0 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +int main(void) { + return 0; +} diff --git a/bindings/cpp/tests/elf_fixtures/undefined_host_hook.c b/bindings/cpp/tests/elf_fixtures/undefined_host_hook.c new file mode 100644 index 000000000..d1802bb6b --- /dev/null +++ b/bindings/cpp/tests/elf_fixtures/undefined_host_hook.c @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extern void forbidden_host_hook(void); + +void paimon_fixture_call_forbidden_host_hook(void) { + forbidden_host_hook(); +} diff --git a/bindings/cpp/tests/elf_fixtures/undefined_operator_new.c b/bindings/cpp/tests/elf_fixtures/undefined_operator_new.c new file mode 100644 index 000000000..f3999c429 --- /dev/null +++ b/bindings/cpp/tests/elf_fixtures/undefined_operator_new.c @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +extern void *_Znwm(size_t size); + +void *paimon_fixture_force_operator_new(size_t size) { return _Znwm(size); } diff --git a/bindings/cpp/tests/expect_elf_rejected.cmake b/bindings/cpp/tests/expect_elf_rejected.cmake new file mode 100644 index 000000000..dcd05f011 --- /dev/null +++ b/bindings/cpp/tests/expect_elf_rejected.cmake @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +foreach(required IN ITEMS VERIFIER LIBRARY EXPECTED) + if(NOT DEFINED ${required}) + message(FATAL_ERROR "missing -D${required}=...") + endif() +endforeach() + +execute_process( + COMMAND "${VERIFIER}" "${LIBRARY}" + RESULT_VARIABLE verifier_result + OUTPUT_VARIABLE verifier_stdout + ERROR_VARIABLE verifier_stderr) +set(verifier_output "${verifier_stdout}\n${verifier_stderr}") + +if(verifier_result EQUAL 0) + message(FATAL_ERROR + "ELF verifier accepted forbidden fixture ${LIBRARY}:\n${verifier_output}") +endif() +string(FIND "${verifier_output}" "${EXPECTED}" expected_index) +if(expected_index EQUAL -1) + message(FATAL_ERROR + "ELF verifier did not report '${EXPECTED}':\n${verifier_output}") +endif() diff --git a/bindings/cpp/tests/header_smoke.cpp b/bindings/cpp/tests/header_smoke.cpp new file mode 100644 index 000000000..5d1503dc9 --- /dev/null +++ b/bindings/cpp/tests/header_smoke.cpp @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +static_assert(std::is_move_constructible::value, "moveable"); +static_assert(!std::is_copy_constructible::value, + "not copyable"); +static_assert(std::is_nothrow_destructible::value, + "noexcept Arrow ownership"); +static_assert(std::is_nothrow_destructible::value, + "noexcept committer ownership"); + +void paimon_cpp_header_smoke(const paimon::Option* options, + std::size_t option_count, + void* arrow_array, void* arrow_schema) { + const auto abi = paimon::abi_version(); + auto version = paimon::library_version(); + (void)abi; + (void)version; + auto catalog = paimon::Catalog::create(options, option_count); + auto identifier = paimon::Identifier::create("default", "table"); + auto direct_table = paimon::Table::from_schema_json( + "/tmp/table", "{}", "default", "table"); + auto split_plan = paimon::Plan::from_split_bytes(nullptr, 0); + (void)direct_table; + (void)split_plan; + if (!catalog || !identifier) { + return; + } + + auto table = catalog.value().get_table(identifier.value()); + if (!table) { + return; + } + + auto read_builder = table.value().new_read_builder(); + if (read_builder) { + const char* projection[] = {"id", nullptr}; + auto projection_status = read_builder.value().with_projection(projection); + auto case_status = read_builder.value().with_case_sensitive(true); + auto scan = read_builder.value().new_scan(); + auto read = read_builder.value().new_read(); + auto stream_options = paimon::StreamScanOptions::defaults(); + (void)projection_status; + (void)case_status; + if (scan && read) { + auto plan = scan.value().plan(); + if (plan) { + auto reader = read.value().to_arrow(plan.value()); + if (reader) { + auto batch = reader.value().next(); + (void)batch; + } + } + if (stream_options) { + stream_options.value().with_startup( + paimon::StreamStartupMode::latest); + stream_options.value().with_follow_up( + paimon::StreamFollowUpMode::automatic); + auto stream_scan = read_builder.value().new_stream_scan( + stream_options.value()); + if (stream_scan) { + const auto checkpoint = stream_scan.value().checkpoint(); + auto restore = stream_scan.value().restore(checkpoint); + auto poll = stream_scan.value().poll(); + (void)restore; + if (poll && poll.value().has_data()) { + auto plan_bytes = poll.value().plan().serialize(); + if (plan_bytes) { + auto restored_plan = paimon::StreamPlan::deserialize( + plan_bytes.value().data(), plan_bytes.value().size()); + (void)restored_plan; + } + auto stream_reader = poll.value().plan().read_to_arrow( + read.value(), paimon::StreamReadMode::data); + (void)stream_reader; + } + } + } + } + } + + auto write_builder = table.value().new_write_builder("stable-writer"); + if (!write_builder) { + return; + } + auto overwrite_status = write_builder.value().with_overwrite(); + auto writer = write_builder.value().new_write(); + auto committer = write_builder.value().new_commit(); + (void)overwrite_status; + if (!writer || !committer) { + return; + } + auto write_status = writer.value().write_arrow(arrow_array, arrow_schema); + auto prepared = writer.value().prepare_commit(); + (void)write_status; + if (prepared) { + auto durable = prepared.value().prepare(1); + if (durable) { + auto serialized = durable.value().serialize(); + if (serialized) { + auto restored = paimon::PreparedCommit::deserialize( + serialized.value().data(), serialized.value().size()); + if (restored) { + auto merge_status = durable.value().merge(restored.value()); + auto commit_status = committer.value().commit_prepared(durable.value()); + (void)merge_status; + (void)commit_status; + } + } + } + } +} diff --git a/bindings/cpp/tests/helper_config/paimon_cpp_helper_config.h b/bindings/cpp/tests/helper_config/paimon_cpp_helper_config.h new file mode 100644 index 000000000..47648349a --- /dev/null +++ b/bindings/cpp/tests/helper_config/paimon_cpp_helper_config.h @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PAIMON_CPP_HELPER_CONFIG_H +#define PAIMON_CPP_HELPER_CONFIG_H + +#define PAIMON_CPP_HELPER_CONFIG_VALUE 73 + +#endif diff --git a/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt new file mode 100644 index 000000000..cff091859 --- /dev/null +++ b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 3.15) +project(PaimonCppInstallTreeConsumer LANGUAGES C CXX) + +find_package(PaimonCpp CONFIG REQUIRED) +paimon_add_no_runtime_plugin( + paimon_install_tree_consumer + SOURCES plugin.cpp) + +option(PAIMON_INJECT_FORBIDDEN_RUNTIME "Test the installed ELF guard" OFF) +if(PAIMON_INJECT_FORBIDDEN_RUNTIME) + target_link_options( + paimon_install_tree_consumer PRIVATE -Wl,--no-as-needed) + target_link_libraries(paimon_install_tree_consumer PRIVATE gcc_s) +endif() diff --git a/bindings/cpp/tests/install_tree_consumer/plugin.cpp b/bindings/cpp/tests/install_tree_consumer/plugin.cpp new file mode 100644 index 000000000..2b5bcd878 --- /dev/null +++ b/bindings/cpp/tests/install_tree_consumer/plugin.cpp @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +PAIMON_CPP_PLUGIN_EXPORT std::uint32_t +paimon_install_tree_consumer_abi() noexcept { + return paimon::abi_version(); +} diff --git a/bindings/cpp/tests/no_cpp_runtime_plugin.cpp b/bindings/cpp/tests/no_cpp_runtime_plugin.cpp new file mode 100644 index 000000000..2fd534988 --- /dev/null +++ b/bindings/cpp/tests/no_cpp_runtime_plugin.cpp @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "paimon_cpp_helper_config.h" + +#ifndef PAIMON_CPP_HELPER_COMPILE_DEFINITION +#error "no-runtime helper did not forward compile definitions" +#endif + +static_assert(PAIMON_CPP_HELPER_COMPILE_DEFINITION == + PAIMON_CPP_HELPER_CONFIG_VALUE, + "no-runtime helper configuration mismatch"); + +// These C-linkage exports make this a realistic C++ implementation plugin that +// can itself be loaded on a host without libstdc++ or libc++. +PAIMON_CPP_PLUGIN_EXPORT std::uint32_t +paimon_cpp_plugin_abi_version() noexcept { + return paimon::abi_version(); +} + +PAIMON_CPP_PLUGIN_EXPORT std::size_t paimon_cpp_plugin_library_version( + char* output, std::size_t capacity) noexcept { + auto version = paimon::library_version(); + const auto copied = output == nullptr + ? 0 + : (version.size() < capacity ? version.size() + : capacity); + for (std::size_t index = 0; index < copied; ++index) { + output[index] = static_cast(version.data()[index]); + } + return version.size(); +} + +PAIMON_CPP_PLUGIN_EXPORT std::int32_t paimon_cpp_plugin_open_catalog( + const paimon::Option* options, std::size_t options_len) noexcept { + auto catalog = paimon::Catalog::create(options, options_len); + if (!catalog) { + return static_cast(catalog.error().code()) + 1; + } + // The move-only Catalog is deliberately closed by its noexcept destructor. + return 0; +} + +PAIMON_CPP_PLUGIN_EXPORT std::int32_t +paimon_cpp_plugin_error_self_reset() noexcept { + paimon::Error error( + paimon::adopt_handle, + ::paimon_stream_scan_restore(nullptr, 0)); + if (!error) { + return -1; + } + auto* same = error.native_handle(); + error.reset(same); + return error.native_handle() == same ? 0 : -2; +} diff --git a/bindings/cpp/tests/paimon_test_stub.h b/bindings/cpp/tests/paimon_test_stub.h new file mode 100644 index 000000000..00253f084 --- /dev/null +++ b/bindings/cpp/tests/paimon_test_stub.h @@ -0,0 +1,314 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef PAIMON_CPP_TEST_PAIMON_STUB_H +#define PAIMON_CPP_TEST_PAIMON_STUB_H + +#include +#include +#include + +#define PAIMON_ERROR_UNEXPECTED 0 +#define PAIMON_ERROR_UNSUPPORTED 1 +#define PAIMON_ERROR_NOT_FOUND 2 +#define PAIMON_ERROR_ALREADY_EXISTS 3 +#define PAIMON_ERROR_INVALID_INPUT 4 +#define PAIMON_ERROR_IO 5 +#define PAIMON_ERROR_OUT_OF_RANGE 6 + +#define PAIMON_STREAM_STARTUP_LATEST_FULL 0 +#define PAIMON_STREAM_STARTUP_LATEST 1 +#define PAIMON_STREAM_STARTUP_FROM_SNAPSHOT 2 +#define PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL 3 +#define PAIMON_STREAM_FOLLOW_UP_AUTO 0 +#define PAIMON_STREAM_FOLLOW_UP_DELTA 1 +#define PAIMON_STREAM_FOLLOW_UP_CHANGELOG 2 +#define PAIMON_STREAM_POLL_DATA 0 +#define PAIMON_STREAM_POLL_WAITING 1 +#define PAIMON_STREAM_POLL_END 2 +#define PAIMON_STREAM_READ_DATA 0 +#define PAIMON_STREAM_READ_AUDIT_LOG 1 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct paimon_bytes { + uint8_t* data; + size_t len; +} paimon_bytes; + +typedef struct paimon_error { + int32_t code; + paimon_bytes message; +} paimon_error; + +typedef struct paimon_option { + const char* key; + const char* value; +} paimon_option; + +typedef struct paimon_catalog paimon_catalog; +typedef struct paimon_identifier paimon_identifier; +typedef struct paimon_table paimon_table; +typedef struct paimon_read_builder paimon_read_builder; +typedef struct paimon_table_scan paimon_table_scan; +typedef struct paimon_plan paimon_plan; +typedef struct paimon_table_read paimon_table_read; +typedef struct paimon_record_batch_reader paimon_record_batch_reader; +typedef struct paimon_write_builder paimon_write_builder; +typedef struct paimon_table_write paimon_table_write; +typedef struct paimon_commit_messages paimon_commit_messages; +typedef struct paimon_table_commit paimon_table_commit; +typedef struct paimon_prepared_commit paimon_prepared_commit; +typedef struct paimon_stream_scan paimon_stream_scan; +typedef struct paimon_stream_plan paimon_stream_plan; + +typedef struct paimon_stream_scan_options { + uint32_t struct_size; + int32_t startup_mode; + int32_t follow_up_mode; + int64_t snapshot_id; + uint64_t reserved[4]; +} paimon_stream_scan_options; + +typedef struct paimon_arrow_batch { + void* array; + void* schema; +} paimon_arrow_batch; + +typedef struct paimon_result_catalog_new { + paimon_catalog* catalog; + paimon_error* error; +} paimon_result_catalog_new; + +typedef struct paimon_result_identifier_new { + paimon_identifier* identifier; + paimon_error* error; +} paimon_result_identifier_new; + +typedef struct paimon_result_get_table { + paimon_table* table; + paimon_error* error; +} paimon_result_get_table; + +typedef struct paimon_result_read_builder { + paimon_read_builder* read_builder; + paimon_error* error; +} paimon_result_read_builder; + +typedef struct paimon_result_table_scan { + paimon_table_scan* scan; + paimon_error* error; +} paimon_result_table_scan; + +typedef struct paimon_result_new_read { + paimon_table_read* read; + paimon_error* error; +} paimon_result_new_read; + +typedef struct paimon_result_plan { + paimon_plan* plan; + paimon_error* error; +} paimon_result_plan; + +typedef struct paimon_result_record_batch_reader { + paimon_record_batch_reader* reader; + paimon_error* error; +} paimon_result_record_batch_reader; + +typedef struct paimon_result_next_batch { + paimon_arrow_batch batch; + paimon_error* error; +} paimon_result_next_batch; + +typedef struct paimon_result_write_builder { + paimon_write_builder* write_builder; + paimon_error* error; +} paimon_result_write_builder; + +typedef struct paimon_result_table_write { + paimon_table_write* write; + paimon_error* error; +} paimon_result_table_write; + +typedef struct paimon_result_table_commit { + paimon_table_commit* commit; + paimon_error* error; +} paimon_result_table_commit; + +typedef struct paimon_result_prepare_commit { + paimon_commit_messages* messages; + paimon_error* error; +} paimon_result_prepare_commit; + +typedef struct paimon_result_prepared_commit { + paimon_prepared_commit* prepared; + paimon_error* error; +} paimon_result_prepared_commit; + +typedef struct paimon_result_bytes { + paimon_bytes bytes; + paimon_error* error; +} paimon_result_bytes; + +typedef struct paimon_result_stream_scan { + paimon_stream_scan* scan; + paimon_error* error; +} paimon_result_stream_scan; + +typedef struct paimon_result_stream_poll { + int32_t status; + paimon_stream_plan* plan; + int64_t snapshot_id; + int64_t next_snapshot_id; + int64_t watermark; + uint8_t has_watermark; + uint8_t reserved[7]; + paimon_error* error; +} paimon_result_stream_poll; + +void paimon_error_free(paimon_error* error); +void paimon_bytes_free(paimon_bytes bytes); +uint32_t paimon_abi_version(void); +paimon_bytes paimon_library_version(void); +paimon_result_catalog_new paimon_catalog_create(const paimon_option* options, + size_t options_len); +void paimon_catalog_free(paimon_catalog* catalog); +paimon_result_get_table paimon_catalog_get_table( + const paimon_catalog* catalog, const paimon_identifier* identifier); +paimon_result_identifier_new paimon_identifier_new(const char* database, + const char* object); +void paimon_identifier_free(paimon_identifier* identifier); + +paimon_result_get_table paimon_table_from_schema_json( + const char* table_path, const char* table_schema_json, + const char* database, const char* table_name, const char* branch, + const paimon_option* storage_options, size_t storage_options_len); +void paimon_table_free(paimon_table* table); +paimon_result_read_builder paimon_table_new_read_builder( + const paimon_table* table); +paimon_result_read_builder paimon_table_new_read_builder_with_options( + const paimon_table* table, const paimon_option* options, + size_t options_len); +void paimon_read_builder_free(paimon_read_builder* builder); +paimon_error* paimon_read_builder_with_projection( + paimon_read_builder* builder, const char* const* columns); +paimon_error* paimon_read_builder_with_case_sensitive( + paimon_read_builder* builder, bool case_sensitive); +paimon_result_table_scan paimon_read_builder_new_scan( + const paimon_read_builder* builder); +paimon_result_new_read paimon_read_builder_new_read( + const paimon_read_builder* builder); +paimon_error* paimon_stream_scan_options_init( + paimon_stream_scan_options* options); +paimon_result_stream_scan paimon_read_builder_new_stream_scan( + const paimon_read_builder* builder, + const paimon_stream_scan_options* options); +paimon_result_stream_poll paimon_stream_scan_poll(paimon_stream_scan* scan); +int64_t paimon_stream_scan_checkpoint(const paimon_stream_scan* scan); +paimon_error* paimon_stream_scan_restore(paimon_stream_scan* scan, + int64_t next_snapshot_id); +void paimon_stream_scan_free(paimon_stream_scan* scan); +uint8_t paimon_stream_plan_is_full(const paimon_stream_plan* plan); +size_t paimon_stream_plan_num_splits(const paimon_stream_plan* plan); +paimon_result_bytes paimon_stream_plan_serialize( + const paimon_stream_plan* plan); +paimon_result_stream_poll paimon_stream_plan_deserialize( + const uint8_t* data, size_t size); +paimon_result_record_batch_reader paimon_stream_plan_read_to_arrow( + const paimon_table_read* read, const paimon_stream_plan* plan, + size_t offset, size_t length, int32_t read_mode); +void paimon_stream_plan_free(paimon_stream_plan* plan); +void paimon_table_scan_free(paimon_table_scan* scan); +paimon_result_plan paimon_table_scan_plan(const paimon_table_scan* scan); +paimon_result_plan paimon_plan_from_split_bytes(const uint8_t* data, + size_t size); +void paimon_plan_free(paimon_plan* plan); +size_t paimon_plan_num_splits(const paimon_plan* plan); +void paimon_table_read_free(paimon_table_read* read); +paimon_result_record_batch_reader paimon_table_read_to_arrow( + const paimon_table_read* read, const paimon_plan* plan, size_t offset, + size_t length); +paimon_result_next_batch paimon_record_batch_reader_next( + paimon_record_batch_reader* reader); +void paimon_record_batch_reader_free(paimon_record_batch_reader* reader); +void paimon_arrow_batch_free(paimon_arrow_batch batch); + +paimon_result_write_builder paimon_table_new_write_builder( + const paimon_table* table); +paimon_result_write_builder paimon_table_new_write_builder_with_commit_user( + const paimon_table* table, const char* commit_user); +void paimon_write_builder_free(paimon_write_builder* builder); +paimon_error* paimon_write_builder_with_overwrite( + paimon_write_builder* builder); +paimon_result_table_write paimon_write_builder_new_write( + const paimon_write_builder* builder); +paimon_result_table_commit paimon_write_builder_new_commit( + const paimon_write_builder* builder); +void paimon_table_write_free(paimon_table_write* writer); +paimon_error* paimon_table_write_write_arrow_batch(paimon_table_write* writer, + void* array, + void* schema); +paimon_result_prepare_commit paimon_table_write_prepare_commit( + paimon_table_write* writer); +void paimon_commit_messages_free(paimon_commit_messages* messages); +paimon_result_prepared_commit paimon_commit_messages_prepare( + const paimon_commit_messages* messages, int64_t checkpoint_id); +paimon_result_bytes paimon_prepared_commit_serialize( + const paimon_prepared_commit* prepared); +paimon_result_prepared_commit paimon_prepared_commit_deserialize( + const uint8_t* data, size_t size); +int64_t paimon_prepared_commit_identifier( + const paimon_prepared_commit* prepared); +void paimon_prepared_commit_free(paimon_prepared_commit* prepared); +paimon_error* paimon_commit_messages_merge( + paimon_commit_messages* target, const paimon_commit_messages* source); +paimon_error* paimon_prepared_commit_merge( + paimon_prepared_commit* target, const paimon_prepared_commit* source); +void paimon_table_commit_free(paimon_table_commit* committer); +paimon_error* paimon_table_commit_commit( + const paimon_table_commit* committer, paimon_commit_messages* messages); +paimon_error* paimon_table_commit_commit_with_identifier( + const paimon_table_commit* committer, paimon_commit_messages* messages, + int64_t checkpoint_id); +paimon_error* paimon_table_commit_filter_and_commit_with_identifier( + const paimon_table_commit* committer, paimon_commit_messages* messages, + int64_t checkpoint_id); +paimon_error* paimon_table_commit_commit_prepared( + const paimon_table_commit* committer, + const paimon_prepared_commit* prepared); +paimon_error* paimon_table_commit_overwrite( + const paimon_table_commit* committer, paimon_commit_messages* messages); +paimon_error* paimon_table_commit_overwrite_with_identifier( + const paimon_table_commit* committer, paimon_commit_messages* messages, + int64_t checkpoint_id); +paimon_error* paimon_table_commit_truncate_table( + const paimon_table_commit* committer); +paimon_error* paimon_table_commit_truncate_table_with_identifier( + const paimon_table_commit* committer, int64_t checkpoint_id); +paimon_error* paimon_table_commit_abort( + const paimon_table_commit* committer, paimon_commit_messages* messages); +paimon_error* paimon_table_commit_abort_prepared( + const paimon_table_commit* committer, + const paimon_prepared_commit* prepared); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // PAIMON_CPP_TEST_PAIMON_STUB_H diff --git a/bindings/cpp/tests/relink_probe.cpp b/bindings/cpp/tests/relink_probe.cpp new file mode 100644 index 000000000..ad667a167 --- /dev/null +++ b/bindings/cpp/tests/relink_probe.cpp @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +PAIMON_CPP_PLUGIN_EXPORT std::uint32_t paimon_cpp_relink_probe() noexcept { + return paimon::abi_version() + 1001; +} diff --git a/bindings/cpp/tests/run_install_tree_consumer.cmake b/bindings/cpp/tests/run_install_tree_consumer.cmake new file mode 100644 index 000000000..8ae377a4d --- /dev/null +++ b/bindings/cpp/tests/run_install_tree_consumer.cmake @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +foreach(required IN ITEMS MAIN_BUILD_DIR CONSUMER_SOURCE_DIR TEST_ROOT + C_COMPILER CXX_COMPILER PLUGIN_FILENAME VERIFIER) + if(NOT DEFINED ${required}) + message(FATAL_ERROR "missing -D${required}=...") + endif() +endforeach() + +set(test_prefix "${TEST_ROOT}/prefix") +set(consumer_build "${TEST_ROOT}/build") +if(NOT DEFINED EXPECT_BUILD_FAILURE) + set(EXPECT_BUILD_FAILURE OFF) +endif() +file(REMOVE_RECURSE "${TEST_ROOT}") + +execute_process( + COMMAND "${CMAKE_COMMAND}" --install "${MAIN_BUILD_DIR}" + --prefix "${test_prefix}" + RESULT_VARIABLE install_result + OUTPUT_VARIABLE install_stdout + ERROR_VARIABLE install_stderr) +if(NOT install_result EQUAL 0) + message(FATAL_ERROR + "install-tree setup failed:\n${install_stdout}\n${install_stderr}") +endif() + +execute_process( + COMMAND "${CMAKE_COMMAND}" + -S "${CONSUMER_SOURCE_DIR}" + -B "${consumer_build}" + "-DCMAKE_PREFIX_PATH=${test_prefix}" + "-DCMAKE_C_COMPILER=${C_COMPILER}" + "-DCMAKE_CXX_COMPILER=${CXX_COMPILER}" + "-DPAIMON_INJECT_FORBIDDEN_RUNTIME=${EXPECT_BUILD_FAILURE}" + RESULT_VARIABLE configure_result + OUTPUT_VARIABLE configure_stdout + ERROR_VARIABLE configure_stderr) +if(NOT configure_result EQUAL 0) + message(FATAL_ERROR + "install-tree consumer configure failed:\n${configure_stdout}\n${configure_stderr}") +endif() + +execute_process( + COMMAND "${CMAKE_COMMAND}" --build "${consumer_build}" + RESULT_VARIABLE build_result + OUTPUT_VARIABLE build_stdout + ERROR_VARIABLE build_stderr) +if(EXPECT_BUILD_FAILURE) + if(build_result EQUAL 0) + message(FATAL_ERROR + "installed helper accepted a forbidden runtime dependency") + endif() + set(build_output "${build_stdout}\n${build_stderr}") + if(NOT build_output MATCHES "forbidden non-C runtime dependency") + message(FATAL_ERROR + "consumer failed for the wrong reason:\n${build_output}") + endif() + return() +endif() +if(NOT build_result EQUAL 0) + message(FATAL_ERROR + "install-tree consumer build failed:\n${build_stdout}\n${build_stderr}") +endif() + +set(plugin "${consumer_build}/${PLUGIN_FILENAME}") +execute_process( + COMMAND "${VERIFIER}" "${plugin}" + RESULT_VARIABLE verifier_result + OUTPUT_VARIABLE verifier_stdout + ERROR_VARIABLE verifier_stderr) +if(NOT verifier_result EQUAL 0) + message(FATAL_ERROR + "installed no-runtime plugin failed ELF verification:\n${verifier_stdout}\n${verifier_stderr}") +endif() diff --git a/bindings/cpp/tests/run_isolated_load.cmake b/bindings/cpp/tests/run_isolated_load.cmake new file mode 100644 index 000000000..9eeb85235 --- /dev/null +++ b/bindings/cpp/tests/run_isolated_load.cmake @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +foreach(required IN ITEMS LOADER PLUGIN PAIMON_C_LIBRARY TEST_ROOT) + if(NOT DEFINED ${required}) + message(FATAL_ERROR "missing -D${required}=...") + endif() +endforeach() + +file(REMOVE_RECURSE "${TEST_ROOT}") +file(MAKE_DIRECTORY "${TEST_ROOT}") +file(COPY "${PLUGIN}" "${PAIMON_C_LIBRARY}" DESTINATION "${TEST_ROOT}") +get_filename_component(plugin_name "${PLUGIN}" NAME) +execute_process( + COMMAND "${LOADER}" "./${plugin_name}" + WORKING_DIRECTORY "${TEST_ROOT}" + RESULT_VARIABLE load_result + OUTPUT_VARIABLE load_stdout + ERROR_VARIABLE load_stderr) +if(NOT load_result EQUAL 0) + message(FATAL_ERROR + "isolated plugin load failed:\n${load_stdout}\n${load_stderr}") +endif() diff --git a/bindings/go/DEPENDENCIES.rust.tsv b/bindings/go/DEPENDENCIES.rust.tsv index f50b9243b..9cd511fcc 100644 --- a/bindings/go/DEPENDENCIES.rust.tsv +++ b/bindings/go/DEPENDENCIES.rust.tsv @@ -102,8 +102,6 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X -foreign-types@0.3.2 X X -foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -136,7 +134,6 @@ httpdate@1.0.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X -hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -193,7 +190,6 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X -native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-bigint-dig@0.8.6 X X @@ -214,10 +210,7 @@ opendal-service-gcs@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X -openssl@0.10.81 X -openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X -openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -276,6 +269,7 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X +ring@0.17.14 X X roaring@0.11.4 X X rsa@0.9.10 X X rust-ini@0.21.3 X @@ -350,7 +344,6 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X -tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -372,7 +365,6 @@ url@2.5.8 X X urlencoding@2.1.3 X utf8_iter@1.0.4 X X uuid@1.24.0 X X -vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/bindings/python/DEPENDENCIES.rust.tsv b/bindings/python/DEPENDENCIES.rust.tsv index f77beb384..fa1715942 100644 --- a/bindings/python/DEPENDENCIES.rust.tsv +++ b/bindings/python/DEPENDENCIES.rust.tsv @@ -174,8 +174,6 @@ flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.1.5 X foldhash@0.2.0 X -foreign-types@0.3.2 X X -foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X @@ -218,7 +216,6 @@ humantime@2.4.0 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X -hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -296,7 +293,6 @@ mio@1.2.2 X murmurhash32@0.3.1 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X -native-tls@0.2.18 X X no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X @@ -326,10 +322,7 @@ opendal-service-hdfs-native@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X -openssl@0.10.81 X -openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X -openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-float@5.3.0 X @@ -419,6 +412,7 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X +ring@0.17.14 X X rle-decode-fast@1.0.3 X X roaring@0.11.4 X X roxmltree@0.21.1 X X @@ -519,7 +513,6 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X -tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-stream@0.1.18 X tokio-util@0.7.18 X @@ -550,7 +543,6 @@ urlencoding@2.1.3 X utf8-ranges@1.0.5 X X utf8_iter@1.0.4 X X uuid@1.24.0 X X -vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/crates/integration_tests/DEPENDENCIES.rust.tsv b/crates/integration_tests/DEPENDENCIES.rust.tsv index 565b00e51..e3ca727d3 100644 --- a/crates/integration_tests/DEPENDENCIES.rust.tsv +++ b/crates/integration_tests/DEPENDENCIES.rust.tsv @@ -94,8 +94,6 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X -foreign-types@0.3.2 X X -foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -128,7 +126,6 @@ httpdate@1.0.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X -hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -183,7 +180,6 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X -native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-complex@0.4.6 X X @@ -197,10 +193,7 @@ opendal-http-transport-reqwest@0.58.2 X opendal-layer-retry@0.58.2 X opendal-service-fs@0.58.2 X opendal-service-oss@0.58.2 X -openssl@0.10.81 X -openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X -openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -247,6 +240,7 @@ reqsign-core@3.3.1 X reqsign-file-read-tokio@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X +ring@0.17.14 X X roaring@0.11.4 X X rust-ini@0.21.3 X rustc_version@0.4.1 X X @@ -314,7 +308,6 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X -tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -336,7 +329,6 @@ url@2.5.8 X X urlencoding@2.1.3 X utf8_iter@1.0.4 X X uuid@1.24.0 X X -vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv index 09433b81d..e8784ca9f 100644 --- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv +++ b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv @@ -181,8 +181,6 @@ flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.1.5 X foldhash@0.2.0 X -foreign-types@0.3.2 X X -foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X @@ -226,7 +224,6 @@ humantime@2.4.0 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X -hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -309,7 +306,6 @@ moka@0.12.15 X X murmurhash32@0.3.1 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X -native-tls@0.2.18 X X never-say-never@6.6.666 X X X no_std_io2@0.9.4 X X nom@7.1.3 X @@ -335,10 +331,7 @@ opendal-http-transport-reqwest@0.58.2 X opendal-layer-retry@0.58.2 X opendal-service-fs@0.58.2 X opendal-service-oss@0.58.2 X -openssl@0.10.81 X -openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X -openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-float@5.3.0 X @@ -416,6 +409,7 @@ reqsign-core@3.3.1 X reqsign-file-read-tokio@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X +ring@0.17.14 X X rle-decode-fast@1.0.3 X X roaring@0.11.4 X X rust-ini@0.21.3 X @@ -511,7 +505,6 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X -tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-stream@0.1.18 X tokio-util@0.7.18 X @@ -539,7 +532,6 @@ urlencoding@2.1.3 X utf8-ranges@1.0.5 X X utf8_iter@1.0.4 X X uuid@1.24.0 X X -vcpkg@0.2.15 X X version_check@0.9.5 X X vortex@0.75.0 X vortex-alp@0.75.0 X diff --git a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv index 67a578223..05a660898 100644 --- a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv +++ b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv @@ -97,8 +97,6 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X -foreign-types@0.3.2 X X -foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -131,7 +129,6 @@ httpdate@1.0.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X -hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -187,7 +184,6 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X -native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-complex@0.4.6 X X @@ -201,10 +197,7 @@ opendal-http-transport-reqwest@0.58.2 X opendal-layer-retry@0.58.2 X opendal-service-fs@0.58.2 X opendal-service-oss@0.58.2 X -openssl@0.10.81 X -openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X -openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -251,6 +244,7 @@ reqsign-core@3.3.1 X reqsign-file-read-tokio@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X +ring@0.17.14 X X roaring@0.11.4 X X rust-ini@0.21.3 X rustc_version@0.4.1 X X @@ -320,7 +314,6 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X -tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -342,7 +335,6 @@ url@2.5.8 X X urlencoding@2.1.3 X utf8_iter@1.0.4 X X uuid@1.24.0 X X -vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index dc09b9644..e9b4fd31b 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -118,7 +118,15 @@ tokio-util = { workspace = true, features = ["compat", "io-util"] } parquet = { workspace = true, features = ["async", "zstd", "lz4", "snap"] } orc-rust = "0.8.0" async-stream = "0.3.6" -reqwest = { version = "0.12", features = ["json"] } +# Use the rustls TLS stack with the platform trust store so native artifacts do +# not depend on libssl/libcrypto while enterprise and system CAs remain usable. +reqwest = { version = "0.12", default-features = false, features = [ + "charset", + "http2", + "json", + "rustls-tls-native-roots", + "system-proxy", +] } # DLF authentication dependencies base64 = "0.22" hex = "0.4" diff --git a/crates/paimon/DEPENDENCIES.rust.tsv b/crates/paimon/DEPENDENCIES.rust.tsv index 9225deb96..1990717db 100644 --- a/crates/paimon/DEPENDENCIES.rust.tsv +++ b/crates/paimon/DEPENDENCIES.rust.tsv @@ -156,8 +156,6 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X -foreign-types@0.3.2 X X -foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X @@ -203,7 +201,6 @@ humansize@2.1.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X -hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -286,7 +283,6 @@ moka@0.12.15 X X murmurhash32@0.3.1 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X -native-tls@0.2.18 X X never-say-never@6.6.666 X X X no_std_io2@0.9.4 X X nom@7.1.3 X @@ -320,10 +316,7 @@ opendal-service-hdfs-native@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X -openssl@0.10.81 X -openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X -openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-float@5.3.0 X @@ -406,6 +399,7 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X +ring@0.17.14 X X rle-decode-fast@1.0.3 X X roaring@0.11.4 X X roxmltree@0.21.1 X X @@ -506,7 +500,6 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X -tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -533,7 +526,6 @@ urlencoding@2.1.3 X utf8-ranges@1.0.5 X X utf8_iter@1.0.4 X X uuid@1.24.0 X X -vcpkg@0.2.15 X X version_check@0.9.5 X X vortex@0.75.0 X vortex-alp@0.75.0 X diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 1257f6328..ccf0603b9 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -29,7 +29,7 @@ use chrono::{DateTime, Utc}; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use opendal::raw::normalize_root; -use opendal::Operator; +use opendal::{ErrorKind as OpendalErrorKind, Operator}; use snafu::ResultExt; use tokio_util::compat::FuturesAsyncWriteCompatExt; use url::Url; @@ -442,6 +442,125 @@ impl FileIO { Ok(()) } + + /// Publish a fully written temporary file without replacing an existing + /// destination. + /// + /// The operation uses only backend capabilities whose destination + /// precondition is atomic. It never falls back to `exists + write`, which + /// would allow two committers to overwrite the same snapshot. Backends + /// without a conditional rename, copy, or write must use REST commit or an + /// external lock. + pub(crate) async fn publish_if_not_exists( + &self, + src: &str, + dst: &str, + contents: Bytes, + ) -> Result { + let (op_src, relative_src) = self.create(src).await?; + let (op_dst, relative_dst) = self.create(dst).await?; + let src_cache_path = cache_object_path(&op_src, &relative_src); + let dst_cache_path = cache_object_path(&op_dst, &relative_dst); + let capability = op_src.info().capability(); + + if capability.rename_with_if_not_exists { + match op_src + .rename_with(&relative_src, &relative_dst) + .if_not_exists(true) + .await + { + Ok(_) => { + self.invalidate_publish_cache(&src_cache_path, &dst_cache_path) + .await; + return Ok(true); + } + Err(error) if destination_already_exists(&error) => { + let _ = op_src.delete(&relative_src).await; + return Ok(false); + } + Err(error) if error.kind() != OpendalErrorKind::Unsupported => { + let _ = op_src.delete(&relative_src).await; + return Err(error).context(IoUnexpectedSnafu { + message: format!("Failed to publish '{src}' as '{dst}'"), + }); + } + Err(_) => {} + } + } + + if capability.copy_with_if_not_exists { + match op_src + .copy_with(&relative_src, &relative_dst) + .if_not_exists(true) + .await + { + Ok(_) => { + let _ = op_src.delete(&relative_src).await; + self.invalidate_publish_cache(&src_cache_path, &dst_cache_path) + .await; + return Ok(true); + } + Err(error) if destination_already_exists(&error) => { + let _ = op_src.delete(&relative_src).await; + return Ok(false); + } + Err(error) if error.kind() != OpendalErrorKind::Unsupported => { + let _ = op_src.delete(&relative_src).await; + return Err(error).context(IoUnexpectedSnafu { + message: format!("Failed to publish '{src}' as '{dst}'"), + }); + } + Err(_) => {} + } + } + + let write_capability = op_dst.info().capability(); + if write_capability.write_with_if_not_exists { + match op_dst + .write_with(&relative_dst, contents) + .if_not_exists(true) + .await + { + Ok(_) => { + let _ = op_src.delete(&relative_src).await; + self.invalidate_publish_cache(&src_cache_path, &dst_cache_path) + .await; + return Ok(true); + } + Err(error) if destination_already_exists(&error) => { + let _ = op_src.delete(&relative_src).await; + return Ok(false); + } + Err(error) => { + let _ = op_src.delete(&relative_src).await; + return Err(error).context(IoUnexpectedSnafu { + message: format!("Failed to publish '{src}' as '{dst}'"), + }); + } + } + } + + let _ = op_src.delete(&relative_src).await; + Err(Error::Unsupported { + message: format!( + "Storage backend for '{dst}' has no atomic publish-if-absent operation; use REST commit or an external lock" + ), + }) + } + + async fn invalidate_publish_cache(&self, src: &str, dst: &str) { + if let Some(cache) = &self.cache { + cache.invalidate_prefix(src).await; + cache.invalidate_prefix(dst).await; + } + } +} + +fn destination_already_exists(error: &opendal::Error) -> bool { + matches!( + error.kind(), + OpendalErrorKind::ConditionNotMatch | OpendalErrorKind::AlreadyExists + ) } fn status_path(base_path: &str, entry_path: &str) -> String { @@ -1295,6 +1414,39 @@ mod file_action_test { common_test_list_status_paths(&file_io, "file:/tmp/test_list_status_paths_fs/").await; } + #[tokio::test] + async fn test_publish_if_not_exists_fs_has_one_winner_and_cleans_temps() { + let directory = tempdir().unwrap(); + let file_io = setup_fs_file_io(); + let first = local_file_path(&directory.path().join("first.tmp")); + let second = local_file_path(&directory.path().join("second.tmp")); + let target = local_file_path(&directory.path().join("snapshot-1")); + let first_bytes = Bytes::from_static(b"first"); + let second_bytes = Bytes::from_static(b"second"); + file_io + .new_output(&first) + .unwrap() + .write(first_bytes.clone()) + .await + .unwrap(); + file_io + .new_output(&second) + .unwrap() + .write(second_bytes.clone()) + .await + .unwrap(); + + let (first_result, second_result) = tokio::join!( + file_io.publish_if_not_exists(&first, &target, first_bytes), + file_io.publish_if_not_exists(&second, &target, second_bytes) + ); + assert_ne!(first_result.unwrap(), second_result.unwrap()); + let committed = file_io.new_input(&target).unwrap().read().await.unwrap(); + assert!(committed.as_ref() == b"first" || committed.as_ref() == b"second"); + assert!(!file_io.exists(&first).await.unwrap()); + assert!(!file_io.exists(&second).await.unwrap()); + } + #[test] fn test_from_path_detects_local_fs_path() { let dir = tempdir().unwrap(); diff --git a/crates/paimon/src/lib.rs b/crates/paimon/src/lib.rs index 9e6b13400..9c407cf0f 100644 --- a/crates/paimon/src/lib.rs +++ b/crates/paimon/src/lib.rs @@ -54,7 +54,8 @@ pub use table::{ IncrementalScanMode, IncrementalSplit, PartitionBucket, Plan, PostponeBucketPlan, PostponeFixedBucketTableCommit, PostponeFixedBucketTableWrite, RESTEnv, RESTSnapshotCommit, ReadBuilder, RenamingSnapshotCommit, RowRange, ScanTrace, SnapshotCommit, SnapshotManager, - Table, TableCommit, TableRead, TableScan, TableUpdate, TableWrite, TagManager, WriteBuilder, + StreamPlan, StreamScan, StreamScanFollowUpMode, StreamScanPoll, StreamScanStartupMode, Table, + TableCommit, TableRead, TableScan, TableUpdate, TableWrite, TagManager, WriteBuilder, }; pub use table::{ diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index 7afff6f67..5b37a3c64 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -95,6 +95,9 @@ pub(crate) const DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL_OPTION: &str = "alter-column-null-to-not-null.disabled"; const MERGE_ENGINE_OPTION: &str = "merge-engine"; pub(crate) const CHANGELOG_PRODUCER_OPTION: &str = "changelog-producer"; +const NUM_LEVELS_OPTION: &str = "num-levels"; +const NUM_SORTED_RUN_COMPACTION_TRIGGER_OPTION: &str = "num-sorted-run.compaction-trigger"; +const DEFAULT_NUM_SORTED_RUN_COMPACTION_TRIGGER: i32 = 5; const ROWKIND_FIELD_OPTION: &str = "rowkind.field"; const IGNORE_DELETE_OPTION: &str = "ignore-delete"; const IGNORE_UPDATE_BEFORE_OPTION: &str = "ignore-update-before"; @@ -614,6 +617,44 @@ impl<'a> CoreOptions<'a> { } } + /// Total number of merge-tree levels. + /// + /// Java defaults this to `num-sorted-run.compaction-trigger + 1` so a + /// compaction always has at least one non-zero target level. + pub fn num_levels(&self) -> crate::Result { + fn positive_i32(raw: &str, option: &str) -> crate::Result { + let value = raw + .parse::() + .map_err(|error| crate::Error::DataInvalid { + message: format!("Option '{option}' must be a positive integer, got: {raw}"), + source: Some(Box::new(error)), + })?; + if value <= 0 { + return Err(crate::Error::DataInvalid { + message: format!("Option '{option}' must be greater than 0, got: {value}"), + source: None, + }); + } + Ok(value) + } + + if let Some(raw) = self.options.get(NUM_LEVELS_OPTION) { + return positive_i32(raw, NUM_LEVELS_OPTION); + } + let trigger = match self.options.get(NUM_SORTED_RUN_COMPACTION_TRIGGER_OPTION) { + Some(raw) => positive_i32(raw, NUM_SORTED_RUN_COMPACTION_TRIGGER_OPTION)?, + None => DEFAULT_NUM_SORTED_RUN_COMPACTION_TRIGGER, + }; + trigger + .checked_add(1) + .ok_or_else(|| crate::Error::DataInvalid { + message: format!( + "Option '{NUM_SORTED_RUN_COMPACTION_TRIGGER_OPTION}' cannot be incremented: {trigger}" + ), + source: None, + }) + } + /// The `rowkind.field` option: a user column whose value encodes the row kind. pub fn rowkind_field(&self) -> Option<&str> { self.options.get(ROWKIND_FIELD_OPTION).map(String::as_str) diff --git a/crates/paimon/src/table/commit_message.rs b/crates/paimon/src/table/commit_message.rs index 55afbf643..d338ddfae 100644 --- a/crates/paimon/src/table/commit_message.rs +++ b/crates/paimon/src/table/commit_message.rs @@ -17,11 +17,12 @@ use crate::spec::DataFileMeta; use crate::spec::IndexFileMeta; +use serde::{Deserialize, Serialize}; /// A commit message representing new files to be committed for a specific partition and bucket. /// /// Reference: [org.apache.paimon.table.sink.CommitMessage](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageImpl.java) -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct CommitMessage { /// Binary row bytes for the partition. pub partition: Vec, diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 92c73a6fb..db3276f02 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -96,6 +96,7 @@ mod sorted_global_index_build_builder; mod sorted_global_index_options; mod source; mod stats_filter; +mod stream_scan; pub(crate) mod table_commit; mod table_read; mod table_scan; @@ -149,6 +150,9 @@ pub use sorted_global_index_build_builder::{ pub use source::{ merge_row_ranges, DataSplit, DataSplitBuilder, DeletionFile, PartitionBucket, Plan, RowRange, }; +pub use stream_scan::{ + StreamPlan, StreamScan, StreamScanFollowUpMode, StreamScanPoll, StreamScanStartupMode, +}; pub use table_commit::TableCommit; pub use table_read::TableRead; pub use table_scan::TableScan; diff --git a/crates/paimon/src/table/read_builder.rs b/crates/paimon/src/table/read_builder.rs index ec8ef966e..51b4f6923 100644 --- a/crates/paimon/src/table/read_builder.rs +++ b/crates/paimon/src/table/read_builder.rs @@ -24,6 +24,7 @@ use super::bucket_filter::{extract_predicate_for_keys, split_partition_and_data_ use super::format_read_builder::FormatReadBuilder; use super::incremental_scan::{IncrementalScan, IncrementalScanMode}; use super::partition_filter::PartitionFilter; +use super::stream_scan::{StreamScan, StreamScanFollowUpMode, StreamScanStartupMode}; use super::table_read::{configured_parquet_read_budget, TableRead}; use super::{Table, TableScan}; use crate::spec::{CoreOptions, DataField, Predicate}; @@ -288,6 +289,33 @@ impl<'a> ReadBuilder<'a> { } } + /// Create an owned, stateful continuous snapshot scanner. + /// + /// The returned scanner clones the table and scan configuration. It remains + /// valid after this builder and its originating table handle are dropped. + /// Filters, projection-driven data-evolution pruning, and row ranges are + /// preserved. Limit pushdown is rejected because advancing a stream cursor + /// past a partially planned snapshot would lose data. + pub async fn new_stream_scan( + &self, + startup_mode: StreamScanStartupMode, + follow_up_mode: StreamScanFollowUpMode, + ) -> Result { + let mut scan = match &self.0 { + ReadBuilderKind::Paimon(builder) => { + builder.new_stream_scan(startup_mode, follow_up_mode) + } + ReadBuilderKind::Format(_) => Err(Error::Unsupported { + message: "Continuous stream scan is not supported for format tables".to_string(), + }), + }?; + // Freeze the `Latest` boundary before returning. If initialization is + // deferred until the first poll, a concurrently committed snapshot can + // be mistaken for pre-existing data and skipped. + scan.initialize().await?; + Ok(scan) + } + /// Create a table read for consuming splits (e.g. from a scan plan). pub fn new_read(&self) -> Result> { match &self.0 { @@ -332,6 +360,37 @@ impl<'a> PaimonReadBuilder<'a> { } } + fn new_stream_scan( + &self, + startup_mode: StreamScanStartupMode, + follow_up_mode: StreamScanFollowUpMode, + ) -> Result { + if self.limit.is_some() { + return Err(Error::Unsupported { + message: "Continuous stream scan does not support limit pushdown".to_string(), + }); + } + let partition_filter = self.filter.partition_predicate.clone().map(|pred| { + PartitionFilter::from_predicate(pred, &self.table.schema().partition_fields()) + }); + let read_type = self.resolve_read_type()?; + let projected_read_field_ids = projected_read_field_ids_with_predicates( + &read_type, + &self.filter.data_predicates, + self.table.schema().fields(), + ); + StreamScan::try_new( + self.table.clone(), + partition_filter, + self.filter.data_predicates.clone(), + self.filter.bucket_predicate.clone(), + self.effective_row_ranges(), + projected_read_field_ids, + startup_mode, + follow_up_mode, + ) + } + /// Set column projection by name. Output order follows the caller-specified order. /// An empty list is a valid zero-column projection. /// diff --git a/crates/paimon/src/table/snapshot_manager.rs b/crates/paimon/src/table/snapshot_manager.rs index 1de8baa9e..1a4185413 100644 --- a/crates/paimon/src/table/snapshot_manager.rs +++ b/crates/paimon/src/table/snapshot_manager.rs @@ -145,9 +145,13 @@ impl SnapshotManager { let hint_path = self.latest_hint_path(); if let Some(hint_id) = self.read_hint(&hint_path).await { if hint_id > 0 { - let next_path = self.snapshot_path(hint_id + 1); - let next_input = self.file_io.new_input(&next_path)?; - if !next_input.exists().await? { + if let Some(next_id) = hint_id.checked_add(1) { + let next_path = self.snapshot_path(next_id); + let next_input = self.file_io.new_input(&next_path)?; + if !next_input.exists().await? { + return Ok(Some(hint_id)); + } + } else { return Ok(Some(hint_id)); } } @@ -244,10 +248,10 @@ impl SnapshotManager { /// Writes the snapshot JSON to the target path. Returns `false` if the /// target already exists (another writer won the race). /// - /// On file systems that support atomic rename, we write to a temp file - /// first then rename. On backends where rename is not supported (e.g. - /// memory, object stores), we fall back to a direct write after an - /// existence check. + /// The snapshot is first written under a unique temporary name and then + /// published with a backend-enforced destination precondition. Backends + /// without an atomic publish-if-absent primitive fail closed instead of + /// using a racy `exists + write` fallback. pub async fn commit_snapshot(&self, snapshot: &Snapshot) -> crate::Result { let target_path = self.snapshot_path(snapshot.id()); @@ -256,37 +260,19 @@ impl SnapshotManager { source: Some(Box::new(e)), })?; - // Try rename-based atomic commit first, fall back to check-and-write. - // - // TODO: opendal's rename uses POSIX semantics which silently overwrites the target. - // The exists() check below narrows the race window but does not eliminate it. - // Java Paimon uses `lock.runWithLock(() -> !fileIO.exists(newPath) && callable.call())` - // for full mutual exclusion. We need an external lock mechanism (like Java's Lock - // interface) for backends without atomic rename-no-replace support. let tmp_path = format!("{}.tmp-{}", target_path, uuid::Uuid::new_v4()); let output = self.file_io.new_output(&tmp_path)?; - output.write(bytes::Bytes::from(json.clone())).await?; - - // Check before rename to avoid silent overwrite (opendal uses POSIX rename semantics) - if self.file_io.exists(&target_path).await? { - let _ = self.file_io.delete_file(&tmp_path).await; + let json = bytes::Bytes::from(json); + output.write(json.clone()).await?; + + if !self + .file_io + .publish_if_not_exists(&tmp_path, &target_path, json) + .await? + { return Ok(false); } - match self.file_io.rename(&tmp_path, &target_path).await { - Ok(()) => {} - Err(_) => { - // Rename not supported (e.g. memory/object store). - // Clean up temp file, then check-and-write. - let _ = self.file_io.delete_file(&tmp_path).await; - if self.file_io.exists(&target_path).await? { - return Ok(false); - } - let output = self.file_io.new_output(&target_path)?; - output.write(bytes::Bytes::from(json)).await?; - } - } - // Update LATEST hint (best-effort) let _ = self.write_latest_hint(snapshot.id()).await; Ok(true) @@ -638,6 +624,35 @@ mod tests { assert!(!result); } + #[tokio::test] + async fn test_concurrent_snapshot_publish_has_one_winner() { + let (_, sm) = setup("memory:/test_commit_race").await; + let first = test_snapshot(1); + let second = Snapshot::builder() + .version(3) + .id(1) + .schema_id(0) + .base_manifest_list("other-base-list".to_string()) + .delta_manifest_list("other-delta-list".to_string()) + .commit_user("other-user".to_string()) + .commit_identifier(1) + .commit_kind(CommitKind::APPEND) + .time_millis(1001) + .build(); + + let (first_result, second_result) = + tokio::join!(sm.commit_snapshot(&first), sm.commit_snapshot(&second)); + let first_won = first_result.unwrap(); + let second_won = second_result.unwrap(); + assert_ne!(first_won, second_won); + + let committed = sm.get_snapshot(1).await.unwrap(); + assert!( + (first_won && committed.commit_user() == "test-user") + || (second_won && committed.commit_user() == "other-user") + ); + } + #[tokio::test] async fn test_commit_updates_latest_hint() { let (_, sm) = setup("memory:/test_commit_hint").await; @@ -656,6 +671,13 @@ mod tests { assert_eq!(hint, Some(42)); } + #[tokio::test] + async fn test_latest_hint_at_max_id_does_not_overflow() { + let (_, sm) = setup("memory:/test_latest_hint_max").await; + sm.write_latest_hint(i64::MAX).await.unwrap(); + assert_eq!(sm.get_latest_snapshot_id().await.unwrap(), Some(i64::MAX)); + } + #[tokio::test] async fn test_list_all_ids_empty() { let (_, sm) = setup("memory:/test_list_empty").await; diff --git a/crates/paimon/src/table/source.rs b/crates/paimon/src/table/source.rs index aaaf66cfc..65697ec75 100644 --- a/crates/paimon/src/table/source.rs +++ b/crates/paimon/src/table/source.rs @@ -23,6 +23,71 @@ use crate::spec::{BinaryRow, DataFileMeta, DataFileMetaRowLayout}; use crate::table::stats_filter::group_by_overlapping_row_id; use serde::{Deserialize, Serialize}; use std::sync::Arc; +use url::Url; + +const MAX_RESTORED_FILE_NAME_BYTES: usize = 4 * 1024; + +fn safe_restored_file_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= MAX_RESTORED_FILE_NAME_BYTES + && name != "." + && name != ".." + && !name.contains('/') + && !name.contains('\\') + && !name.contains('\0') +} + +fn url_authority_matches(left: &Url, right: &Url) -> bool { + left.scheme() == right.scheme() + && left.username() == right.username() + && left.password() == right.password() + && left.host_str() == right.host_str() + && left.port_or_known_default() == right.port_or_known_default() +} + +fn path_has_root(root: &str, candidate: &str) -> bool { + match (Url::parse(root), Url::parse(candidate)) { + (Ok(root), Ok(candidate)) => { + if !url_authority_matches(&root, &candidate) + || root.query().is_some() + || root.fragment().is_some() + || candidate.query().is_some() + || candidate.fragment().is_some() + { + return false; + } + path_text_has_root(root.path(), candidate.path()) + } + (Err(_), Err(_)) => path_text_has_root(root, candidate), + _ => false, + } +} + +fn path_text_has_root(root: &str, candidate: &str) -> bool { + fn components(path: &str) -> Option> { + let mut result = Vec::new(); + for component in path.split(['/', '\\']) { + match component { + "" | "." => {} + ".." => return None, + value => result.push(value), + } + } + Some(result) + } + + if root.is_empty() + || candidate.is_empty() + || root.starts_with('/') != candidate.starts_with('/') + || root.starts_with('\\') != candidate.starts_with('\\') + { + return false; + } + let (Some(root), Some(candidate)) = (components(root), components(candidate)) else { + return false; + }; + candidate.len() >= root.len() && candidate[..root.len()] == root +} fn is_vector_store_file_name(file_name: &str) -> bool { file_name.to_ascii_lowercase().contains(".vector.") @@ -574,6 +639,65 @@ impl DataSplit { file.data_file_path(&self.bucket_path) } + /// Validate that a deserialized split cannot escape its table root. + /// + /// Planned splits are trusted Rust objects, but persisted split bytes can + /// cross an FFI/process boundary. Version 1 recovery therefore rejects + /// external data paths and requires every referenced path to remain under + /// the table location. + pub fn validate_restored_containment(&self, table_location: &str) -> crate::Result<()> { + if !path_has_root(table_location, self.bucket_path()) { + return Err(crate::Error::DataInvalid { + message: format!( + "Restored split bucket path '{}' is outside table root '{table_location}'", + self.bucket_path() + ), + source: None, + }); + } + for file in self.data_files() { + if file.external_path.is_some() { + return Err(crate::Error::Unsupported { + message: + "Restored stream plans with external data-file paths are not supported" + .to_string(), + }); + } + if !safe_restored_file_name(&file.file_name) + || file + .extra_files + .iter() + .any(|name| !safe_restored_file_name(name)) + { + return Err(crate::Error::DataInvalid { + message: "Restored stream plan contains an unsafe data-file name".to_string(), + source: None, + }); + } + } + if let Some(deletion_files) = self.data_deletion_files() { + for deletion_file in deletion_files.iter().flatten() { + if deletion_file.offset() < 0 + || deletion_file.length() < 0 + || deletion_file + .offset() + .checked_add(deletion_file.length()) + .is_none() + || !path_has_root(table_location, deletion_file.path()) + { + return Err(crate::Error::DataInvalid { + message: format!( + "Restored deletion file '{}' is invalid or outside table root '{table_location}'", + deletion_file.path() + ), + source: None, + }); + } + } + } + Ok(()) + } + /// Sum of the physical row counts this split knows about. /// /// Files whose count is [`DataFileMeta::ROW_COUNT_UNKNOWN`] contribute @@ -1376,6 +1500,61 @@ mod tests { .unwrap() } + #[test] + fn restored_split_paths_are_confined_to_table() { + let safe = split(vec![file("data.orc", 1, None)], true); + assert!(safe.validate_restored_containment("file:/tmp").is_ok()); + assert!(safe + .validate_restored_containment("file:/tmp-other") + .is_err()); + + let outside_bucket = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path("s3://warehouse/table-evil/bucket-0".to_string()) + .with_total_buckets(1) + .with_data_files(vec![file("data.orc", 1, None)]) + .build() + .unwrap(); + assert!(outside_bucket + .validate_restored_containment("s3://warehouse/table") + .is_err()); + } + + #[test] + fn restored_split_rejects_external_and_unsafe_files() { + let mut external = file("data.orc", 1, None); + external.external_path = Some("file:/etc/passwd".to_string()); + assert!(split(vec![external], true) + .validate_restored_containment("file:/tmp") + .is_err()); + + let unsafe_name = file("../data.orc", 1, None); + assert!(split(vec![unsafe_name], true) + .validate_restored_containment("file:/tmp") + .is_err()); + + let with_outside_deletion = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path("file:/tmp/bucket-0".to_string()) + .with_total_buckets(1) + .with_data_files(vec![file("data.orc", 1, None)]) + .with_data_deletion_files(vec![Some(DeletionFile::new( + "file:/elsewhere/dv.idx".to_string(), + 0, + 8, + Some(1), + ))]) + .build() + .unwrap(); + assert!(with_outside_deletion + .validate_restored_containment("file:/tmp") + .is_err()); + } + #[test] fn data_split_clone_shares_planned_metadata() { let split = DataSplitBuilder::new() diff --git a/crates/paimon/src/table/stream_scan.rs b/crates/paimon/src/table/stream_scan.rs new file mode 100644 index 000000000..d1b8016da --- /dev/null +++ b/crates/paimon/src/table/stream_scan.rs @@ -0,0 +1,685 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Stateful continuous snapshot scan. +//! +//! The cursor has the same meaning as Java `DataTableStreamScan`: planning a +//! snapshot advances `next_snapshot_id` immediately. Callers which hand plans +//! to asynchronous workers must therefore persist their own safe checkpoint +//! only after the planned work has completed. + +use std::collections::HashSet; + +use super::incremental_scan::{IncrementalPlan, IncrementalScanMode, IncrementalSplit}; +use super::partition_filter::PartitionFilter; +use super::table_scan::SnapshotLevelFilter; +use super::{Plan, RowRange, SnapshotManager, Table, TableScan}; +use crate::spec::{ChangelogProducer, CommitKind, Predicate, Snapshot}; + +const FIRST_SNAPSHOT_ID: i64 = 1; +const RANGE_READ_ATTEMPTS: usize = 2; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AvailableRange { + Empty, + Range { earliest: i64, latest: i64 }, + Transient, +} + +/// How a continuous scan chooses its first snapshot. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamScanStartupMode { + /// Read the latest table state in full, then follow later snapshots. + LatestFull, + /// Ignore snapshots which already exist when the scan starts and follow + /// snapshots committed afterwards. + /// + /// When the table has no snapshot at startup, snapshot 1 is consumed as an + /// incremental snapshot once it appears, matching Java Paimon. + Latest, + /// Read `snapshot_id` inclusively as the first incremental snapshot. + FromSnapshot(i64), + /// Read `snapshot_id` as a full table state, then follow later snapshots. + FromSnapshotFull(i64), +} + +/// How snapshots after the startup phase are planned. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamScanFollowUpMode { + /// Use delta manifests for `changelog-producer=none`; otherwise use + /// changelog manifests. + Auto, + /// Read APPEND snapshot delta manifests. + Delta, + /// Read changelog manifests. + Changelog, +} + +/// A plan emitted by a continuous scan. +#[derive(Debug)] +pub enum StreamPlan { + /// A complete table state at one snapshot. + Full { + snapshot_id: i64, + watermark: Option, + next_snapshot_id: i64, + plan: Plan, + }, + /// Delta or changelog work for one snapshot. + Incremental { + snapshot_id: i64, + watermark: Option, + next_snapshot_id: i64, + plan: IncrementalPlan, + }, +} + +impl StreamPlan { + /// Snapshot represented by this plan. + pub fn snapshot_id(&self) -> i64 { + match self { + Self::Full { snapshot_id, .. } | Self::Incremental { snapshot_id, .. } => *snapshot_id, + } + } + + /// Snapshot watermark, when one was committed. + pub fn watermark(&self) -> Option { + match self { + Self::Full { watermark, .. } | Self::Incremental { watermark, .. } => *watermark, + } + } + + /// Cursor immediately after this plan was produced. + pub fn next_snapshot_id(&self) -> i64 { + match self { + Self::Full { + next_snapshot_id, .. + } + | Self::Incremental { + next_snapshot_id, .. + } => *next_snapshot_id, + } + } + + pub fn full_plan(&self) -> Option<&Plan> { + match self { + Self::Full { plan, .. } => Some(plan), + Self::Incremental { .. } => None, + } + } + + pub fn incremental_plan(&self) -> Option<&IncrementalPlan> { + match self { + Self::Incremental { plan, .. } => Some(plan), + Self::Full { .. } => None, + } + } + + pub fn into_full_plan(self) -> Option { + match self { + Self::Full { plan, .. } => Some(plan), + Self::Incremental { .. } => None, + } + } + + pub fn into_incremental_plan(self) -> Option { + match self { + Self::Incremental { plan, .. } => Some(plan), + Self::Full { .. } => None, + } + } +} + +/// Result of one non-blocking continuous-scan poll. +#[derive(Debug)] +pub enum StreamScanPoll { + /// Work is available. + Data(StreamPlan), + /// The next expected snapshot has not been committed yet. + Waiting, + /// The configured bounded scan is complete. + /// + /// Bounded-watermark configuration is not implemented in the first + /// version, so this variant is reserved for forward-compatible consumers. + End, +} + +/// An owned, stateful continuous scanner. +/// +/// The scanner clones the table and all scan-time predicates at construction, +/// so it remains valid after the originating [`Table`] or read builder is +/// dropped. It does not spawn a background task; callers control polling and +/// backpressure. +#[derive(Debug)] +pub struct StreamScan { + table: Table, + snapshot_manager: SnapshotManager, + partition_filter: Option, + data_predicates: Vec, + bucket_predicate: Option, + row_ranges: Option>, + projected_read_field_ids: Option>, + startup_mode: StreamScanStartupMode, + follow_up_mode: IncrementalScanMode, + startup_complete: bool, + next_snapshot_id: Option, + current_watermark: Option, +} + +impl StreamScan { + #[allow(clippy::too_many_arguments)] + pub(crate) fn try_new( + table: Table, + partition_filter: Option, + data_predicates: Vec, + bucket_predicate: Option, + row_ranges: Option>, + projected_read_field_ids: Option>, + startup_mode: StreamScanStartupMode, + follow_up_mode: StreamScanFollowUpMode, + ) -> crate::Result { + match startup_mode { + StreamScanStartupMode::FromSnapshot(snapshot_id) + | StreamScanStartupMode::FromSnapshotFull(snapshot_id) + if snapshot_id < FIRST_SNAPSHOT_ID => + { + return Err(crate::Error::DataInvalid { + message: format!( + "Stream scan starting snapshot id must be at least {FIRST_SNAPSHOT_ID}, got {snapshot_id}" + ), + source: None, + }); + } + _ => {} + } + + if table.is_format_table() { + return Err(crate::Error::Unsupported { + message: "Continuous stream scan is not supported for format tables".to_string(), + }); + } + + let core_options = table.schema().core_options(); + let changelog_producer = core_options.try_changelog_producer()?; + let deletion_vectors_enabled = core_options.deletion_vectors_enabled(); + let follow_up_mode = match (follow_up_mode, changelog_producer) { + (StreamScanFollowUpMode::Delta, ChangelogProducer::Lookup) + if deletion_vectors_enabled + && matches!( + startup_mode, + StreamScanStartupMode::LatestFull + | StreamScanStartupMode::FromSnapshotFull(_) + ) => + { + return Err(crate::Error::Unsupported { + message: "Deletion-vector lookup tables require changelog follow-up" + .to_string(), + }); + } + (StreamScanFollowUpMode::Delta, _) => IncrementalScanMode::Delta, + (StreamScanFollowUpMode::Changelog, ChangelogProducer::None) => { + return Err(crate::Error::Unsupported { + message: "Changelog stream follow-up requires a changelog producer".to_string(), + }); + } + (StreamScanFollowUpMode::Changelog, _) => IncrementalScanMode::Changelog, + (StreamScanFollowUpMode::Auto, ChangelogProducer::None) => IncrementalScanMode::Delta, + (StreamScanFollowUpMode::Auto, _) => IncrementalScanMode::Changelog, + }; + let snapshot_manager = table.snapshot_manager(); + + Ok(Self { + table, + snapshot_manager, + partition_filter, + data_predicates, + bucket_predicate, + row_ranges, + projected_read_field_ids, + startup_mode, + follow_up_mode, + startup_complete: false, + next_snapshot_id: None, + current_watermark: None, + }) + } + + /// Resolved follow-up mode. `Auto` is collapsed during construction. + pub fn follow_up_mode(&self) -> IncrementalScanMode { + self.follow_up_mode + } + + /// The next snapshot which will be considered, suitable for checkpointing. + /// + /// This cursor advances when a plan is produced, not when its splits finish. + pub fn checkpoint(&self) -> Option { + self.next_snapshot_id + } + + /// Restore a previously checkpointed next snapshot id. + /// + /// `Some(id)` bypasses startup selection. `None` resets the scanner and + /// applies its configured startup mode again. + pub fn restore(&mut self, next_snapshot_id: Option) -> crate::Result<()> { + if next_snapshot_id.is_some_and(|id| id < FIRST_SNAPSHOT_ID) { + return Err(crate::Error::DataInvalid { + message: format!( + "Stream scan checkpoint must be at least {FIRST_SNAPSHOT_ID}, got {}", + next_snapshot_id.unwrap() + ), + source: None, + }); + } + self.next_snapshot_id = next_snapshot_id; + self.startup_complete = next_snapshot_id.is_some(); + self.current_watermark = None; + Ok(()) + } + + /// Most recent watermark observed on a planned (including empty) snapshot. + pub fn watermark(&self) -> Option { + self.current_watermark + } + + /// Freeze startup state which is observable at scanner creation time. + /// + /// `Latest` must remember whether the table was empty when the source was + /// created. Without this explicit async initialization, a snapshot + /// committed between construction and the first poll could be mistaken for + /// pre-existing data and skipped. Other startup modes resolve their first + /// plan during polling and need no eager IO. + pub async fn initialize(&mut self) -> crate::Result<()> { + if self.startup_complete || self.startup_mode != StreamScanStartupMode::Latest { + return Ok(()); + } + self.table + .schema() + .core_options() + .ensure_read_authorized()?; + let next_snapshot_id = match self.snapshot_manager.get_latest_snapshot_id().await? { + Some(latest) => next_id(latest)?, + None => FIRST_SNAPSHOT_ID, + }; + self.next_snapshot_id = Some(next_snapshot_id); + self.startup_complete = true; + Ok(()) + } + + /// Poll once without waiting for future snapshots. + pub async fn poll_next(&mut self) -> crate::Result { + self.table + .schema() + .core_options() + .ensure_read_authorized()?; + self.initialize().await?; + if !self.startup_complete { + return self.poll_startup().await; + } + self.poll_follow_up().await + } + + fn table_scan(&self) -> TableScan<'_> { + TableScan::new( + &self.table, + self.partition_filter.clone(), + self.data_predicates.clone(), + self.bucket_predicate.clone(), + None, + self.row_ranges.clone(), + ) + .with_projected_read_field_ids(self.projected_read_field_ids.clone()) + } + + async fn poll_startup(&mut self) -> crate::Result { + match self.startup_mode { + StreamScanStartupMode::LatestFull => { + let Some(snapshot) = self.snapshot_manager.get_latest_snapshot().await? else { + return Ok(StreamScanPoll::Waiting); + }; + self.plan_full_startup(snapshot).await + } + StreamScanStartupMode::Latest => { + unreachable!("Latest startup is resolved by initialize") + } + StreamScanStartupMode::FromSnapshot(snapshot_id) => { + let (earliest, latest) = match self.available_range().await? { + AvailableRange::Empty | AvailableRange::Transient => { + return Ok(StreamScanPoll::Waiting) + } + AvailableRange::Range { earliest, latest } => (earliest, latest), + }; + validate_incremental_start(snapshot_id, earliest, latest)?; + self.next_snapshot_id = Some(snapshot_id); + self.startup_complete = true; + Ok(StreamScanPoll::Waiting) + } + StreamScanStartupMode::FromSnapshotFull(snapshot_id) => { + let (earliest, latest) = match self.available_range().await? { + AvailableRange::Empty | AvailableRange::Transient => { + return Ok(StreamScanPoll::Waiting) + } + AvailableRange::Range { earliest, latest } => (earliest, latest), + }; + validate_full_start(snapshot_id, earliest, latest)?; + let Some(snapshot) = self.try_get_snapshot(snapshot_id).await? else { + return Ok(StreamScanPoll::Waiting); + }; + self.plan_full_startup(snapshot).await + } + } + } + + async fn plan_full_startup(&mut self, snapshot: Snapshot) -> crate::Result { + // Reading a full state at an overwrite snapshot is well-defined. The + // overwrite restriction applies only to follow-up change plans. + let snapshot_id = snapshot.id(); + let watermark = snapshot.watermark(); + let next_snapshot_id = self.full_start_next_snapshot_id(snapshot_id)?; + let level_filter = self.full_start_level_filter()?; + let plan = self + .table_scan() + .plan_snapshot_full(&snapshot, level_filter) + .await?; + self.next_snapshot_id = Some(next_snapshot_id); + self.current_watermark = watermark; + self.startup_complete = true; + Ok(StreamScanPoll::Data(StreamPlan::Full { + snapshot_id, + watermark, + next_snapshot_id, + plan, + })) + } + + fn full_start_level_filter(&self) -> crate::Result> { + let options = self.table.schema().core_options(); + // Lookup-style tables expose their stable materialized state above + // level 0. Deletion-vector-only tables replay the starting snapshot in + // the incremental phase so its un-compacted level-0 changes are not + // lost. + if options.deletion_vectors_enabled() { + return Ok(Some(SnapshotLevelFilter::GreaterThan(0))); + } + if self.follow_up_mode != IncrementalScanMode::Changelog { + return Ok(None); + } + match options.try_changelog_producer()? { + // Lookup compaction will emit level-0 input through a later + // changelog. Reading it in the full phase would emit it twice. + ChangelogProducer::Lookup => Ok(Some(SnapshotLevelFilter::GreaterThan(0))), + // Full-compaction changelog covers all changes since the previous + // last-level state. Start from that materialized state only. + ChangelogProducer::FullCompaction => { + Ok(Some(SnapshotLevelFilter::Equal(options.num_levels()? - 1))) + } + ChangelogProducer::None | ChangelogProducer::Input => Ok(None), + } + } + + fn full_start_next_snapshot_id(&self, snapshot_id: i64) -> crate::Result { + let options = self.table.schema().core_options(); + if options.deletion_vectors_enabled() + && options.try_changelog_producer()? != ChangelogProducer::Lookup + { + // The full plan deliberately excludes level 0. Revisit this same + // snapshot once through Delta/Changelog to emit those changes. + Ok(snapshot_id) + } else { + next_id(snapshot_id) + } + } + + async fn poll_follow_up(&mut self) -> crate::Result { + loop { + let snapshot_id = + self.next_snapshot_id + .ok_or_else(|| crate::Error::UnexpectedError { + message: "Stream scan startup completed without a next snapshot id" + .to_string(), + source: None, + })?; + let Some(snapshot) = self.next_snapshot(snapshot_id).await? else { + return Ok(StreamScanPoll::Waiting); + }; + + if snapshot.commit_kind() == &CommitKind::OVERWRITE { + return Err(crate::Error::Unsupported { + message: format!( + "Streaming follow-up scan cannot safely consume OVERWRITE snapshot {snapshot_id}" + ), + }); + } + + let should_scan = match self.follow_up_mode { + IncrementalScanMode::Delta => snapshot.commit_kind() == &CommitKind::APPEND, + IncrementalScanMode::Changelog => snapshot.changelog_manifest_list().is_some(), + IncrementalScanMode::Auto | IncrementalScanMode::Diff => { + unreachable!("stream follow-up mode must resolve to Delta or Changelog") + } + }; + + let following_snapshot_id = next_id(snapshot_id)?; + if !should_scan { + self.next_snapshot_id = Some(following_snapshot_id); + continue; + } + + let raw_plan = match self.follow_up_mode { + IncrementalScanMode::Delta => { + self.table_scan() + .plan_snapshot_delta_streaming(&snapshot) + .await? + } + IncrementalScanMode::Changelog => { + self.table_scan() + .plan_snapshot_changelog_streaming(&snapshot) + .await? + } + IncrementalScanMode::Auto | IncrementalScanMode::Diff => unreachable!(), + }; + let splits = raw_plan + .into_splits() + .into_iter() + .map(IncrementalSplit::Data) + .collect(); + let plan = IncrementalPlan::try_new(self.follow_up_mode, splits)?; + + // Match Java DataTableStreamScan: the checkpoint cursor advances as + // soon as planning succeeds. Empty snapshots also advance and the + // same poll keeps looking for useful work. + self.next_snapshot_id = Some(following_snapshot_id); + self.current_watermark = snapshot.watermark(); + if plan.splits().is_empty() { + continue; + } + + return Ok(StreamScanPoll::Data(StreamPlan::Incremental { + snapshot_id, + watermark: snapshot.watermark(), + next_snapshot_id: following_snapshot_id, + plan, + })); + } + } + + async fn next_snapshot(&mut self, snapshot_id: i64) -> crate::Result> { + if let Some(snapshot) = self.try_get_snapshot(snapshot_id).await? { + return Ok(Some(snapshot)); + } + + // The snapshot may be committed after the first lookup but before the + // range observation. Re-read it before classifying the miss as a gap. + let range = self.available_range().await?; + if let Some(snapshot) = self.try_get_snapshot(snapshot_id).await? { + return Ok(Some(snapshot)); + } + + match range { + AvailableRange::Transient => Ok(None), + AvailableRange::Empty if snapshot_id == FIRST_SNAPSHOT_ID => Ok(None), + AvailableRange::Empty => Err(crate::Error::DataInvalid { + message: format!( + "Next expected snapshot {snapshot_id} is out of range because the table currently has no snapshots" + ), + source: None, + }), + AvailableRange::Range { earliest, latest } if snapshot_id < earliest => { + Err(crate::Error::DataInvalid { + message: format!( + "Next expected snapshot {snapshot_id} has expired; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }) + } + // A range hint/listing can become visible before the snapshot + // object itself on an eventually consistent backend. Polling + // frequency must never turn that transient state into data loss. + AvailableRange::Range { latest, .. } if snapshot_id <= latest => Ok(None), + AvailableRange::Range { latest, .. } + if latest.checked_add(1) == Some(snapshot_id) => + { + Ok(None) + } + AvailableRange::Range { earliest, latest } => Err(crate::Error::DataInvalid { + message: format!( + "Next expected snapshot {snapshot_id} is too large; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }), + } + } + + async fn try_get_snapshot(&self, snapshot_id: i64) -> crate::Result> { + match self.snapshot_manager.get_snapshot(snapshot_id).await { + Ok(snapshot) => Ok(Some(snapshot)), + Err(crate::Error::SnapshotNotExist { + snapshot_id: missing, + }) if missing == snapshot_id => Ok(None), + Err(error) => Err(error), + } + } + + async fn available_range(&mut self) -> crate::Result { + let mut last_observation = (None, None); + for _ in 0..RANGE_READ_ATTEMPTS { + let earliest = self.snapshot_manager.earliest_snapshot_id().await?; + let latest = self.snapshot_manager.get_latest_snapshot_id().await?; + match (earliest, latest) { + (None, None) => { + return Ok(AvailableRange::Empty); + } + (Some(earliest), Some(latest)) if earliest <= latest => { + return Ok(AvailableRange::Range { earliest, latest }); + } + observation => last_observation = observation, + } + } + + let _ = last_observation; + Ok(AvailableRange::Transient) + } +} + +fn next_id(snapshot_id: i64) -> crate::Result { + snapshot_id + .checked_add(1) + .ok_or_else(|| crate::Error::DataInvalid { + message: format!("Snapshot id {snapshot_id} cannot be advanced"), + source: None, + }) +} + +fn validate_incremental_start(snapshot_id: i64, earliest: i64, latest: i64) -> crate::Result<()> { + if snapshot_id < earliest { + return Err(crate::Error::DataInvalid { + message: format!( + "Stream starting snapshot {snapshot_id} has expired; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }); + } + if snapshot_id > latest.saturating_add(1) { + return Err(crate::Error::DataInvalid { + message: format!( + "Stream starting snapshot {snapshot_id} is too large; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }); + } + Ok(()) +} + +fn validate_full_start(snapshot_id: i64, earliest: i64, latest: i64) -> crate::Result<()> { + if snapshot_id < earliest { + return Err(crate::Error::DataInvalid { + message: format!( + "Full stream starting snapshot {snapshot_id} has expired; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }); + } + if snapshot_id > latest { + return Err(crate::Error::SnapshotNotExist { snapshot_id }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + validate_full_start, validate_incremental_start, StreamPlan, StreamScanFollowUpMode, + StreamScanPoll, StreamScanStartupMode, + }; + + #[test] + fn start_range_validation_is_explicit() { + assert!(validate_incremental_start(4, 4, 6).is_ok()); + assert!(validate_incremental_start(7, 4, 6).is_ok()); + assert!(matches!( + validate_incremental_start(3, 4, 6), + Err(crate::Error::DataInvalid { .. }) + )); + assert!(matches!( + validate_incremental_start(8, 4, 6), + Err(crate::Error::DataInvalid { .. }) + )); + assert!(matches!( + validate_full_start(7, 4, 6), + Err(crate::Error::SnapshotNotExist { snapshot_id: 7 }) + )); + } + + #[test] + fn public_modes_and_poll_are_matchable() { + let _ = [ + StreamScanStartupMode::LatestFull, + StreamScanStartupMode::Latest, + StreamScanStartupMode::FromSnapshot(1), + StreamScanStartupMode::FromSnapshotFull(1), + ]; + let _ = [ + StreamScanFollowUpMode::Auto, + StreamScanFollowUpMode::Delta, + StreamScanFollowUpMode::Changelog, + ]; + let waiting = StreamScanPoll::Waiting; + assert!(matches!(waiting, StreamScanPoll::Waiting)); + let end = StreamScanPoll::End; + assert!(matches!(end, StreamScanPoll::End)); + let _: Option = None; + } +} diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index 9f3f707d1..72ca64840 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -40,7 +40,7 @@ use crate::Result; use apache_avro::{to_value, Schema}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; /// Batch commit identifier (i64::MAX), same as Python's BATCH_COMMIT_IDENTIFIER. const BATCH_COMMIT_IDENTIFIER: i64 = i64::MAX; @@ -48,10 +48,50 @@ const BATCH_COMMIT_IDENTIFIER: i64 = i64::MAX; const CHECK_ROLLING_RECORD_COUNT: usize = 1000; const DELETION_VECTORS_INDEX_TYPE: &str = "DELETION_VECTORS"; +fn checked_next_snapshot_id(snapshot_id: i64) -> Result { + snapshot_id + .checked_add(1) + .ok_or_else(|| crate::Error::DataInvalid { + message: format!("Snapshot id {snapshot_id} cannot be incremented"), + source: None, + }) +} + +fn validate_commit_identifier(commit_identifier: i64) -> Result<()> { + if commit_identifier < 0 { + return Err(crate::Error::DataInvalid { + message: format!( + "Streaming commit identifier must be non-negative, got {commit_identifier}" + ), + source: None, + }); + } + Ok(()) +} + +fn validate_streaming_commit_identifier(commit_identifier: i64) -> Result<()> { + validate_commit_identifier(commit_identifier)?; + if commit_identifier == BATCH_COMMIT_IDENTIFIER { + return Err(crate::Error::DataInvalid { + message: format!( + "Streaming commit identifier {BATCH_COMMIT_IDENTIFIER} is reserved for batch commits" + ), + source: None, + }); + } + Ok(()) +} + type PartitionBucketKey = (Vec, i32); type RowIdRange = (i64, i64); type ExistingRowIdRanges = HashMap>; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum IdentifierCommitStatus { + Known(bool), + HistoryTruncated { earliest_snapshot_id: i64 }, +} + fn validate_bucket_ownership(messages: &[CommitMessage]) -> Result<()> { let mut owners = HashSet::new(); for message in messages { @@ -164,13 +204,14 @@ impl TableCommit { /// Commit new files in APPEND mode. pub async fn commit(&self, commit_messages: Vec) -> Result<()> { - self.commit_with_identifier(commit_messages, BATCH_COMMIT_IDENTIFIER) + self.commit_with_identifier_impl(commit_messages, BATCH_COMMIT_IDENTIFIER, false) .await } /// Commit new files with a caller-provided commit identifier. /// /// Identifiers must increase monotonically for a given `commit_user`. + /// `i64::MAX` is reserved for unidentified batch commits. /// All messages for one identifier must be submitted in a single call. /// This method does not filter previously committed identifiers. Use /// [`Self::filter_and_commit_with_identifier`] when retrying an uncertain @@ -180,20 +221,23 @@ impl TableCommit { commit_messages: Vec, commit_identifier: i64, ) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.commit_with_identifier_impl(commit_messages, commit_identifier, false) .await } /// Filter a previously committed identifier, then commit if it is new. /// - /// Identifiers must increase monotonically for a given `commit_user`. This - /// method is intended for retrying the same uncertain result; regular + /// Identifiers must increase monotonically for a given `commit_user`. + /// `i64::MAX` is reserved for unidentified batch commits. + /// This method is intended for retrying the same uncertain result; regular /// commits should use [`Self::commit_with_identifier`]. pub async fn filter_and_commit_with_identifier( &self, commit_messages: Vec, commit_identifier: i64, ) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.commit_with_identifier_impl(commit_messages, commit_identifier, true) .await } @@ -204,6 +248,7 @@ impl TableCommit { commit_identifier: i64, filter_committed: bool, ) -> Result<()> { + validate_commit_identifier(commit_identifier)?; // A commit validates against the existing snapshot. CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; @@ -325,6 +370,7 @@ impl TableCommit { static_partitions: Option>>, commit_identifier: i64, ) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.overwrite_impl(commit_messages, static_partitions, commit_identifier, true) .await } @@ -336,6 +382,7 @@ impl TableCommit { commit_identifier: i64, filter_committed: bool, ) -> Result<()> { + validate_commit_identifier(commit_identifier)?; // A commit validates against the existing snapshot. CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; @@ -582,6 +629,7 @@ impl TableCommit { partitions: Vec>>, commit_identifier: i64, ) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.truncate_partitions_impl(partitions, commit_identifier, true) .await } @@ -592,6 +640,7 @@ impl TableCommit { commit_identifier: i64, filter_committed: bool, ) -> Result<()> { + validate_commit_identifier(commit_identifier)?; // A commit validates against the existing snapshot. CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; @@ -665,6 +714,7 @@ impl TableCommit { /// A previously committed identifier is filtered so retrying an uncertain /// result cannot delete data committed in between. pub async fn truncate_table_with_identifier(&self, commit_identifier: i64) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.truncate_table_impl(commit_identifier, true).await } @@ -673,6 +723,7 @@ impl TableCommit { commit_identifier: i64, filter_committed: bool, ) -> Result<()> { + validate_commit_identifier(commit_identifier)?; // A commit validates against the existing snapshot. CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; @@ -695,6 +746,50 @@ impl TableCommit { .await } + /// Return whether this commit user has already published `commit_identifier` + /// or a later identifier. + /// + /// Streaming identifiers are monotonically increasing, so a later snapshot + /// also proves that an older checkpoint must not be committed or aborted. + pub async fn is_identifier_committed(&self, commit_identifier: i64) -> Result { + validate_streaming_commit_identifier(commit_identifier)?; + CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; + self.table.ensure_not_branch_reference_for_write()?; + let latest_snapshot = self.snapshot_manager.get_latest_snapshot().await?; + match self + .commit_identifier_status(&latest_snapshot, commit_identifier) + .await? + { + IdentifierCommitStatus::Known(committed) => Ok(committed), + IdentifierCommitStatus::HistoryTruncated { + earliest_snapshot_id, + } => Err(crate::Error::DataInvalid { + message: format!( + "Commit identifier {commit_identifier} is indeterminate because retained snapshot history starts at {earliest_snapshot_id}; the required snapshot history is out of range" + ), + source: None, + }), + } + } + + /// Abort prepared files only if their streaming identifier has not already + /// been published. + /// + /// This makes recovery after a lost commit acknowledgement safe: retrying + /// or accidentally aborting a committed prepared checkpoint is a no-op. + /// The caller must still serialize commit and abort operations for one + /// `commit_user` so a new commit cannot race this check. + pub async fn abort_if_uncommitted( + &self, + commit_messages: &[CommitMessage], + commit_identifier: i64, + ) -> Result<()> { + if self.is_identifier_committed(commit_identifier).await? { + return Ok(()); + } + self.abort(commit_messages).await + } + /// Abort a prepared commit by deleting newly written data, changelog and index files. /// /// Deletion is best-effort and mirrors Python `FileStoreCommit.abort`: missing @@ -778,12 +873,14 @@ impl TableCommit { let mut retry_count = 0u32; let mut duplicate_check_start_snapshot_id: Option = None; let mut retry_state: Option> = None; - let start_time_ms = current_time_millis(); + let start_time = Instant::now(); // An identified destructive no-op must still record its identifier. // Otherwise a retry after an intervening write can execute the operation // for the first time and delete data which was not present originally. let commit_empty_overwrite = filter_committed && plan.commit_kind_hint() == CommitKind::OVERWRITE; + let enforce_monotonic_identifier = + !filter_committed && commit_identifier != BATCH_COMMIT_IDENTIFIER; let mut filter_committed = filter_committed; loop { @@ -810,6 +907,19 @@ impl TableCommit { break; } } + if enforce_monotonic_identifier + && self + .is_committed_identifier(&latest_snapshot, commit_identifier) + .await? + { + return Err(crate::Error::DataInvalid { + message: format!( + "Commit identifier {commit_identifier} is not greater than the latest identifier retained for commit_user '{}'", + self.commit_user + ), + source: None, + }); + } validate_expected_latest_snapshot(expected_snapshot_id, &latest_snapshot)?; let resolved = self .resolve_commit(&mut plan, &latest_snapshot, retry_state.as_deref()) @@ -830,14 +940,17 @@ impl TableCommit { match result { CommitAttemptResult::Success => break, CommitAttemptResult::Retry(state) => { - duplicate_check_start_snapshot_id.get_or_insert_with(|| { - latest_snapshot.as_ref().map(|s| s.id() + 1).unwrap_or(1) - }); + if duplicate_check_start_snapshot_id.is_none() { + duplicate_check_start_snapshot_id = Some(match &latest_snapshot { + Some(snapshot) => checked_next_snapshot_id(snapshot.id())?, + None => 1, + }); + } retry_state = Some(state); } } - let elapsed_ms = current_time_millis() - start_time_ms; + let elapsed_ms = u64::try_from(start_time.elapsed().as_millis()).unwrap_or(u64::MAX); if elapsed_ms > self.commit_timeout_ms || retry_count >= self.commit_max_retries { let snap_id = duplicate_check_start_snapshot_id.unwrap_or(1); return Err(crate::Error::DataInvalid { @@ -864,7 +977,10 @@ impl TableCommit { latest_snapshot: &Option, commit_identifier: i64, ) -> Result { - let new_snapshot_id = latest_snapshot.as_ref().map(|s| s.id() + 1).unwrap_or(1); + let new_snapshot_id = match latest_snapshot { + Some(snapshot) => checked_next_snapshot_id(snapshot.id())?, + None => 1, + }; // Row tracking let mut next_row_id: Option = None; @@ -1274,8 +1390,26 @@ impl TableCommit { latest_snapshot: &Option, commit_identifier: i64, ) -> Result { + match self + .commit_identifier_status(latest_snapshot, commit_identifier) + .await? + { + IdentifierCommitStatus::Known(committed) => Ok(committed), + // A fresh, globally unique commit_user must be able to begin on a + // table whose early snapshots have expired. Retry safety across + // that retention boundary cannot be proven; abort uses the public + // fail-closed path above instead. + IdentifierCommitStatus::HistoryTruncated { .. } => Ok(false), + } + } + + async fn commit_identifier_status( + &self, + latest_snapshot: &Option, + commit_identifier: i64, + ) -> Result { let Some(latest) = latest_snapshot else { - return Ok(false); + return Ok(IdentifierCommitStatus::Known(false)); }; let earliest_snapshot_id = self .snapshot_manager @@ -1288,11 +1422,19 @@ impl TableCommit { } else { self.snapshot_manager.get_snapshot(snapshot_id).await? }; - if snapshot.commit_user() == self.commit_user { - return Ok(commit_identifier <= snapshot.commit_identifier()); + if snapshot.commit_user() == self.commit_user + && snapshot.commit_identifier() != BATCH_COMMIT_IDENTIFIER + && commit_identifier <= snapshot.commit_identifier() + { + return Ok(IdentifierCommitStatus::Known(true)); } } - Ok(false) + if earliest_snapshot_id > 1 { + return Ok(IdentifierCommitStatus::HistoryTruncated { + earliest_snapshot_id, + }); + } + Ok(IdentifierCommitStatus::Known(false)) } /// Check if this commit was already completed during an in-process retry. @@ -1792,7 +1934,13 @@ impl TableCommit { return Ok(false); }; - for snapshot_id in cached_snapshot.id() + 1..=latest_snapshot.id() { + if cached_snapshot.id() > latest_snapshot.id() { + return Ok(false); + } + if cached_snapshot.id() == latest_snapshot.id() { + return Ok(true); + } + for snapshot_id in checked_next_snapshot_id(cached_snapshot.id())?..=latest_snapshot.id() { *delta_probe_count += 1; let snapshot = match self.snapshot_manager.get_snapshot(snapshot_id).await { Ok(snapshot) => snapshot, @@ -1897,7 +2045,10 @@ impl TableCommit { let entry_refs = commit_entries.iter().collect::>(); let partition_filter = self.build_entries_partition_filter(&entry_refs)?; let mut entries = Vec::new(); - for snapshot_id in from_snapshot.id() + 1..=to_snapshot.id() { + if from_snapshot.id() >= to_snapshot.id() { + return Ok(Some(entries)); + } + for snapshot_id in checked_next_snapshot_id(from_snapshot.id())?..=to_snapshot.id() { let snapshot = match self.snapshot_manager.get_snapshot(snapshot_id).await { Ok(snapshot) => snapshot, Err(_) => return Ok(None), @@ -2037,7 +2188,11 @@ impl TableCommit { .collect::>(); let partition_filter = self.build_entries_partition_filter(&fixed_entries)?; - for snapshot_id in check_from_snapshot.max(0) + 1..=latest_snapshot.id() { + let check_from_snapshot = check_from_snapshot.max(0); + if check_from_snapshot >= latest_snapshot.id() { + return Ok(()); + } + for snapshot_id in checked_next_snapshot_id(check_from_snapshot)?..=latest_snapshot.id() { let snapshot = self.snapshot_manager.get_snapshot(snapshot_id).await?; let concurrent_entries = self .read_delta_entries(partition_filter.as_ref(), &snapshot) @@ -2351,7 +2506,10 @@ impl TableCommit { let delta_entry_refs = delta_entries.iter().collect::>(); let partition_filter = self.build_entries_partition_filter(&delta_entry_refs)?; - for snapshot_id in check_from_snapshot + 1..=latest_snapshot.id() { + if check_from_snapshot >= latest_snapshot.id() { + return Ok(()); + } + for snapshot_id in checked_next_snapshot_id(check_from_snapshot)?..=latest_snapshot.id() { let snapshot = self.snapshot_manager.get_snapshot(snapshot_id).await?; if snapshot.commit_kind() == &CommitKind::COMPACT { continue; @@ -3159,6 +3317,44 @@ fn rand_f64() -> f64 { mod tests { use super::*; + #[test] + fn test_snapshot_successor_rejects_overflow() { + assert_eq!(checked_next_snapshot_id(1).unwrap(), 2); + assert!(checked_next_snapshot_id(i64::MAX).is_err()); + } + + #[tokio::test] + async fn test_invalid_streaming_identifiers_are_rejected_even_for_noops() { + let file_io = test_file_io(); + let table_path = "memory:/test_negative_commit_identifier"; + setup_dirs(&file_io, table_path).await; + let commit = setup_commit(&file_io, table_path); + + for invalid in [-1, BATCH_COMMIT_IDENTIFIER] { + assert!(commit + .commit_with_identifier(Vec::new(), invalid) + .await + .is_err()); + assert!(commit + .filter_and_commit_with_identifier(Vec::new(), invalid) + .await + .is_err()); + assert!(commit + .overwrite_with_identifier(Vec::new(), None, invalid) + .await + .is_err()); + assert!(commit + .truncate_partitions_with_identifier(Vec::new(), invalid) + .await + .is_err()); + assert!(commit + .truncate_table_with_identifier(invalid) + .await + .is_err()); + } + assert!(latest_snapshot(&file_io, table_path).await.is_none()); + } + #[tokio::test] async fn abort_still_cleans_up_for_a_query_auth_table() { let table = crate::table::query_auth_table(); @@ -3617,6 +3813,84 @@ mod tests { ); } + #[tokio::test] + async fn test_non_filtering_identifiers_must_increase() { + let file_io = test_file_io(); + let table_path = "memory:/test_monotonic_commit_identifier"; + setup_dirs(&file_io, table_path).await; + let commit = setup_commit(&file_io, table_path); + + commit + .commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file("data-7.parquet", 100)], + )], + 7, + ) + .await + .unwrap(); + for identifier in [7, 6] { + let error = commit + .commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file( + &format!("invalid-{identifier}.parquet"), + 100, + )], + )], + identifier, + ) + .await + .expect_err("non-filtering identifiers must increase"); + assert!(error.to_string().contains("not greater")); + } + commit + .commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file("data-8.parquet", 100)], + )], + 8, + ) + .await + .unwrap(); + + let latest = latest_snapshot(&file_io, table_path).await.unwrap(); + assert_eq!(latest.id(), 2); + assert_eq!(latest.commit_identifier(), 8); + } + + #[tokio::test] + async fn test_identifier_lookup_checks_all_retained_snapshots() { + let file_io = test_file_io(); + let table_path = "memory:/test_legacy_out_of_order_identifiers"; + setup_dirs(&file_io, table_path).await; + let snapshot_manager = SnapshotManager::new(file_io.clone(), table_path.to_string()); + for (snapshot_id, commit_identifier) in [(1, 10), (2, 5)] { + let snapshot = Snapshot::builder() + .version(3) + .id(snapshot_id) + .schema_id(0) + .base_manifest_list("base-list".to_string()) + .delta_manifest_list("delta-list".to_string()) + .commit_user("test-user".to_string()) + .commit_identifier(commit_identifier) + .commit_kind(CommitKind::APPEND) + .time_millis(snapshot_id as u64) + .build(); + assert!(snapshot_manager.commit_snapshot(&snapshot).await.unwrap()); + } + + let commit = setup_commit(&file_io, table_path); + assert!(commit.is_identifier_committed(7).await.unwrap()); + assert!(!commit.is_identifier_committed(11).await.unwrap()); + } + #[tokio::test] async fn test_filter_and_commit_rejects_expired_older_identifier() { let file_io = test_file_io(); @@ -3650,6 +3924,61 @@ mod tests { ); } + #[tokio::test] + async fn test_abort_fails_closed_when_commit_history_is_truncated() { + let file_io = test_file_io(); + let table_path = "memory:/test_abort_truncated_identifier_history"; + setup_dirs(&file_io, table_path).await; + + let commit = setup_commit(&file_io, table_path); + let first = CommitMessage::new( + vec![], + 0, + vec![test_data_file("possibly-live.parquet", 100)], + ); + commit + .commit_with_identifier(vec![first.clone()], 1) + .await + .unwrap(); + let other_commit = TableCommit::new(test_table(&file_io, table_path), "other-user".into()); + other_commit + .commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file("other.parquet", 100)], + )], + 1, + ) + .await + .unwrap(); + + let snapshot_manager = SnapshotManager::new(file_io.clone(), table_path.to_string()); + snapshot_manager.delete_snapshot(1).await.unwrap(); + let error = commit + .abort_if_uncommitted(&[first], 1) + .await + .expect_err("truncated history cannot prove that abort is safe"); + assert!(error.to_string().contains("indeterminate")); + assert!(error.to_string().contains("out of range")); + + let new_job = TableCommit::new(test_table(&file_io, table_path), "brand-new-user".into()); + new_job + .filter_and_commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file("new-job.parquet", 100)], + )], + 1, + ) + .await + .expect("a new commit_user must be able to start after history truncation"); + let latest = latest_snapshot(&file_io, table_path).await.unwrap(); + assert_eq!(latest.id(), 3); + assert_eq!(latest.commit_user(), "brand-new-user"); + } + #[tokio::test] async fn test_overwrite_retry_preserves_intervening_commit() { let file_io = test_file_io(); diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index d5de9048a..10230e266 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -61,6 +61,22 @@ const MANIFEST_DIR: &str = "manifest"; /// Path segment for index directory under table. const DELETION_VECTORS_INDEX_TYPE: &str = "DELETION_VECTORS"; +/// Additional file-level restriction used by streaming full-start scans. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SnapshotLevelFilter { + GreaterThan(i32), + Equal(i32), +} + +impl SnapshotLevelFilter { + fn matches(self, level: i32) -> bool { + match self { + Self::GreaterThan(bound) => level > bound, + Self::Equal(expected) => level == expected, + } + } +} + #[derive(Debug, Default)] struct ManifestReadCounters { entries_read: usize, @@ -118,6 +134,7 @@ async fn read_all_manifest_entries( table_path: &str, snapshot: &Snapshot, skip_level_zero: bool, + level_filter: Option, scan_all_files: bool, has_primary_keys: bool, partition_filter: Option<&PartitionFilter>, @@ -237,7 +254,9 @@ async fn read_all_manifest_entries( // Post-filter: level-0 and data predicates (need DataFileMeta) let mut filtered = Vec::with_capacity(entries.len()); for entry in entries { - if skip_level_zero && has_primary_keys && entry.file().level == 0 { + if (skip_level_zero && has_primary_keys && entry.file().level == 0) + || level_filter.is_some_and(|filter| !filter.matches(entry.file().level)) + { counters.pruned_by_level += 1; continue; } @@ -538,6 +557,14 @@ fn merge_manifest_entries(mut entries: Vec) -> Vec entries } +/// Keep change files without netting DELETE against ADD. A stream/batch +/// incremental manifest describes events, so a rewrite's ADD must survive even +/// when the same identity also appears as DELETE in that manifest. +fn retain_incremental_add_entries(mut entries: Vec) -> Vec { + entries.retain(|entry| *entry.kind() == FileKind::Add); + entries +} + /// Whether scan-owned pruning still preserves `merged_row_count()` as a safe /// row-count hint. /// @@ -968,6 +995,19 @@ impl<'a> TableScan<'a> { } } + /// Plan stream changes without attaching post-commit index/DV state. + pub(crate) async fn plan_snapshot_delta_streaming( + &self, + snapshot: &Snapshot, + ) -> crate::Result { + match &self.0 { + TableScanKind::Paimon(scan) => scan.plan_snapshot_delta_streaming(snapshot).await, + TableScanKind::Format(_) => Err(crate::Error::Unsupported { + message: "Format tables do not support incremental delta scan".to_string(), + }), + } + } + /// Plan data splits from a snapshot's changelog manifest list only. pub(crate) async fn plan_snapshot_changelog(&self, snapshot: &Snapshot) -> crate::Result { match &self.0 { @@ -978,6 +1018,33 @@ impl<'a> TableScan<'a> { } } + /// Plan stream changelog work without current-state index/DV pruning. + pub(crate) async fn plan_snapshot_changelog_streaming( + &self, + snapshot: &Snapshot, + ) -> crate::Result { + match &self.0 { + TableScanKind::Paimon(scan) => scan.plan_snapshot_changelog_streaming(snapshot).await, + TableScanKind::Format(_) => Err(crate::Error::Unsupported { + message: "Format tables do not support incremental changelog scan".to_string(), + }), + } + } + + /// Plan the complete table state at an already resolved snapshot. + pub(crate) async fn plan_snapshot_full( + &self, + snapshot: &Snapshot, + level_filter: Option, + ) -> crate::Result { + match &self.0 { + TableScanKind::Paimon(scan) => scan.plan_snapshot_full(snapshot, level_filter).await, + TableScanKind::Format(_) => Err(crate::Error::Unsupported { + message: "Format tables do not support Paimon snapshot stream scan".to_string(), + }), + } + } + /// Plan before/after full-snapshot splits for batch incremental Diff. pub(crate) async fn plan_snapshot_diff( &self, @@ -1103,7 +1170,7 @@ impl<'a> PaimonTableScan<'a> { Some(snapshot) => snapshot, None => return Ok(Plan::new(Vec::new())), }; - self.plan_snapshot(snapshot, data_evolution_read_field_ids.as_ref(), None) + self.plan_snapshot(snapshot, data_evolution_read_field_ids.as_ref(), None, None) .await } @@ -1124,6 +1191,7 @@ impl<'a> PaimonTableScan<'a> { .plan_snapshot( snapshot, data_evolution_read_field_ids.as_ref(), + None, Some(&mut trace), ) .await?; @@ -1177,7 +1245,7 @@ impl<'a> PaimonTableScan<'a> { &self, snapshot: &Snapshot, ) -> crate::Result> { - self.plan_manifest_entries_with_trace(snapshot, None, None) + self.plan_manifest_entries_with_trace(snapshot, None, None, None) .await } @@ -1185,6 +1253,7 @@ impl<'a> PaimonTableScan<'a> { &self, snapshot: &Snapshot, row_range_index: Option<&RowRangeIndex>, + level_filter: Option, trace: Option<&mut ScanTrace>, ) -> crate::Result> { let file_io = self.table.file_io(); @@ -1251,6 +1320,7 @@ impl<'a> PaimonTableScan<'a> { table_path, snapshot, skip_level_zero, + level_filter, self.scan_all_files, has_primary_keys, self.partition_filter.as_ref(), @@ -1474,6 +1544,22 @@ impl<'a> PaimonTableScan<'a> { snapshot, snapshot.delta_manifest_list(), data_evolution_read_field_ids.as_ref(), + false, + ) + .await + } + + pub(crate) async fn plan_snapshot_delta_streaming( + &self, + snapshot: &Snapshot, + ) -> crate::Result { + self.ensure_query_auth_allowed()?; + let data_evolution_read_field_ids = self.projected_read_field_ids()?; + self.plan_snapshot_manifest_list( + snapshot, + snapshot.delta_manifest_list(), + data_evolution_read_field_ids.as_ref(), + true, ) .await } @@ -1493,6 +1579,42 @@ impl<'a> PaimonTableScan<'a> { snapshot, list_name, data_evolution_read_field_ids.as_ref(), + false, + ) + .await + } + + pub(crate) async fn plan_snapshot_changelog_streaming( + &self, + snapshot: &Snapshot, + ) -> crate::Result { + self.ensure_query_auth_allowed()?; + let Some(list_name) = snapshot.changelog_manifest_list() else { + return Ok(Plan::new(Vec::new())); + }; + let data_evolution_read_field_ids = self.projected_read_field_ids()?; + self.plan_snapshot_manifest_list( + snapshot, + list_name, + data_evolution_read_field_ids.as_ref(), + true, + ) + .await + } + + /// Plan the complete table state at an already resolved snapshot. + pub(crate) async fn plan_snapshot_full( + &self, + snapshot: &Snapshot, + level_filter: Option, + ) -> crate::Result { + self.ensure_query_auth_allowed()?; + let data_evolution_read_field_ids = self.projected_read_field_ids()?; + self.plan_snapshot( + snapshot.clone(), + data_evolution_read_field_ids.as_ref(), + level_filter, + None, ) .await } @@ -1502,24 +1624,34 @@ impl<'a> PaimonTableScan<'a> { snapshot: &Snapshot, manifest_list_name: &str, data_evolution_read_field_ids: Option<&HashSet>, + streaming_changes: bool, ) -> crate::Result { if matches!(self.limit, Some(0)) { return Ok(Plan::new(Vec::new())); } let core_options = CoreOptions::new(self.table.schema().options()); let data_evolution_enabled = core_options.data_evolution_enabled(); - let global_index_settings = - self.global_index_scan_settings(&core_options, data_evolution_enabled)?; - let index_entries = self - .read_index_manifest_entries( - snapshot, - global_index_settings.is_some(), - core_options.deletion_vectors_enabled(), - ) - .await?; - let manifest_row_ranges = self - .manifest_row_ranges(snapshot, index_entries.as_deref(), global_index_settings) - .await?; + let (index_entries, global_index_settings, manifest_row_ranges) = if streaming_changes { + // Stream plans represent changes, not the post-commit table state. + // A current-state global index can prune a required retract or + // UPDATE_BEFORE event, and a current deletion vector can mask the + // very row the stream must emit. Keep only explicit row ranges. + (None, None, self.row_ranges.clone()) + } else { + let settings = + self.global_index_scan_settings(&core_options, data_evolution_enabled)?; + let entries = self + .read_index_manifest_entries( + snapshot, + settings.is_some(), + core_options.deletion_vectors_enabled(), + ) + .await?; + let ranges = self + .manifest_row_ranges(snapshot, entries.as_deref(), settings) + .await?; + (entries, settings, ranges) + }; if manifest_row_ranges.as_ref().is_some_and(Vec::is_empty) { return Ok(Plan::new(Vec::new())); } @@ -1703,7 +1835,13 @@ impl<'a> PaimonTableScan<'a> { let manifest_entries = crate::spec::avro::from_manifest_bytes_filtered_shared( &bytes, &shared_cache, - &mut |_kind, partition_bytes, bucket, total_buckets| { + &mut |kind, partition_bytes, bucket, total_buckets| { + // Java's incremental reader uses readAndNoMergeFileEntries + // and then selects ADD. Merging a DELETE+ADD rewrite here + // would cancel the new change before it can be emitted. + if kind != FileKind::Add { + return false; + } if has_primary_keys && !scan_all_files && bucket < 0 { return false; } @@ -1734,7 +1872,7 @@ impl<'a> PaimonTableScan<'a> { )?; entries.extend(manifest_entries); } - let entries = merge_manifest_entries(entries); + let entries = retain_incremental_add_entries(entries); let entries = if let Some(index) = row_range_index { retain_manifest_entry_row_ranges(entries, index) } else { @@ -1747,6 +1885,7 @@ impl<'a> PaimonTableScan<'a> { &self, snapshot: Snapshot, data_evolution_read_field_ids: Option<&HashSet>, + level_filter: Option, mut trace: Option<&mut ScanTrace>, ) -> crate::Result { if matches!(self.limit, Some(0)) { @@ -1784,6 +1923,7 @@ impl<'a> PaimonTableScan<'a> { .plan_manifest_entries_with_trace( &snapshot, row_range_index.as_ref(), + level_filter, trace.as_deref_mut(), ) .await?; @@ -2164,11 +2304,11 @@ mod tests { use super::{ data_evolution_row_range_groups, data_file_overlaps_row_range_index, group_data_files_by_partition_bucket, manifest_file_overlaps_row_range_index, - prune_data_evolution_group_by_read_fields, retain_index_manifest_entry, - retain_index_manifest_entry_for_scan, retain_manifest_entry_row_ranges, - retain_manifest_row_ranges, scan_predicate_field_ids, should_skip_level_zero_for_scan, - split_row_ranges_for_files, LimitPushdownAccumulator, PaimonTableScan, RowRangeIndex, - TableScan, + prune_data_evolution_group_by_read_fields, retain_incremental_add_entries, + retain_index_manifest_entry, retain_index_manifest_entry_for_scan, + retain_manifest_entry_row_ranges, retain_manifest_row_ranges, scan_predicate_field_ids, + should_skip_level_zero_for_scan, split_row_ranges_for_files, LimitPushdownAccumulator, + PaimonTableScan, RowRangeIndex, TableScan, }; use crate::catalog::Identifier; use crate::io::FileIOBuilder; @@ -2530,6 +2670,25 @@ mod tests { ); } + #[test] + fn test_incremental_entries_do_not_net_delete_against_add() { + let entry = |kind: FileKind| { + ManifestEntry::new( + kind, + Vec::new(), + 0, + 1, + make_evo_file("same.parquet", 1, 1, 1, None), + 2, + ) + }; + let changes = + retain_incremental_add_entries(vec![entry(FileKind::Delete), entry(FileKind::Add)]); + assert_eq!(changes.len(), 1); + assert_eq!(*changes[0].kind(), FileKind::Add); + assert_eq!(changes[0].file().file_name, "same.parquet"); + } + fn file_names(groups: &[Vec]) -> Vec> { groups .iter() diff --git a/crates/paimon/tests/stream_scan_test.rs b/crates/paimon/tests/stream_scan_test.rs new file mode 100644 index 000000000..e66ccf466 --- /dev/null +++ b/crates/paimon/tests/stream_scan_test.rs @@ -0,0 +1,524 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod common; + +use common::incremental_helpers::{ + make_batch, make_partitioned_batch, memory_table, partitioned_pk_schema, persist_table_schema, + pk_schema, setup_dirs, write_batch, write_partitioned, +}; +use paimon::spec::{CommitKind, Datum, PredicateBuilder, Snapshot}; +use paimon::table::{ + IncrementalScanMode, IncrementalSplit, StreamPlan, StreamScanFollowUpMode, StreamScanPoll, + StreamScanStartupMode, +}; + +async fn commit_metadata_snapshot(table: &paimon::Table, snapshot_id: i64, kind: CommitKind) { + let snapshot = Snapshot::builder() + .version(3) + .id(snapshot_id) + .schema_id(table.schema().id()) + .base_manifest_list(String::new()) + .delta_manifest_list(String::new()) + .commit_user("stream-test".to_string()) + .commit_identifier(snapshot_id) + .commit_kind(kind) + .time_millis(snapshot_id as u64) + .watermark(Some(snapshot_id * 100)) + .build(); + assert!(table + .snapshot_manager() + .commit_snapshot(&snapshot) + .await + .unwrap()); +} + +async fn write_batch_at_level(table: &paimon::Table, ids: Vec, values: Vec, level: i32) { + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + writer + .write_arrow_batch(&make_batch(ids, values)) + .await + .unwrap(); + let mut messages = writer.prepare_commit().await.unwrap(); + for message in &mut messages { + for file in &mut message.new_files { + file.level = level; + } + } + builder.new_commit().commit(messages).await.unwrap(); +} + +async fn full_start_levels(table: &paimon::Table) -> Vec { + let mut scan = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::LatestFull, + StreamScanFollowUpMode::Auto, + ) + .await + .unwrap(); + let plan = expect_data(scan.poll_next().await.unwrap()); + let mut levels = plan + .full_plan() + .unwrap() + .splits() + .iter() + .flat_map(|split| split.data_files()) + .map(|file| file.level) + .collect::>(); + levels.sort_unstable(); + levels +} + +fn expect_data(poll: StreamScanPoll) -> StreamPlan { + match poll { + StreamScanPoll::Data(plan) => plan, + other => panic!("expected stream data, got {other:?}"), + } +} + +#[tokio::test] +async fn latest_full_is_owned_and_then_follows_new_delta_snapshots() { + let table_path = "memory:/stream_scan/latest_full"; + let (file_io, table) = memory_table(table_path, partitioned_pk_schema("1")); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_partitioned( + &table, + make_partitioned_batch(vec!["a", "b"], vec![1, 2], vec![10, 20]), + ) + .await; + + let writer_table = table.clone(); + let mut builder = table.new_read_builder(); + let filter = PredicateBuilder::new(table.schema().fields()) + .equal("pt", Datum::String("a".to_string())) + .unwrap(); + builder.with_filter(filter); + builder.with_projection(&["pt", "id"]).unwrap(); + let mut scan = builder + .new_stream_scan( + StreamScanStartupMode::LatestFull, + StreamScanFollowUpMode::Auto, + ) + .await + .unwrap(); + drop(builder); + drop(table); + + let first = expect_data(scan.poll_next().await.unwrap()); + assert_eq!(first.snapshot_id(), 1); + assert_eq!(first.next_snapshot_id(), 2); + assert_eq!(scan.checkpoint(), Some(2)); + assert!(matches!(first, StreamPlan::Full { .. })); + assert_eq!(first.full_plan().unwrap().splits().len(), 1); + assert_eq!(scan.follow_up_mode(), IncrementalScanMode::Delta); + + write_partitioned( + &writer_table, + make_partitioned_batch(vec!["a", "b"], vec![3, 4], vec![30, 40]), + ) + .await; + let second = expect_data(scan.poll_next().await.unwrap()); + assert_eq!(second.snapshot_id(), 2); + assert_eq!(second.next_snapshot_id(), 3); + assert_eq!(scan.checkpoint(), Some(3)); + let incremental = second.incremental_plan().unwrap(); + assert_eq!(incremental.mode(), IncrementalScanMode::Delta); + assert_eq!(incremental.splits().len(), 1); + + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); +} + +#[tokio::test] +async fn changelog_full_start_uses_java_level_filters() { + let lookup_path = "memory:/stream_scan/lookup_full_levels"; + let (lookup_io, lookup_table) = + memory_table(lookup_path, pk_schema(&[("changelog-producer", "lookup")])); + setup_dirs(&lookup_io, lookup_path).await; + persist_table_schema(&lookup_io, lookup_path, lookup_table.schema()).await; + write_batch_at_level(&lookup_table, vec![1], vec![10], 0).await; + write_batch_at_level(&lookup_table, vec![2], vec![20], 1).await; + assert_eq!(full_start_levels(&lookup_table).await, vec![1]); + + let full_compaction_path = "memory:/stream_scan/full_compaction_levels"; + let (full_compaction_io, full_compaction_table) = memory_table( + full_compaction_path, + pk_schema(&[ + ("changelog-producer", "full-compaction"), + ("num-levels", "3"), + ]), + ); + setup_dirs(&full_compaction_io, full_compaction_path).await; + persist_table_schema( + &full_compaction_io, + full_compaction_path, + full_compaction_table.schema(), + ) + .await; + write_batch_at_level(&full_compaction_table, vec![1], vec![10], 0).await; + write_batch_at_level(&full_compaction_table, vec![2], vec![20], 1).await; + write_batch_at_level(&full_compaction_table, vec![3], vec![30], 2).await; + assert_eq!(full_start_levels(&full_compaction_table).await, vec![2]); +} + +#[tokio::test] +async fn explicit_follow_up_validation_rejects_only_unsafe_combinations() { + let input_path = "memory:/stream_scan/input_explicit_delta"; + let (input_io, input_table) = memory_table( + input_path, + pk_schema(&[("changelog-producer", "input"), ("bucket", "1")]), + ); + setup_dirs(&input_io, input_path).await; + persist_table_schema(&input_io, input_path, input_table.schema()).await; + let scan = input_table + .new_read_builder() + .new_stream_scan(StreamScanStartupMode::Latest, StreamScanFollowUpMode::Delta) + .await + .expect("input changelog tables may be consumed explicitly as delta"); + assert_eq!(scan.follow_up_mode(), IncrementalScanMode::Delta); + + let dv_lookup_path = "memory:/stream_scan/dv_lookup_explicit_delta"; + let (dv_lookup_io, dv_lookup_table) = memory_table( + dv_lookup_path, + pk_schema(&[ + ("changelog-producer", "lookup"), + ("deletion-vectors.enabled", "true"), + ("bucket", "1"), + ]), + ); + setup_dirs(&dv_lookup_io, dv_lookup_path).await; + persist_table_schema(&dv_lookup_io, dv_lookup_path, dv_lookup_table.schema()).await; + let error = dv_lookup_table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::LatestFull, + StreamScanFollowUpMode::Delta, + ) + .await + .expect_err("DV lookup requires future compaction changelog"); + assert!(matches!(error, paimon::Error::Unsupported { .. })); + let scan = dv_lookup_table + .new_read_builder() + .new_stream_scan(StreamScanStartupMode::Latest, StreamScanFollowUpMode::Delta) + .await + .expect("non-full startup may explicitly consume future lookup deltas"); + assert_eq!(scan.follow_up_mode(), IncrementalScanMode::Delta); + + let none_path = "memory:/stream_scan/none_explicit_changelog"; + let (none_io, none_table) = memory_table( + none_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&none_io, none_path).await; + persist_table_schema(&none_io, none_path, none_table.schema()).await; + let error = none_table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::Latest, + StreamScanFollowUpMode::Changelog, + ) + .await + .expect_err("a table without changelog files cannot use changelog follow-up"); + assert!(matches!(error, paimon::Error::Unsupported { .. })); +} + +#[tokio::test] +async fn deletion_vector_full_start_replays_starting_level_zero() { + let table_path = "memory:/stream_scan/dv_full_start"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[ + ("changelog-producer", "none"), + ("deletion-vectors.enabled", "true"), + ("bucket", "1"), + ]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + + for startup in [ + StreamScanStartupMode::LatestFull, + StreamScanStartupMode::FromSnapshotFull(1), + ] { + let mut scan = table + .new_read_builder() + .new_stream_scan(startup, StreamScanFollowUpMode::Auto) + .await + .unwrap(); + let full = expect_data(scan.poll_next().await.unwrap()); + assert!(full.full_plan().unwrap().splits().is_empty()); + assert_eq!(full.next_snapshot_id(), 1); + assert_eq!(scan.checkpoint(), Some(1)); + + let incremental = expect_data(scan.poll_next().await.unwrap()); + assert_eq!(incremental.snapshot_id(), 1); + assert_eq!(incremental.next_snapshot_id(), 2); + let splits = incremental.incremental_plan().unwrap().splits(); + assert!(!splits.is_empty()); + for split in splits { + let IncrementalSplit::Data(split) = split else { + panic!("stream delta must contain data splits"); + }; + assert!(split.data_files().iter().all(|file| file.level == 0)); + assert!(split.data_deletion_files().is_none()); + } + } +} + +#[tokio::test] +async fn latest_on_initially_empty_table_includes_first_snapshot() { + let table_path = "memory:/stream_scan/latest_empty"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + + let mut scan = table + .new_read_builder() + .new_stream_scan(StreamScanStartupMode::Latest, StreamScanFollowUpMode::Delta) + .await + .unwrap(); + assert_eq!(scan.checkpoint(), Some(1)); + + // Commit after construction but before the first poll. The async + // constructor freezes the empty-table boundary, so snapshot 1 is retained. + write_batch(&table, &make_batch(vec![1], vec![10])).await; + let first = expect_data(scan.poll_next().await.unwrap()); + assert_eq!(first.snapshot_id(), 1); + assert_eq!(scan.checkpoint(), Some(2)); +} + +#[tokio::test] +async fn inconsistent_range_observation_recovers_across_polls() { + let table_path = "memory:/stream_scan/transient_range"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + + // Model a commit between earliest/latest observations: the latest hint is + // visible while the corresponding snapshot is not yet visible. + table.snapshot_manager().write_latest_hint(1).await.unwrap(); + let mut scan = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(1), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + + commit_metadata_snapshot(&table, 1, CommitKind::APPEND).await; + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert_eq!(scan.checkpoint(), Some(2)); +} + +#[tokio::test] +async fn empty_append_snapshot_advances_cursor_and_restore_replays_from_checkpoint() { + let table_path = "memory:/stream_scan/empty_snapshot"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + commit_metadata_snapshot(&table, 2, CommitKind::APPEND).await; + + let mut scan = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(1), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert_eq!(scan.checkpoint(), Some(1)); + assert_eq!( + expect_data(scan.poll_next().await.unwrap()).snapshot_id(), + 1 + ); + + // Snapshot 2 is an APPEND with an empty delta manifest. One poll consumes + // it and waits for snapshot 3 instead of returning the same empty plan. + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert_eq!(scan.checkpoint(), Some(3)); + assert_eq!(scan.watermark(), Some(200)); + + scan.restore(Some(1)).unwrap(); + assert_eq!( + expect_data(scan.poll_next().await.unwrap()).snapshot_id(), + 1 + ); + assert_eq!(scan.checkpoint(), Some(2)); +} + +#[tokio::test] +async fn transient_missing_snapshot_waits_but_range_errors_are_explicit() { + let table_path = "memory:/stream_scan/range_errors"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + commit_metadata_snapshot(&table, 3, CommitKind::APPEND).await; + + let mut missing = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(2), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + // Poll frequency must not turn an eventually consistent in-range miss into + // a permanent gap. It remains Waiting until the object becomes visible. + for _ in 0..16 { + assert!(matches!( + missing.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + } + + table.snapshot_manager().delete_snapshot(1).await.unwrap(); + let mut expired = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(1), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + let expired_error = expired.poll_next().await.unwrap_err(); + assert!(matches!(expired_error, paimon::Error::DataInvalid { .. })); + assert!(expired_error.to_string().contains("expired")); + + let mut too_large = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(5), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + let too_large_error = too_large.poll_next().await.unwrap_err(); + assert!(matches!(too_large_error, paimon::Error::DataInvalid { .. })); + assert!(too_large_error.to_string().contains("too large")); +} + +#[tokio::test] +async fn overwrite_follow_up_is_not_silently_skipped() { + let table_path = "memory:/stream_scan/overwrite"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + + let mut scan = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(1), + StreamScanFollowUpMode::Auto, + ) + .await + .unwrap(); + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert_eq!( + expect_data(scan.poll_next().await.unwrap()).snapshot_id(), + 1 + ); + commit_metadata_snapshot(&table, 2, CommitKind::OVERWRITE).await; + + let error = scan.poll_next().await.unwrap_err(); + assert!(matches!(error, paimon::Error::Unsupported { .. })); + assert!(error.to_string().contains("OVERWRITE snapshot 2")); + assert_eq!(scan.checkpoint(), Some(2)); +} + +#[tokio::test] +async fn from_snapshot_full_reads_exact_snapshot_and_reports_missing_target() { + let table_path = "memory:/stream_scan/from_full"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + + let mut full = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshotFull(1), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + let plan = expect_data(full.poll_next().await.unwrap()); + assert!(matches!(plan, StreamPlan::Full { .. })); + assert_eq!(plan.snapshot_id(), 1); + assert_eq!(full.checkpoint(), Some(2)); + + let mut missing = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshotFull(2), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + assert!(matches!( + missing.poll_next().await.unwrap_err(), + paimon::Error::SnapshotNotExist { snapshot_id: 2 } + )); +} diff --git a/docs/src/c-binding.md b/docs/src/c-binding.md index 1cd25ad94..0b520b474 100644 --- a/docs/src/c-binding.md +++ b/docs/src/c-binding.md @@ -24,14 +24,15 @@ catalog and table access, scan planning, predicate push-down, streaming reads, writes and commits, and vector search. Record batches cross the ABI through the [Arrow C Data Interface](https://arrow.apache.org/docs/format/CDataInterface.html). -The C binding is currently built from source. The repository does not check in -a generated header or publish pre-built C packages. +The C binding is currently built from source. Its generated, C++-compatible +public header is checked in at `bindings/c/include/paimon.h`; releases do not +yet publish pre-built native packages. ## Prerequisites - A Rust toolchain supported by this repository - A C11-compatible compiler -- [`cbindgen`](https://github.com/mozilla/cbindgen) for generating the C header +- [`cbindgen`](https://github.com/mozilla/cbindgen) only when updating the C ABI - An Arrow implementation if the application reads or writes record batches Install `cbindgen` when it is not already available: @@ -46,7 +47,8 @@ Run the following commands from the repository root: ```bash cargo build --release -p paimon-c -cbindgen bindings/c --lang c --output target/release/paimon.h +cbindgen --config bindings/c/cbindgen.toml bindings/c \ + --output bindings/c/include/paimon.h ``` The build produces a dynamic library and a static library under @@ -58,11 +60,11 @@ The build produces a dynamic library and a static library under | macOS | `libpaimon_c.dylib` | | Windows | `paimon_c.dll` | -Link the generated header and library into an application: +Link the checked-in header and library into an application: ```bash cc -std=c11 example.c \ - -Itarget/release \ + -Ibindings/c/include \ -Ltarget/release \ -lpaimon_c \ -o example @@ -78,6 +80,11 @@ LD_LIBRARY_PATH=target/release ./example /path/to/warehouse DYLD_LIBRARY_PATH=target/release ./example /path/to/warehouse ``` +The header-only C++17 facade under `bindings/cpp` adds move-only RAII handles +without creating a C++ shared library. The only Paimon binary remains +`libpaimon_c`, and release validation rejects dependencies on `libstdc++`, +`libc++`, `GLIBCXX_*`, or `CXXABI_*` symbols. + ## Opening and Scanning a Table The following program opens `default.my_table` from a filesystem catalog and @@ -275,6 +282,60 @@ range is clamped to the number of available splits. reversed: `paimon_table_write_write_arrow_batch` consumes the exported Arrow structures, so the caller must not release them again. +## Continuous Stream Reading + +`paimon_read_builder_new_stream_scan` creates an owned pull-based scanner. It +does not start a callback thread. Each call to `paimon_stream_scan_poll` returns +one of `PAIMON_STREAM_POLL_DATA`, `PAIMON_STREAM_POLL_WAITING`, or +`PAIMON_STREAM_POLL_END`: + +```c +paimon_stream_scan_options options; +paimon_error *error = paimon_stream_scan_options_init(&options); +options.startup_mode = PAIMON_STREAM_STARTUP_LATEST; +options.follow_up_mode = PAIMON_STREAM_FOLLOW_UP_AUTO; + +paimon_result_stream_scan created = + paimon_read_builder_new_stream_scan(read_builder, &options); + +for (;;) { + paimon_result_stream_poll poll = paimon_stream_scan_poll(created.scan); + if (poll.error != NULL) { + /* Inspect and free poll.error. */ + break; + } + if (poll.status == PAIMON_STREAM_POLL_WAITING) { + /* Schedule the next poll with application-controlled backoff. */ + continue; + } + if (poll.status == PAIMON_STREAM_POLL_END) { + break; + } + + paimon_result_record_batch_reader batches = + paimon_stream_plan_read_to_arrow( + read, poll.plan, 0, SIZE_MAX, PAIMON_STREAM_READ_DATA); + /* Drain batches before checkpointing poll.next_snapshot_id. */ + paimon_stream_plan_free(poll.plan); +} +``` + +The scan checkpoint is the next snapshot ID and advances when planning +succeeds. Persist it only after all returned work is durably accounted for. +`paimon_stream_plan_serialize` preserves a pending plan across restart. The +current format recovers at plan boundaries, so a partially consumed plan may be +replayed. A stream-scan handle is single-thread-confined; poll, checkpoint, +restore, and free calls for one handle must be externally serialized. +Persisted stream plans with external data-file paths are not supported; plan +serialization fails before the checkpoint is persisted. +Audit-log mode is available for incremental plans and prepends the UTF-8 +`rowkind` column (`+I`, `-U`, `+U`, `-D`). Follow-up `OVERWRITE` snapshots are +reported as unsupported in version 1 rather than silently skipped. + +Decoupled changelog fallback and consumer snapshot-retention registration are +not implemented yet. Configure snapshot retention to exceed the maximum reader +lag; an expired cursor fails explicitly instead of skipping data. + ## Projection and Predicates Projection uses a null-terminated array of column names: @@ -355,6 +416,32 @@ values. `paimon_commit_messages_free` even after a successful commit. The caller retains message ownership and may retry a failed commit. +For a recoverable streaming checkpoint, bind the messages to a non-negative, +monotonically increasing identifier with `paimon_commit_messages_prepare`. +`INT64_MAX` is reserved for unidentified batch commits and is not a valid +streaming checkpoint identifier. +Persist the bytes returned by `paimon_prepared_commit_serialize` before +committing. After a crash or an indeterminate commit response, deserialize the +same bytes and call `paimon_table_commit_commit_prepared`; this retry-safe path +filters an identifier which was already committed. Parallel writers may merge +prepared commits only when table, `commit_user`, overwrite mode, and identifier +all match. Do not call `paimon_table_commit_abort_prepared` after an +indeterminate response: retry first so files from a successful commit are not +deleted. Commit and abort for the same `(table, commit_user)` must also be +fenced across processes; truncated snapshot history makes abort fail closed. +Duplicate filtering is stored in retained snapshots, so snapshot retention +must cover the maximum writer-recovery horizon. Do not retry a prepared commit +older than that horizon, and use a new globally unique `commit_user` for each +fresh job. +Serialized plan/commit blobs are trusted checkpoint state and are not +cryptographically authenticated, so persist them with appropriate integrity +and access controls. + +Filesystem-catalog snapshot publication requires an atomic +publish-if-not-exists capability. If the storage backend cannot provide a +conditional rename, copy, or write, commit returns `Unsupported`; use REST +commit or an external lock rather than a racy check-then-write fallback. + ## Error Handling and Resource Ownership Functions that can fail use one of two conventions: diff --git a/scripts/release_licenses.py b/scripts/release_licenses.py index 5785cf803..b17b33ce4 100644 --- a/scripts/release_licenses.py +++ b/scripts/release_licenses.py @@ -192,32 +192,6 @@ class BundledComponent: crate_version="0.4.7", required_features=("static",), ), - BundledComponent( - crate="openssl-sys", - license_path="third-party-licenses/openssl-1.1.1.LICENSE", - component="OpenSSL 1.1.1k FIPS libssl and libcrypto shared libraries", - component_url="https://github.com/openssl/openssl/tree/OpenSSL_1_1_1k", - license_name="OpenSSL 1.1.1 and Original SSLeay Licenses", - anchor="bundled-openssl-1.1.1", - components=("python",), - targets=("x86_64-unknown-linux-gnu",), - license_from_repository=True, - required=True, - relationship="linked through", - ), - BundledComponent( - crate="openssl-sys", - license_path="third-party-licenses/openssl-1.1.1.LICENSE", - component="OpenSSL 1.1.1w libssl and libcrypto shared libraries", - component_url="https://github.com/openssl/openssl/tree/OpenSSL_1_1_1w", - license_name="OpenSSL 1.1.1 and Original SSLeay Licenses", - anchor="bundled-openssl-1.1.1", - components=("python",), - targets=("aarch64-unknown-linux-gnu",), - license_from_repository=True, - required=True, - relationship="linked through", - ), ) ALLOC_PLACEHOLDER = "Copyright (c) <year> <owner>." diff --git a/scripts/verify_python_wheels.py b/scripts/verify_python_wheels.py index a01242f33..40be17637 100644 --- a/scripts/verify_python_wheels.py +++ b/scripts/verify_python_wheels.py @@ -403,7 +403,6 @@ def verify_wheel(path: Path) -> tuple[str, str]: require(record_member in names, f"wheel has no RECORD: {path.name}") verify_record(archive, names, record_member, path.name) - license_report = None for license_file in metadata.get_all("License-File", []): relative = normalized_relative_path(license_file, "License-File entry") members = ( @@ -436,7 +435,6 @@ def verify_wheel(path: Path) -> tuple[str, str]: if relative.name == "NOTICE": verify_avro_notice(actual, f"{path.name}:{member}") if relative.name == "THIRD-PARTY-LICENSES.html": - license_report = actual verify_license_report(actual, target, f"{path.name}:{member}") native_members = [ @@ -465,50 +463,10 @@ def verify_wheel(path: Path) -> tuple[str, str]: extra_native_members = [ name for name in names if name != native_member and is_native_library(name) ] - if target.endswith("linux-gnu"): - openssl_version = ( - b"OpenSSL 1.1.1k FIPS" - if target.startswith("x86_64") - else b"OpenSSL 1.1.1w" - ) - openssl_libraries = ( - (r"pypaimon_rust\.libs/libcrypto-[^/]+\.so\.1\.1", openssl_version), - (r"pypaimon_rust\.libs/libssl-[^/]+\.so\.1\.1", None), - ) - for pattern, version_marker in openssl_libraries: - matches = [ - name for name in extra_native_members if re.fullmatch(pattern, name) - ] - require( - len(matches) == 1, - f"wheel must contain one {pattern}: {extra_native_members}", - ) - with archive.open(matches[0]) as native_file: - content = native_file.read() - verify_native_header( - content, - target, - f"{path.name}:{matches[0]}", - ) - if version_marker is not None: - require( - version_marker in content, - f"wheel libcrypto is missing {version_marker!r}", - ) - require( - len(extra_native_members) == len(openssl_libraries), - f"wheel has unexpected native libraries: {extra_native_members}", - ) - require( - license_report is not None - and b'id="bundled-openssl-1.1.1"' in license_report, - "Linux wheel license report is missing the OpenSSL anchor", - ) - else: - require( - not extra_native_members, - f"wheel has unexpected native libraries: {extra_native_members}", - ) + require( + not extra_native_members, + f"wheel has unexpected native libraries: {extra_native_members}", + ) return target, version From 99d2e8fe33737497b97b21ca20fad71a00b90720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 14:57:08 +0800 Subject: [PATCH 02/21] feat(bindings): expose catalog table DDL --- bindings/c/include/paimon.h | 35 +++++++- bindings/c/src/catalog.rs | 118 ++++++++++++++++++++++++- bindings/c/src/tests.rs | 95 ++++++++++++++++++++ bindings/cpp/README.md | 13 +++ bindings/cpp/include/paimon/paimon.hpp | 22 +++++ bindings/cpp/tests/header_smoke.cpp | 5 ++ bindings/cpp/tests/paimon_test_stub.h | 6 ++ 7 files changed, 289 insertions(+), 5 deletions(-) diff --git a/bindings/c/include/paimon.h b/bindings/c/include/paimon.h index 8aaba187a..c26723aae 100644 --- a/bindings/c/include/paimon.h +++ b/bindings/c/include/paimon.h @@ -159,6 +159,10 @@ typedef struct paimon_result_catalog_new { struct paimon_error *error; } paimon_result_catalog_new; +typedef struct paimon_identifier { + void *inner; +} paimon_identifier; + typedef struct paimon_table { void *inner; } paimon_table; @@ -168,10 +172,6 @@ typedef struct paimon_result_get_table { struct paimon_error *error; } paimon_result_get_table; -typedef struct paimon_identifier { - void *inner; -} paimon_identifier; - /** * Opaque container for commit messages and their originating write context. */ @@ -571,6 +571,33 @@ void paimon_bytes_free(struct paimon_bytes bytes); struct paimon_result_catalog_new paimon_catalog_create(const struct paimon_option *options, size_t options_len); +/** + * Create a table from a logical Paimon `Schema` JSON document. + * + * The input is normalized and validated through `SchemaBuilder` before it is + * sent to the catalog. Field IDs in the JSON are therefore treated as input + * ordering hints and reassigned canonically from zero. + * + * # Safety + * `catalog` and `identifier` must be valid Paimon handles. `schema_json` must + * point to a valid null-terminated UTF-8 string. + */ +struct paimon_error *paimon_catalog_create_table_from_schema_json(const struct paimon_catalog *catalog, + const struct paimon_identifier *identifier, + const char *schema_json, + bool ignore_if_exists); + +/** + * Drop a table from the catalog. + * + * # Safety + * `catalog` and `identifier` must be valid Paimon handles, or null (returns an + * error). + */ +struct paimon_error *paimon_catalog_drop_table(const struct paimon_catalog *catalog, + const struct paimon_identifier *identifier, + bool ignore_if_not_exists); + /** * Free a paimon_catalog. * diff --git a/bindings/c/src/catalog.rs b/bindings/c/src/catalog.rs index 9c5c683b7..a21dc078a 100644 --- a/bindings/c/src/catalog.rs +++ b/bindings/c/src/catalog.rs @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. -use std::ffi::c_void; +use std::ffi::{c_char, c_void}; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::Arc; use paimon::catalog::Identifier; +use paimon::spec::Schema; use paimon::{Catalog, CatalogFactory, Options}; use crate::error::{check_non_null, paimon_error, validate_cstr}; @@ -26,6 +28,38 @@ use crate::result::{paimon_result_catalog_new, paimon_result_get_table}; use crate::runtime; use crate::types::{paimon_catalog, paimon_option, paimon_table}; +fn catalog_panic_error(operation: &str) -> *mut paimon_error { + paimon_error::new( + crate::error::PaimonErrorCode::Unexpected, + format!("Rust panic while executing {operation}"), + ) +} + +fn validate_creation_schema_json(schema_json: &str) -> Result { + let parsed = serde_json::from_str::(schema_json).map_err(|error| { + paimon_error::new( + crate::error::PaimonErrorCode::InvalidInput, + format!("Failed to parse creation schema JSON: {error}"), + ) + })?; + + let mut builder = Schema::builder(); + for field in parsed.fields() { + builder = builder.column_with_description( + field.name(), + field.data_type().clone(), + field.description().map(str::to_string), + ); + } + builder + .partition_keys(parsed.partition_keys().iter().cloned()) + .primary_key(parsed.primary_keys().iter().cloned()) + .options(parsed.options().clone()) + .comment(parsed.comment().map(str::to_string)) + .build() + .map_err(paimon_error::from_paimon) +} + /// Create a catalog using CatalogFactory with the given options. /// /// # Safety @@ -136,3 +170,85 @@ pub unsafe extern "C" fn paimon_catalog_get_table( }, } } + +/// Create a table from a logical Paimon `Schema` JSON document. +/// +/// The input is normalized and validated through `SchemaBuilder` before it is +/// sent to the catalog. Field IDs in the JSON are therefore treated as input +/// ordering hints and reassigned canonically from zero. +/// +/// # Safety +/// `catalog` and `identifier` must be valid Paimon handles. `schema_json` must +/// point to a valid null-terminated UTF-8 string. +#[no_mangle] +pub unsafe extern "C" fn paimon_catalog_create_table_from_schema_json( + catalog: *const paimon_catalog, + identifier: *const crate::types::paimon_identifier, + schema_json: *const c_char, + ignore_if_exists: bool, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(catalog, "catalog") { + return error; + } + if let Err(error) = check_non_null(identifier, "identifier") { + return error; + } + let schema_json = match validate_cstr(schema_json, "schema_json") { + Ok(value) => value, + Err(error) => return error, + }; + let schema = match validate_creation_schema_json(&schema_json) { + Ok(value) => value, + Err(error) => return error, + }; + let catalog_ref = &*((*catalog).inner as *const Arc); + let identifier_ref = &*((*identifier).inner as *const Identifier); + match runtime().block_on(catalog_ref.create_table(identifier_ref, schema, ignore_if_exists)) + { + Ok(()) => std::ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| catalog_panic_error("paimon_catalog_create_table_from_schema_json")) +} + +/// Drop a table from the catalog. +/// +/// # Safety +/// `catalog` and `identifier` must be valid Paimon handles, or null (returns an +/// error). +#[no_mangle] +pub unsafe extern "C" fn paimon_catalog_drop_table( + catalog: *const paimon_catalog, + identifier: *const crate::types::paimon_identifier, + ignore_if_not_exists: bool, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(catalog, "catalog") { + return error; + } + if let Err(error) = check_non_null(identifier, "identifier") { + return error; + } + let catalog_ref = &*((*catalog).inner as *const Arc); + let identifier_ref = &*((*identifier).inner as *const Identifier); + match runtime().block_on(catalog_ref.drop_table(identifier_ref, ignore_if_not_exists)) { + Ok(()) => std::ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| catalog_panic_error("paimon_catalog_drop_table")) +} + +const _: unsafe extern "C" fn( + *const paimon_catalog, + *const crate::types::paimon_identifier, + *const c_char, + bool, +) -> *mut paimon_error = paimon_catalog_create_table_from_schema_json; +const _: unsafe extern "C" fn( + *const paimon_catalog, + *const crate::types::paimon_identifier, + bool, +) -> *mut paimon_error = paimon_catalog_drop_table; diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs index f2bcfa69a..39eace0c2 100644 --- a/bindings/c/src/tests.rs +++ b/bindings/c/src/tests.rs @@ -46,14 +46,109 @@ use paimon::spec::{ use paimon::table::{SnapshotManager, Table}; use crate::blob_reader::*; +use crate::catalog::*; use crate::error::*; use crate::file_io::*; +use crate::identifier::*; use crate::stream::*; use crate::table::*; use crate::types::*; use crate::vector_search::*; use crate::write::*; +#[test] +fn test_catalog_create_and_drop_table_from_schema_json() { + let directory = tempfile::tempdir().unwrap(); + let warehouse = CString::new(directory.path().to_string_lossy().as_bytes()).unwrap(); + let warehouse_key = CString::new("warehouse").unwrap(); + let options = [paimon_option { + key: warehouse_key.as_ptr(), + value: warehouse.as_ptr(), + }]; + let catalog_result = unsafe { paimon_catalog_create(options.as_ptr(), options.len()) }; + assert!(catalog_result.error.is_null()); + assert!(!catalog_result.catalog.is_null()); + + let catalog = unsafe { &*((*catalog_result.catalog).inner as *const Arc) }; + crate::runtime() + .block_on(catalog.create_database("default", true, HashMap::new())) + .unwrap(); + + let database = CString::new("default").unwrap(); + let table_name = CString::new("ffi_ddl").unwrap(); + let identifier_result = + unsafe { paimon_identifier_new(database.as_ptr(), table_name.as_ptr()) }; + assert!(identifier_result.error.is_null()); + assert!(!identifier_result.identifier.is_null()); + + let schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .option("bucket", "1") + .option("bucket-key", "id") + .build() + .unwrap(); + let schema_json = CString::new(serde_json::to_string(&schema).unwrap()).unwrap(); + + let create_error = unsafe { + paimon_catalog_create_table_from_schema_json( + catalog_result.catalog, + identifier_result.identifier, + schema_json.as_ptr(), + false, + ) + }; + assert!(create_error.is_null()); + + let table_result = + unsafe { paimon_catalog_get_table(catalog_result.catalog, identifier_result.identifier) }; + assert!(table_result.error.is_null()); + assert!(!table_result.table.is_null()); + unsafe { paimon_table_free(table_result.table) }; + + let duplicate_error = unsafe { + paimon_catalog_create_table_from_schema_json( + catalog_result.catalog, + identifier_result.identifier, + schema_json.as_ptr(), + false, + ) + }; + assert!(!duplicate_error.is_null()); + assert_eq!( + unsafe { (*duplicate_error).code }, + PAIMON_ERROR_ALREADY_EXISTS + ); + unsafe { paimon_error_free(duplicate_error) }; + assert!(unsafe { + paimon_catalog_create_table_from_schema_json( + catalog_result.catalog, + identifier_result.identifier, + schema_json.as_ptr(), + true, + ) + } + .is_null()); + + assert!(unsafe { + paimon_catalog_drop_table(catalog_result.catalog, identifier_result.identifier, false) + } + .is_null()); + let missing = + unsafe { paimon_catalog_get_table(catalog_result.catalog, identifier_result.identifier) }; + assert!(missing.table.is_null()); + assert!(!missing.error.is_null()); + unsafe { paimon_error_free(missing.error) }; + assert!(unsafe { + paimon_catalog_drop_table(catalog_result.catalog, identifier_result.identifier, true) + } + .is_null()); + + unsafe { + paimon_identifier_free(identifier_result.identifier); + paimon_catalog_free(catalog_result.catalog); + } +} + // ========================================================================= // Helpers // ========================================================================= diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index 795831a31..2b1c5fcac 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -58,6 +58,19 @@ Persisted stream plans currently reject external data-file paths. The failure is reported by `StreamPlan::serialize` before a checkpoint can be acknowledged, instead of producing a checkpoint that cannot be restored. +Catalog DDL is available directly from the facade. Creation accepts the JSON +form of Paimon's logical `Schema`; it validates and canonically reassigns field +IDs before calling the catalog. Both operations return `Status`, so callers can +choose strict or idempotent create/drop semantics without a Java helper: + +```cpp +auto identifier = paimon::Identifier::create("default", "events"); +auto created = catalog.create_table_from_schema_json( + identifier.value(), schema_json, /*ignore_if_exists=*/false); +auto dropped = catalog.drop_table( + identifier.value(), /*ignore_if_not_exists=*/true); +``` + ## Build Generate the C header and build the Rust library first: diff --git a/bindings/cpp/include/paimon/paimon.hpp b/bindings/cpp/include/paimon/paimon.hpp index fc456dd01..c5dab0087 100644 --- a/bindings/cpp/include/paimon/paimon.hpp +++ b/bindings/cpp/include/paimon/paimon.hpp @@ -501,6 +501,14 @@ class Catalog final { [[nodiscard]] Result
get_table( const Identifier& identifier) const noexcept; + [[nodiscard]] Status create_table_from_schema_json( + const Identifier& identifier, const char* schema_json, + bool ignore_if_exists = false) const noexcept; + + [[nodiscard]] Status drop_table( + const Identifier& identifier, + bool ignore_if_not_exists = false) const noexcept; + [[nodiscard]] ::paimon_catalog* native_handle() const noexcept { return handle_.get(); } @@ -1179,6 +1187,20 @@ inline Result
Catalog::get_table( return Result
::success(Table(adopt_handle, result.table)); } +inline Status Catalog::create_table_from_schema_json( + const Identifier& identifier, const char* schema_json, + bool ignore_if_exists) const noexcept { + return detail::status_from(::paimon_catalog_create_table_from_schema_json( + handle_.get(), identifier.native_handle(), schema_json, + ignore_if_exists)); +} + +inline Status Catalog::drop_table(const Identifier& identifier, + bool ignore_if_not_exists) const noexcept { + return detail::status_from(::paimon_catalog_drop_table( + handle_.get(), identifier.native_handle(), ignore_if_not_exists)); +} + inline Result
Table::from_schema_json( const char* table_path, const char* table_schema_json, const char* database, const char* table_name, const char* branch, const Option* storage_options, diff --git a/bindings/cpp/tests/header_smoke.cpp b/bindings/cpp/tests/header_smoke.cpp index 5d1503dc9..6bbee3e79 100644 --- a/bindings/cpp/tests/header_smoke.cpp +++ b/bindings/cpp/tests/header_smoke.cpp @@ -46,6 +46,11 @@ void paimon_cpp_header_smoke(const paimon::Option* options, } auto table = catalog.value().get_table(identifier.value()); + auto create_table_status = catalog.value().create_table_from_schema_json( + identifier.value(), "{}", true); + auto drop_table_status = catalog.value().drop_table(identifier.value(), true); + (void)create_table_status; + (void)drop_table_status; if (!table) { return; } diff --git a/bindings/cpp/tests/paimon_test_stub.h b/bindings/cpp/tests/paimon_test_stub.h index 00253f084..2da449030 100644 --- a/bindings/cpp/tests/paimon_test_stub.h +++ b/bindings/cpp/tests/paimon_test_stub.h @@ -191,6 +191,12 @@ paimon_result_catalog_new paimon_catalog_create(const paimon_option* options, void paimon_catalog_free(paimon_catalog* catalog); paimon_result_get_table paimon_catalog_get_table( const paimon_catalog* catalog, const paimon_identifier* identifier); +paimon_error* paimon_catalog_create_table_from_schema_json( + const paimon_catalog* catalog, const paimon_identifier* identifier, + const char* schema_json, bool ignore_if_exists); +paimon_error* paimon_catalog_drop_table( + const paimon_catalog* catalog, const paimon_identifier* identifier, + bool ignore_if_not_exists); paimon_result_identifier_new paimon_identifier_new(const char* database, const char* object); void paimon_identifier_free(paimon_identifier* identifier); From e94bdc13480dfc8d5eff716d20f300aa7d50c008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 15:20:28 +0800 Subject: [PATCH 03/21] fix(bindings): build portable Linux C ABI library --- bindings/cpp/README.md | 10 +++- bindings/cpp/scripts/build_linux_release.sh | 51 +++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100755 bindings/cpp/scripts/build_linux_release.sh diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index 2b1c5fcac..a361fbfc6 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -138,12 +138,18 @@ select an externally installed `libpaimon_c`. ## Linux runtime guard -Run the ELF guard on every release artifact: +Install Zig and `cargo-zigbuild`, then produce every Linux release artifact +through the checked build script. It targets glibc 2.17 and immediately runs +the ELF guard, preventing a newer build host from silently raising the runtime +glibc requirement or introducing a shared compiler/C++ runtime: ```bash -bindings/cpp/scripts/verify_linux_elf.sh target/release/libpaimon_c.so +bindings/cpp/scripts/build_linux_release.sh ``` +The validated library is written to +`target//release/libpaimon_c.so`. + It prints the build host's `ldd --version` and applies a `DT_NEEDED` allowlist containing glibc components and `libpaimon_c`. It rejects C++ runtimes, `libgcc_s`, `libunwind`, `libatomic`, `GLIBCXX`/`CXXABI`/`GCC` symbol versions, diff --git a/bindings/cpp/scripts/build_linux_release.sh b/bindings/cpp/scripts/build_linux_release.sh new file mode 100755 index 000000000..56fe0e136 --- /dev/null +++ b/bindings/cpp/scripts/build_linux_release.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +repository_root=$(CDPATH= cd -- "${script_dir}/../../.." && pwd) + +case "$(uname -m)" in + x86_64|amd64) + rust_target=x86_64-unknown-linux-gnu + ;; + aarch64|arm64) + rust_target=aarch64-unknown-linux-gnu + ;; + *) + echo "unsupported Linux architecture: $(uname -m)" >&2 + exit 2 + ;; +esac + +if ! command -v cargo-zigbuild >/dev/null 2>&1; then + echo "cargo-zigbuild is required to build the glibc 2.17 artifact" >&2 + exit 2 +fi + +cd "${repository_root}" +cargo zigbuild --locked --release -p paimon-c \ + --target "${rust_target}.2.17" + +target_dir=${CARGO_TARGET_DIR:-${repository_root}/target} +case "${target_dir}" in + /*) ;; + *) target_dir="${repository_root}/${target_dir}" ;; +esac +library="${target_dir}/${rust_target}/release/libpaimon_c.so" +"${script_dir}/verify_linux_elf.sh" "${library}" +printf 'validated-library=%s\n' "${library}" From 73fd1c7bd96f495341a4f94985c624411ece1ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 17:27:58 +0800 Subject: [PATCH 04/21] chore(bindings): organize C helper scripts --- bindings/c/{ => scripts}/check-header.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) rename bindings/c/{ => scripts}/check-header.sh (78%) diff --git a/bindings/c/check-header.sh b/bindings/c/scripts/check-header.sh similarity index 78% rename from bindings/c/check-header.sh rename to bindings/c/scripts/check-header.sh index 9626484c8..79ec9efdd 100755 --- a/bindings/c/check-header.sh +++ b/bindings/c/scripts/check-header.sh @@ -18,16 +18,17 @@ set -eu script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -repo_dir=$(CDPATH= cd -- "$script_dir/../.." && pwd) +binding_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +repo_dir=$(CDPATH= cd -- "$binding_dir/../.." && pwd) generated=$(mktemp) trap 'rm -f "$generated"' EXIT HUP INT TERM -cbindgen --quiet --config "$script_dir/cbindgen.toml" \ - "$script_dir" --output "$generated" +cbindgen --quiet --config "$binding_dir/cbindgen.toml" \ + "$binding_dir" --output "$generated" -if ! cmp -s "$generated" "$script_dir/include/paimon.h"; then +if ! cmp -s "$generated" "$binding_dir/include/paimon.h"; then echo "bindings/c/include/paimon.h is stale; regenerate it with cbindgen" >&2 - diff -u "$script_dir/include/paimon.h" "$generated" || true + diff -u "$binding_dir/include/paimon.h" "$generated" || true exit 1 fi From 546f959b75a73f3bbcb5574ef01bc52327cbf3fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 17:50:24 +0800 Subject: [PATCH 05/21] build(cpp): compile the in-tree C ABI automatically --- .../{cpp => c}/scripts/build_linux_release.sh | 39 +++-- bindings/cpp/CMakeLists.txt | 148 +++++++++++------- bindings/cpp/README.md | 37 +++-- bindings/cpp/cmake/PaimonCppConfig.cmake.in | 55 ++++--- .../install_tree_consumer/CMakeLists.txt | 4 + bindings/cpp/tests/run_isolated_load.cmake | 5 +- 6 files changed, 179 insertions(+), 109 deletions(-) rename bindings/{cpp => c}/scripts/build_linux_release.sh (67%) diff --git a/bindings/cpp/scripts/build_linux_release.sh b/bindings/c/scripts/build_linux_release.sh similarity index 67% rename from bindings/cpp/scripts/build_linux_release.sh rename to bindings/c/scripts/build_linux_release.sh index 56fe0e136..6be527efd 100755 --- a/bindings/cpp/scripts/build_linux_release.sh +++ b/bindings/c/scripts/build_linux_release.sh @@ -18,19 +18,32 @@ set -eu script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) repository_root=$(CDPATH= cd -- "${script_dir}/../../.." && pwd) +elf_verifier="${repository_root}/bindings/cpp/scripts/verify_linux_elf.sh" -case "$(uname -m)" in - x86_64|amd64) - rust_target=x86_64-unknown-linux-gnu - ;; - aarch64|arm64) - rust_target=aarch64-unknown-linux-gnu - ;; - *) - echo "unsupported Linux architecture: $(uname -m)" >&2 - exit 2 - ;; -esac +if [ -n "${PAIMON_LINUX_RUST_TARGET:-}" ]; then + case "${PAIMON_LINUX_RUST_TARGET}" in + x86_64-unknown-linux-gnu|aarch64-unknown-linux-gnu) + rust_target=${PAIMON_LINUX_RUST_TARGET} + ;; + *) + echo "unsupported Linux Rust target: ${PAIMON_LINUX_RUST_TARGET}" >&2 + exit 2 + ;; + esac +else + case "$(uname -m)" in + x86_64|amd64) + rust_target=x86_64-unknown-linux-gnu + ;; + aarch64|arm64) + rust_target=aarch64-unknown-linux-gnu + ;; + *) + echo "unsupported Linux architecture: $(uname -m)" >&2 + exit 2 + ;; + esac +fi if ! command -v cargo-zigbuild >/dev/null 2>&1; then echo "cargo-zigbuild is required to build the glibc 2.17 artifact" >&2 @@ -47,5 +60,5 @@ case "${target_dir}" in *) target_dir="${repository_root}/${target_dir}" ;; esac library="${target_dir}/${rust_target}/release/libpaimon_c.so" -"${script_dir}/verify_linux_elf.sh" "${library}" +"${elf_verifier}" "${library}" printf 'validated-library=%s\n' "${library}" diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index b4339360a..435c112f7 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -23,44 +23,95 @@ include(GNUInstallDirs) option(PAIMON_CPP_BUILD_EXAMPLES "Build the C++ facade examples" OFF) option(PAIMON_CPP_BUILD_TESTS "Build the header compile smoke test" OFF) -set(PAIMON_C_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../c/include" CACHE PATH - "Directory containing the cbindgen-generated paimon.h") -set(PAIMON_C_LIBRARY "" CACHE FILEPATH "Path to libpaimon_c") +get_filename_component( + paimon_rust_root "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE) +set(paimon_c_include_dir "${CMAKE_CURRENT_SOURCE_DIR}/../c/include") -if(PAIMON_C_LIBRARY AND NOT EXISTS "${PAIMON_C_LIBRARY}") - message(FATAL_ERROR "PAIMON_C_LIBRARY does not exist: ${PAIMON_C_LIBRARY}") +if(TARGET Paimon::c) + message( + FATAL_ERROR + "bindings/cpp owns Paimon::c and does not accept an external C ABI target") endif() -set(PAIMON_C_INSTALL_FILENAME - "${CMAKE_SHARED_LIBRARY_PREFIX}paimon_c${CMAKE_SHARED_LIBRARY_SUFFIX}") -if(PAIMON_C_LIBRARY) - get_filename_component( - PAIMON_C_INSTALL_FILENAME "${PAIMON_C_LIBRARY}" NAME) +if(DEFINED ENV{CARGO} AND EXISTS "$ENV{CARGO}") + set(paimon_cargo_executable "$ENV{CARGO}") +elseif(DEFINED ENV{HOME} AND EXISTS "$ENV{HOME}/.cargo/bin/cargo") + set(paimon_cargo_executable "$ENV{HOME}/.cargo/bin/cargo") +else() + set(paimon_find_appbundle "${CMAKE_FIND_APPBUNDLE}") + set(CMAKE_FIND_APPBUNDLE NEVER) + find_program(paimon_cargo_executable NAMES cargo) + set(CMAKE_FIND_APPBUNDLE "${paimon_find_appbundle}") endif() - -# Keep the C ABI dependency as a target in both the build and install trees. -# PaimonCppConfig.cmake recreates this imported target for installed consumers. -if(NOT TARGET Paimon::c) - if(PAIMON_C_LIBRARY) - add_library(Paimon::c SHARED IMPORTED GLOBAL) - set_target_properties( - Paimon::c - PROPERTIES - IMPORTED_LOCATION "${PAIMON_C_LIBRARY}" - IMPORTED_NO_SONAME TRUE - INTERFACE_INCLUDE_DIRECTORIES "${PAIMON_C_INCLUDE_DIR}") +if(NOT paimon_cargo_executable) + message(FATAL_ERROR "Rust Cargo was not found") +endif() +execute_process( + COMMAND "${paimon_cargo_executable}" --version + RESULT_VARIABLE paimon_cargo_version_result + OUTPUT_VARIABLE paimon_cargo_version + OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT paimon_cargo_version_result EQUAL 0 OR + NOT paimon_cargo_version MATCHES "^cargo [0-9]") + message(FATAL_ERROR "Not a Rust Cargo executable: ${paimon_cargo_executable}") +endif() +if(APPLE) + set(paimon_c_library "${paimon_rust_root}/target/release/libpaimon_c.dylib") + set(paimon_c_build_command + "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" "${paimon_cargo_executable}" + build --locked --release -p paimon-c) +elseif(UNIX) + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" paimon_system_processor) + if(paimon_system_processor MATCHES "^(x86_64|amd64)$") + set(paimon_linux_rust_target x86_64-unknown-linux-gnu) + elseif(paimon_system_processor MATCHES "^(aarch64|arm64)$") + set(paimon_linux_rust_target aarch64-unknown-linux-gnu) else() - add_library(Paimon::c INTERFACE IMPORTED GLOBAL) - set_target_properties( - Paimon::c - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${PAIMON_C_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES paimon_c) + message( + FATAL_ERROR + "Unsupported Linux architecture for paimon-c: ${CMAKE_SYSTEM_PROCESSOR}") endif() + set(paimon_c_library + "${paimon_rust_root}/target/${paimon_linux_rust_target}/release/libpaimon_c.so") + set(paimon_c_build_command + "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" + "PAIMON_LINUX_RUST_TARGET=${paimon_linux_rust_target}" + "${CMAKE_CURRENT_SOURCE_DIR}/../c/scripts/build_linux_release.sh") +elseif(WIN32) + set(paimon_c_library "${paimon_rust_root}/target/release/paimon_c.dll") + set(paimon_c_build_command + "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" "${paimon_cargo_executable}" + build --locked --release -p paimon-c) +else() + message(FATAL_ERROR "Unsupported platform for automatic paimon-c build") endif() +add_custom_target( + paimon_c_cargo_build ALL + COMMAND ${paimon_c_build_command} + WORKING_DIRECTORY "${paimon_rust_root}" + BYPRODUCTS "${paimon_c_library}" + COMMENT "Building the in-tree Rust paimon-c library" + VERBATIM + USES_TERMINAL) + +get_filename_component( + PAIMON_C_INSTALL_FILENAME "${paimon_c_library}" NAME) + +# Keep the C ABI dependency as a target in both the build and install trees. +# PaimonCppConfig.cmake recreates this imported target for installed consumers. +add_library(Paimon::c SHARED IMPORTED GLOBAL) +set_target_properties( + Paimon::c + PROPERTIES + IMPORTED_LOCATION "${paimon_c_library}" + IMPORTED_NO_SONAME TRUE + INTERFACE_INCLUDE_DIRECTORIES "${paimon_c_include_dir}") +add_dependencies(Paimon::c paimon_c_cargo_build) + add_library(paimon_cpp INTERFACE) add_library(Paimon::cpp ALIAS paimon_cpp) +add_dependencies(paimon_cpp paimon_c_cargo_build) set_target_properties(paimon_cpp PROPERTIES EXPORT_NAME cpp) target_compile_features(paimon_cpp INTERFACE cxx_std_17) target_include_directories( @@ -69,19 +120,14 @@ target_include_directories( "$" "$") -if(PAIMON_C_INCLUDE_DIR) - target_include_directories( - paimon_cpp INTERFACE "$") -endif() +target_include_directories( + paimon_cpp INTERFACE "$") target_link_libraries(paimon_cpp INTERFACE Paimon::c) include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/PaimonNoRuntimePlugin.cmake") if(PAIMON_CPP_BUILD_EXAMPLES) - if(NOT PAIMON_C_INCLUDE_DIR) - message(FATAL_ERROR "PAIMON_C_INCLUDE_DIR is required for examples") - endif() add_executable(paimon_cpp_batch_read examples/batch_read.cpp) target_link_libraries(paimon_cpp_batch_read PRIVATE Paimon::cpp) add_executable(paimon_cpp_streaming_write examples/streaming_write.cpp) @@ -113,7 +159,7 @@ if(PAIMON_CPP_BUILD_TESTS) COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target paimon_cpp_header_smoke) - if(PAIMON_C_LIBRARY AND UNIX AND NOT APPLE) + if(UNIX AND NOT APPLE) # Compile as C++, but deliberately invoke the C linker driver. This proves # the facade itself needs no libstdc++/libc++ symbols while resolving every # wrapped function against the real libpaimon_c artifact. @@ -122,29 +168,29 @@ if(PAIMON_CPP_BUILD_TESTS) set(paimon_cpp_smoke_library "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_cpp_real_link_smoke${CMAKE_SHARED_LIBRARY_SUFFIX}") get_filename_component( - PAIMON_C_LIBRARY_DIR "${PAIMON_C_LIBRARY}" DIRECTORY) + paimon_c_library_dir "${paimon_c_library}" DIRECTORY) add_custom_command( OUTPUT "${paimon_cpp_smoke_object}" COMMAND "${CMAKE_CXX_COMPILER}" -std=c++17 -fPIC -fno-exceptions -fno-rtti -fvisibility=hidden -fvisibility-inlines-hidden "-I${CMAKE_CURRENT_SOURCE_DIR}/include" - "-I${PAIMON_C_INCLUDE_DIR}" + "-I${paimon_c_include_dir}" -c "${CMAKE_CURRENT_SOURCE_DIR}/tests/header_smoke.cpp" -o "${paimon_cpp_smoke_object}" DEPENDS tests/header_smoke.cpp include/paimon/paimon.hpp - "${PAIMON_C_INCLUDE_DIR}/paimon.h" + "${paimon_c_include_dir}/paimon.h" VERBATIM) add_custom_command( OUTPUT "${paimon_cpp_smoke_library}" COMMAND "${CMAKE_C_COMPILER}" -shared "${paimon_cpp_smoke_object}" - "-L${PAIMON_C_LIBRARY_DIR}" -lpaimon_c -Wl,-z,defs + "-L${paimon_c_library_dir}" -lpaimon_c -Wl,-z,defs -o "${paimon_cpp_smoke_library}" - DEPENDS "${paimon_cpp_smoke_object}" "${PAIMON_C_LIBRARY}" + DEPENDS "${paimon_cpp_smoke_object}" "${paimon_c_library}" VERBATIM) add_custom_target( paimon_cpp_real_link_smoke ALL DEPENDS "${paimon_cpp_smoke_library}") @@ -172,7 +218,7 @@ if(PAIMON_CPP_BUILD_TESTS) add_test( NAME paimon_c_no_cpp_runtime COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "${PAIMON_C_LIBRARY}") + "${paimon_c_library}") add_test( NAME paimon_cpp_facade_no_cpp_runtime COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" @@ -250,7 +296,7 @@ if(PAIMON_CPP_BUILD_TESTS) "${CMAKE_COMMAND}" "-DLOADER=$" "-DPLUGIN=$" - "-DPAIMON_C_LIBRARY=${PAIMON_C_LIBRARY}" + "-DPAIMON_C_LIBRARY_UNDER_TEST=${paimon_c_library}" "-DTEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/isolated-load-test" -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_isolated_load.cmake") @@ -302,16 +348,12 @@ if(PAIMON_CPP_BUILD_TESTS) endif() install(DIRECTORY include/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") -if(EXISTS "${PAIMON_C_INCLUDE_DIR}/paimon.h") - install( - FILES "${PAIMON_C_INCLUDE_DIR}/paimon.h" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") -endif() -if(PAIMON_C_LIBRARY) - install( - FILES "${PAIMON_C_LIBRARY}" - DESTINATION "${CMAKE_INSTALL_LIBDIR}") -endif() +install( + FILES "${paimon_c_include_dir}/paimon.h" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +install( + FILES "${paimon_c_library}" + DESTINATION "${CMAKE_INSTALL_LIBDIR}") install( TARGETS paimon_cpp EXPORT PaimonCppTargets diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index a361fbfc6..86cd4441a 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -73,14 +73,25 @@ auto dropped = catalog.drop_table( ## Build -Generate the C header and build the Rust library first: +Configure and build the C++ facade directly. The build always compiles the +in-tree `bindings/c` crate first, so the C and C++ layers come from the same +source revision: ```bash -cargo build --release -p paimon-c -cbindgen --config bindings/c/cbindgen.toml bindings/c \ - --output bindings/c/include/paimon.h +cmake -S bindings/cpp -B target/cpp-build \ + -DPAIMON_CPP_BUILD_EXAMPLES=ON +cmake --build target/cpp-build ``` +macOS and Windows use Cargo's release profile. Linux uses +`bindings/c/scripts/build_linux_release.sh`, which requires Zig and +`cargo-zigbuild`, targets glibc 2.17, and rejects non-C runtime dependencies. +External prebuilt paimon-c libraries and parent-provided `Paimon::c` targets are +deliberately unsupported. + +When changing the C ABI, regenerate the checked header separately with +`cbindgen`; ordinary builds consume the checked-in `bindings/c/include/paimon.h`. + For a shared plugin that must load without `libstdc++`, `libc++`, or `libgcc_s`, compile the C++ source without exceptions/RTTI and use the C linker driver for the final link: @@ -99,19 +110,15 @@ The plugin must expose each public entry point with no-exceptions subset. Linking the final `.so` with a C++ driver can add a C++ runtime even when the source does not call that runtime directly. -Or install its CMake interface target: +Install its CMake interface target: ```bash -cmake -S bindings/cpp -B target/cpp-build \ - -DPAIMON_C_LIBRARY="$PWD/target/release/libpaimon_c.so" \ - -DPAIMON_CPP_BUILD_EXAMPLES=ON -cmake --build target/cpp-build cmake --install target/cpp-build --prefix /your/prefix ``` -When `PAIMON_C_LIBRARY` is set, installation copies `libpaimon_c` into the -prefix and the package exports imported target `Paimon::c`. Installed consumers -can build a verified no-runtime plugin with the provided helper: +Installation always bundles the just-built `libpaimon_c` and exports imported +target `Paimon::c`. Installed consumers can build a verified no-runtime plugin +with the provided helper: ```cmake cmake_minimum_required(VERSION 3.15) @@ -133,8 +140,8 @@ only `$ORIGIN` as its runtime search path, and runs the installed ELF guard after every successful link. `Paimon::cpp` remains the header-only facade target for consumers that manage -their own final link. `PaimonCpp_C_LIBRARY` may be set before `find_package` to -select an externally installed `libpaimon_c`. +their own final link. The installed package always resolves `Paimon::c` to the +library bundled in the same installation prefix. ## Linux runtime guard @@ -144,7 +151,7 @@ the ELF guard, preventing a newer build host from silently raising the runtime glibc requirement or introducing a shared compiler/C++ runtime: ```bash -bindings/cpp/scripts/build_linux_release.sh +bindings/c/scripts/build_linux_release.sh ``` The validated library is written to diff --git a/bindings/cpp/cmake/PaimonCppConfig.cmake.in b/bindings/cpp/cmake/PaimonCppConfig.cmake.in index f37f23993..f6e960995 100644 --- a/bindings/cpp/cmake/PaimonCppConfig.cmake.in +++ b/bindings/cpp/cmake/PaimonCppConfig.cmake.in @@ -17,39 +17,42 @@ @PACKAGE_INIT@ -if(NOT TARGET Paimon::c) - set(_paimon_c_bundled - "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_LIBDIR@/@PAIMON_C_INSTALL_FILENAME@") - if(DEFINED PaimonCpp_C_LIBRARY) - set(_paimon_c_library "${PaimonCpp_C_LIBRARY}") - elseif(EXISTS "${_paimon_c_bundled}") - set(_paimon_c_library "${_paimon_c_bundled}") - else() - unset(_PaimonCpp_DISCOVERED_C_LIBRARY CACHE) - find_library( - _PaimonCpp_DISCOVERED_C_LIBRARY - NAMES paimon_c - HINTS "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_LIBDIR@") - set(_paimon_c_library "${_PaimonCpp_DISCOVERED_C_LIBRARY}") - unset(_PaimonCpp_DISCOVERED_C_LIBRARY CACHE) - endif() - - if(NOT _paimon_c_library OR NOT EXISTS "${_paimon_c_library}") +if(TARGET Paimon::cpp) + if(NOT TARGET Paimon::c) set(PaimonCpp_FOUND FALSE) set(PaimonCpp_NOT_FOUND_MESSAGE - "libpaimon_c was not found; install it under the package prefix or set PaimonCpp_C_LIBRARY") + "Paimon::cpp exists without its bundled Paimon::c target") return() endif() + include("${CMAKE_CURRENT_LIST_DIR}/PaimonNoRuntimePlugin.cmake") + check_required_components(PaimonCpp) + return() +endif() - add_library(Paimon::c SHARED IMPORTED) - set_target_properties( - Paimon::c - PROPERTIES - IMPORTED_LOCATION "${_paimon_c_library}" - IMPORTED_NO_SONAME TRUE - INTERFACE_INCLUDE_DIRECTORIES "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") +if(TARGET Paimon::c) + set(PaimonCpp_FOUND FALSE) + set(PaimonCpp_NOT_FOUND_MESSAGE + "Paimon::c already exists; PaimonCpp requires its bundled paimon-c library") + return() endif() +set(_paimon_c_library + "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_LIBDIR@/@PAIMON_C_INSTALL_FILENAME@") +if(NOT EXISTS "${_paimon_c_library}") + set(PaimonCpp_FOUND FALSE) + set(PaimonCpp_NOT_FOUND_MESSAGE + "the bundled paimon-c library is missing: ${_paimon_c_library}") + return() +endif() + +add_library(Paimon::c SHARED IMPORTED) +set_target_properties( + Paimon::c + PROPERTIES + IMPORTED_LOCATION "${_paimon_c_library}" + IMPORTED_NO_SONAME TRUE + INTERFACE_INCLUDE_DIRECTORIES "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") + include("${CMAKE_CURRENT_LIST_DIR}/PaimonCppTargets.cmake") include("${CMAKE_CURRENT_LIST_DIR}/PaimonNoRuntimePlugin.cmake") check_required_components(PaimonCpp) diff --git a/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt index cff091859..83ab16f71 100644 --- a/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt +++ b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt @@ -18,6 +18,10 @@ cmake_minimum_required(VERSION 3.15) project(PaimonCppInstallTreeConsumer LANGUAGES C CXX) +find_package(PaimonCpp CONFIG REQUIRED) +# Package discovery may happen through more than one dependency. A repeated +# lookup must retain the same bundled Paimon::c target instead of treating it +# as an external override. find_package(PaimonCpp CONFIG REQUIRED) paimon_add_no_runtime_plugin( paimon_install_tree_consumer diff --git a/bindings/cpp/tests/run_isolated_load.cmake b/bindings/cpp/tests/run_isolated_load.cmake index 9eeb85235..2514e3627 100644 --- a/bindings/cpp/tests/run_isolated_load.cmake +++ b/bindings/cpp/tests/run_isolated_load.cmake @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -foreach(required IN ITEMS LOADER PLUGIN PAIMON_C_LIBRARY TEST_ROOT) +foreach(required IN ITEMS LOADER PLUGIN PAIMON_C_LIBRARY_UNDER_TEST TEST_ROOT) if(NOT DEFINED ${required}) message(FATAL_ERROR "missing -D${required}=...") endif() @@ -23,7 +23,8 @@ endforeach() file(REMOVE_RECURSE "${TEST_ROOT}") file(MAKE_DIRECTORY "${TEST_ROOT}") -file(COPY "${PLUGIN}" "${PAIMON_C_LIBRARY}" DESTINATION "${TEST_ROOT}") +file(COPY "${PLUGIN}" "${PAIMON_C_LIBRARY_UNDER_TEST}" + DESTINATION "${TEST_ROOT}") get_filename_component(plugin_name "${PLUGIN}" NAME) execute_process( COMMAND "${LOADER}" "./${plugin_name}" From f0ce636326799d344258277033c00df5b7aff089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 18:19:55 +0800 Subject: [PATCH 06/21] build(cpp): stage a ready-to-use SDK --- .gitignore | 1 + bindings/cpp/CMakeLists.txt | 58 +++++++++++++++++++++++++++++++++---- bindings/cpp/README.md | 14 +++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 2ed1d6f45..45d9e2707 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ # under the License. /target +/bindings/cpp/target/ .idea .vscode **/.DS_Store diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index 435c112f7..dbe616e17 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -347,13 +347,42 @@ if(PAIMON_CPP_BUILD_TESTS) endif() endif() -install(DIRECTORY include/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +set(paimon_cpp_install_component PaimonCppSdk) +install( + DIRECTORY include/ + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT ${paimon_cpp_install_component}) install( FILES "${paimon_c_include_dir}/paimon.h" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT ${paimon_cpp_install_component}) install( FILES "${paimon_c_library}" - DESTINATION "${CMAKE_INSTALL_LIBDIR}") + DESTINATION "${CMAKE_INSTALL_LIBDIR}" + COMPONENT ${paimon_cpp_install_component}) +if(APPLE) + find_program(paimon_install_name_tool NAMES install_name_tool) + find_program(paimon_codesign NAMES codesign) + if(NOT paimon_install_name_tool OR NOT paimon_codesign) + message(FATAL_ERROR "install_name_tool and codesign are required on macOS") + endif() + install( + CODE + "set(paimon_installed_library \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${PAIMON_C_INSTALL_FILENAME}\") + execute_process( + COMMAND \"${paimon_install_name_tool}\" -id \"@rpath/${PAIMON_C_INSTALL_FILENAME}\" \"\${paimon_installed_library}\" + RESULT_VARIABLE paimon_install_name_result) + if(NOT paimon_install_name_result EQUAL 0) + message(FATAL_ERROR \"failed to set the paimon-c install name\") + endif() + execute_process( + COMMAND \"${paimon_codesign}\" --force --sign - \"\${paimon_installed_library}\" + RESULT_VARIABLE paimon_codesign_result) + if(NOT paimon_codesign_result EQUAL 0) + message(FATAL_ERROR \"failed to sign the installed paimon-c library\") + endif()" + COMPONENT ${paimon_cpp_install_component}) +endif() install( TARGETS paimon_cpp EXPORT PaimonCppTargets @@ -362,7 +391,8 @@ install( EXPORT PaimonCppTargets FILE PaimonCppTargets.cmake NAMESPACE Paimon:: - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp") + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp" + COMPONENT ${paimon_cpp_install_component}) configure_package_config_file( cmake/PaimonCppConfig.cmake.in @@ -377,7 +407,23 @@ install( "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfigVersion.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/PaimonNoRuntimePlugin.cmake" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp") + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp" + COMPONENT ${paimon_cpp_install_component}) install( PROGRAMS "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp") + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp" + COMPONENT ${paimon_cpp_install_component}) + +# Keep the normal CMake install target, and also materialize a ready-to-use SDK +# beside the build files so `cmake --build` has an obvious deliverable. +set(paimon_cpp_sdk_dir "${CMAKE_CURRENT_BINARY_DIR}/sdk") +add_custom_target( + paimon_cpp_sdk ALL + COMMAND + "${CMAKE_COMMAND}" --install "${CMAKE_BINARY_DIR}" + --prefix "${paimon_cpp_sdk_dir}" + --component ${paimon_cpp_install_component} + DEPENDS paimon_c_cargo_build + COMMENT "Staging the Paimon C++ SDK in ${paimon_cpp_sdk_dir}" + VERBATIM + USES_TERMINAL) diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index 86cd4441a..16112ad78 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -83,6 +83,20 @@ cmake -S bindings/cpp -B target/cpp-build \ cmake --build target/cpp-build ``` +The default build stages a complete SDK under `target/cpp-build/sdk`: + +```text +sdk/ +├── include/paimon.h +├── include/paimon/paimon.hpp +├── /libpaimon_c.so # Linux +└── /cmake/PaimonCpp/ +``` + +`` follows GNUInstallDirs and is normally `lib` or `lib64`. macOS uses +`libpaimon_c.dylib` in the same location. The facade is header-only, so there is +intentionally no separate `libpaimon_cpp` shared library. + macOS and Windows use Cargo's release profile. Linux uses `bindings/c/scripts/build_linux_release.sh`, which requires Zig and `cargo-zigbuild`, targets glibc 2.17, and rejects non-C runtime dependencies. From 799588f3a657c92e8b5ba0e9544f72ecd2cde98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 18:49:23 +0800 Subject: [PATCH 07/21] build(cpp): stage artifacts in the build target --- bindings/cpp/CMakeLists.txt | 11 +++++------ bindings/cpp/README.md | 5 +++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index dbe616e17..397941ead 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -414,16 +414,15 @@ install( DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp" COMPONENT ${paimon_cpp_install_component}) -# Keep the normal CMake install target, and also materialize a ready-to-use SDK -# beside the build files so `cmake --build` has an obvious deliverable. -set(paimon_cpp_sdk_dir "${CMAKE_CURRENT_BINARY_DIR}/sdk") +# Keep the normal CMake install target, and also materialize the ready-to-use +# headers, library, and package files directly in the CMake build directory. add_custom_target( - paimon_cpp_sdk ALL + paimon_cpp_artifacts ALL COMMAND "${CMAKE_COMMAND}" --install "${CMAKE_BINARY_DIR}" - --prefix "${paimon_cpp_sdk_dir}" + --prefix "${CMAKE_CURRENT_BINARY_DIR}" --component ${paimon_cpp_install_component} DEPENDS paimon_c_cargo_build - COMMENT "Staging the Paimon C++ SDK in ${paimon_cpp_sdk_dir}" + COMMENT "Staging the Paimon C++ artifacts in ${CMAKE_CURRENT_BINARY_DIR}" VERBATIM USES_TERMINAL) diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index 16112ad78..a5070da17 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -83,10 +83,11 @@ cmake -S bindings/cpp -B target/cpp-build \ cmake --build target/cpp-build ``` -The default build stages a complete SDK under `target/cpp-build/sdk`: +The CMake build directory itself is a complete, directly consumable artifact +tree: ```text -sdk/ +target/cpp-build/ ├── include/paimon.h ├── include/paimon/paimon.hpp ├── /libpaimon_c.so # Linux From 97eb443810ca5930e6cdf80da30856d76e7f8f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 20:21:51 +0800 Subject: [PATCH 08/21] build(cpp): use native Cargo for Linux --- bindings/c/scripts/build_linux_release.sh | 64 ------------------- bindings/cpp/CMakeLists.txt | 28 ++------ bindings/cpp/README.md | 40 ++++++------ bindings/cpp/scripts/verify_linux_elf.sh | 26 ++------ .../install_tree_consumer/CMakeLists.txt | 2 +- 5 files changed, 32 insertions(+), 128 deletions(-) delete mode 100755 bindings/c/scripts/build_linux_release.sh diff --git a/bindings/c/scripts/build_linux_release.sh b/bindings/c/scripts/build_linux_release.sh deleted file mode 100755 index 6be527efd..000000000 --- a/bindings/c/scripts/build_linux_release.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env sh -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to you under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -repository_root=$(CDPATH= cd -- "${script_dir}/../../.." && pwd) -elf_verifier="${repository_root}/bindings/cpp/scripts/verify_linux_elf.sh" - -if [ -n "${PAIMON_LINUX_RUST_TARGET:-}" ]; then - case "${PAIMON_LINUX_RUST_TARGET}" in - x86_64-unknown-linux-gnu|aarch64-unknown-linux-gnu) - rust_target=${PAIMON_LINUX_RUST_TARGET} - ;; - *) - echo "unsupported Linux Rust target: ${PAIMON_LINUX_RUST_TARGET}" >&2 - exit 2 - ;; - esac -else - case "$(uname -m)" in - x86_64|amd64) - rust_target=x86_64-unknown-linux-gnu - ;; - aarch64|arm64) - rust_target=aarch64-unknown-linux-gnu - ;; - *) - echo "unsupported Linux architecture: $(uname -m)" >&2 - exit 2 - ;; - esac -fi - -if ! command -v cargo-zigbuild >/dev/null 2>&1; then - echo "cargo-zigbuild is required to build the glibc 2.17 artifact" >&2 - exit 2 -fi - -cd "${repository_root}" -cargo zigbuild --locked --release -p paimon-c \ - --target "${rust_target}.2.17" - -target_dir=${CARGO_TARGET_DIR:-${repository_root}/target} -case "${target_dir}" in - /*) ;; - *) target_dir="${repository_root}/${target_dir}" ;; -esac -library="${target_dir}/${rust_target}/release/libpaimon_c.so" -"${elf_verifier}" "${library}" -printf 'validated-library=%s\n' "${library}" diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index 397941ead..efdb05ee5 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -61,22 +61,10 @@ if(APPLE) "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" "${paimon_cargo_executable}" build --locked --release -p paimon-c) elseif(UNIX) - string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" paimon_system_processor) - if(paimon_system_processor MATCHES "^(x86_64|amd64)$") - set(paimon_linux_rust_target x86_64-unknown-linux-gnu) - elseif(paimon_system_processor MATCHES "^(aarch64|arm64)$") - set(paimon_linux_rust_target aarch64-unknown-linux-gnu) - else() - message( - FATAL_ERROR - "Unsupported Linux architecture for paimon-c: ${CMAKE_SYSTEM_PROCESSOR}") - endif() - set(paimon_c_library - "${paimon_rust_root}/target/${paimon_linux_rust_target}/release/libpaimon_c.so") + set(paimon_c_library "${paimon_rust_root}/target/release/libpaimon_c.so") set(paimon_c_build_command - "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" - "PAIMON_LINUX_RUST_TARGET=${paimon_linux_rust_target}" - "${CMAKE_CURRENT_SOURCE_DIR}/../c/scripts/build_linux_release.sh") + "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" "${paimon_cargo_executable}" + build --locked --release -p paimon-c) elseif(WIN32) set(paimon_c_library "${paimon_rust_root}/target/release/paimon_c.dll") set(paimon_c_build_command @@ -308,13 +296,9 @@ if(PAIMON_CPP_BUILD_TESTS) paimon_elf_fixture_libgcc PRIVATE -Wl,--no-as-needed) target_link_libraries(paimon_elf_fixture_libgcc PRIVATE gcc_s) add_test( - NAME paimon_elf_guard_rejects_libgcc - COMMAND - "${CMAKE_COMMAND}" - "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "-DLIBRARY=$" - "-DEXPECTED=libgcc_s" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") + NAME paimon_elf_guard_allows_libgcc + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" + "$") endif() set(paimon_cpp_install_test_root diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index a5070da17..63d6ace64 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -98,18 +98,18 @@ target/cpp-build/ `libpaimon_c.dylib` in the same location. The facade is header-only, so there is intentionally no separate `libpaimon_cpp` shared library. -macOS and Windows use Cargo's release profile. Linux uses -`bindings/c/scripts/build_linux_release.sh`, which requires Zig and -`cargo-zigbuild`, targets glibc 2.17, and rejects non-C runtime dependencies. -External prebuilt paimon-c libraries and parent-provided `Paimon::c` targets are -deliberately unsupported. +All platforms use `cargo build --locked --release -p paimon-c`. Build Linux +release artifacts on the oldest glibc version that must be supported; glibc is +backward compatible with binaries built against older symbol versions. +External prebuilt paimon-c libraries and parent-provided `Paimon::c` targets +are deliberately unsupported. When changing the C ABI, regenerate the checked header separately with `cbindgen`; ordinary builds consume the checked-in `bindings/c/include/paimon.h`. -For a shared plugin that must load without `libstdc++`, `libc++`, or -`libgcc_s`, compile the C++ source without exceptions/RTTI and use the C linker -driver for the final link: +For a shared plugin that must load without `libstdc++` or `libc++`, compile the +C++ source without exceptions/RTTI and use the C linker driver for the final +link: ```bash c++ -std=c++17 -fPIC -fno-exceptions -fno-rtti \ @@ -160,26 +160,24 @@ library bundled in the same installation prefix. ## Linux runtime guard -Install Zig and `cargo-zigbuild`, then produce every Linux release artifact -through the checked build script. It targets glibc 2.17 and immediately runs -the ELF guard, preventing a newer build host from silently raising the runtime -glibc requirement or introducing a shared compiler/C++ runtime: +Run the ELF guard on the library staged by CMake when validating a Linux +release artifact: ```bash -bindings/c/scripts/build_linux_release.sh +bindings/cpp/scripts/verify_linux_elf.sh \ + target/cpp-build/lib/libpaimon_c.so ``` -The validated library is written to -`target//release/libpaimon_c.so`. +Some distributions use `lib64` instead of `lib`. The build host determines the +minimum glibc version; build on glibc 2.17 when 2.17 is the deployment baseline. It prints the build host's `ldd --version` and applies a `DT_NEEDED` allowlist -containing glibc components and `libpaimon_c`. It rejects C++ runtimes, -`libgcc_s`, `libunwind`, `libatomic`, `GLIBCXX`/`CXXABI`/`GCC` symbol versions, +containing glibc components, `libgcc_s`, and `libpaimon_c`. It rejects C++ +runtimes, `libunwind`, `libatomic`, `GLIBCXX`/`CXXABI` symbol versions, undefined or exported C++ mangled symbols, unversioned host hooks, operator -new/delete, RTTI/dynamic-cast support, absolute runtime paths, and -private/non-baseline glibc ABI versions. Glibc's C-level `__cxa_atexit`, -`__cxa_finalize`, and `__cxa_thread_atexit_impl` remain allowed. The highest -referenced numeric `GLIBC_*` symbol version must not exceed 2.17. +new/delete, RTTI/dynamic-cast support, absolute runtime paths, and private +glibc ABI versions. Glibc's C-level `__cxa_atexit`, `__cxa_finalize`, and +`__cxa_thread_atexit_impl` remain allowed. `Scan::plan()` remains a bounded scan. Use `StreamScanOptions` and `ReadBuilder::new_stream_scan` for a stateful continuous scan. Persist diff --git a/bindings/cpp/scripts/verify_linux_elf.sh b/bindings/cpp/scripts/verify_linux_elf.sh index 89a144271..52c1c2106 100755 --- a/bindings/cpp/scripts/verify_linux_elf.sh +++ b/bindings/cpp/scripts/verify_linux_elf.sh @@ -102,11 +102,11 @@ needed_names=$(printf '%s\n' "$needed" | sed -n 's/.*Shared library: \[\([^]]*\)\].*/\1/p') for dependency in $needed_names; do case "$dependency" in - libstdc++*|libc++*|libsupc++*|libgcc_s*|libunwind*|libatomic*) + libstdc++*|libc++*|libsupc++*|libunwind*|libatomic*) echo "forbidden non-C runtime dependency in DT_NEEDED: $dependency" >&2 exit 1 ;; - libc.so.*|libm.so.*|libpthread.so.*|libdl.so.*|librt.so.*|libutil.so.*|libresolv.so.*|libanl.so.*|libBrokenLocale.so.*|libcrypt.so.*|libnss_*.so.*|ld-linux*.so.*|ld64.so.*|ld.so.*|libpaimon_c.so*) + libc.so.*|libm.so.*|libpthread.so.*|libdl.so.*|librt.so.*|libutil.so.*|libresolv.so.*|libanl.so.*|libBrokenLocale.so.*|libcrypt.so.*|libnss_*.so.*|libgcc_s.so.*|ld-linux*.so.*|ld64.so.*|ld.so.*|libpaimon_c.so*) ;; *) echo "dependency is outside the glibc/libpaimon_c allowlist: $dependency" >&2 @@ -122,7 +122,7 @@ unexpected_unversioned=$(printf '%s\n' "$undefined" | awk ' name = $8 base = name sub(/@.*/, "", base) - if (base == "" || name ~ /@GLIBC_[0-9]/ || base ~ /^paimon_/ || + if (base == "" || name ~ /@(GLIBC|GCC)_[0-9]/ || base ~ /^paimon_/ || base ~ /^_Z/) { next } @@ -167,10 +167,10 @@ fi version_info=$($readelf_cmd --version-info --wide "$library" || true) symbol_versions=$(printf '%s\n%s\n' "$undefined" "$version_info") -if printf '%s\n' "$symbol_versions" | grep -Eq 'GLIBCXX_|CXXABI_|GCC_[0-9]'; then - echo "forbidden C++/compiler runtime symbol version" >&2 +if printf '%s\n' "$symbol_versions" | grep -Eq 'GLIBCXX_|CXXABI_'; then + echo "forbidden C++ runtime symbol version" >&2 printf '%s\n' "$symbol_versions" | - grep -E 'GLIBCXX_|CXXABI_|GCC_[0-9]' >&2 + grep -E 'GLIBCXX_|CXXABI_' >&2 exit 1 fi @@ -200,20 +200,6 @@ if printf '%s\n' "$symbol_versions" | grep -Eq 'GLIBC_(PRIVATE|ABI_)'; then exit 1 fi -max_glibc=$(printf '%s\n' "$symbol_versions" | - grep -Eo 'GLIBC_[0-9][0-9.]*' | - sed 's/^GLIBC_//' | - sort -V | - tail -n 1 || true) -if [ -n "$max_glibc" ]; then - newest=$(printf '%s\n' 2.17 "$max_glibc" | sort -V | tail -n 1) - if [ "$newest" != "2.17" ]; then - echo "GLIBC symbol version $max_glibc exceeds supported baseline 2.17" >&2 - printf '%s\n' "$symbol_versions" | grep "GLIBC_$max_glibc" >&2 - exit 1 - fi -fi - if ! command -v c++filt >/dev/null 2>&1; then echo "c++filt is required to inspect demangled undefined symbols" >&2 exit 2 diff --git a/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt index 83ab16f71..1300b8959 100644 --- a/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt +++ b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt @@ -31,5 +31,5 @@ option(PAIMON_INJECT_FORBIDDEN_RUNTIME "Test the installed ELF guard" OFF) if(PAIMON_INJECT_FORBIDDEN_RUNTIME) target_link_options( paimon_install_tree_consumer PRIVATE -Wl,--no-as-needed) - target_link_libraries(paimon_install_tree_consumer PRIVATE gcc_s) + target_link_libraries(paimon_install_tree_consumer PRIVATE stdc++) endif() From 8087528961e6bf244ad0cae6026e14de2ced690d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 20:27:14 +0800 Subject: [PATCH 09/21] refactor(cpp): remove no-runtime plugin guard --- bindings/cpp/CMakeLists.txt | 211 +---------------- bindings/cpp/README.md | 67 +----- bindings/cpp/cmake/PaimonCppConfig.cmake.in | 2 - .../cpp/cmake/PaimonNoRuntimePlugin.cmake | 140 ----------- bindings/cpp/include/paimon/paimon.hpp | 12 - bindings/cpp/scripts/verify_linux_elf.sh | 217 ------------------ .../cpp/tests/check_incremental_relink.cmake | 49 ---- bindings/cpp/tests/dlopen_smoke.c | 63 ----- .../cpp/tests/elf_fixtures/executable_stack.c | 19 -- .../cpp/tests/elf_fixtures/needs_libgcc.c | 24 -- .../cpp/tests/elf_fixtures/pie_executable.c | 19 -- .../tests/elf_fixtures/undefined_host_hook.c | 21 -- .../elf_fixtures/undefined_operator_new.c | 24 -- bindings/cpp/tests/expect_elf_rejected.cmake | 39 ---- .../helper_config/paimon_cpp_helper_config.h | 22 -- .../install_tree_consumer/CMakeLists.txt | 14 +- .../{plugin.cpp => main.cpp} | 9 +- bindings/cpp/tests/no_cpp_runtime_plugin.cpp | 71 ------ bindings/cpp/tests/relink_probe.cpp | 21 -- .../cpp/tests/run_install_tree_consumer.cmake | 30 +-- bindings/cpp/tests/run_isolated_load.cmake | 38 --- 21 files changed, 27 insertions(+), 1085 deletions(-) delete mode 100644 bindings/cpp/cmake/PaimonNoRuntimePlugin.cmake delete mode 100755 bindings/cpp/scripts/verify_linux_elf.sh delete mode 100644 bindings/cpp/tests/check_incremental_relink.cmake delete mode 100644 bindings/cpp/tests/dlopen_smoke.c delete mode 100644 bindings/cpp/tests/elf_fixtures/executable_stack.c delete mode 100644 bindings/cpp/tests/elf_fixtures/needs_libgcc.c delete mode 100644 bindings/cpp/tests/elf_fixtures/pie_executable.c delete mode 100644 bindings/cpp/tests/elf_fixtures/undefined_host_hook.c delete mode 100644 bindings/cpp/tests/elf_fixtures/undefined_operator_new.c delete mode 100644 bindings/cpp/tests/expect_elf_rejected.cmake delete mode 100644 bindings/cpp/tests/helper_config/paimon_cpp_helper_config.h rename bindings/cpp/tests/install_tree_consumer/{plugin.cpp => main.cpp} (85%) delete mode 100644 bindings/cpp/tests/no_cpp_runtime_plugin.cpp delete mode 100644 bindings/cpp/tests/relink_probe.cpp delete mode 100644 bindings/cpp/tests/run_isolated_load.cmake diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index efdb05ee5..bc6416a52 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -16,7 +16,7 @@ # under the License. cmake_minimum_required(VERSION 3.15) -project(PaimonCpp VERSION 0.1.0 LANGUAGES C CXX) +project(PaimonCpp VERSION 0.1.0 LANGUAGES CXX) include(CMakePackageConfigHelpers) include(GNUInstallDirs) @@ -113,8 +113,6 @@ target_include_directories( target_link_libraries(paimon_cpp INTERFACE Paimon::c) -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/PaimonNoRuntimePlugin.cmake") - if(PAIMON_CPP_BUILD_EXAMPLES) add_executable(paimon_cpp_batch_read examples/batch_read.cpp) target_link_libraries(paimon_cpp_batch_read PRIVATE Paimon::cpp) @@ -132,203 +130,23 @@ if(PAIMON_CPP_BUILD_TESTS) PRIVATE PAIMON_C_HEADER="paimon_test_stub.h") target_include_directories( paimon_cpp_header_smoke PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/tests") - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") - target_compile_options( - paimon_cpp_header_smoke - PRIVATE - -fno-exceptions - -fno-rtti - -fvisibility=hidden - -fvisibility-inlines-hidden) - endif() target_link_libraries(paimon_cpp_header_smoke PRIVATE Paimon::cpp) add_test( NAME paimon_cpp_header_compile_smoke COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target paimon_cpp_header_smoke) - if(UNIX AND NOT APPLE) - # Compile as C++, but deliberately invoke the C linker driver. This proves - # the facade itself needs no libstdc++/libc++ symbols while resolving every - # wrapped function against the real libpaimon_c artifact. - set(paimon_cpp_smoke_object - "${CMAKE_CURRENT_BINARY_DIR}/paimon_cpp_real_link_smoke.o") - set(paimon_cpp_smoke_library - "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}paimon_cpp_real_link_smoke${CMAKE_SHARED_LIBRARY_SUFFIX}") - get_filename_component( - paimon_c_library_dir "${paimon_c_library}" DIRECTORY) - add_custom_command( - OUTPUT "${paimon_cpp_smoke_object}" - COMMAND - "${CMAKE_CXX_COMPILER}" -std=c++17 -fPIC -fno-exceptions -fno-rtti - -fvisibility=hidden -fvisibility-inlines-hidden - "-I${CMAKE_CURRENT_SOURCE_DIR}/include" - "-I${paimon_c_include_dir}" - -c "${CMAKE_CURRENT_SOURCE_DIR}/tests/header_smoke.cpp" - -o "${paimon_cpp_smoke_object}" - DEPENDS - tests/header_smoke.cpp - include/paimon/paimon.hpp - "${paimon_c_include_dir}/paimon.h" - VERBATIM) - add_custom_command( - OUTPUT "${paimon_cpp_smoke_library}" - COMMAND - "${CMAKE_C_COMPILER}" -shared - "${paimon_cpp_smoke_object}" - "-L${paimon_c_library_dir}" -lpaimon_c -Wl,-z,defs - -o "${paimon_cpp_smoke_library}" - DEPENDS "${paimon_cpp_smoke_object}" "${paimon_c_library}" - VERBATIM) - add_custom_target( - paimon_cpp_real_link_smoke ALL DEPENDS "${paimon_cpp_smoke_library}") - - # The helper always compiles with the configured CXX compiler and links the - # resulting object with the configured C compiler. This remains correct when - # those drivers are different versions, as on an older deployment host. - paimon_add_no_runtime_plugin( - paimon_cpp_no_runtime_plugin - SOURCES tests/no_cpp_runtime_plugin.cpp - INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/tests/helper_config" - COMPILE_DEFINITIONS PAIMON_CPP_HELPER_COMPILE_DEFINITION=73) - - configure_file( - tests/relink_probe.cpp - "${CMAKE_CURRENT_BINARY_DIR}/relink_probe.cpp" - COPYONLY) - paimon_add_no_runtime_plugin( - paimon_cpp_relink_probe - SOURCES "${CMAKE_CURRENT_BINARY_DIR}/relink_probe.cpp") - - add_executable(paimon_cpp_dlopen_smoke tests/dlopen_smoke.c) - target_link_libraries(paimon_cpp_dlopen_smoke PRIVATE ${CMAKE_DL_LIBS}) - - add_test( - NAME paimon_c_no_cpp_runtime - COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "${paimon_c_library}") - add_test( - NAME paimon_cpp_facade_no_cpp_runtime - COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "${paimon_cpp_smoke_library}") - add_test( - NAME paimon_cpp_plugin_no_cpp_runtime - COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "$") - - add_library( - paimon_elf_fixture_undefined_operator_new SHARED - tests/elf_fixtures/undefined_operator_new.c) - add_test( - NAME paimon_elf_guard_rejects_operator_new - COMMAND - "${CMAKE_COMMAND}" - "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "-DLIBRARY=$" - "-DEXPECTED=C++ mangled" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") - - add_library( - paimon_elf_fixture_undefined_host_hook SHARED - tests/elf_fixtures/undefined_host_hook.c) - add_test( - NAME paimon_elf_guard_rejects_host_hook - COMMAND - "${CMAKE_COMMAND}" - "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "-DLIBRARY=$" - "-DEXPECTED=unversioned undefined symbol" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") - - add_library( - paimon_elf_fixture_executable_stack SHARED - tests/elf_fixtures/executable_stack.c) - target_link_options( - paimon_elf_fixture_executable_stack PRIVATE -Wl,-z,execstack) - add_test( - NAME paimon_elf_guard_rejects_executable_stack - COMMAND - "${CMAKE_COMMAND}" - "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "-DLIBRARY=$" - "-DEXPECTED=executable GNU_STACK" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") - - add_executable( - paimon_elf_fixture_pie - tests/elf_fixtures/pie_executable.c) - target_compile_options(paimon_elf_fixture_pie PRIVATE -fPIE) - target_link_options(paimon_elf_fixture_pie PRIVATE -pie) - add_test( - NAME paimon_elf_guard_rejects_pie_executable - COMMAND - "${CMAKE_COMMAND}" - "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "-DLIBRARY=$" - "-DEXPECTED=PIE executable" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/expect_elf_rejected.cmake") - - add_test( - NAME paimon_cpp_plugin_incremental_relink - COMMAND - "${CMAKE_COMMAND}" - "-DBUILD_DIR=${CMAKE_BINARY_DIR}" - "-DSOURCE=${CMAKE_CURRENT_BINARY_DIR}/relink_probe.cpp" - "-DLIBRARY=$" - "-DTARGET=paimon_cpp_relink_probe" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/check_incremental_relink.cmake") - - add_test( - NAME paimon_cpp_plugin_isolated_load - COMMAND - "${CMAKE_COMMAND}" - "-DLOADER=$" - "-DPLUGIN=$" - "-DPAIMON_C_LIBRARY_UNDER_TEST=${paimon_c_library}" - "-DTEST_ROOT=${CMAKE_CURRENT_BINARY_DIR}/isolated-load-test" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_isolated_load.cmake") - - if(CMAKE_C_COMPILER_ID MATCHES "Clang|GNU") - add_library( - paimon_elf_fixture_libgcc SHARED - tests/elf_fixtures/needs_libgcc.c) - target_link_options( - paimon_elf_fixture_libgcc PRIVATE -Wl,--no-as-needed) - target_link_libraries(paimon_elf_fixture_libgcc PRIVATE gcc_s) - add_test( - NAME paimon_elf_guard_allows_libgcc - COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - "$") - endif() - - set(paimon_cpp_install_test_root - "${CMAKE_CURRENT_BINARY_DIR}/install-tree-consumer-test") - add_test( - NAME paimon_cpp_install_tree_consumer - COMMAND - "${CMAKE_COMMAND}" - "-DMAIN_BUILD_DIR=${CMAKE_BINARY_DIR}" - "-DCONSUMER_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/tests/install_tree_consumer" - "-DTEST_ROOT=${paimon_cpp_install_test_root}" - "-DC_COMPILER=${CMAKE_C_COMPILER}" - "-DCXX_COMPILER=${CMAKE_CXX_COMPILER}" - "-DPLUGIN_FILENAME=${CMAKE_SHARED_LIBRARY_PREFIX}paimon_install_tree_consumer${CMAKE_SHARED_LIBRARY_SUFFIX}" - "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_install_tree_consumer.cmake") - add_test( - NAME paimon_cpp_install_tree_guard_rejects_runtime - COMMAND - "${CMAKE_COMMAND}" - "-DMAIN_BUILD_DIR=${CMAKE_BINARY_DIR}" - "-DCONSUMER_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/tests/install_tree_consumer" - "-DTEST_ROOT=${paimon_cpp_install_test_root}-forbidden" - "-DC_COMPILER=${CMAKE_C_COMPILER}" - "-DCXX_COMPILER=${CMAKE_CXX_COMPILER}" - "-DPLUGIN_FILENAME=${CMAKE_SHARED_LIBRARY_PREFIX}paimon_install_tree_consumer${CMAKE_SHARED_LIBRARY_SUFFIX}" - "-DVERIFIER=${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" - -DEXPECT_BUILD_FAILURE=ON - -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_install_tree_consumer.cmake") - endif() + set(paimon_cpp_install_test_root + "${CMAKE_CURRENT_BINARY_DIR}/install-tree-consumer-test") + add_test( + NAME paimon_cpp_install_tree_consumer + COMMAND + "${CMAKE_COMMAND}" + "-DMAIN_BUILD_DIR=${CMAKE_BINARY_DIR}" + "-DCONSUMER_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/tests/install_tree_consumer" + "-DTEST_ROOT=${paimon_cpp_install_test_root}" + "-DCXX_COMPILER=${CMAKE_CXX_COMPILER}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_install_tree_consumer.cmake") endif() set(paimon_cpp_install_component PaimonCppSdk) @@ -390,11 +208,6 @@ install( FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfigVersion.cmake" - "${CMAKE_CURRENT_SOURCE_DIR}/cmake/PaimonNoRuntimePlugin.cmake" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp" - COMPONENT ${paimon_cpp_install_component}) -install( - PROGRAMS "${CMAKE_CURRENT_SOURCE_DIR}/scripts/verify_linux_elf.sh" DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp" COMPONENT ${paimon_cpp_install_component}) diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index 63d6ace64..ea47e24b7 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -107,78 +107,21 @@ are deliberately unsupported. When changing the C ABI, regenerate the checked header separately with `cbindgen`; ordinary builds consume the checked-in `bindings/c/include/paimon.h`. -For a shared plugin that must load without `libstdc++` or `libc++`, compile the -C++ source without exceptions/RTTI and use the C linker driver for the final -link: - -```bash -c++ -std=c++17 -fPIC -fno-exceptions -fno-rtti \ - -Ibindings/cpp/include -Ibindings/c/include \ - -c plugin.cpp -o plugin.o -cc -shared plugin.o -Ltarget/release -lpaimon_c -Wl,-z,defs \ - -o libplugin.so -bindings/cpp/scripts/verify_linux_elf.sh libplugin.so -``` - -The plugin must expose each public entry point with -`PAIMON_CPP_PLUGIN_EXPORT` and stay within the facade's allocation-free, -no-exceptions subset. Linking the final `.so` with a C++ driver can add a C++ -runtime even when the source does not call that runtime directly. - -Install its CMake interface target: +Install the CMake interface target elsewhere when needed: ```bash cmake --install target/cpp-build --prefix /your/prefix ``` -Installation always bundles the just-built `libpaimon_c` and exports imported -target `Paimon::c`. Installed consumers can build a verified no-runtime plugin -with the provided helper: +Installation always bundles the just-built `libpaimon_c` and exports +`Paimon::c` plus the header-only `Paimon::cpp` target. A consumer only needs: ```cmake -cmake_minimum_required(VERSION 3.15) -project(MyPaimonPlugin LANGUAGES C CXX) find_package(PaimonCpp CONFIG REQUIRED) -paimon_add_no_runtime_plugin( - my_paimon_plugin - SOURCES plugin.cpp - INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/include" - COMPILE_DEFINITIONS MY_PLUGIN_ABI=1) -``` - -Configure C++ compilation through the helper's `SOURCES`, -`INCLUDE_DIRECTORIES`, `COMPILE_DEFINITIONS`, `COMPILE_OPTIONS`, and -`LINK_LIBRARIES` arguments. Do not add C++ sources to the returned C-link -target with `target_sources`; doing so bypasses the split compile/link model. -The helper hides all non-exported C++ symbols, links with the C driver, embeds -only `$ORIGIN` as its runtime search path, and runs the installed ELF guard -after every successful link. - -`Paimon::cpp` remains the header-only facade target for consumers that manage -their own final link. The installed package always resolves `Paimon::c` to the -library bundled in the same installation prefix. - -## Linux runtime guard - -Run the ELF guard on the library staged by CMake when validating a Linux -release artifact: - -```bash -bindings/cpp/scripts/verify_linux_elf.sh \ - target/cpp-build/lib/libpaimon_c.so +add_executable(my_paimon_app main.cpp) +target_link_libraries(my_paimon_app PRIVATE Paimon::cpp) ``` -Some distributions use `lib64` instead of `lib`. The build host determines the -minimum glibc version; build on glibc 2.17 when 2.17 is the deployment baseline. - -It prints the build host's `ldd --version` and applies a `DT_NEEDED` allowlist -containing glibc components, `libgcc_s`, and `libpaimon_c`. It rejects C++ -runtimes, `libunwind`, `libatomic`, `GLIBCXX`/`CXXABI` symbol versions, -undefined or exported C++ mangled symbols, unversioned host hooks, operator -new/delete, RTTI/dynamic-cast support, absolute runtime paths, and private -glibc ABI versions. Glibc's C-level `__cxa_atexit`, `__cxa_finalize`, and -`__cxa_thread_atexit_impl` remain allowed. - `Scan::plan()` remains a bounded scan. Use `StreamScanOptions` and `ReadBuilder::new_stream_scan` for a stateful continuous scan. Persist `StreamScan::checkpoint()` only after every split in the returned plan has been diff --git a/bindings/cpp/cmake/PaimonCppConfig.cmake.in b/bindings/cpp/cmake/PaimonCppConfig.cmake.in index f6e960995..987391140 100644 --- a/bindings/cpp/cmake/PaimonCppConfig.cmake.in +++ b/bindings/cpp/cmake/PaimonCppConfig.cmake.in @@ -24,7 +24,6 @@ if(TARGET Paimon::cpp) "Paimon::cpp exists without its bundled Paimon::c target") return() endif() - include("${CMAKE_CURRENT_LIST_DIR}/PaimonNoRuntimePlugin.cmake") check_required_components(PaimonCpp) return() endif() @@ -54,5 +53,4 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") include("${CMAKE_CURRENT_LIST_DIR}/PaimonCppTargets.cmake") -include("${CMAKE_CURRENT_LIST_DIR}/PaimonNoRuntimePlugin.cmake") check_required_components(PaimonCpp) diff --git a/bindings/cpp/cmake/PaimonNoRuntimePlugin.cmake b/bindings/cpp/cmake/PaimonNoRuntimePlugin.cmake deleted file mode 100644 index 977f4121e..000000000 --- a/bindings/cpp/cmake/PaimonNoRuntimePlugin.cmake +++ /dev/null @@ -1,140 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -include_guard(GLOBAL) - -# Capture this while the module is included. CMAKE_CURRENT_LIST_DIR inside a -# function can otherwise refer to the consumer's calling list file. -if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/verify_linux_elf.sh") - set(_PAIMON_NO_RUNTIME_ELF_VERIFIER - "${CMAKE_CURRENT_LIST_DIR}/verify_linux_elf.sh") -else() - set(_PAIMON_NO_RUNTIME_ELF_VERIFIER - "${CMAKE_CURRENT_LIST_DIR}/../scripts/verify_linux_elf.sh") -endif() - -# Build a C-linkage plugin from C++17 sources without linking a C++ runtime. -# The source must itself stay within the no-exceptions/no-RTTI subset used by -# the Paimon facade. The final target is linked by the configured C driver. -function(paimon_add_no_runtime_plugin target) - if(NOT TARGET Paimon::cpp OR NOT TARGET Paimon::c) - message(FATAL_ERROR - "paimon_add_no_runtime_plugin requires Paimon::cpp and Paimon::c") - endif() - set(multi_value_args - SOURCES INCLUDE_DIRECTORIES COMPILE_DEFINITIONS COMPILE_OPTIONS - LINK_LIBRARIES) - cmake_parse_arguments(PAIMON_PLUGIN "" "" "${multi_value_args}" ${ARGN}) - if(PAIMON_PLUGIN_KEYWORDS_MISSING_VALUES) - message(FATAL_ERROR - "missing values for: ${PAIMON_PLUGIN_KEYWORDS_MISSING_VALUES}") - endif() - if(PAIMON_PLUGIN_SOURCES) - if(PAIMON_PLUGIN_UNPARSED_ARGUMENTS) - message(FATAL_ERROR - "unexpected no-runtime plugin arguments: ${PAIMON_PLUGIN_UNPARSED_ARGUMENTS}") - endif() - set(plugin_sources ${PAIMON_PLUGIN_SOURCES}) - else() - # Preserve the original positional-source form. - set(plugin_sources ${PAIMON_PLUGIN_UNPARSED_ARGUMENTS}) - endif() - if(NOT plugin_sources) - message(FATAL_ERROR - "paimon_add_no_runtime_plugin(${target}) requires source files") - endif() - if(TARGET "${target}" OR TARGET "${target}__paimon_cpp_objects") - message(FATAL_ERROR "target already exists: ${target}") - endif() - if(NOT UNIX OR APPLE OR - NOT CMAKE_C_COMPILER_ID MATCHES "Clang|GNU" OR - NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") - message(FATAL_ERROR - "no-runtime plugins currently require GNU/Clang C and C++ compilers on Linux") - endif() - - if(NOT EXISTS "${_PAIMON_NO_RUNTIME_ELF_VERIFIER}") - message(FATAL_ERROR - "Paimon ELF verifier is missing: ${_PAIMON_NO_RUNTIME_ELF_VERIFIER}") - endif() - - add_library("${target}__paimon_cpp_objects" OBJECT ${plugin_sources}) - set_target_properties( - "${target}__paimon_cpp_objects" - PROPERTIES - POSITION_INDEPENDENT_CODE ON - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CXX_EXTENSIONS OFF) - target_compile_options( - "${target}__paimon_cpp_objects" - PRIVATE - -fno-exceptions - -fno-rtti - -fvisibility=hidden - -fvisibility-inlines-hidden - ${PAIMON_PLUGIN_COMPILE_OPTIONS}) - if(PAIMON_PLUGIN_INCLUDE_DIRECTORIES) - target_include_directories( - "${target}__paimon_cpp_objects" - PRIVATE ${PAIMON_PLUGIN_INCLUDE_DIRECTORIES}) - endif() - if(PAIMON_PLUGIN_COMPILE_DEFINITIONS) - target_compile_definitions( - "${target}__paimon_cpp_objects" - PRIVATE ${PAIMON_PLUGIN_COMPILE_DEFINITIONS}) - endif() - target_link_libraries( - "${target}__paimon_cpp_objects" - PRIVATE Paimon::cpp ${PAIMON_PLUGIN_LINK_LIBRARIES}) - - # Hide the C++ object language from the final C target. CMake otherwise adds - # its configured implicit C++ libraries even when LINKER_LANGUAGE is C. - set(archive_target "${target}__paimon_cpp_archive") - add_library( - "${archive_target}" STATIC - $) - set_target_properties("${archive_target}" PROPERTIES LINKER_LANGUAGE CXX) - - set(link_stub "${CMAKE_CURRENT_BINARY_DIR}/${target}__paimon_link_stub.c") - file(GENERATE OUTPUT "${link_stub}" - CONTENT "/* Generated C link anchor for a Paimon no-runtime plugin. */\n") - add_library("${target}" SHARED "${link_stub}") - set_target_properties( - "${target}" - PROPERTIES - LINKER_LANGUAGE C - BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH "\$ORIGIN") - add_dependencies("${target}" "${archive_target}") - set_property( - TARGET "${target}" APPEND PROPERTY - LINK_DEPENDS "$") - target_link_options( - "${target}" - PRIVATE - "-Wl,--whole-archive,$,--no-whole-archive" - -Wl,-z,defs) - target_link_libraries( - "${target}" PRIVATE Paimon::c ${PAIMON_PLUGIN_LINK_LIBRARIES}) - add_custom_command( - TARGET "${target}" - POST_BUILD - COMMAND "${_PAIMON_NO_RUNTIME_ELF_VERIFIER}" "$" - COMMENT "Verifying that ${target} has a C-only dynamic ABI" - VERBATIM) -endfunction() diff --git a/bindings/cpp/include/paimon/paimon.hpp b/bindings/cpp/include/paimon/paimon.hpp index c5dab0087..2d864a159 100644 --- a/bindings/cpp/include/paimon/paimon.hpp +++ b/bindings/cpp/include/paimon/paimon.hpp @@ -39,18 +39,6 @@ extern "C" { #include PAIMON_C_HEADER } -// Mark the deliberately small C ABI exported by a no-runtime plugin. The -// CMake helper hides every other C++ symbol so inline facade implementation -// details cannot leak into the plugin's dynamic ABI. -#if defined(_WIN32) -#define PAIMON_CPP_PLUGIN_EXPORT extern "C" __declspec(dllexport) -#elif defined(__GNUC__) || defined(__clang__) -#define PAIMON_CPP_PLUGIN_EXPORT \ - extern "C" __attribute__((visibility("default"))) -#else -#define PAIMON_CPP_PLUGIN_EXPORT extern "C" -#endif - namespace paimon { struct adopt_handle_t { diff --git a/bindings/cpp/scripts/verify_linux_elf.sh b/bindings/cpp/scripts/verify_linux_elf.sh deleted file mode 100755 index 52c1c2106..000000000 --- a/bindings/cpp/scripts/verify_linux_elf.sh +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env sh -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -eu - -if [ "$#" -ne 1 ]; then - echo "usage: $0 /path/to/libpaimon_c.so" >&2 - exit 2 -fi - -library=$1 -if [ ! -f "$library" ]; then - echo "not a file: $library" >&2 - exit 2 -fi - -if command -v readelf >/dev/null 2>&1; then - readelf_cmd=readelf -elif command -v llvm-readelf >/dev/null 2>&1; then - readelf_cmd=llvm-readelf -else - echo "readelf or llvm-readelf is required" >&2 - exit 2 -fi - -elf_header=$($readelf_cmd -h "$library") -if ! printf '%s\n' "$elf_header" | grep -q 'ELF'; then - echo "not an ELF shared object: $library" >&2 - exit 1 -fi -if ! printf '%s\n' "$elf_header" | grep -Eq 'Type:[[:space:]]+DYN'; then - echo "ELF artifact is not a shared object: $library" >&2 - exit 1 -fi - -program_headers=$($readelf_cmd -W -l "$library") -if printf '%s\n' "$program_headers" | grep -q 'INTERP'; then - echo "ELF artifact is a PIE executable, not a shared object" >&2 - printf '%s\n' "$program_headers" | grep 'INTERP' >&2 - exit 1 -fi -if printf '%s\n' "$program_headers" | grep -Eq \ - 'GNU_STACK.*W.*E|GNU_STACK.*E.*W'; then - echo "forbidden executable GNU_STACK segment" >&2 - printf '%s\n' "$program_headers" | grep 'GNU_STACK' >&2 - exit 1 -fi -if printf '%s\n' "$program_headers" | grep -Eq \ - 'LOAD.*W.*E|LOAD.*E.*W'; then - echo "forbidden writable and executable LOAD segment" >&2 - printf '%s\n' "$program_headers" | grep 'LOAD' >&2 - exit 1 -fi - -dynamic_section=$($readelf_cmd -d "$library") -if printf '%s\n' "$dynamic_section" | grep -q 'TEXTREL'; then - echo "forbidden text relocation" >&2 - printf '%s\n' "$dynamic_section" | grep 'TEXTREL' >&2 - exit 1 -fi - -if command -v ldd >/dev/null 2>&1; then - ldd --version 2>&1 | sed -n '1,2p' -fi - -needed=$(printf '%s\n' "$dynamic_section" | grep 'NEEDED' || true) -printf '%s\n' "$needed" - -runtime_paths=$(printf '%s\n' "$dynamic_section" | - sed -n 's/.*(RPATH).*Library rpath: \[\([^]]*\)\].*/\1/p; - s/.*(RUNPATH).*Library runpath: \[\([^]]*\)\].*/\1/p') -old_ifs=$IFS -IFS=: -for runtime_path in $runtime_paths; do - case "$runtime_path" in - '$ORIGIN'|'${ORIGIN}') - ;; - *) - echo "forbidden runtime search path: $runtime_path" >&2 - exit 1 - ;; - esac -done -IFS=$old_ifs - -needed_names=$(printf '%s\n' "$needed" | - sed -n 's/.*Shared library: \[\([^]]*\)\].*/\1/p') -for dependency in $needed_names; do - case "$dependency" in - libstdc++*|libc++*|libsupc++*|libunwind*|libatomic*) - echo "forbidden non-C runtime dependency in DT_NEEDED: $dependency" >&2 - exit 1 - ;; - libc.so.*|libm.so.*|libpthread.so.*|libdl.so.*|librt.so.*|libutil.so.*|libresolv.so.*|libanl.so.*|libBrokenLocale.so.*|libcrypt.so.*|libnss_*.so.*|libgcc_s.so.*|ld-linux*.so.*|ld64.so.*|ld.so.*|libpaimon_c.so*) - ;; - *) - echo "dependency is outside the glibc/libpaimon_c allowlist: $dependency" >&2 - exit 1 - ;; - esac -done - -undefined=$($readelf_cmd --dyn-syms --wide "$library" | grep ' UND ' || true) -unexpected_unversioned=$(printf '%s\n' "$undefined" | awk ' - { - bind = $5 - name = $8 - base = name - sub(/@.*/, "", base) - if (base == "" || name ~ /@(GLIBC|GCC)_[0-9]/ || base ~ /^paimon_/ || - base ~ /^_Z/) { - next - } - if (bind == "WEAK" && - (base == "_ITM_deregisterTMCloneTable" || - base == "_ITM_registerTMCloneTable" || - base == "__gmon_start__" || - base == "_Jv_RegisterClasses" || - base == "ZSTD_trace_compress_begin" || - base == "ZSTD_trace_compress_end" || - base == "ZSTD_trace_decompress_begin" || - base == "ZSTD_trace_decompress_end" || - base == "OPENSSL_memory_alloc" || - base == "OPENSSL_memory_free" || - base == "OPENSSL_memory_get_size" || - base == "OPENSSL_memory_realloc" || - base == "sdallocx" || - base == "gettid" || - base == "statx" || - base == "getrandom" || - base == "copy_file_range" || - base == "__cxa_thread_atexit_impl")) { - next - } - print - }') -if [ -n "$unexpected_unversioned" ]; then - echo "forbidden unversioned undefined symbol; only paimon_* and narrow weak CRT hooks are allowed" >&2 - printf '%s\n' "$unexpected_unversioned" >&2 - exit 1 -fi - -defined=$($readelf_cmd --dyn-syms --wide "$library" | - awk '$7 != "UND" && $8 != "" { print }') -if printf '%s\n' "$defined" | grep -Eq \ - '[[:space:]]_Z[A-Za-z0-9_$.@]*'; then - echo "forbidden exported C++ mangled symbol" >&2 - printf '%s\n' "$defined" | - grep -E '[[:space:]]_Z[A-Za-z0-9_$.@]*' >&2 - exit 1 -fi - -version_info=$($readelf_cmd --version-info --wide "$library" || true) -symbol_versions=$(printf '%s\n%s\n' "$undefined" "$version_info") -if printf '%s\n' "$symbol_versions" | grep -Eq 'GLIBCXX_|CXXABI_'; then - echo "forbidden C++ runtime symbol version" >&2 - printf '%s\n' "$symbol_versions" | - grep -E 'GLIBCXX_|CXXABI_' >&2 - exit 1 -fi - -if printf '%s\n' "$undefined" | grep -Eq \ - '[[:space:]]_Z[A-Za-z0-9_$.@]*'; then - echo "forbidden C++ mangled undefined symbol" >&2 - printf '%s\n' "$undefined" | - grep -E '[[:space:]]_Z[A-Za-z0-9_$.@]*' >&2 - exit 1 -fi - -cxa_symbols=$(printf '%s\n' "$undefined" | - grep '__cxa_' | - grep -Ev '__cxa_(atexit|finalize|thread_atexit_impl)(@|$)' || true) -if [ -n "$cxa_symbols" ] || - printf '%s\n' "$undefined" | grep -Eq '__gxx_personality_v0|__dynamic_cast'; then - echo "forbidden C++ ABI undefined symbol" >&2 - printf '%s\n' "$cxa_symbols" >&2 - printf '%s\n' "$undefined" | - grep -E '__gxx_personality_v0|__dynamic_cast' >&2 || true - exit 1 -fi - -if printf '%s\n' "$symbol_versions" | grep -Eq 'GLIBC_(PRIVATE|ABI_)'; then - echo "private or non-baseline glibc ABI requirement" >&2 - printf '%s\n' "$symbol_versions" | grep -E 'GLIBC_(PRIVATE|ABI_)' >&2 - exit 1 -fi - -if ! command -v c++filt >/dev/null 2>&1; then - echo "c++filt is required to inspect demangled undefined symbols" >&2 - exit 2 -fi - -demangled=$(printf '%s\n' "$undefined" | c++filt) -if printf '%s\n' "$demangled" | grep -Eq \ - 'std::|__gnu_cxx::|typeinfo for|vtable for|operator (new|delete)(\[\])?\(|__dynamic_cast'; then - echo "forbidden demangled C++ undefined symbol" >&2 - printf '%s\n' "$demangled" | grep -E \ - 'std::|__gnu_cxx::|typeinfo for|vtable for|operator (new|delete)(\[\])?\(|__dynamic_cast' >&2 - exit 1 -fi - -echo "ELF C++ runtime check passed: $library" diff --git a/bindings/cpp/tests/check_incremental_relink.cmake b/bindings/cpp/tests/check_incremental_relink.cmake deleted file mode 100644 index b18e4277d..000000000 --- a/bindings/cpp/tests/check_incremental_relink.cmake +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -foreach(required IN ITEMS BUILD_DIR SOURCE LIBRARY TARGET) - if(NOT DEFINED ${required}) - message(FATAL_ERROR "missing -D${required}=...") - endif() -endforeach() - -file(SHA256 "${LIBRARY}" before_hash) -file(READ "${SOURCE}" source_text) -if(source_text MATCHES "\\+ 1001") - string(REPLACE "+ 1001" "+ 1002" source_text "${source_text}") -elseif(source_text MATCHES "\\+ 1002") - string(REPLACE "+ 1002" "+ 1001" source_text "${source_text}") -else() - message(FATAL_ERROR "relink probe source does not contain its toggle") -endif() -file(WRITE "${SOURCE}" "${source_text}") - -execute_process( - COMMAND "${CMAKE_COMMAND}" --build "${BUILD_DIR}" --target "${TARGET}" - RESULT_VARIABLE build_result - OUTPUT_VARIABLE build_stdout - ERROR_VARIABLE build_stderr) -if(NOT build_result EQUAL 0) - message(FATAL_ERROR - "incremental plugin rebuild failed:\n${build_stdout}\n${build_stderr}") -endif() - -file(SHA256 "${LIBRARY}" after_hash) -if(before_hash STREQUAL after_hash) - message(FATAL_ERROR - "plugin did not relink after its C++ object archive changed") -endif() diff --git a/bindings/cpp/tests/dlopen_smoke.c b/bindings/cpp/tests/dlopen_smoke.c deleted file mode 100644 index 4b4a18d68..000000000 --- a/bindings/cpp/tests/dlopen_smoke.c +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -typedef uint32_t (*abi_version_fn)(void); -typedef int32_t (*self_reset_fn)(void); - -static void *required_symbol(void *library, const char *name) { - void *symbol; - dlerror(); - symbol = dlsym(library, name); - if (symbol == NULL || dlerror() != NULL) { - fprintf(stderr, "missing plugin symbol: %s\n", name); - return NULL; - } - return symbol; -} - -int main(int argc, char **argv) { - void *library; - void *symbol; - abi_version_fn abi_version; - self_reset_fn self_reset; - if (argc != 2) { - return 2; - } - library = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); - if (library == NULL) { - fprintf(stderr, "dlopen failed: %s\n", dlerror()); - return 1; - } - symbol = required_symbol(library, "paimon_cpp_plugin_abi_version"); - if (symbol == NULL) { - return 1; - } - memcpy(&abi_version, &symbol, sizeof(abi_version)); - symbol = required_symbol(library, "paimon_cpp_plugin_error_self_reset"); - if (symbol == NULL) { - return 1; - } - memcpy(&self_reset, &symbol, sizeof(self_reset)); - if (abi_version() != 1 || self_reset() != 0) { - return 1; - } - return dlclose(library) == 0 ? 0 : 1; -} diff --git a/bindings/cpp/tests/elf_fixtures/executable_stack.c b/bindings/cpp/tests/elf_fixtures/executable_stack.c deleted file mode 100644 index 3528d690b..000000000 --- a/bindings/cpp/tests/elf_fixtures/executable_stack.c +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -int paimon_elf_fixture_executable_stack(void) { - return 0; -} diff --git a/bindings/cpp/tests/elf_fixtures/needs_libgcc.c b/bindings/cpp/tests/elf_fixtures/needs_libgcc.c deleted file mode 100644 index 4c460b24a..000000000 --- a/bindings/cpp/tests/elf_fixtures/needs_libgcc.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -extern void _Unwind_Resume(void *exception_object); - -void paimon_fixture_force_libgcc(void *exception_object) { - _Unwind_Resume(exception_object); -} diff --git a/bindings/cpp/tests/elf_fixtures/pie_executable.c b/bindings/cpp/tests/elf_fixtures/pie_executable.c deleted file mode 100644 index 3a621d61f..000000000 --- a/bindings/cpp/tests/elf_fixtures/pie_executable.c +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -int main(void) { - return 0; -} diff --git a/bindings/cpp/tests/elf_fixtures/undefined_host_hook.c b/bindings/cpp/tests/elf_fixtures/undefined_host_hook.c deleted file mode 100644 index d1802bb6b..000000000 --- a/bindings/cpp/tests/elf_fixtures/undefined_host_hook.c +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -extern void forbidden_host_hook(void); - -void paimon_fixture_call_forbidden_host_hook(void) { - forbidden_host_hook(); -} diff --git a/bindings/cpp/tests/elf_fixtures/undefined_operator_new.c b/bindings/cpp/tests/elf_fixtures/undefined_operator_new.c deleted file mode 100644 index f3999c429..000000000 --- a/bindings/cpp/tests/elf_fixtures/undefined_operator_new.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include - -extern void *_Znwm(size_t size); - -void *paimon_fixture_force_operator_new(size_t size) { return _Znwm(size); } diff --git a/bindings/cpp/tests/expect_elf_rejected.cmake b/bindings/cpp/tests/expect_elf_rejected.cmake deleted file mode 100644 index dcd05f011..000000000 --- a/bindings/cpp/tests/expect_elf_rejected.cmake +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -foreach(required IN ITEMS VERIFIER LIBRARY EXPECTED) - if(NOT DEFINED ${required}) - message(FATAL_ERROR "missing -D${required}=...") - endif() -endforeach() - -execute_process( - COMMAND "${VERIFIER}" "${LIBRARY}" - RESULT_VARIABLE verifier_result - OUTPUT_VARIABLE verifier_stdout - ERROR_VARIABLE verifier_stderr) -set(verifier_output "${verifier_stdout}\n${verifier_stderr}") - -if(verifier_result EQUAL 0) - message(FATAL_ERROR - "ELF verifier accepted forbidden fixture ${LIBRARY}:\n${verifier_output}") -endif() -string(FIND "${verifier_output}" "${EXPECTED}" expected_index) -if(expected_index EQUAL -1) - message(FATAL_ERROR - "ELF verifier did not report '${EXPECTED}':\n${verifier_output}") -endif() diff --git a/bindings/cpp/tests/helper_config/paimon_cpp_helper_config.h b/bindings/cpp/tests/helper_config/paimon_cpp_helper_config.h deleted file mode 100644 index 47648349a..000000000 --- a/bindings/cpp/tests/helper_config/paimon_cpp_helper_config.h +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef PAIMON_CPP_HELPER_CONFIG_H -#define PAIMON_CPP_HELPER_CONFIG_H - -#define PAIMON_CPP_HELPER_CONFIG_VALUE 73 - -#endif diff --git a/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt index 1300b8959..ad6ed378c 100644 --- a/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt +++ b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt @@ -16,20 +16,12 @@ # under the License. cmake_minimum_required(VERSION 3.15) -project(PaimonCppInstallTreeConsumer LANGUAGES C CXX) +project(PaimonCppInstallTreeConsumer LANGUAGES CXX) find_package(PaimonCpp CONFIG REQUIRED) # Package discovery may happen through more than one dependency. A repeated # lookup must retain the same bundled Paimon::c target instead of treating it # as an external override. find_package(PaimonCpp CONFIG REQUIRED) -paimon_add_no_runtime_plugin( - paimon_install_tree_consumer - SOURCES plugin.cpp) - -option(PAIMON_INJECT_FORBIDDEN_RUNTIME "Test the installed ELF guard" OFF) -if(PAIMON_INJECT_FORBIDDEN_RUNTIME) - target_link_options( - paimon_install_tree_consumer PRIVATE -Wl,--no-as-needed) - target_link_libraries(paimon_install_tree_consumer PRIVATE stdc++) -endif() +add_executable(paimon_install_tree_consumer main.cpp) +target_link_libraries(paimon_install_tree_consumer PRIVATE Paimon::cpp) diff --git a/bindings/cpp/tests/install_tree_consumer/plugin.cpp b/bindings/cpp/tests/install_tree_consumer/main.cpp similarity index 85% rename from bindings/cpp/tests/install_tree_consumer/plugin.cpp rename to bindings/cpp/tests/install_tree_consumer/main.cpp index 2b5bcd878..275a99739 100644 --- a/bindings/cpp/tests/install_tree_consumer/plugin.cpp +++ b/bindings/cpp/tests/install_tree_consumer/main.cpp @@ -17,7 +17,10 @@ #include -PAIMON_CPP_PLUGIN_EXPORT std::uint32_t -paimon_install_tree_consumer_abi() noexcept { - return paimon::abi_version(); +int main() { + const auto abi = paimon::abi_version(); + auto version = paimon::library_version(); + (void)abi; + (void)version; + return 0; } diff --git a/bindings/cpp/tests/no_cpp_runtime_plugin.cpp b/bindings/cpp/tests/no_cpp_runtime_plugin.cpp deleted file mode 100644 index 2fd534988..000000000 --- a/bindings/cpp/tests/no_cpp_runtime_plugin.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include - -#include "paimon_cpp_helper_config.h" - -#ifndef PAIMON_CPP_HELPER_COMPILE_DEFINITION -#error "no-runtime helper did not forward compile definitions" -#endif - -static_assert(PAIMON_CPP_HELPER_COMPILE_DEFINITION == - PAIMON_CPP_HELPER_CONFIG_VALUE, - "no-runtime helper configuration mismatch"); - -// These C-linkage exports make this a realistic C++ implementation plugin that -// can itself be loaded on a host without libstdc++ or libc++. -PAIMON_CPP_PLUGIN_EXPORT std::uint32_t -paimon_cpp_plugin_abi_version() noexcept { - return paimon::abi_version(); -} - -PAIMON_CPP_PLUGIN_EXPORT std::size_t paimon_cpp_plugin_library_version( - char* output, std::size_t capacity) noexcept { - auto version = paimon::library_version(); - const auto copied = output == nullptr - ? 0 - : (version.size() < capacity ? version.size() - : capacity); - for (std::size_t index = 0; index < copied; ++index) { - output[index] = static_cast(version.data()[index]); - } - return version.size(); -} - -PAIMON_CPP_PLUGIN_EXPORT std::int32_t paimon_cpp_plugin_open_catalog( - const paimon::Option* options, std::size_t options_len) noexcept { - auto catalog = paimon::Catalog::create(options, options_len); - if (!catalog) { - return static_cast(catalog.error().code()) + 1; - } - // The move-only Catalog is deliberately closed by its noexcept destructor. - return 0; -} - -PAIMON_CPP_PLUGIN_EXPORT std::int32_t -paimon_cpp_plugin_error_self_reset() noexcept { - paimon::Error error( - paimon::adopt_handle, - ::paimon_stream_scan_restore(nullptr, 0)); - if (!error) { - return -1; - } - auto* same = error.native_handle(); - error.reset(same); - return error.native_handle() == same ? 0 : -2; -} diff --git a/bindings/cpp/tests/relink_probe.cpp b/bindings/cpp/tests/relink_probe.cpp deleted file mode 100644 index ad667a167..000000000 --- a/bindings/cpp/tests/relink_probe.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -PAIMON_CPP_PLUGIN_EXPORT std::uint32_t paimon_cpp_relink_probe() noexcept { - return paimon::abi_version() + 1001; -} diff --git a/bindings/cpp/tests/run_install_tree_consumer.cmake b/bindings/cpp/tests/run_install_tree_consumer.cmake index 8ae377a4d..9ddbbe54c 100644 --- a/bindings/cpp/tests/run_install_tree_consumer.cmake +++ b/bindings/cpp/tests/run_install_tree_consumer.cmake @@ -16,7 +16,7 @@ # under the License. foreach(required IN ITEMS MAIN_BUILD_DIR CONSUMER_SOURCE_DIR TEST_ROOT - C_COMPILER CXX_COMPILER PLUGIN_FILENAME VERIFIER) + CXX_COMPILER) if(NOT DEFINED ${required}) message(FATAL_ERROR "missing -D${required}=...") endif() @@ -24,9 +24,6 @@ endforeach() set(test_prefix "${TEST_ROOT}/prefix") set(consumer_build "${TEST_ROOT}/build") -if(NOT DEFINED EXPECT_BUILD_FAILURE) - set(EXPECT_BUILD_FAILURE OFF) -endif() file(REMOVE_RECURSE "${TEST_ROOT}") execute_process( @@ -45,9 +42,7 @@ execute_process( -S "${CONSUMER_SOURCE_DIR}" -B "${consumer_build}" "-DCMAKE_PREFIX_PATH=${test_prefix}" - "-DCMAKE_C_COMPILER=${C_COMPILER}" "-DCMAKE_CXX_COMPILER=${CXX_COMPILER}" - "-DPAIMON_INJECT_FORBIDDEN_RUNTIME=${EXPECT_BUILD_FAILURE}" RESULT_VARIABLE configure_result OUTPUT_VARIABLE configure_stdout ERROR_VARIABLE configure_stderr) @@ -61,30 +56,7 @@ execute_process( RESULT_VARIABLE build_result OUTPUT_VARIABLE build_stdout ERROR_VARIABLE build_stderr) -if(EXPECT_BUILD_FAILURE) - if(build_result EQUAL 0) - message(FATAL_ERROR - "installed helper accepted a forbidden runtime dependency") - endif() - set(build_output "${build_stdout}\n${build_stderr}") - if(NOT build_output MATCHES "forbidden non-C runtime dependency") - message(FATAL_ERROR - "consumer failed for the wrong reason:\n${build_output}") - endif() - return() -endif() if(NOT build_result EQUAL 0) message(FATAL_ERROR "install-tree consumer build failed:\n${build_stdout}\n${build_stderr}") endif() - -set(plugin "${consumer_build}/${PLUGIN_FILENAME}") -execute_process( - COMMAND "${VERIFIER}" "${plugin}" - RESULT_VARIABLE verifier_result - OUTPUT_VARIABLE verifier_stdout - ERROR_VARIABLE verifier_stderr) -if(NOT verifier_result EQUAL 0) - message(FATAL_ERROR - "installed no-runtime plugin failed ELF verification:\n${verifier_stdout}\n${verifier_stderr}") -endif() diff --git a/bindings/cpp/tests/run_isolated_load.cmake b/bindings/cpp/tests/run_isolated_load.cmake deleted file mode 100644 index 2514e3627..000000000 --- a/bindings/cpp/tests/run_isolated_load.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -foreach(required IN ITEMS LOADER PLUGIN PAIMON_C_LIBRARY_UNDER_TEST TEST_ROOT) - if(NOT DEFINED ${required}) - message(FATAL_ERROR "missing -D${required}=...") - endif() -endforeach() - -file(REMOVE_RECURSE "${TEST_ROOT}") -file(MAKE_DIRECTORY "${TEST_ROOT}") -file(COPY "${PLUGIN}" "${PAIMON_C_LIBRARY_UNDER_TEST}" - DESTINATION "${TEST_ROOT}") -get_filename_component(plugin_name "${PLUGIN}" NAME) -execute_process( - COMMAND "${LOADER}" "./${plugin_name}" - WORKING_DIRECTORY "${TEST_ROOT}" - RESULT_VARIABLE load_result - OUTPUT_VARIABLE load_stdout - ERROR_VARIABLE load_stderr) -if(NOT load_result EQUAL 0) - message(FATAL_ERROR - "isolated plugin load failed:\n${load_stdout}\n${load_stderr}") -endif() From 0af830b43afd5ea39a4c290475514488e6dd5e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 20:56:50 +0800 Subject: [PATCH 10/21] fix(bindings): harden C++ build and CI --- .github/workflows/ci.yml | 34 ++++++++++++++++++++++++++++++++++ bindings/cpp/CMakeLists.txt | 31 ++++++++++++++++++------------- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 772d15b53..8c46be4a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,14 @@ jobs: - name: Format run: cargo fmt --all -- --check + - name: Install cbindgen + uses: taiki-e/install-action@065d6a08a14e61e89fb0a4c10eecdbdef39c7d8e # v2.85.4 + with: + tool: cbindgen@0.29.4 + + - name: Check C header + run: ./bindings/c/scripts/check-header.sh + - name: Clippy run: cargo clippy --locked --all-targets --workspace --features fulltext,vortex -- -D warnings @@ -159,6 +167,32 @@ jobs: - name: Build run: cargo build --locked --features fulltext,vortex + cpp: + name: cpp (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + steps: + - uses: actions/checkout@v7 + + - name: Test C ABI + run: cargo test --locked -p paimon-c + + - name: Configure C++ facade + run: > + cmake -S bindings/cpp -B target/cpp-ci + -DPAIMON_CPP_BUILD_EXAMPLES=ON + -DPAIMON_CPP_BUILD_TESTS=ON + + - name: Build C++ facade + run: cmake --build target/cpp-ci --parallel 4 + + - name: Test C++ facade + run: ctest --test-dir target/cpp-ci --output-on-failure + unit: runs-on: ${{ matrix.os }} strategy: diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index bc6416a52..a44355972 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -47,7 +47,7 @@ if(NOT paimon_cargo_executable) message(FATAL_ERROR "Rust Cargo was not found") endif() execute_process( - COMMAND "${paimon_cargo_executable}" --version + COMMAND "${paimon_cargo_executable}" -vV RESULT_VARIABLE paimon_cargo_version_result OUTPUT_VARIABLE paimon_cargo_version OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -55,24 +55,29 @@ if(NOT paimon_cargo_version_result EQUAL 0 OR NOT paimon_cargo_version MATCHES "^cargo [0-9]") message(FATAL_ERROR "Not a Rust Cargo executable: ${paimon_cargo_executable}") endif() +string(REGEX MATCH "host: ([^\n\r]+)" paimon_cargo_host_match + "${paimon_cargo_version}") +if(NOT paimon_cargo_host_match) + message(FATAL_ERROR "Cargo did not report its host target") +endif() +set(paimon_rust_host "${CMAKE_MATCH_1}") +set(paimon_cargo_target_dir "${paimon_rust_root}/target") +set(paimon_c_release_dir + "${paimon_cargo_target_dir}/${paimon_rust_host}/release") if(APPLE) - set(paimon_c_library "${paimon_rust_root}/target/release/libpaimon_c.dylib") - set(paimon_c_build_command - "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" "${paimon_cargo_executable}" - build --locked --release -p paimon-c) + set(paimon_c_library "${paimon_c_release_dir}/libpaimon_c.dylib") elseif(UNIX) - set(paimon_c_library "${paimon_rust_root}/target/release/libpaimon_c.so") - set(paimon_c_build_command - "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" "${paimon_cargo_executable}" - build --locked --release -p paimon-c) + set(paimon_c_library "${paimon_c_release_dir}/libpaimon_c.so") elseif(WIN32) - set(paimon_c_library "${paimon_rust_root}/target/release/paimon_c.dll") - set(paimon_c_build_command - "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" "${paimon_cargo_executable}" - build --locked --release -p paimon-c) + set(paimon_c_library "${paimon_c_release_dir}/paimon_c.dll") else() message(FATAL_ERROR "Unsupported platform for automatic paimon-c build") endif() +set(paimon_c_build_command + "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" + "CARGO_TARGET_DIR=${paimon_cargo_target_dir}" + "${paimon_cargo_executable}" build --locked --release -p paimon-c + --target "${paimon_rust_host}") add_custom_target( paimon_c_cargo_build ALL From 3f65b880023944fa29eb20609537b60f73308e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Tue, 1 Sep 2026 21:48:19 +0800 Subject: [PATCH 11/21] build: keep TLS stack changes out of C++ bindings --- Cargo.lock | 110 +++++++++++++++++- DEPENDENCIES.rust.tsv | 10 +- benchmarks/tpcds/DEPENDENCIES.rust.tsv | 10 +- bindings/c/DEPENDENCIES.rust.tsv | 10 +- bindings/go/DEPENDENCIES.rust.tsv | 10 +- bindings/python/DEPENDENCIES.rust.tsv | 10 +- .../integration_tests/DEPENDENCIES.rust.tsv | 10 +- .../datafusion/DEPENDENCIES.rust.tsv | 10 +- .../paimon-rest-server/DEPENDENCIES.rust.tsv | 10 +- crates/paimon/Cargo.toml | 10 +- crates/paimon/DEPENDENCIES.rust.tsv | 10 +- scripts/release_licenses.py | 26 +++++ scripts/verify_python_wheels.py | 50 +++++++- 13 files changed, 258 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a338b39a1..3570b2938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2647,6 +2647,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -3145,12 +3160,27 @@ dependencies = [ "hyper", "hyper-util", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -4043,6 +4073,23 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "never-say-never" version = "6.6.666" @@ -4489,12 +4536,49 @@ dependencies = [ "url", ] +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "orc-rust" version = "0.8.0" @@ -5776,22 +5860,21 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", "mime", + "native-tls", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-native-tls", "tower", "tower-http", "tower-service", @@ -5955,7 +6038,6 @@ checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ "aws-lc-rs", "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -7041,6 +7123,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -7325,6 +7417,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/DEPENDENCIES.rust.tsv b/DEPENDENCIES.rust.tsv index c0db34674..98277510c 100644 --- a/DEPENDENCIES.rust.tsv +++ b/DEPENDENCIES.rust.tsv @@ -217,6 +217,8 @@ flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.1.5 X foldhash@0.2.0 X +foreign-types@0.3.2 X X +foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X @@ -264,6 +266,7 @@ humantime@2.4.0 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X +hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -351,6 +354,7 @@ moka@0.12.15 X X murmurhash32@0.3.1 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X +native-tls@0.2.18 X X never-say-never@6.6.666 X X X no_std_io2@0.9.4 X X nom@7.1.3 X @@ -387,7 +391,10 @@ opendal-service-hdfs-native@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X +openssl@0.10.81 X +openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X +openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-float@5.3.0 X @@ -488,7 +495,6 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X -ring@0.17.14 X X rle-decode-fast@1.0.3 X X roaring@0.11.4 X X roxmltree@0.21.1 X X @@ -598,6 +604,7 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X +tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-stream@0.1.18 X tokio-util@0.7.18 X @@ -629,6 +636,7 @@ utf8-ranges@1.0.5 X X utf8_iter@1.0.4 X X utf8parse@0.2.2 X X uuid@1.24.0 X X +vcpkg@0.2.15 X X version_check@0.9.5 X X vortex@0.75.0 X vortex-alp@0.75.0 X diff --git a/benchmarks/tpcds/DEPENDENCIES.rust.tsv b/benchmarks/tpcds/DEPENDENCIES.rust.tsv index 099fb4091..0da960875 100644 --- a/benchmarks/tpcds/DEPENDENCIES.rust.tsv +++ b/benchmarks/tpcds/DEPENDENCIES.rust.tsv @@ -148,6 +148,8 @@ flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.1.5 X foldhash@0.2.0 X +foreign-types@0.3.2 X X +foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -183,6 +185,7 @@ humantime@2.4.0 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X +hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -243,6 +246,7 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X +native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-complex@0.4.6 X X @@ -259,7 +263,10 @@ opendal-http-transport-reqwest@0.58.2 X opendal-layer-retry@0.58.2 X opendal-service-fs@0.58.2 X opendal-service-oss@0.58.2 X +openssl@0.10.81 X +openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X +openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -316,7 +323,6 @@ reqsign-core@3.3.1 X reqsign-file-read-tokio@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X -ring@0.17.14 X X roaring@0.11.4 X X rust-ini@0.21.3 X rustc_version@0.4.1 X X @@ -388,6 +394,7 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X +tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-stream@0.1.18 X tokio-util@0.7.18 X @@ -412,6 +419,7 @@ urlencoding@2.1.3 X utf8_iter@1.0.4 X X utf8parse@0.2.2 X X uuid@1.24.0 X X +vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/bindings/c/DEPENDENCIES.rust.tsv b/bindings/c/DEPENDENCIES.rust.tsv index 9cd511fcc..f50b9243b 100644 --- a/bindings/c/DEPENDENCIES.rust.tsv +++ b/bindings/c/DEPENDENCIES.rust.tsv @@ -102,6 +102,8 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X +foreign-types@0.3.2 X X +foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -134,6 +136,7 @@ httpdate@1.0.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X +hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -190,6 +193,7 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X +native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-bigint-dig@0.8.6 X X @@ -210,7 +214,10 @@ opendal-service-gcs@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X +openssl@0.10.81 X +openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X +openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -269,7 +276,6 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X -ring@0.17.14 X X roaring@0.11.4 X X rsa@0.9.10 X X rust-ini@0.21.3 X @@ -344,6 +350,7 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X +tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -365,6 +372,7 @@ url@2.5.8 X X urlencoding@2.1.3 X utf8_iter@1.0.4 X X uuid@1.24.0 X X +vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/bindings/go/DEPENDENCIES.rust.tsv b/bindings/go/DEPENDENCIES.rust.tsv index 9cd511fcc..f50b9243b 100644 --- a/bindings/go/DEPENDENCIES.rust.tsv +++ b/bindings/go/DEPENDENCIES.rust.tsv @@ -102,6 +102,8 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X +foreign-types@0.3.2 X X +foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -134,6 +136,7 @@ httpdate@1.0.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X +hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -190,6 +193,7 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X +native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-bigint-dig@0.8.6 X X @@ -210,7 +214,10 @@ opendal-service-gcs@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X +openssl@0.10.81 X +openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X +openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -269,7 +276,6 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X -ring@0.17.14 X X roaring@0.11.4 X X rsa@0.9.10 X X rust-ini@0.21.3 X @@ -344,6 +350,7 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X +tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -365,6 +372,7 @@ url@2.5.8 X X urlencoding@2.1.3 X utf8_iter@1.0.4 X X uuid@1.24.0 X X +vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/bindings/python/DEPENDENCIES.rust.tsv b/bindings/python/DEPENDENCIES.rust.tsv index fa1715942..f77beb384 100644 --- a/bindings/python/DEPENDENCIES.rust.tsv +++ b/bindings/python/DEPENDENCIES.rust.tsv @@ -174,6 +174,8 @@ flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.1.5 X foldhash@0.2.0 X +foreign-types@0.3.2 X X +foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X @@ -216,6 +218,7 @@ humantime@2.4.0 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X +hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -293,6 +296,7 @@ mio@1.2.2 X murmurhash32@0.3.1 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X +native-tls@0.2.18 X X no_std_io2@0.9.4 X X nom@7.1.3 X num@0.4.3 X X @@ -322,7 +326,10 @@ opendal-service-hdfs-native@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X +openssl@0.10.81 X +openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X +openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-float@5.3.0 X @@ -412,7 +419,6 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X -ring@0.17.14 X X rle-decode-fast@1.0.3 X X roaring@0.11.4 X X roxmltree@0.21.1 X X @@ -513,6 +519,7 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X +tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-stream@0.1.18 X tokio-util@0.7.18 X @@ -543,6 +550,7 @@ urlencoding@2.1.3 X utf8-ranges@1.0.5 X X utf8_iter@1.0.4 X X uuid@1.24.0 X X +vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/crates/integration_tests/DEPENDENCIES.rust.tsv b/crates/integration_tests/DEPENDENCIES.rust.tsv index e3ca727d3..565b00e51 100644 --- a/crates/integration_tests/DEPENDENCIES.rust.tsv +++ b/crates/integration_tests/DEPENDENCIES.rust.tsv @@ -94,6 +94,8 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X +foreign-types@0.3.2 X X +foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -126,6 +128,7 @@ httpdate@1.0.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X +hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -180,6 +183,7 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X +native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-complex@0.4.6 X X @@ -193,7 +197,10 @@ opendal-http-transport-reqwest@0.58.2 X opendal-layer-retry@0.58.2 X opendal-service-fs@0.58.2 X opendal-service-oss@0.58.2 X +openssl@0.10.81 X +openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X +openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -240,7 +247,6 @@ reqsign-core@3.3.1 X reqsign-file-read-tokio@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X -ring@0.17.14 X X roaring@0.11.4 X X rust-ini@0.21.3 X rustc_version@0.4.1 X X @@ -308,6 +314,7 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X +tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -329,6 +336,7 @@ url@2.5.8 X X urlencoding@2.1.3 X utf8_iter@1.0.4 X X uuid@1.24.0 X X +vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv index e8784ca9f..09433b81d 100644 --- a/crates/integrations/datafusion/DEPENDENCIES.rust.tsv +++ b/crates/integrations/datafusion/DEPENDENCIES.rust.tsv @@ -181,6 +181,8 @@ flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.1.5 X foldhash@0.2.0 X +foreign-types@0.3.2 X X +foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X @@ -224,6 +226,7 @@ humantime@2.4.0 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X +hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -306,6 +309,7 @@ moka@0.12.15 X X murmurhash32@0.3.1 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X +native-tls@0.2.18 X X never-say-never@6.6.666 X X X no_std_io2@0.9.4 X X nom@7.1.3 X @@ -331,7 +335,10 @@ opendal-http-transport-reqwest@0.58.2 X opendal-layer-retry@0.58.2 X opendal-service-fs@0.58.2 X opendal-service-oss@0.58.2 X +openssl@0.10.81 X +openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X +openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-float@5.3.0 X @@ -409,7 +416,6 @@ reqsign-core@3.3.1 X reqsign-file-read-tokio@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X -ring@0.17.14 X X rle-decode-fast@1.0.3 X X roaring@0.11.4 X X rust-ini@0.21.3 X @@ -505,6 +511,7 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X +tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-stream@0.1.18 X tokio-util@0.7.18 X @@ -532,6 +539,7 @@ urlencoding@2.1.3 X utf8-ranges@1.0.5 X X utf8_iter@1.0.4 X X uuid@1.24.0 X X +vcpkg@0.2.15 X X version_check@0.9.5 X X vortex@0.75.0 X vortex-alp@0.75.0 X diff --git a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv index 05a660898..67a578223 100644 --- a/crates/paimon-rest-server/DEPENDENCIES.rust.tsv +++ b/crates/paimon-rest-server/DEPENDENCIES.rust.tsv @@ -97,6 +97,8 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X +foreign-types@0.3.2 X X +foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs_extra@1.3.0 X futures@0.3.33 X X @@ -129,6 +131,7 @@ httpdate@1.0.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X +hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -184,6 +187,7 @@ miniz_oxide@0.8.9 X X X mio@1.2.2 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X +native-tls@0.2.18 X X num@0.4.3 X X num-bigint@0.4.8 X X num-complex@0.4.6 X X @@ -197,7 +201,10 @@ opendal-http-transport-reqwest@0.58.2 X opendal-layer-retry@0.58.2 X opendal-service-fs@0.58.2 X opendal-service-oss@0.58.2 X +openssl@0.10.81 X +openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X +openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-multimap@0.7.3 X @@ -244,7 +251,6 @@ reqsign-core@3.3.1 X reqsign-file-read-tokio@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X -ring@0.17.14 X X roaring@0.11.4 X X rust-ini@0.21.3 X rustc_version@0.4.1 X X @@ -314,6 +320,7 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X +tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -335,6 +342,7 @@ url@2.5.8 X X urlencoding@2.1.3 X utf8_iter@1.0.4 X X uuid@1.24.0 X X +vcpkg@0.2.15 X X version_check@0.9.5 X X walkdir@2.5.0 X X want@0.3.1 X diff --git a/crates/paimon/Cargo.toml b/crates/paimon/Cargo.toml index e9b4fd31b..dc09b9644 100644 --- a/crates/paimon/Cargo.toml +++ b/crates/paimon/Cargo.toml @@ -118,15 +118,7 @@ tokio-util = { workspace = true, features = ["compat", "io-util"] } parquet = { workspace = true, features = ["async", "zstd", "lz4", "snap"] } orc-rust = "0.8.0" async-stream = "0.3.6" -# Use the rustls TLS stack with the platform trust store so native artifacts do -# not depend on libssl/libcrypto while enterprise and system CAs remain usable. -reqwest = { version = "0.12", default-features = false, features = [ - "charset", - "http2", - "json", - "rustls-tls-native-roots", - "system-proxy", -] } +reqwest = { version = "0.12", features = ["json"] } # DLF authentication dependencies base64 = "0.22" hex = "0.4" diff --git a/crates/paimon/DEPENDENCIES.rust.tsv b/crates/paimon/DEPENDENCIES.rust.tsv index 1990717db..9225deb96 100644 --- a/crates/paimon/DEPENDENCIES.rust.tsv +++ b/crates/paimon/DEPENDENCIES.rust.tsv @@ -156,6 +156,8 @@ flatbuffers@25.12.19 X flate2@1.1.9 X X fnv@1.0.7 X X foldhash@0.2.0 X +foreign-types@0.3.2 X X +foreign-types-shared@0.1.1 X X form_urlencoded@1.2.2 X X fs4@0.13.1 X X fs_extra@1.3.0 X @@ -201,6 +203,7 @@ humansize@2.1.3 X X hybrid-array@0.4.13 X X hyper@1.10.1 X hyper-rustls@0.27.9 X X X +hyper-tls@0.6.0 X X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X @@ -283,6 +286,7 @@ moka@0.12.15 X X murmurhash32@0.3.1 X nalgebra@0.33.3 X nalgebra-macros@0.2.2 X +native-tls@0.2.18 X X never-say-never@6.6.666 X X X no_std_io2@0.9.4 X X nom@7.1.3 X @@ -316,7 +320,10 @@ opendal-service-hdfs-native@0.58.2 X opendal-service-obs@0.58.2 X opendal-service-oss@0.58.2 X opendal-service-s3@0.58.2 X +openssl@0.10.81 X +openssl-macros@0.1.1 X X openssl-probe@0.2.1 X X +openssl-sys@0.9.117 X orc-rust@0.8.0 X ordered-float@2.10.1 X ordered-float@5.3.0 X @@ -399,7 +406,6 @@ reqsign-huaweicloud-obs@3.0.6 X reqsign-tencent-cos@3.0.6 X reqwest@0.12.28 X X reqwest@0.13.4 X X -ring@0.17.14 X X rle-decode-fast@1.0.3 X X roaring@0.11.4 X X roxmltree@0.21.1 X X @@ -500,6 +506,7 @@ tiny-keccak@2.0.2 X tinystr@0.8.3 X tokio@1.53.0 X tokio-macros@2.7.1 X +tokio-native-tls@0.3.1 X tokio-rustls@0.26.4 X X tokio-util@0.7.18 X tower@0.5.3 X @@ -526,6 +533,7 @@ urlencoding@2.1.3 X utf8-ranges@1.0.5 X X utf8_iter@1.0.4 X X uuid@1.24.0 X X +vcpkg@0.2.15 X X version_check@0.9.5 X X vortex@0.75.0 X vortex-alp@0.75.0 X diff --git a/scripts/release_licenses.py b/scripts/release_licenses.py index b17b33ce4..5785cf803 100644 --- a/scripts/release_licenses.py +++ b/scripts/release_licenses.py @@ -192,6 +192,32 @@ class BundledComponent: crate_version="0.4.7", required_features=("static",), ), + BundledComponent( + crate="openssl-sys", + license_path="third-party-licenses/openssl-1.1.1.LICENSE", + component="OpenSSL 1.1.1k FIPS libssl and libcrypto shared libraries", + component_url="https://github.com/openssl/openssl/tree/OpenSSL_1_1_1k", + license_name="OpenSSL 1.1.1 and Original SSLeay Licenses", + anchor="bundled-openssl-1.1.1", + components=("python",), + targets=("x86_64-unknown-linux-gnu",), + license_from_repository=True, + required=True, + relationship="linked through", + ), + BundledComponent( + crate="openssl-sys", + license_path="third-party-licenses/openssl-1.1.1.LICENSE", + component="OpenSSL 1.1.1w libssl and libcrypto shared libraries", + component_url="https://github.com/openssl/openssl/tree/OpenSSL_1_1_1w", + license_name="OpenSSL 1.1.1 and Original SSLeay Licenses", + anchor="bundled-openssl-1.1.1", + components=("python",), + targets=("aarch64-unknown-linux-gnu",), + license_from_repository=True, + required=True, + relationship="linked through", + ), ) ALLOC_PLACEHOLDER = "Copyright (c) <year> <owner>." diff --git a/scripts/verify_python_wheels.py b/scripts/verify_python_wheels.py index 40be17637..a01242f33 100644 --- a/scripts/verify_python_wheels.py +++ b/scripts/verify_python_wheels.py @@ -403,6 +403,7 @@ def verify_wheel(path: Path) -> tuple[str, str]: require(record_member in names, f"wheel has no RECORD: {path.name}") verify_record(archive, names, record_member, path.name) + license_report = None for license_file in metadata.get_all("License-File", []): relative = normalized_relative_path(license_file, "License-File entry") members = ( @@ -435,6 +436,7 @@ def verify_wheel(path: Path) -> tuple[str, str]: if relative.name == "NOTICE": verify_avro_notice(actual, f"{path.name}:{member}") if relative.name == "THIRD-PARTY-LICENSES.html": + license_report = actual verify_license_report(actual, target, f"{path.name}:{member}") native_members = [ @@ -463,10 +465,50 @@ def verify_wheel(path: Path) -> tuple[str, str]: extra_native_members = [ name for name in names if name != native_member and is_native_library(name) ] - require( - not extra_native_members, - f"wheel has unexpected native libraries: {extra_native_members}", - ) + if target.endswith("linux-gnu"): + openssl_version = ( + b"OpenSSL 1.1.1k FIPS" + if target.startswith("x86_64") + else b"OpenSSL 1.1.1w" + ) + openssl_libraries = ( + (r"pypaimon_rust\.libs/libcrypto-[^/]+\.so\.1\.1", openssl_version), + (r"pypaimon_rust\.libs/libssl-[^/]+\.so\.1\.1", None), + ) + for pattern, version_marker in openssl_libraries: + matches = [ + name for name in extra_native_members if re.fullmatch(pattern, name) + ] + require( + len(matches) == 1, + f"wheel must contain one {pattern}: {extra_native_members}", + ) + with archive.open(matches[0]) as native_file: + content = native_file.read() + verify_native_header( + content, + target, + f"{path.name}:{matches[0]}", + ) + if version_marker is not None: + require( + version_marker in content, + f"wheel libcrypto is missing {version_marker!r}", + ) + require( + len(extra_native_members) == len(openssl_libraries), + f"wheel has unexpected native libraries: {extra_native_members}", + ) + require( + license_report is not None + and b'id="bundled-openssl-1.1.1"' in license_report, + "Linux wheel license report is missing the OpenSSL anchor", + ) + else: + require( + not extra_native_members, + f"wheel has unexpected native libraries: {extra_native_members}", + ) return target, version From b262c4b3f093963d145b94b96b0dd630c4951439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 10:48:34 +0800 Subject: [PATCH 12/21] ci(cpp): remove unrelated C ABI test --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c46be4a4..9b6094e14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,9 +178,6 @@ jobs: steps: - uses: actions/checkout@v7 - - name: Test C ABI - run: cargo test --locked -p paimon-c - - name: Configure C++ facade run: > cmake -S bindings/cpp -B target/cpp-ci From f1e032cd7e15a5dd7c4112cc1a977c3a71a156a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 11:05:38 +0800 Subject: [PATCH 13/21] feat(cpp): package SDK with CPack --- bindings/cpp/CMakeLists.txt | 63 +++++++++++++++- bindings/cpp/README.md | 26 +++++++ .../cpp/cmake/PaimonCppCPackOptions.cmake.in | 71 +++++++++++++++++++ 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 bindings/cpp/cmake/PaimonCppCPackOptions.cmake.in diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index a44355972..89b8349d6 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -164,7 +164,7 @@ install( DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" COMPONENT ${paimon_cpp_install_component}) install( - FILES "${paimon_c_library}" + PROGRAMS "${paimon_c_library}" DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT ${paimon_cpp_install_component}) if(APPLE) @@ -215,12 +215,17 @@ install( "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfigVersion.cmake" DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp" COMPONENT ${paimon_cpp_install_component}) +install( + FILES "${paimon_rust_root}/LICENSE" "${paimon_rust_root}/NOTICE" + DESTINATION "${CMAKE_INSTALL_DATADIR}/doc/paimon-cpp" + COMPONENT ${paimon_cpp_install_component}) # Keep the normal CMake install target, and also materialize the ready-to-use # headers, library, and package files directly in the CMake build directory. add_custom_target( paimon_cpp_artifacts ALL COMMAND + "${CMAKE_COMMAND}" -E env "DESTDIR=" "${CMAKE_COMMAND}" --install "${CMAKE_BINARY_DIR}" --prefix "${CMAKE_CURRENT_BINARY_DIR}" --component ${paimon_cpp_install_component} @@ -228,3 +233,59 @@ add_custom_target( COMMENT "Staging the Paimon C++ artifacts in ${CMAKE_CURRENT_BINARY_DIR}" VERBATIM USES_TERMINAL) + +# A Linux package build emits the native package formats plus a portable +# archive in one CPack invocation. All three contain the same binary, so build +# on the oldest Linux/OpenSSL ABI baseline that the resulting packages support. +set(CPACK_PACKAGE_NAME "paimon-cpp-sdk") +set(CPACK_PACKAGE_VENDOR "Apache Software Foundation") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY + "Apache Paimon C and header-only C++ SDK") +set(CPACK_PACKAGE_HOMEPAGE_URL "https://paimon.apache.org/") +set(CPACK_PACKAGE_CONTACT "dev@paimon.apache.org") +set(CPACK_PACKAGE_VERSION "${PROJECT_VERSION}") +set(CPACK_RESOURCE_FILE_LICENSE "${paimon_rust_root}/LICENSE") +set(CPACK_PACKAGE_DIRECTORY "${CMAKE_BINARY_DIR}/packages") +set(CPACK_PACKAGE_CHECKSUM SHA256) +set(CPACK_MONOLITHIC_INSTALL ON) +set(CPACK_PACKAGE_RELOCATABLE FALSE) + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(CPACK_GENERATOR "DEB;RPM;TGZ") + set(CPACK_PACKAGING_INSTALL_PREFIX "/usr") + + set(CPACK_DEBIAN_PACKAGE_NAME "paimon-cpp-dev") + set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Apache Paimon Developers") + set(CPACK_DEBIAN_PACKAGE_SECTION "libdevel") + set(CPACK_DEBIAN_PACKAGE_RELEASE "1") + set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) + if(paimon_rust_host MATCHES "^x86_64-") + set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE amd64) + elseif(paimon_rust_host MATCHES "^aarch64-") + set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE arm64) + else() + message( + FATAL_ERROR + "No Debian architecture mapping for Rust host ${paimon_rust_host}") + endif() + + set(CPACK_RPM_PACKAGE_NAME "paimon-cpp-devel") + set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") + set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries") + set(CPACK_RPM_PACKAGE_RELEASE "1") + set(CPACK_RPM_PACKAGE_RELEASE_DIST ON) + set(CPACK_RPM_PACKAGE_AUTOREQ ON) + set(CPACK_RPM_PACKAGE_RELOCATABLE FALSE) + set(CPACK_RPM_FILE_NAME RPM-DEFAULT) + + configure_file( + cmake/PaimonCppCPackOptions.cmake.in + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppCPackOptions.cmake" + @ONLY) + set(CPACK_PROJECT_CONFIG_FILE + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppCPackOptions.cmake") +else() + set(CPACK_GENERATOR TGZ) +endif() + +include(CPack) diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index ea47e24b7..f2d6f1ac4 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -122,6 +122,32 @@ add_executable(my_paimon_app main.cpp) target_link_libraries(my_paimon_app PRIVATE Paimon::cpp) ``` +## Linux packages + +The CPack `package` target builds all supported Linux package formats in one +run after compiling the in-tree Rust library: + +```bash +cmake -S bindings/cpp -B target/cpp-build +cmake --build target/cpp-build --target package +ls target/cpp-build/packages +``` + +It produces a Debian/Ubuntu `paimon-cpp-dev` DEB, an RPM-family +`paimon-cpp-devel` RPM, and a `paimon-cpp-sdk` TGZ, plus a SHA-256 checksum for +each package. Building the RPM requires the distribution's `rpmbuild` tool. +Install the native package with, for example: + +```bash +sudo apt install ./paimon-cpp-dev_*.deb +sudo dnf install ./paimon-cpp-devel-*.rpm +``` + +All formats from one run contain the same `libpaimon_c.so`. Package format does +not change its glibc or OpenSSL ABI: build on each binary compatibility baseline +that customers need. The TGZ is the format-neutral fallback and contains the +same `/usr` installation tree. + `Scan::plan()` remains a bounded scan. Use `StreamScanOptions` and `ReadBuilder::new_stream_scan` for a stateful continuous scan. Persist `StreamScan::checkpoint()` only after every split in the returned plan has been diff --git a/bindings/cpp/cmake/PaimonCppCPackOptions.cmake.in b/bindings/cpp/cmake/PaimonCppCPackOptions.cmake.in new file mode 100644 index 000000000..2de9a4b30 --- /dev/null +++ b/bindings/cpp/cmake/PaimonCppCPackOptions.cmake.in @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# CPack loads this file once per generator. The RPM generator derives ELF +# requirements itself; for a DEB built on a non-Debian host, derive the Ubuntu +# OpenSSL package from the actual SONAME linked by libpaimon_c. +if(CPACK_GENERATOR STREQUAL "DEB") + execute_process( + COMMAND "@CMAKE_READELF@" --dynamic "@paimon_c_library@" + RESULT_VARIABLE paimon_readelf_result + OUTPUT_VARIABLE paimon_dynamic_section + ERROR_VARIABLE paimon_readelf_error) + if(NOT paimon_readelf_result EQUAL 0) + message( + FATAL_ERROR + "Cannot inspect libpaimon_c dependencies: ${paimon_readelf_error}") + endif() + + execute_process( + COMMAND "@CMAKE_READELF@" --version-info "@paimon_c_library@" + RESULT_VARIABLE paimon_version_info_result + OUTPUT_VARIABLE paimon_version_info + ERROR_VARIABLE paimon_version_info_error) + if(NOT paimon_version_info_result EQUAL 0) + message( + FATAL_ERROR + "Cannot inspect libpaimon_c symbol versions: ${paimon_version_info_error}") + endif() + string( + REGEX MATCHALL "GLIBC_[0-9]+\\.[0-9]+(\\.[0-9]+)?" + paimon_glibc_symbols "${paimon_version_info}") + set(paimon_minimum_glibc 0) + foreach(paimon_glibc_symbol IN LISTS paimon_glibc_symbols) + string(REPLACE "GLIBC_" "" paimon_glibc_version "${paimon_glibc_symbol}") + if(paimon_glibc_version VERSION_GREATER paimon_minimum_glibc) + set(paimon_minimum_glibc "${paimon_glibc_version}") + endif() + endforeach() + if(paimon_minimum_glibc STREQUAL 0) + message(FATAL_ERROR "No GLIBC symbol versions found in libpaimon_c") + endif() + + set(paimon_debian_dependencies "libc6 (>= ${paimon_minimum_glibc})") + if(paimon_dynamic_section MATCHES "libssl\\.so\\.3") + list(APPEND paimon_debian_dependencies libssl3) + elseif(paimon_dynamic_section MATCHES "libssl\\.so\\.1\\.1") + list(APPEND paimon_debian_dependencies libssl1.1) + elseif(paimon_dynamic_section MATCHES "libssl\\.so") + message( + FATAL_ERROR + "Unsupported OpenSSL SONAME in libpaimon_c; set an explicit DEB mapping") + endif() + if(paimon_dynamic_section MATCHES "libgcc_s\\.so") + list(APPEND paimon_debian_dependencies "libgcc-s1 | libgcc1") + endif() + list(JOIN paimon_debian_dependencies ", " CPACK_DEBIAN_PACKAGE_DEPENDS) +endif() From 22c59ad85dababe1eef7155a4f82c945fe1f71a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 11:31:15 +0800 Subject: [PATCH 14/21] refactor(bindings): remove unused version ABI --- bindings/c/include/paimon.h | 16 ------- bindings/c/src/lib.rs | 1 - bindings/c/src/version.rs | 46 ------------------- bindings/cpp/include/paimon/paimon.hpp | 8 ---- bindings/cpp/tests/header_smoke.cpp | 4 -- .../cpp/tests/install_tree_consumer/main.cpp | 4 -- bindings/cpp/tests/paimon_test_stub.h | 2 - 7 files changed, 81 deletions(-) delete mode 100644 bindings/c/src/version.rs diff --git a/bindings/c/include/paimon.h b/bindings/c/include/paimon.h index c26723aae..bf98d4226 100644 --- a/bindings/c/include/paimon.h +++ b/bindings/c/include/paimon.h @@ -471,14 +471,6 @@ typedef struct paimon_result_table_write { extern "C" { #endif // __cplusplus -/** - * ABI version for the native C boundary. - * - * Version 1 is additive: callers must still feature-detect newer symbols when - * loading the shared library dynamically. - */ -uint32_t paimon_abi_version(void); - /** * Free the ArrowArray and ArrowSchema container structs for a single batch. * @@ -686,14 +678,6 @@ void paimon_identifier_free(struct paimon_identifier *id); */ struct paimon_result_identifier_new paimon_identifier_new(const char *database, const char *object); -/** - * Return the paimon-rust package version as an owned UTF-8 byte buffer. - * - * The returned bytes are not NUL terminated and must be released with - * `paimon_bytes_free`. - */ -struct paimon_bytes paimon_library_version(void); - /** * Free a paimon_plan. * diff --git a/bindings/c/src/lib.rs b/bindings/c/src/lib.rs index 297b9e355..46908cd4b 100644 --- a/bindings/c/src/lib.rs +++ b/bindings/c/src/lib.rs @@ -31,7 +31,6 @@ mod table; mod tests; mod types; mod vector_search; -mod version; mod write; use std::sync::OnceLock; diff --git a/bindings/c/src/version.rs b/bindings/c/src/version.rs deleted file mode 100644 index c3f277ab0..000000000 --- a/bindings/c/src/version.rs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use crate::types::paimon_bytes; -use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::ptr; - -/// ABI version for the native C boundary. -/// -/// Version 1 is additive: callers must still feature-detect newer symbols when -/// loading the shared library dynamically. -#[no_mangle] -pub extern "C" fn paimon_abi_version() -> u32 { - 1 -} - -/// Return the paimon-rust package version as an owned UTF-8 byte buffer. -/// -/// The returned bytes are not NUL terminated and must be released with -/// `paimon_bytes_free`. -#[no_mangle] -pub extern "C" fn paimon_library_version() -> paimon_bytes { - catch_unwind(AssertUnwindSafe(|| { - paimon_bytes::new(env!("CARGO_PKG_VERSION").as_bytes().to_vec()) - })) - .unwrap_or(paimon_bytes { - data: ptr::null_mut(), - len: 0, - }) -} - -const _: extern "C" fn() -> u32 = paimon_abi_version; -const _: extern "C" fn() -> paimon_bytes = paimon_library_version; diff --git a/bindings/cpp/include/paimon/paimon.hpp b/bindings/cpp/include/paimon/paimon.hpp index 2d864a159..de2a4eab8 100644 --- a/bindings/cpp/include/paimon/paimon.hpp +++ b/bindings/cpp/include/paimon/paimon.hpp @@ -327,14 +327,6 @@ class [[nodiscard]] Result final { using Status = Result; using Option = ::paimon_option; -[[nodiscard]] inline std::uint32_t abi_version() noexcept { - return ::paimon_abi_version(); -} - -[[nodiscard]] inline Bytes library_version() noexcept { - return Bytes(adopt_handle, ::paimon_library_version()); -} - namespace detail { inline Status status_from(::paimon_error* error) noexcept { diff --git a/bindings/cpp/tests/header_smoke.cpp b/bindings/cpp/tests/header_smoke.cpp index 6bbee3e79..9797ed590 100644 --- a/bindings/cpp/tests/header_smoke.cpp +++ b/bindings/cpp/tests/header_smoke.cpp @@ -30,10 +30,6 @@ static_assert(std::is_nothrow_destructible::value, void paimon_cpp_header_smoke(const paimon::Option* options, std::size_t option_count, void* arrow_array, void* arrow_schema) { - const auto abi = paimon::abi_version(); - auto version = paimon::library_version(); - (void)abi; - (void)version; auto catalog = paimon::Catalog::create(options, option_count); auto identifier = paimon::Identifier::create("default", "table"); auto direct_table = paimon::Table::from_schema_json( diff --git a/bindings/cpp/tests/install_tree_consumer/main.cpp b/bindings/cpp/tests/install_tree_consumer/main.cpp index 275a99739..ba9590ad0 100644 --- a/bindings/cpp/tests/install_tree_consumer/main.cpp +++ b/bindings/cpp/tests/install_tree_consumer/main.cpp @@ -18,9 +18,5 @@ #include int main() { - const auto abi = paimon::abi_version(); - auto version = paimon::library_version(); - (void)abi; - (void)version; return 0; } diff --git a/bindings/cpp/tests/paimon_test_stub.h b/bindings/cpp/tests/paimon_test_stub.h index 2da449030..1a7303443 100644 --- a/bindings/cpp/tests/paimon_test_stub.h +++ b/bindings/cpp/tests/paimon_test_stub.h @@ -184,8 +184,6 @@ typedef struct paimon_result_stream_poll { void paimon_error_free(paimon_error* error); void paimon_bytes_free(paimon_bytes bytes); -uint32_t paimon_abi_version(void); -paimon_bytes paimon_library_version(void); paimon_result_catalog_new paimon_catalog_create(const paimon_option* options, size_t options_len); void paimon_catalog_free(paimon_catalog* catalog); From a2f574e147e5edfb8e7c2e305667ef6f75e2cace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 11:45:58 +0800 Subject: [PATCH 15/21] build(cpp): generate C header in build tree --- .github/workflows/ci.yml | 13 +- bindings/c/include/paimon.h | 1848 ---------------------------- bindings/c/scripts/check-header.sh | 38 - bindings/cpp/CMakeLists.txt | 53 +- bindings/cpp/README.md | 6 +- docs/src/c-binding.md | 19 +- 6 files changed, 64 insertions(+), 1913 deletions(-) delete mode 100644 bindings/c/include/paimon.h delete mode 100755 bindings/c/scripts/check-header.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b6094e14..27b3c0f60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,14 +121,6 @@ jobs: - name: Format run: cargo fmt --all -- --check - - name: Install cbindgen - uses: taiki-e/install-action@065d6a08a14e61e89fb0a4c10eecdbdef39c7d8e # v2.85.4 - with: - tool: cbindgen@0.29.4 - - - name: Check C header - run: ./bindings/c/scripts/check-header.sh - - name: Clippy run: cargo clippy --locked --all-targets --workspace --features fulltext,vortex -- -D warnings @@ -178,6 +170,11 @@ jobs: steps: - uses: actions/checkout@v7 + - name: Install cbindgen + uses: taiki-e/install-action@065d6a08a14e61e89fb0a4c10eecdbdef39c7d8e # v2.85.4 + with: + tool: cbindgen@0.29.4 + - name: Configure C++ facade run: > cmake -S bindings/cpp -B target/cpp-ci diff --git a/bindings/c/include/paimon.h b/bindings/c/include/paimon.h deleted file mode 100644 index bf98d4226..000000000 --- a/bindings/c/include/paimon.h +++ /dev/null @@ -1,1848 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -#ifndef PAIMON_C_H -#define PAIMON_C_H - -#include -#include -#include -#include -#include - -#define PAIMON_ERROR_ALREADY_EXISTS 3 - -#define PAIMON_ERROR_INVALID_INPUT 4 - -#define PAIMON_ERROR_IO 5 - -#define PAIMON_ERROR_NOT_FOUND 2 - -#define PAIMON_ERROR_OUT_OF_RANGE 6 - -#define PAIMON_ERROR_UNEXPECTED 0 - -#define PAIMON_ERROR_UNSUPPORTED 1 - -#define PAIMON_STREAM_FOLLOW_UP_AUTO 0 - -#define PAIMON_STREAM_FOLLOW_UP_CHANGELOG 2 - -#define PAIMON_STREAM_FOLLOW_UP_DELTA 1 - -#define PAIMON_STREAM_POLL_DATA 0 - -#define PAIMON_STREAM_POLL_END 2 - -#define PAIMON_STREAM_POLL_WAITING 1 - -#define PAIMON_STREAM_READ_AUDIT_LOG 1 - -#define PAIMON_STREAM_READ_DATA 0 - -#define PAIMON_STREAM_STARTUP_FROM_SNAPSHOT 2 - -#define PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL 3 - -#define PAIMON_STREAM_STARTUP_LATEST 1 - -#define PAIMON_STREAM_STARTUP_LATEST_FULL 0 - -/** - * A single Arrow record batch exported via the Arrow C Data Interface. - * - * `array` and `schema` point to heap-allocated ArrowArray and ArrowSchema - * structs. After importing the data, call `paimon_arrow_batch_free` to free - * the container structs. - */ -typedef struct paimon_arrow_batch { - /** - * Pointer to a heap-allocated ArrowArray. - */ - void *array; - /** - * Pointer to a heap-allocated ArrowSchema. - */ - void *schema; -} paimon_arrow_batch; - -typedef struct paimon_blob_reader { - void *inner; -} paimon_blob_reader; - -/** - * C-compatible byte buffer. - */ -typedef struct paimon_bytes { - uint8_t *data; - size_t len; -} paimon_bytes; - -/** - * C-compatible error type. - */ -typedef struct paimon_error { - int32_t code; - struct paimon_bytes message; -} paimon_error; - -typedef struct paimon_result_blob_reader { - struct paimon_blob_reader *reader; - struct paimon_error *error; -} paimon_result_blob_reader; - -/** - * C-compatible key-value pair for options. - */ -typedef struct paimon_option { - const char *key; - const char *value; -} paimon_option; - -typedef struct paimon_blob_stream { - void *inner; -} paimon_blob_stream; - -typedef struct paimon_result_blob_stream { - struct paimon_blob_stream *stream; - struct paimon_error *error; -} paimon_result_blob_stream; - -typedef struct paimon_bytes_array { - struct paimon_bytes *data; - size_t len; -} paimon_bytes_array; - -typedef struct paimon_result_read_blobs { - struct paimon_bytes_array blobs; - struct paimon_error *error; -} paimon_result_read_blobs; - -typedef struct paimon_byte_slice { - const uint8_t *data; - size_t len; -} paimon_byte_slice; - -typedef struct paimon_result_blob_stream_read { - size_t bytes_read; - struct paimon_error *error; -} paimon_result_blob_stream_read; - -typedef struct paimon_result_blob_stream_seek { - uint64_t position; - struct paimon_error *error; -} paimon_result_blob_stream_seek; - -/** - * Opaque wrapper around a heap-allocated Rust object. - */ -typedef struct paimon_catalog { - void *inner; -} paimon_catalog; - -typedef struct paimon_result_catalog_new { - struct paimon_catalog *catalog; - struct paimon_error *error; -} paimon_result_catalog_new; - -typedef struct paimon_identifier { - void *inner; -} paimon_identifier; - -typedef struct paimon_table { - void *inner; -} paimon_table; - -typedef struct paimon_result_get_table { - struct paimon_table *table; - struct paimon_error *error; -} paimon_result_get_table; - -/** - * Opaque container for commit messages and their originating write context. - */ -typedef struct paimon_commit_messages { - void *inner; -} paimon_commit_messages; - -/** - * Opaque durable prepared-commit handle for a standard table write. - */ -typedef struct paimon_prepared_commit { - void *inner; -} paimon_prepared_commit; - -typedef struct paimon_result_prepared_commit { - struct paimon_prepared_commit *prepared; - struct paimon_error *error; -} paimon_result_prepared_commit; - -/** - * Opaque wrapper around a cloneable Paimon FileIO. - */ -typedef struct paimon_file_io { - void *inner; -} paimon_file_io; - -typedef struct paimon_result_file_io_new { - struct paimon_file_io *file_io; - struct paimon_error *error; -} paimon_result_file_io_new; - -/** - * Version 1 callbacks for an externally managed file-block cache. - * - * Callbacks may run concurrently on arbitrary Rust runtime blocking threads. - * They must not unwind across the C ABI. `get` returns the number of bytes - * copied into `output`; return `-1` for a miss and any value other than the - * requested length for a fail-open miss. All callback buffers and paths are - * borrowed only for the duration of the call. Paths use pointer-plus-length - * because canonical storage keys may contain embedded NUL separators. - */ -typedef struct paimon_file_cache_callbacks_v1 { - void *context; - int64_t (*get)(void *context, - const uint8_t *path_data, - size_t path_length, - uint64_t offset, - size_t length, - uint8_t *output); - int32_t (*put)(void *context, - const uint8_t *path_data, - size_t path_length, - uint64_t offset, - const uint8_t *data, - size_t length); - int32_t (*invalidate_path)(void *context, const uint8_t *path_data, size_t path_length); - int32_t (*invalidate_prefix)(void *context, const uint8_t *prefix_data, size_t prefix_length); - /** - * Releases `context` after the last FileIO/table clone is dropped. - */ - void (*destroy)(void *context); -} paimon_file_cache_callbacks_v1; - -typedef struct paimon_result_identifier_new { - struct paimon_identifier *identifier; - struct paimon_error *error; -} paimon_result_identifier_new; - -typedef struct paimon_plan { - void *inner; -} paimon_plan; - -typedef struct paimon_result_plan { - struct paimon_plan *plan; - struct paimon_error *error; -} paimon_result_plan; - -typedef struct paimon_postpone_fixed_bucket_commit_messages { - void *inner; -} paimon_postpone_fixed_bucket_commit_messages; - -typedef struct paimon_postpone_fixed_bucket_table_commit { - void *inner; -} paimon_postpone_fixed_bucket_table_commit; - -typedef struct paimon_postpone_fixed_bucket_table_write { - void *inner; -} paimon_postpone_fixed_bucket_table_write; - -typedef struct paimon_result_postpone_fixed_bucket_prepare_commit { - struct paimon_postpone_fixed_bucket_commit_messages *messages; - struct paimon_error *error; -} paimon_result_postpone_fixed_bucket_prepare_commit; - -typedef struct paimon_postpone_fixed_bucket_write_builder { - void *inner; -} paimon_postpone_fixed_bucket_write_builder; - -typedef struct paimon_result_postpone_fixed_bucket_table_commit { - struct paimon_postpone_fixed_bucket_table_commit *commit; - struct paimon_error *error; -} paimon_result_postpone_fixed_bucket_table_commit; - -typedef struct paimon_result_postpone_fixed_bucket_table_write { - struct paimon_postpone_fixed_bucket_table_write *write; - struct paimon_error *error; -} paimon_result_postpone_fixed_bucket_table_write; - -/** - * Opaque wrapper around a Predicate. - */ -typedef struct paimon_predicate { - void *inner; -} paimon_predicate; - -typedef struct paimon_result_predicate { - struct paimon_predicate *predicate; - struct paimon_error *error; -} paimon_result_predicate; - -/** - * A typed literal value for predicate comparison, passed across FFI. - * - * # Design - * - * We use a tagged flat struct instead of opaque heap-allocated handles - * (like DuckDB's `duckdb_value`). The trade-off: - * - * - **Pro**: Zero allocation — the entire datum is passed by value on the - * stack, with no heap round-trips or free calls needed. This keeps the - * FFI surface minimal and the Go/C caller simple. - * - **Con**: The struct is larger than any single variant needs, wasting - * some bytes per datum (currently ~56 bytes vs. ~16 for the largest - * single variant). - * - * Since datums are only used for predicate construction (not a hot path), - * the extra size is acceptable. - * - * # Tags - * - * - 0: Bool, 1: TinyInt, 2: SmallInt, 3: Int, 4: Long - * - 5: Float, 6: Double, 7: String, 8: Date, 9: Time - * - 10: Timestamp, 11: LocalZonedTimestamp, 12: Decimal, 13: Bytes - * - * `tag` determines which value fields are valid: - * - `Bool`/`TinyInt`/`SmallInt`/`Int`/`Long`/`Date`/`Time` → `int_val` - * - `Float`/`Double` → `double_val` - * - `String`/`Bytes` → `str_data` + `str_len` - * - `Timestamp`/`LocalZonedTimestamp` → `int_val` (millis) + `int_val2` (nanos) - * - `Decimal` → `int_val` + `int_val2` (unscaled i128) + `uint_val` (precision) + `uint_val2` (scale) - */ -typedef struct paimon_datum { - int32_t tag; - int64_t int_val; - double double_val; - const uint8_t *str_data; - size_t str_len; - int64_t int_val2; - uint32_t uint_val; - uint32_t uint_val2; -} paimon_datum; - -typedef struct paimon_result_bytes { - struct paimon_bytes bytes; - struct paimon_error *error; -} paimon_result_bytes; - -typedef struct paimon_read_builder { - void *inner; -} paimon_read_builder; - -typedef struct paimon_table_read { - void *inner; -} paimon_table_read; - -typedef struct paimon_result_new_read { - struct paimon_table_read *read; - struct paimon_error *error; -} paimon_result_new_read; - -typedef struct paimon_table_scan { - void *inner; -} paimon_table_scan; - -typedef struct paimon_result_table_scan { - struct paimon_table_scan *scan; - struct paimon_error *error; -} paimon_result_table_scan; - -typedef struct paimon_stream_scan { - void *inner; -} paimon_stream_scan; - -typedef struct paimon_result_stream_scan { - struct paimon_stream_scan *scan; - struct paimon_error *error; -} paimon_result_stream_scan; - -/** - * Extensible options for a continuous scan. - * - * Initialize this with `paimon_stream_scan_options_init`; future versions may - * consume fields from `reserved` while preserving this prefix. - */ -typedef struct paimon_stream_scan_options { - uint32_t struct_size; - int32_t startup_mode; - int32_t follow_up_mode; - int64_t snapshot_id; - uint64_t reserved[4]; -} paimon_stream_scan_options; - -typedef struct paimon_record_batch_reader { - void *inner; -} paimon_record_batch_reader; - -typedef struct paimon_result_next_batch { - struct paimon_arrow_batch batch; - struct paimon_error *error; -} paimon_result_next_batch; - -typedef struct paimon_stream_plan { - void *inner; -} paimon_stream_plan; - -typedef struct paimon_result_stream_poll { - int32_t status; - struct paimon_stream_plan *plan; - int64_t snapshot_id; - int64_t next_snapshot_id; - int64_t watermark; - uint8_t has_watermark; - uint8_t reserved[7]; - struct paimon_error *error; -} paimon_result_stream_poll; - -typedef struct paimon_result_record_batch_reader { - struct paimon_record_batch_reader *reader; - struct paimon_error *error; -} paimon_result_record_batch_reader; - -typedef struct paimon_table_commit { - void *inner; -} paimon_table_commit; - -typedef struct paimon_result_postpone_fixed_bucket_write_builder { - struct paimon_postpone_fixed_bucket_write_builder *write_builder; - struct paimon_error *error; -} paimon_result_postpone_fixed_bucket_write_builder; - -typedef struct paimon_result_read_builder { - struct paimon_read_builder *read_builder; - struct paimon_error *error; -} paimon_result_read_builder; - -/** - * Opaque wrapper around a vector-search builder. - */ -typedef struct paimon_vector_search_builder { - void *inner; -} paimon_vector_search_builder; - -typedef struct paimon_result_vector_search_builder { - struct paimon_vector_search_builder *builder; - struct paimon_error *error; -} paimon_result_vector_search_builder; - -typedef struct paimon_write_builder { - void *inner; -} paimon_write_builder; - -typedef struct paimon_result_write_builder { - struct paimon_write_builder *write_builder; - struct paimon_error *error; -} paimon_result_write_builder; - -typedef struct paimon_table_write { - void *inner; -} paimon_table_write; - -typedef struct paimon_result_prepare_commit { - struct paimon_commit_messages *messages; - struct paimon_error *error; -} paimon_result_prepare_commit; - -typedef struct paimon_result_table_commit { - struct paimon_table_commit *commit; - struct paimon_error *error; -} paimon_result_table_commit; - -typedef struct paimon_result_table_write { - struct paimon_table_write *write; - struct paimon_error *error; -} paimon_result_table_write; - -#ifdef __cplusplus -extern "C" { -#endif // __cplusplus - -/** - * Free the ArrowArray and ArrowSchema container structs for a single batch. - * - * # Safety - * `batch` must contain valid pointers returned by `paimon_record_batch_reader_next`. - */ -void paimon_arrow_batch_free(struct paimon_arrow_batch batch); - -/** - * # Safety - * `reader` is null or was returned by `paimon_blob_reader_new`. - */ -void paimon_blob_reader_free(struct paimon_blob_reader *reader); - -/** - * # Safety - * `options` is null for zero length or points to valid UTF-8 C-string pairs. - */ -struct paimon_result_blob_reader paimon_blob_reader_new(const struct paimon_option *options, - size_t options_len); - -/** - * Open one descriptor for incremental reads. - * - * # Safety - * `reader` is valid and `descriptor` points to `descriptor_len` bytes. - */ -struct paimon_result_blob_stream paimon_blob_reader_open_blob(const struct paimon_blob_reader *reader, - const uint8_t *descriptor, - size_t descriptor_len); - -/** - * # Safety - * The handle and input slices are valid for this call. Free the output with - * `paimon_bytes_array_free`. - */ -struct paimon_result_read_blobs paimon_blob_reader_read_blobs(const struct paimon_blob_reader *reader, - const struct paimon_byte_slice *descriptors, - size_t descriptors_len); - -/** - * # Safety - * `stream` is null or was returned by `paimon_blob_reader_open_blob`. - */ -void paimon_blob_stream_free(struct paimon_blob_stream *stream); - -/** - * Read at most `buffer_len` bytes into caller-owned memory. - * - * A zero `bytes_read` result means end of stream when `buffer_len` is nonzero. - * - * # Safety - * `stream` is valid and `buffer` points to `buffer_len` writable bytes. - */ -struct paimon_result_blob_stream_read paimon_blob_stream_read(struct paimon_blob_stream *stream, - uint8_t *buffer, - size_t buffer_len); - -/** - * Seek within the descriptor's range. `whence` uses the standard 0, 1, 2 values. - * - * # Safety - * `stream` is valid. - */ -struct paimon_result_blob_stream_seek paimon_blob_stream_seek(struct paimon_blob_stream *stream, - int64_t offset, - int32_t whence); - -/** - * # Safety - * `array` was returned by `paimon_blob_reader_read_blobs`. - */ -void paimon_bytes_array_free(struct paimon_bytes_array array); - -/** - * Free a paimon_bytes buffer. - * - * # Safety - * Only call with bytes returned from paimon C functions. - */ -void paimon_bytes_free(struct paimon_bytes bytes); - -/** - * Create a catalog using CatalogFactory with the given options. - * - * # Safety - * `options` must be a valid pointer to an array of `paimon_option` with `options_len` elements. - * Each key and value in the options must be valid null-terminated C strings. - */ -struct paimon_result_catalog_new paimon_catalog_create(const struct paimon_option *options, - size_t options_len); - -/** - * Create a table from a logical Paimon `Schema` JSON document. - * - * The input is normalized and validated through `SchemaBuilder` before it is - * sent to the catalog. Field IDs in the JSON are therefore treated as input - * ordering hints and reassigned canonically from zero. - * - * # Safety - * `catalog` and `identifier` must be valid Paimon handles. `schema_json` must - * point to a valid null-terminated UTF-8 string. - */ -struct paimon_error *paimon_catalog_create_table_from_schema_json(const struct paimon_catalog *catalog, - const struct paimon_identifier *identifier, - const char *schema_json, - bool ignore_if_exists); - -/** - * Drop a table from the catalog. - * - * # Safety - * `catalog` and `identifier` must be valid Paimon handles, or null (returns an - * error). - */ -struct paimon_error *paimon_catalog_drop_table(const struct paimon_catalog *catalog, - const struct paimon_identifier *identifier, - bool ignore_if_not_exists); - -/** - * Free a paimon_catalog. - * - * # Safety - * Only call with a catalog returned from `paimon_catalog_create`. - */ -void paimon_catalog_free(struct paimon_catalog *catalog); - -/** - * Get a table from the catalog. - * - * # Safety - * `catalog` and `identifier` must be valid pointers from previous paimon C calls, or null (returns error). - */ -struct paimon_result_get_table paimon_catalog_get_table(const struct paimon_catalog *catalog, - const struct paimon_identifier *identifier); - -/** - * Free standard commit messages. - */ -void paimon_commit_messages_free(struct paimon_commit_messages *msgs); - -/** - * Merge standard commit messages for one logical commit. - */ -struct paimon_error *paimon_commit_messages_merge(struct paimon_commit_messages *target, - const struct paimon_commit_messages *source); - -/** - * Bind standard commit messages to a monotonically increasing streaming - * commit identifier. The returned prepared commit owns a clone of the - * messages, so the source handle remains valid. Valid identifiers are in - * `[0, INT64_MAX)`; `INT64_MAX` is reserved for unidentified batch commits. - */ -struct paimon_result_prepared_commit paimon_commit_messages_prepare(const struct paimon_commit_messages *msgs, - int64_t commit_identifier); - -/** - * Free a paimon_error. - * - * # Safety - * Only call with errors returned from paimon C functions. - */ -void paimon_error_free(struct paimon_error *err); - -/** - * Create a reusable FileIO from a representative storage path and options. - */ -struct paimon_result_file_io_new paimon_file_io_create(const char *path, - const struct paimon_option *options, - size_t options_len); - -/** - * Create a reusable FileIO backed by a caller-managed block cache. - * - * A non-null `callbacks->get` and a non-zero `block_size` are required. - * `whitelist` may be null to use `meta,global-index`. Once validation and - * storage construction succeed, Rust owns `callbacks->context` and invokes - * `destroy` exactly once after the last derived FileIO/table is dropped. - */ -struct paimon_result_file_io_new paimon_file_io_create_with_cache_v1(const char *path, - const struct paimon_option *options, - size_t options_len, - const struct paimon_file_cache_callbacks_v1 *callbacks, - uint64_t block_size, - const char *whitelist); - -/** - * Free a FileIO handle. Tables created from it retain their own clone. - */ -void paimon_file_io_free(struct paimon_file_io *file_io); - -/** - * Free a paimon_identifier. - * - * # Safety - * Only call with an identifier returned from `paimon_identifier_new`. - */ -void paimon_identifier_free(struct paimon_identifier *id); - -/** - * Create a new Identifier. - * - * # Safety - * `database` and `object` must be valid null-terminated C strings, or null (returns error). - */ -struct paimon_result_identifier_new paimon_identifier_new(const char *database, const char *object); - -/** - * Free a paimon_plan. - * - * # Safety - * Only call with a plan returned from `paimon_table_scan_plan`. - * A plan returned from `paimon_plan_from_split_bytes` is also a valid source. - */ -void paimon_plan_free(struct paimon_plan *plan); - -/** - * Build a one-split `paimon_plan` from a serialized Paimon-native `DataSplit` - * byte buffer (the wire form produced by `DataSplit::serialize` / Java - * `DataSplit#serialize`). `data` must be raw bytes (Base64 already decoded by - * the caller). - * - * The returned plan is usable with `paimon_table_read_to_arrow` and must be - * freed with `paimon_plan_free`. - * - * # Safety - * `data` must point to `len` valid bytes, or be null when `len == 0`. - */ -struct paimon_result_plan paimon_plan_from_split_bytes(const uint8_t *data, size_t len); - -/** - * Return the number of data splits in a plan. - * - * # Safety - * `plan` must be a valid pointer from `paimon_table_scan_plan`, or null (returns 0). - * A plan returned from `paimon_plan_from_split_bytes` is also a valid source. - */ -size_t paimon_plan_num_splits(const struct paimon_plan *plan); - -/** - * Free postpone fixed-bucket commit messages. - */ -void paimon_postpone_fixed_bucket_commit_messages_free(struct paimon_postpone_fixed_bucket_commit_messages *msgs); - -/** - * Merge postpone fixed-bucket messages for one logical commit. - */ -struct paimon_error *paimon_postpone_fixed_bucket_commit_messages_merge(struct paimon_postpone_fixed_bucket_commit_messages *target, - const struct paimon_postpone_fixed_bucket_commit_messages *source); - -/** - * Abort postpone fixed-bucket commit messages. - */ -struct paimon_error *paimon_postpone_fixed_bucket_table_commit_abort(const struct paimon_postpone_fixed_bucket_table_commit *tc, - struct paimon_postpone_fixed_bucket_commit_messages *msgs); - -/** - * Commit postpone fixed-bucket messages using the builder's mode. - */ -struct paimon_error *paimon_postpone_fixed_bucket_table_commit_commit(const struct paimon_postpone_fixed_bucket_table_commit *tc, - struct paimon_postpone_fixed_bucket_commit_messages *msgs); - -/** - * Commit postpone fixed-bucket messages with an identifier. - */ -struct paimon_error *paimon_postpone_fixed_bucket_table_commit_commit_with_identifier(const struct paimon_postpone_fixed_bucket_table_commit *tc, - struct paimon_postpone_fixed_bucket_commit_messages *msgs, - int64_t commit_identifier); - -/** - * Filter a committed identifier before committing fixed-bucket messages. - */ -struct paimon_error *paimon_postpone_fixed_bucket_table_commit_filter_and_commit_with_identifier(const struct paimon_postpone_fixed_bucket_table_commit *tc, - struct paimon_postpone_fixed_bucket_commit_messages *msgs, - int64_t commit_identifier); - -/** - * Free a postpone fixed-bucket TableCommit. - */ -void paimon_postpone_fixed_bucket_table_commit_free(struct paimon_postpone_fixed_bucket_table_commit *tc); - -/** - * Truncate a table with a postpone fixed-bucket TableCommit. - */ -struct paimon_error *paimon_postpone_fixed_bucket_table_commit_truncate_table(const struct paimon_postpone_fixed_bucket_table_commit *tc); - -/** - * Truncate a table with a stable identifier. - */ -struct paimon_error *paimon_postpone_fixed_bucket_table_commit_truncate_table_with_identifier(const struct paimon_postpone_fixed_bucket_table_commit *tc, - int64_t commit_identifier); - -/** - * Free a postpone fixed-bucket TableWrite. - * - * # Safety - * Only call with a write returned from - * paimon_postpone_fixed_bucket_write_builder_new_write. - */ -void paimon_postpone_fixed_bucket_table_write_free(struct paimon_postpone_fixed_bucket_table_write *tw); - -/** - * Prepare postpone fixed-bucket commit messages. - * - * The returned handle remains owned by the caller. - */ -struct paimon_result_postpone_fixed_bucket_prepare_commit paimon_postpone_fixed_bucket_table_write_prepare_commit(struct paimon_postpone_fixed_bucket_table_write *tw); - -/** - * Write one Arrow record batch with a postpone fixed-bucket TableWrite. - * - * Ownership of array and schema is transferred once Arrow import starts. - */ -struct paimon_error *paimon_postpone_fixed_bucket_table_write_write_arrow_batch(struct paimon_postpone_fixed_bucket_table_write *tw, - void *array, - void *schema); - -/** - * Free a postpone fixed-bucket write builder. - * - * # Safety - * Only call with a builder returned from - * `paimon_table_new_postpone_fixed_bucket_write_builder`. - */ -void paimon_postpone_fixed_bucket_write_builder_free(struct paimon_postpone_fixed_bucket_write_builder *wb); - -/** - * Create a postpone fixed-bucket TableCommit. - */ -struct paimon_result_postpone_fixed_bucket_table_commit paimon_postpone_fixed_bucket_write_builder_new_commit(const struct paimon_postpone_fixed_bucket_write_builder *wb); - -/** - * Create a postpone fixed-bucket TableWrite. - * - * # Safety - * wb must be a valid fixed-bucket builder, or null (returns error). - */ -struct paimon_result_postpone_fixed_bucket_table_write paimon_postpone_fixed_bucket_write_builder_new_write(const struct paimon_postpone_fixed_bucket_write_builder *wb); - -/** - * Set a shared `partition -> total_buckets` plan. - * The caller retains ownership when pointer or builder validation fails. Once - * Arrow import starts, this call consumes both structs even if plan validation - * returns an error. - * - * # Safety - * `wb` must be a valid postpone fixed-bucket builder. `array` and - * `schema` must point to initialized Arrow C Data structs. - */ -struct paimon_error *paimon_postpone_fixed_bucket_write_builder_with_bucket_plan(struct paimon_postpone_fixed_bucket_write_builder *wb, - void *array, - void *schema); - -/** - * Enable overwrite mode for a postpone fixed-bucket write operation. - * - * # Safety - * `wb` must be a valid fixed-bucket builder, or null (returns error). - */ -struct paimon_error *paimon_postpone_fixed_bucket_write_builder_with_overwrite(struct paimon_postpone_fixed_bucket_write_builder *wb); - -/** - * Combine two predicates with AND. Consumes both inputs. - * - * # Safety - * `a` and `b` must be valid pointers from predicate functions. - */ -struct paimon_predicate *paimon_predicate_and(struct paimon_predicate *a, - struct paimon_predicate *b); - -/** - * Create a BETWEEN predicate: `low <= column <= high` (inclusive, case-sensitive - * column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_between(const struct paimon_table *table, - const char *column, - struct paimon_datum low, - struct paimon_datum high); - -/** - * Create a BETWEEN predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_between_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum low, - struct paimon_datum high, - bool case_sensitive); - -/** - * Create a contains predicate: `column LIKE '%datum%'` (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_contains(const struct paimon_table *table, - const char *column, - struct paimon_datum datum); - -/** - * Create a contains predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_contains_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum datum, - bool case_sensitive); - -/** - * Create an ends-with predicate: `column LIKE '%datum'` (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_ends_with(const struct paimon_table *table, - const char *column, - struct paimon_datum datum); - -/** - * Create an ends-with predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_ends_with_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum datum, - bool case_sensitive); - -/** - * Create an equality predicate: `column = datum` (case-sensitive column match). - * - * For case-insensitive column matching use - * `paimon_predicate_equal_with_case_sensitive`. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_equal(const struct paimon_table *table, - const char *column, - struct paimon_datum datum); - -/** - * Create an equality predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_equal_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum datum, - bool case_sensitive); - -/** - * Free a paimon_predicate. - * - * # Safety - * Only call with a predicate returned from paimon predicate functions. - */ -void paimon_predicate_free(struct paimon_predicate *p); - -/** - * Create a greater-or-equal predicate: `column >= datum` (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_greater_or_equal(const struct paimon_table *table, - const char *column, - struct paimon_datum datum); - -/** - * Create a greater-or-equal predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_greater_or_equal_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum datum, - bool case_sensitive); - -/** - * Create a greater-than predicate: `column > datum` (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_greater_than(const struct paimon_table *table, - const char *column, - struct paimon_datum datum); - -/** - * Create a greater-than predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_greater_than_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum datum, - bool case_sensitive); - -/** - * Create an IN predicate: `column IN (datum1, datum2, ...)` (case-sensitive column match). - * - * # Safety - * `table`, `column`, and `datums` must be valid pointers. `datums_len` must be the length. - */ -struct paimon_result_predicate paimon_predicate_is_in(const struct paimon_table *table, - const char *column, - const struct paimon_datum *datums, - size_t datums_len); - -/** - * Create an IN predicate with configurable column-name case sensitivity. - * - * # Safety - * `table`, `column`, and `datums` must be valid pointers. `datums_len` must be the length. - */ -struct paimon_result_predicate paimon_predicate_is_in_with_case_sensitive(const struct paimon_table *table, - const char *column, - const struct paimon_datum *datums, - size_t datums_len, - bool case_sensitive); - -/** - * Create a NOT IN predicate: `column NOT IN (datum1, datum2, ...)` (case-sensitive column match). - * - * # Safety - * `table`, `column`, and `datums` must be valid pointers. `datums_len` must be the length. - */ -struct paimon_result_predicate paimon_predicate_is_not_in(const struct paimon_table *table, - const char *column, - const struct paimon_datum *datums, - size_t datums_len); - -/** - * Create a NOT IN predicate with configurable column-name case sensitivity. - * - * # Safety - * `table`, `column`, and `datums` must be valid pointers. `datums_len` must be the length. - */ -struct paimon_result_predicate paimon_predicate_is_not_in_with_case_sensitive(const struct paimon_table *table, - const char *column, - const struct paimon_datum *datums, - size_t datums_len, - bool case_sensitive); - -/** - * Create an IS NOT NULL predicate (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_is_not_null(const struct paimon_table *table, - const char *column); - -/** - * Create an IS NOT NULL predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_is_not_null_with_case_sensitive(const struct paimon_table *table, - const char *column, - bool case_sensitive); - -/** - * Create an IS NULL predicate (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_is_null(const struct paimon_table *table, - const char *column); - -/** - * Create an IS NULL predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_is_null_with_case_sensitive(const struct paimon_table *table, - const char *column, - bool case_sensitive); - -/** - * Create a less-or-equal predicate: `column <= datum` (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_less_or_equal(const struct paimon_table *table, - const char *column, - struct paimon_datum datum); - -/** - * Create a less-or-equal predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_less_or_equal_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum datum, - bool case_sensitive); - -/** - * Create a less-than predicate: `column < datum` (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_less_than(const struct paimon_table *table, - const char *column, - struct paimon_datum datum); - -/** - * Create a less-than predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_less_than_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum datum, - bool case_sensitive); - -/** - * Create a LIKE predicate: `column LIKE pattern ESCAPE escape` (case-sensitive - * column match). `escape == 0` uses the default escape character. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_like(const struct paimon_table *table, - const char *column, - struct paimon_datum pattern, - char escape); - -/** - * Create a LIKE predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_like_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum pattern, - char escape, - bool case_sensitive); - -/** - * Negate a predicate with NOT. Consumes the input. - * - * # Safety - * `p` must be a valid pointer from a predicate function. - */ -struct paimon_predicate *paimon_predicate_not(struct paimon_predicate *p); - -/** - * Create a NOT BETWEEN predicate: `column < low OR column > high` - * (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_not_between(const struct paimon_table *table, - const char *column, - struct paimon_datum low, - struct paimon_datum high); - -/** - * Create a NOT BETWEEN predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_not_between_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum low, - struct paimon_datum high, - bool case_sensitive); - -/** - * Create a not-equal predicate: `column != datum` (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_not_equal(const struct paimon_table *table, - const char *column, - struct paimon_datum datum); - -/** - * Create a not-equal predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_not_equal_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum datum, - bool case_sensitive); - -/** - * Combine two predicates with OR. Consumes both inputs. - * - * # Safety - * `a` and `b` must be valid pointers from predicate functions. - */ -struct paimon_predicate *paimon_predicate_or(struct paimon_predicate *a, - struct paimon_predicate *b); - -/** - * Create a starts-with predicate: `column LIKE 'datum%'` (case-sensitive column match). - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_starts_with(const struct paimon_table *table, - const char *column, - struct paimon_datum datum); - -/** - * Create a starts-with predicate with configurable column-name case sensitivity. - * - * # Safety - * `table` and `column` must be valid pointers. - */ -struct paimon_result_predicate paimon_predicate_starts_with_with_case_sensitive(const struct paimon_table *table, - const char *column, - struct paimon_datum datum, - bool case_sensitive); - -/** - * Restore a prepared commit serialized by `paimon_prepared_commit_serialize`. - */ -struct paimon_result_prepared_commit paimon_prepared_commit_deserialize(const uint8_t *data, - size_t len); - -/** - * Free a prepared commit. - */ -void paimon_prepared_commit_free(struct paimon_prepared_commit *prepared); - -/** - * Return the commit identifier carried by a prepared commit, or -1 for null. - */ -int64_t paimon_prepared_commit_identifier(const struct paimon_prepared_commit *prepared); - -/** - * Merge two durable prepared commits produced by parallel writers for the - * same table, commit user, mode and identifier. - */ -struct paimon_error *paimon_prepared_commit_merge(struct paimon_prepared_commit *target, - const struct paimon_prepared_commit *source); - -/** - * Serialize a prepared commit into a process-independent, versioned buffer. - * The bytes must be released with `paimon_bytes_free`. - */ -struct paimon_result_bytes paimon_prepared_commit_serialize(const struct paimon_prepared_commit *prepared); - -/** - * Free a paimon_read_builder. - * - * # Safety - * Only call with a read_builder returned from `paimon_table_new_read_builder`. - */ -void paimon_read_builder_free(struct paimon_read_builder *rb); - -/** - * Create a new TableRead from a ReadBuilder. - * - * # Safety - * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). - */ -struct paimon_result_new_read paimon_read_builder_new_read(const struct paimon_read_builder *rb); - -/** - * Create a new TableScan from a ReadBuilder. - * - * # Safety - * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). - */ -struct paimon_result_table_scan paimon_read_builder_new_scan(const struct paimon_read_builder *rb); - -/** - * Create an owned stream scan from a read builder. - * - * The returned scan clones all required Rust state and remains valid after - * the read builder and table handles are freed. A scan handle is - * single-thread-confined: callers must serialize poll/checkpoint/restore/free. - */ -struct paimon_result_stream_scan paimon_read_builder_new_stream_scan(const struct paimon_read_builder *read_builder, - const struct paimon_stream_scan_options *options); - -/** - * Set whether column-name matching for **projection** is case-sensitive for - * this ReadBuilder. Defaults to `true` (exact match). When `false`, projected - * column names are matched by ASCII case-folding and an ambiguous - * (case-colliding) request errors. - * - * This does **not** affect predicate resolution: a predicate is resolved when - * it is constructed, so its case sensitivity is chosen by which constructor - * you call — `paimon_predicate_*` (case-sensitive) or the additive - * `paimon_predicate_*_with_case_sensitive` variant — independently of this - * setting. - * - * # Safety - * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). - */ -struct paimon_error *paimon_read_builder_with_case_sensitive(struct paimon_read_builder *rb, - bool case_sensitive); - -/** - * Set a filter predicate for scan planning. - * - * The predicate is consumed (ownership transferred to the read builder). - * Pass null to clear any previously set filter. - * - * # Safety - * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). - * `predicate` must be a valid pointer from a `paimon_predicate_*` function, or null. - */ -struct paimon_error *paimon_read_builder_with_filter(struct paimon_read_builder *rb, - struct paimon_predicate *predicate); - -/** - * Set column projection for a ReadBuilder. - * - * The `columns` parameter is a null-terminated array of null-terminated C strings. - * Output order follows the caller-specified order. An empty list is a valid - * zero-column projection. An obvious typo — a name that matches no field under - * any case sensitivity — is rejected by this call. Case-dependent resolution - * (a name that matches only case-insensitively, or a case-fold ambiguity) is - * deferred to `paimon_read_builder_new_read`, which uses the case sensitivity - * effective then, so this stays order-independent with - * `paimon_read_builder_with_case_sensitive`. - * - * # Safety - * `rb` must be a valid pointer from `paimon_table_new_read_builder`, or null (returns error). - * `columns` must be a null-terminated array of null-terminated C strings, or null for no projection. - */ -struct paimon_error *paimon_read_builder_with_projection(struct paimon_read_builder *rb, - const char *const *columns); - -/** - * Free a paimon_record_batch_reader. - * - * # Safety - * Only call with a reader returned from `paimon_table_read_to_arrow` or - * `paimon_vector_search_builder_execute_read`. - */ -void paimon_record_batch_reader_free(struct paimon_record_batch_reader *reader); - -/** - * Get the next Arrow record batch from the reader. - * - * When the stream is exhausted, both `batch.array` and `batch.schema` will - * be null. On error, `error` will be non-null. - * - * After importing each batch, call `paimon_arrow_batch_free` to free the - * ArrowArray and ArrowSchema container structs. - * - * # Safety - * `reader` must be a valid pointer from `paimon_table_read_to_arrow`, or null (returns error). - */ -struct paimon_result_next_batch paimon_record_batch_reader_next(struct paimon_record_batch_reader *reader); - -/** - * Restore a stream plan serialized by `paimon_stream_plan_serialize`. - */ -struct paimon_result_stream_poll paimon_stream_plan_deserialize(const uint8_t *data, size_t len); - -/** - * Free a stream plan. It is valid to pass null. - */ -void paimon_stream_plan_free(struct paimon_stream_plan *plan); - -/** - * Return whether a stream plan is an initial full-snapshot plan. - */ -uint8_t paimon_stream_plan_is_full(const struct paimon_stream_plan *plan); - -/** - * Return the number of work splits in a stream plan. - */ -size_t paimon_stream_plan_num_splits(const struct paimon_stream_plan *plan); - -/** - * Read a contiguous split range from a stream plan. - * - * `read_mode=PAIMON_STREAM_READ_AUDIT_LOG` exposes a stable UTF-8 `rowkind` - * column for incremental plans. Full startup plans currently support data - * mode only; callers requiring one fixed audit schema should start at - * `latest` or `from-snapshot`. - */ -struct paimon_result_record_batch_reader paimon_stream_plan_read_to_arrow(const struct paimon_table_read *read, - const struct paimon_stream_plan *plan, - size_t offset, - size_t length, - int32_t read_mode); - -/** - * Serialize planned-but-not-yet-consumed work for an external checkpoint. - * - * The current format checkpoints at plan boundaries. If rows from a plan have already - * been exposed, callers must either replay the plan after recovery or persist - * their own logical rows-to-skip position alongside this buffer. - * Plans containing external data-file paths are rejected because version 1 - * recovery cannot revalidate those paths against a trusted manifest. - */ -struct paimon_result_bytes paimon_stream_plan_serialize(const struct paimon_stream_plan *plan); - -/** - * Return the next-snapshot cursor, or -1 before a startup position exists. - */ -int64_t paimon_stream_scan_checkpoint(const struct paimon_stream_scan *scan); - -/** - * Free a stream scan. It is valid to pass null. - */ -void paimon_stream_scan_free(struct paimon_stream_scan *scan); - -/** - * Fill stream options with forward-compatible defaults (`latest-full`, - * automatic delta/changelog selection). - */ -struct paimon_error *paimon_stream_scan_options_init(struct paimon_stream_scan_options *options); - -/** - * Poll once for a snapshot plan. This call never waits for a future snapshot. - * Calls using the same scan handle must not overlap on different threads. - */ -struct paimon_result_stream_poll paimon_stream_scan_poll(struct paimon_stream_scan *scan); - -/** - * Restore a next-snapshot cursor. Pass -1 to reapply the configured startup - * mode; non-negative values must name a valid Paimon snapshot position. - * This call must not overlap poll/checkpoint/free on the same handle. - */ -struct paimon_error *paimon_stream_scan_restore(struct paimon_stream_scan *scan, - int64_t next_snapshot_id); - -/** - * Abort standard commit messages. - */ -struct paimon_error *paimon_table_commit_abort(const struct paimon_table_commit *tc, - struct paimon_commit_messages *msgs); - -/** - * Abort files referenced by a durable prepared commit. - * - * Do not call this after an indeterminate commit response: retry - * `paimon_table_commit_commit_prepared` first so a successful commit is not - * followed by deletion of its files. The caller must also fence/serialize all - * commit and abort operations for the same `(table, commit_user)` across - * processes. If retained snapshot history cannot prove that abort is safe, - * this function fails closed and deletes nothing. - */ -struct paimon_error *paimon_table_commit_abort_prepared(const struct paimon_table_commit *tc, - const struct paimon_prepared_commit *prepared); - -/** - * Commit standard append messages. - */ -struct paimon_error *paimon_table_commit_commit(const struct paimon_table_commit *tc, - struct paimon_commit_messages *msgs); - -/** - * Commit a durable prepared commit using the retry-safe identifier path. - * - * This is the correct operation after restoring a prepared commit or after a - * previous commit returned an indeterminate transport/IO error. A successful - * earlier commit with the same `(commit_user, commit_identifier)` is filtered. - */ -struct paimon_error *paimon_table_commit_commit_prepared(const struct paimon_table_commit *tc, - const struct paimon_prepared_commit *prepared); - -/** - * Commit standard append messages with an identifier. - */ -struct paimon_error *paimon_table_commit_commit_with_identifier(const struct paimon_table_commit *tc, - struct paimon_commit_messages *msgs, - int64_t commit_identifier); - -/** - * Filter a committed identifier before committing standard append messages. - */ -struct paimon_error *paimon_table_commit_filter_and_commit_with_identifier(const struct paimon_table_commit *tc, - struct paimon_commit_messages *msgs, - int64_t commit_identifier); - -/** - * Free a standard TableCommit. - */ -void paimon_table_commit_free(struct paimon_table_commit *tc); - -/** - * Commit standard overwrite messages. - */ -struct paimon_error *paimon_table_commit_overwrite(const struct paimon_table_commit *tc, - struct paimon_commit_messages *msgs); - -/** - * Commit standard overwrite messages with an identifier. - */ -struct paimon_error *paimon_table_commit_overwrite_with_identifier(const struct paimon_table_commit *tc, - struct paimon_commit_messages *msgs, - int64_t commit_identifier); - -/** - * Truncate a table with a standard TableCommit. - */ -struct paimon_error *paimon_table_commit_truncate_table(const struct paimon_table_commit *tc); - -/** - * Truncate a table with a stable identifier. - */ -struct paimon_error *paimon_table_commit_truncate_table_with_identifier(const struct paimon_table_commit *tc, - int64_t commit_identifier); - -/** - * Free a paimon_table. - * - * # Safety - * Only call with a table returned from `paimon_catalog_get_table`, - * `paimon_table_from_schema_json`, or - * `paimon_table_from_schema_json_with_file_io`. - */ -void paimon_table_free(struct paimon_table *table); - -/** - * Create a table directly from a resolved Paimon table schema JSON. - * - * This constructor does not create a catalog or derive a warehouse. Storage - * options are used only to build FileIO; they are not merged into the supplied - * table schema. `branch` selects the branch-scoped managers while preserving - * the supplied schema; pass null to default to the `main` branch. - * - * # Safety - * All string pointers except `branch` must be valid null-terminated C strings. - * `branch` may be null to select the default `main` branch, or a valid - * null-terminated C string. `storage_options` must point to - * `storage_options_len` valid `paimon_option` values, or be null when - * `storage_options_len` is 0. - */ -struct paimon_result_get_table paimon_table_from_schema_json(const char *table_path, - const char *table_schema_json, - const char *database, - const char *table_name, - const char *branch, - const struct paimon_option *storage_options, - size_t storage_options_len); - -/** - * Create a table from a resolved schema and a caller-created FileIO. - * - * The FileIO is cloned into the table, so its handle may be freed immediately - * after this call. This additive API allows native embedders to share storage - * and an externally managed cache across tables. - * - * # Safety - * `file_io` must be returned by a Paimon FileIO constructor. All string - * pointers except `branch` must be valid null-terminated C strings. `branch` - * may be null to select `main`. - */ -struct paimon_result_get_table paimon_table_from_schema_json_with_file_io(const struct paimon_file_io *file_io, - const char *table_path, - const char *table_schema_json, - const char *database, - const char *table_name, - const char *branch); - -/** - * Create a reader using a table's FileIO. - * - * # Safety - * `table` is a valid handle returned by the Paimon C API. - */ -struct paimon_result_blob_reader paimon_table_new_blob_reader(const struct paimon_table *table); - -/** - * Create a one-shot fixed-bucket WriteBuilder for a postpone table. - * A bucket plan must be set before creating a writer. - * - * # Safety - * `table` must be a valid table pointer, or null (returns error). - */ -struct paimon_result_postpone_fixed_bucket_write_builder paimon_table_new_postpone_fixed_bucket_write_builder(const struct paimon_table *table); - -/** - * Create a fixed-bucket WriteBuilder with a stable commit identity. - * A bucket plan must be set before creating a writer. - * - * # Safety - * `table` must be a valid table pointer. `commit_user` must be a valid UTF-8 - * C string and a safe file-name segment. - */ -struct paimon_result_postpone_fixed_bucket_write_builder paimon_table_new_postpone_fixed_bucket_write_builder_with_commit_user(const struct paimon_table *table, - const char *commit_user); - -/** - * Create a new ReadBuilder from a Table. - * - * # Safety - * `table` must be a valid pointer from `paimon_catalog_get_table` or - * `paimon_table_from_schema_json`, or null (returns error). - */ -struct paimon_result_read_builder paimon_table_new_read_builder(const struct paimon_table *table); - -/** - * Create a ReadBuilder from a Table with scan options (e.g. time-travel - * selectors `scan.snapshot-id` / `scan.tag-name` / `scan.timestamp-millis` / - * `scan.watermark` / `scan.version`). At most one time-travel selector may be - * set. A selector that does not resolve to a snapshot is an error (never a - * silent read-of-latest). - * - * # Safety - * `table` must be a valid pointer. `options` must be a valid pointer to - * `options_len` `paimon_option` values, or null when `options_len` is 0. - */ -struct paimon_result_read_builder paimon_table_new_read_builder_with_options(const struct paimon_table *table, - const struct paimon_option *options, - size_t options_len); - -/** - * Create a new vector-search builder from a Table. - * - * # Safety - * `table` must be a valid pointer from `paimon_catalog_get_table` or - * `paimon_table_from_schema_json`, or null (returns error). - */ -struct paimon_result_vector_search_builder paimon_table_new_vector_search_builder(const struct paimon_table *table); - -/** - * Create a new WriteBuilder from a Table. - * - * The returned WriteBuilder holds a shared `commit_user` (UUID) that will be - * used by both `new_write()` and `new_commit()` for duplicate-commit detection. - * - * # Safety - * `table` must be a valid pointer from `paimon_catalog_get_table` or - * `paimon_table_from_schema_json`, or null (returns error). - */ -struct paimon_result_write_builder paimon_table_new_write_builder(const struct paimon_table *table); - -/** - * Create a WriteBuilder with a caller-provided stable commit identity. - * - * Writers whose messages are merged into one logical commit must use the - * same `commit_user`. - * - * # Safety - * `table` must be a valid table pointer. `commit_user` must be a valid UTF-8 - * C string and a safe file-name segment. - */ -struct paimon_result_write_builder paimon_table_new_write_builder_with_commit_user(const struct paimon_table *table, - const char *commit_user); - -/** - * Free a paimon_table_read. - * - * # Safety - * Only call with a read returned from `paimon_read_builder_new_read`. - */ -void paimon_table_read_free(struct paimon_table_read *read); - -/** - * Read table data as Arrow record batches via a streaming reader. - * - * Returns a `paimon_record_batch_reader` that yields one batch at a time - * via `paimon_record_batch_reader_next`. This avoids loading all batches - * into memory at once. - * - * `offset` and `length` select a contiguous sub-range of splits from the - * plan. The range is clamped to the available splits (out-of-range values - * are silently adjusted). - * - * # Safety - * `read` and `plan` must be valid pointers from previous paimon C calls, or null (returns error). - */ -struct paimon_result_record_batch_reader paimon_table_read_to_arrow(const struct paimon_table_read *read, - const struct paimon_plan *plan, - size_t offset, - size_t length); - -/** - * Free a paimon_table_scan. - * - * # Safety - * Only call with a scan returned from `paimon_read_builder_new_scan`. - */ -void paimon_table_scan_free(struct paimon_table_scan *scan); - -/** - * Execute a scan plan to get splits. - * - * # Safety - * `scan` must be a valid pointer from `paimon_read_builder_new_scan`, or null (returns error). - */ -struct paimon_result_plan paimon_table_scan_plan(const struct paimon_table_scan *scan); - -/** - * Free a standard TableWrite. - * - * # Safety - * Only call with a write returned from paimon_write_builder_new_write. - */ -void paimon_table_write_free(struct paimon_table_write *tw); - -/** - * Prepare standard commit messages. - * - * The returned handle remains owned by the caller. - */ -struct paimon_result_prepare_commit paimon_table_write_prepare_commit(struct paimon_table_write *tw); - -/** - * Write one Arrow record batch with a standard TableWrite. - * - * Ownership of array and schema is transferred once Arrow import starts. - */ -struct paimon_error *paimon_table_write_write_arrow_batch(struct paimon_table_write *tw, - void *array, - void *schema); - -/** - * Execute the vector search and return a streaming Arrow reader over the - * materialized rows (projected user columns plus `__paimon_search_score`). - * Works for both primary-key and data-evolution tables. Consume via - * `paimon_record_batch_reader_next` and free with `paimon_record_batch_reader_free`. - * - * # Safety - * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or - * null (returns an error result). - */ -struct paimon_result_record_batch_reader paimon_vector_search_builder_execute_read(struct paimon_vector_search_builder *b); - -/** - * Free a paimon_vector_search_builder. - * - * # Safety - * Only call with a builder returned from `paimon_table_new_vector_search_builder`. - */ -void paimon_vector_search_builder_free(struct paimon_vector_search_builder *b); - -/** - * Set an optional scalar residual filter for a vector-search builder. - * - * The predicate is consumed (ownership transferred to the builder). Pass null - * to clear any previously set filter. - * - * # Safety - * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or - * null (returns error). `predicate` must be a valid pointer from a - * `paimon_predicate_*` function, or null. - */ -struct paimon_error *paimon_vector_search_builder_with_filter(struct paimon_vector_search_builder *b, - struct paimon_predicate *predicate); - -/** - * Set the maximum number of results for a vector-search builder. - * - * # Safety - * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or - * null (returns error). - */ -struct paimon_error *paimon_vector_search_builder_with_limit(struct paimon_vector_search_builder *b, - size_t limit); - -/** - * Set scan/search options for a vector-search builder. - * - * # Safety - * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or - * null (returns error). `options` must be a valid pointer to `len` - * `paimon_option` values, or null when `len` is 0. - */ -struct paimon_error *paimon_vector_search_builder_with_options(struct paimon_vector_search_builder *b, - const struct paimon_option *options, - size_t len); - -/** - * Restrict the columns materialized by `paimon_vector_search_builder_execute_read` - * to `columns` (plus the always-appended `__paimon_search_score`). Without this - * call `execute_read` materializes every user table column. Only affects - * `execute_read`. - * - * `columns` is a null-terminated array of null-terminated C strings; output - * order follows the caller-specified order. An empty list is a valid zero-column - * projection (only the score column is materialized). Pass null to clear any - * previously set projection. - * - * Unlike `paimon_read_builder_with_projection`, this does not validate column - * names eagerly: the vector builder resolves the projection against the schema - * when the search runs, so an unknown column surfaces as an error from - * `paimon_vector_search_builder_execute_read`. - * - * # Safety - * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or - * null (returns error). `columns` must be a null-terminated array of - * null-terminated C strings, or null to clear the projection. - */ -struct paimon_error *paimon_vector_search_builder_with_projection(struct paimon_vector_search_builder *b, - const char *const *columns); - -/** - * Set the query vector for a vector-search builder. - * - * The `len` floats at `data` are copied into the builder; the caller retains - * ownership of `data`. An empty vector (`len == 0`) is rejected. - * - * # Safety - * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or - * null (returns error). `data` must point to `len` `f32` values when `len > 0`. - */ -struct paimon_error *paimon_vector_search_builder_with_query_vector(struct paimon_vector_search_builder *b, - const float *data, - size_t len); - -/** - * Set the target vector column for a vector-search builder. - * - * # Safety - * `b` must be a valid pointer from `paimon_table_new_vector_search_builder`, or - * null (returns error). `column` must be a valid C string. - */ -struct paimon_error *paimon_vector_search_builder_with_vector_column(struct paimon_vector_search_builder *b, - const char *column); - -/** - * Free a paimon_write_builder. - * - * # Safety - * Only call with a write_builder returned from `paimon_table_new_write_builder`. - */ -void paimon_write_builder_free(struct paimon_write_builder *wb); - -/** - * Create a standard TableCommit from a standard WriteBuilder. - */ -struct paimon_result_table_commit paimon_write_builder_new_commit(const struct paimon_write_builder *wb); - -/** - * Create a standard TableWrite from a standard WriteBuilder. - * - * # Safety - * wb must be a valid standard builder, or null (returns error). - */ -struct paimon_result_table_write paimon_write_builder_new_write(const struct paimon_write_builder *wb); - -/** - * Enable overwrite mode for the WriteBuilder. - * - * # Safety - * `wb` must be a valid pointer from `paimon_table_new_write_builder`, or null (returns error). - */ -struct paimon_error *paimon_write_builder_with_overwrite(struct paimon_write_builder *wb); - -#ifdef __cplusplus -} // extern "C" -#endif // __cplusplus - -#endif /* PAIMON_C_H */ diff --git a/bindings/c/scripts/check-header.sh b/bindings/c/scripts/check-header.sh deleted file mode 100755 index 79ec9efdd..000000000 --- a/bindings/c/scripts/check-header.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env sh -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu - -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -binding_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) -repo_dir=$(CDPATH= cd -- "$binding_dir/../.." && pwd) -generated=$(mktemp) -trap 'rm -f "$generated"' EXIT HUP INT TERM - -cbindgen --quiet --config "$binding_dir/cbindgen.toml" \ - "$binding_dir" --output "$generated" - -if ! cmp -s "$generated" "$binding_dir/include/paimon.h"; then - echo "bindings/c/include/paimon.h is stale; regenerate it with cbindgen" >&2 - diff -u "$binding_dir/include/paimon.h" "$generated" || true - exit 1 -fi - -cc -std=c11 -fsyntax-only -x c "$generated" -c++ -std=c++17 -fno-exceptions -fno-rtti -fsyntax-only -x c++ "$generated" - -echo "C header is current and C/C++ compatible in $repo_dir" diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index 89b8349d6..5483414c3 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -25,7 +25,12 @@ option(PAIMON_CPP_BUILD_EXAMPLES "Build the C++ facade examples" OFF) option(PAIMON_CPP_BUILD_TESTS "Build the header compile smoke test" OFF) get_filename_component( paimon_rust_root "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE) -set(paimon_c_include_dir "${CMAKE_CURRENT_SOURCE_DIR}/../c/include") +get_filename_component( + paimon_c_binding_dir "${CMAKE_CURRENT_SOURCE_DIR}/../c" ABSOLUTE) +set(paimon_c_generated_include_dir + "${CMAKE_CURRENT_BINARY_DIR}/generated/include") +set(paimon_c_generated_header + "${paimon_c_generated_include_dir}/paimon.h") if(TARGET Paimon::c) message( @@ -46,6 +51,17 @@ endif() if(NOT paimon_cargo_executable) message(FATAL_ERROR "Rust Cargo was not found") endif() +if(DEFINED ENV{CBINDGEN} AND EXISTS "$ENV{CBINDGEN}") + set(paimon_cbindgen_executable "$ENV{CBINDGEN}") +else() + set(paimon_find_appbundle "${CMAKE_FIND_APPBUNDLE}") + set(CMAKE_FIND_APPBUNDLE NEVER) + find_program(paimon_cbindgen_executable NAMES cbindgen) + set(CMAKE_FIND_APPBUNDLE "${paimon_find_appbundle}") +endif() +if(NOT paimon_cbindgen_executable) + message(FATAL_ERROR "cbindgen was not found") +endif() execute_process( COMMAND "${paimon_cargo_executable}" -vV RESULT_VARIABLE paimon_cargo_version_result @@ -88,6 +104,29 @@ add_custom_target( VERBATIM USES_TERMINAL) +file( + GLOB_RECURSE paimon_c_header_sources CONFIGURE_DEPENDS + "${paimon_c_binding_dir}/src/*.rs") +add_custom_command( + OUTPUT "${paimon_c_generated_header}" + COMMAND + "${CMAKE_COMMAND}" -E make_directory + "${paimon_c_generated_include_dir}" + COMMAND + "${paimon_cbindgen_executable}" --quiet + --config "${paimon_c_binding_dir}/cbindgen.toml" + "${paimon_c_binding_dir}" + --output "${paimon_c_generated_header}" + DEPENDS + ${paimon_c_header_sources} + "${paimon_c_binding_dir}/Cargo.toml" + "${paimon_c_binding_dir}/cbindgen.toml" + COMMENT "Generating paimon.h from the Rust C ABI" + VERBATIM) +add_custom_target( + paimon_c_header ALL DEPENDS "${paimon_c_generated_header}") +add_dependencies(paimon_c_header paimon_c_cargo_build) + get_filename_component( PAIMON_C_INSTALL_FILENAME "${paimon_c_library}" NAME) @@ -99,12 +138,12 @@ set_target_properties( PROPERTIES IMPORTED_LOCATION "${paimon_c_library}" IMPORTED_NO_SONAME TRUE - INTERFACE_INCLUDE_DIRECTORIES "${paimon_c_include_dir}") -add_dependencies(Paimon::c paimon_c_cargo_build) + INTERFACE_INCLUDE_DIRECTORIES "${paimon_c_generated_include_dir}") +add_dependencies(Paimon::c paimon_c_cargo_build paimon_c_header) add_library(paimon_cpp INTERFACE) add_library(Paimon::cpp ALIAS paimon_cpp) -add_dependencies(paimon_cpp paimon_c_cargo_build) +add_dependencies(paimon_cpp paimon_c_cargo_build paimon_c_header) set_target_properties(paimon_cpp PROPERTIES EXPORT_NAME cpp) target_compile_features(paimon_cpp INTERFACE cxx_std_17) target_include_directories( @@ -114,7 +153,7 @@ target_include_directories( "$") target_include_directories( - paimon_cpp INTERFACE "$") + paimon_cpp INTERFACE "$") target_link_libraries(paimon_cpp INTERFACE Paimon::c) @@ -160,7 +199,7 @@ install( DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" COMPONENT ${paimon_cpp_install_component}) install( - FILES "${paimon_c_include_dir}/paimon.h" + FILES "${paimon_c_generated_header}" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" COMPONENT ${paimon_cpp_install_component}) install( @@ -229,7 +268,7 @@ add_custom_target( "${CMAKE_COMMAND}" --install "${CMAKE_BINARY_DIR}" --prefix "${CMAKE_CURRENT_BINARY_DIR}" --component ${paimon_cpp_install_component} - DEPENDS paimon_c_cargo_build + DEPENDS paimon_c_cargo_build paimon_c_header COMMENT "Staging the Paimon C++ artifacts in ${CMAKE_CURRENT_BINARY_DIR}" VERBATIM USES_TERMINAL) diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index f2d6f1ac4..b4965ed33 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -104,8 +104,10 @@ backward compatible with binaries built against older symbol versions. External prebuilt paimon-c libraries and parent-provided `Paimon::c` targets are deliberately unsupported. -When changing the C ABI, regenerate the checked header separately with -`cbindgen`; ordinary builds consume the checked-in `bindings/c/include/paimon.h`. +Source builds require `cbindgen`. CMake regenerates `paimon.h` from the Rust C +ABI in `target/cpp-build/generated/include`; the generated header is not stored +in Git. The staged build tree and every installed package still contain it at +`include/paimon.h`. Install the CMake interface target elsewhere when needed: diff --git a/docs/src/c-binding.md b/docs/src/c-binding.md index 0b520b474..71ab2608c 100644 --- a/docs/src/c-binding.md +++ b/docs/src/c-binding.md @@ -24,15 +24,14 @@ catalog and table access, scan planning, predicate push-down, streaming reads, writes and commits, and vector search. Record batches cross the ABI through the [Arrow C Data Interface](https://arrow.apache.org/docs/format/CDataInterface.html). -The C binding is currently built from source. Its generated, C++-compatible -public header is checked in at `bindings/c/include/paimon.h`; releases do not -yet publish pre-built native packages. +The C binding is currently built from source. The repository does not check in +its generated public header. ## Prerequisites - A Rust toolchain supported by this repository - A C11-compatible compiler -- [`cbindgen`](https://github.com/mozilla/cbindgen) only when updating the C ABI +- [`cbindgen`](https://github.com/mozilla/cbindgen) for generating the C header - An Arrow implementation if the application reads or writes record batches Install `cbindgen` when it is not already available: @@ -48,7 +47,7 @@ Run the following commands from the repository root: ```bash cargo build --release -p paimon-c cbindgen --config bindings/c/cbindgen.toml bindings/c \ - --output bindings/c/include/paimon.h + --output target/release/paimon.h ``` The build produces a dynamic library and a static library under @@ -60,11 +59,11 @@ The build produces a dynamic library and a static library under | macOS | `libpaimon_c.dylib` | | Windows | `paimon_c.dll` | -Link the checked-in header and library into an application: +Link the generated header and library into an application: ```bash cc -std=c11 example.c \ - -Ibindings/c/include \ + -Itarget/release \ -Ltarget/release \ -lpaimon_c \ -o example @@ -81,9 +80,9 @@ DYLD_LIBRARY_PATH=target/release ./example /path/to/warehouse ``` The header-only C++17 facade under `bindings/cpp` adds move-only RAII handles -without creating a C++ shared library. The only Paimon binary remains -`libpaimon_c`, and release validation rejects dependencies on `libstdc++`, -`libc++`, `GLIBCXX_*`, or `CXXABI_*` symbols. +without creating a C++ shared library. Its CMake build compiles `libpaimon_c` +and generates `paimon.h` automatically; install and CPack targets include the +generated header. ## Opening and Scanning a Table From 3277ee0789563d8a8c802205f0f508353663b671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 12:55:05 +0800 Subject: [PATCH 16/21] test(cpp): execute installed SDK consumer --- bindings/cpp/README.md | 12 ++++++++++-- bindings/cpp/include/paimon/paimon.hpp | 10 +++++----- bindings/cpp/tests/install_tree_consumer/main.cpp | 3 ++- bindings/cpp/tests/run_install_tree_consumer.cmake | 10 ++++++++++ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index b4965ed33..ffff7d8f1 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -104,6 +104,12 @@ backward compatible with binaries built against older symbol versions. External prebuilt paimon-c libraries and parent-provided `Paimon::c` targets are deliberately unsupported. +Linux builds use the OpenSSL selected by the locked `openssl-sys` dependency +and link it dynamically. OpenSSL 1.0.2 is not supported by the current lock; +an old-glibc build host must provide a parallel OpenSSL 1.1 or newer development +installation. The resulting package requires that exact OpenSSL SONAME at +runtime. + Source builds require `cbindgen`. CMake regenerates `paimon.h` from the Rust C ABI in `target/cpp-build/generated/include`; the generated header is not stored in Git. The staged build tree and every installed package still contain it at @@ -147,8 +153,10 @@ sudo dnf install ./paimon-cpp-devel-*.rpm All formats from one run contain the same `libpaimon_c.so`. Package format does not change its glibc or OpenSSL ABI: build on each binary compatibility baseline -that customers need. The TGZ is the format-neutral fallback and contains the -same `/usr` installation tree. +that customers need. Publish a DEB from a Debian/Ubuntu baseline and an RPM from +an RPM-family baseline so native library-directory and dependency conventions +match the target distribution. The TGZ is the format-neutral fallback and +contains the same `/usr` installation tree. `Scan::plan()` remains a bounded scan. Use `StreamScanOptions` and `ReadBuilder::new_stream_scan` for a stateful continuous scan. Persist diff --git a/bindings/cpp/include/paimon/paimon.hpp b/bindings/cpp/include/paimon/paimon.hpp index de2a4eab8..e225037ee 100644 --- a/bindings/cpp/include/paimon/paimon.hpp +++ b/bindings/cpp/include/paimon/paimon.hpp @@ -32,9 +32,9 @@ #define PAIMON_C_HEADER #endif -// The repository's plain `cbindgen --lang c` output does not add a C++ -// compatibility guard. Force C linkage here; nesting is harmless when a -// packaged paimon.h already supplies its own extern "C" block. +// Keep overridden test/embedding headers under C linkage too. Nesting is +// harmless for the generated paimon.h, which has its own C++ compatibility +// guard. extern "C" { #include PAIMON_C_HEADER } @@ -122,8 +122,8 @@ class Error final { ::paimon_error* error_ = nullptr; }; -// Owns a byte buffer allocated by libpaimon_c. This is used by the version and -// durable prepared-commit APIs and never allocates through a C++ runtime. +// Owns a byte buffer allocated by libpaimon_c. This is used by durable stream +// plan and prepared-commit APIs and never allocates through a C++ runtime. class Bytes final { public: constexpr Bytes() noexcept : bytes_{nullptr, 0} {} diff --git a/bindings/cpp/tests/install_tree_consumer/main.cpp b/bindings/cpp/tests/install_tree_consumer/main.cpp index ba9590ad0..a8f6f5ab2 100644 --- a/bindings/cpp/tests/install_tree_consumer/main.cpp +++ b/bindings/cpp/tests/install_tree_consumer/main.cpp @@ -18,5 +18,6 @@ #include int main() { - return 0; + auto options = paimon::StreamScanOptions::defaults(); + return options ? 0 : 1; } diff --git a/bindings/cpp/tests/run_install_tree_consumer.cmake b/bindings/cpp/tests/run_install_tree_consumer.cmake index 9ddbbe54c..5b7b908a2 100644 --- a/bindings/cpp/tests/run_install_tree_consumer.cmake +++ b/bindings/cpp/tests/run_install_tree_consumer.cmake @@ -60,3 +60,13 @@ if(NOT build_result EQUAL 0) message(FATAL_ERROR "install-tree consumer build failed:\n${build_stdout}\n${build_stderr}") endif() + +execute_process( + COMMAND "${consumer_build}/paimon_install_tree_consumer${CMAKE_EXECUTABLE_SUFFIX}" + RESULT_VARIABLE run_result + OUTPUT_VARIABLE run_stdout + ERROR_VARIABLE run_stderr) +if(NOT run_result EQUAL 0) + message(FATAL_ERROR + "install-tree consumer run failed:\n${run_stdout}\n${run_stderr}") +endif() From 3de5fc39639b9faf5e6f0cc4ae724003b50a191f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 15:03:20 +0800 Subject: [PATCH 17/21] fix(cpp): resolve installed package through symlinks --- bindings/cpp/CMakeLists.txt | 1 + bindings/cpp/cmake/PaimonCppConfig.cmake.in | 14 ++++++++++++++ bindings/cpp/tests/run_install_tree_consumer.cmake | 14 ++++++++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index 5483414c3..5c20ece08 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -190,6 +190,7 @@ if(PAIMON_CPP_BUILD_TESTS) "-DCONSUMER_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/tests/install_tree_consumer" "-DTEST_ROOT=${paimon_cpp_install_test_root}" "-DCXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DINSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}" -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_install_tree_consumer.cmake") endif() diff --git a/bindings/cpp/cmake/PaimonCppConfig.cmake.in b/bindings/cpp/cmake/PaimonCppConfig.cmake.in index 987391140..a9e30c487 100644 --- a/bindings/cpp/cmake/PaimonCppConfig.cmake.in +++ b/bindings/cpp/cmake/PaimonCppConfig.cmake.in @@ -17,6 +17,15 @@ @PACKAGE_INIT@ +# CMake may discover /usr/lib64 packages through the /lib64 -> /usr/lib64 +# symlink. Resolve the config directory before deriving the install prefix so +# imported include and library paths still point at /usr. +get_filename_component( + _paimon_cpp_config_dir "${CMAKE_CURRENT_LIST_DIR}" REALPATH) +get_filename_component( + PACKAGE_PREFIX_DIR "${_paimon_cpp_config_dir}/../../.." ABSOLUTE) +unset(_paimon_cpp_config_dir) + if(TARGET Paimon::cpp) if(NOT TARGET Paimon::c) set(PaimonCpp_FOUND FALSE) @@ -53,4 +62,9 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") include("${CMAKE_CURRENT_LIST_DIR}/PaimonCppTargets.cmake") +set_target_properties( + Paimon::cpp + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES + "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") check_required_components(PaimonCpp) diff --git a/bindings/cpp/tests/run_install_tree_consumer.cmake b/bindings/cpp/tests/run_install_tree_consumer.cmake index 5b7b908a2..a99c0319a 100644 --- a/bindings/cpp/tests/run_install_tree_consumer.cmake +++ b/bindings/cpp/tests/run_install_tree_consumer.cmake @@ -16,7 +16,7 @@ # under the License. foreach(required IN ITEMS MAIN_BUILD_DIR CONSUMER_SOURCE_DIR TEST_ROOT - CXX_COMPILER) + CXX_COMPILER INSTALL_LIBDIR) if(NOT DEFINED ${required}) message(FATAL_ERROR "missing -D${required}=...") endif() @@ -37,11 +37,21 @@ if(NOT install_result EQUAL 0) "install-tree setup failed:\n${install_stdout}\n${install_stderr}") endif() +set(config_libdir_alias "${TEST_ROOT}/libdir-alias") +execute_process( + COMMAND "${CMAKE_COMMAND}" -E create_symlink + "${test_prefix}/${INSTALL_LIBDIR}" "${config_libdir_alias}" + RESULT_VARIABLE alias_result + ERROR_VARIABLE alias_stderr) +if(NOT alias_result EQUAL 0) + message(FATAL_ERROR "package config alias setup failed:\n${alias_stderr}") +endif() + execute_process( COMMAND "${CMAKE_COMMAND}" -S "${CONSUMER_SOURCE_DIR}" -B "${consumer_build}" - "-DCMAKE_PREFIX_PATH=${test_prefix}" + "-DPaimonCpp_DIR=${config_libdir_alias}/cmake/PaimonCpp" "-DCMAKE_CXX_COMPILER=${CXX_COMPILER}" RESULT_VARIABLE configure_result OUTPUT_VARIABLE configure_stdout From 37f16895cbfa08883b19f66fe5be160fc2549e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 16:48:57 +0800 Subject: [PATCH 18/21] fix(stream): merge data evolution columns in delta reads --- crates/paimon/src/table/table_read.rs | 11 +++- .../tests/incremental_batch_scan_test.rs | 59 ++++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/crates/paimon/src/table/table_read.rs b/crates/paimon/src/table/table_read.rs index 99c01f7d9..b49d81068 100644 --- a/crates/paimon/src/table/table_read.rs +++ b/crates/paimon/src/table/table_read.rs @@ -319,8 +319,15 @@ impl<'a> PaimonTableRead<'a> { } } // Delta / Changelog rows are read as-is from planned files (no full-table - // merge against historical base versions). - self.new_data_file_reader()?.read(&data_splits) + // merge against historical base versions). Data-evolution tables still + // need their column files merged with the main data file; reading only + // the latter would silently return NULL for BLOB/vector columns. + let core_options = self.table.schema.core_options(); + if core_options.data_evolution_enabled() { + self.read_with_evolution(&data_splits, &core_options) + } else { + self.new_data_file_reader()?.read(&data_splits) + } } fn to_incremental_diff_arrow( diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index 30217e7be..b37cdf3f5 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -17,9 +17,12 @@ mod common; -use arrow_array::{Array, Int32Array, RecordBatch}; +use arrow_array::{Array, BinaryArray, Int32Array, RecordBatch}; +use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; use futures::TryStreamExt; +use paimon::spec::{BlobType, DataType, IntType, Schema, TableSchema}; use paimon::table::IncrementalScanMode; +use std::sync::Arc; use common::incremental_helpers::{ make_batch, make_batch_with_kinds, make_partitioned_batch, memory_table, partitioned_pk_schema, @@ -82,6 +85,60 @@ async fn read_current_pairs(table: &paimon::table::Table) -> Vec<(i32, i32)> { collect_pairs(&batches) } +#[tokio::test] +async fn delta_data_evolution_reads_blob_column_files() { + let table_path = "memory:/incremental_batch/data_evolution_blob"; + let schema = TableSchema::new( + 0, + &Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("payload", DataType::Blob(BlobType::new())) + .option("bucket", "-1") + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .build() + .unwrap(), + ); + let (file_io, table) = memory_table(table_path, schema); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int32, false), + ArrowField::new("payload", ArrowDataType::Binary, true), + ])), + vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(BinaryArray::from(vec![Some(b"blob-data".as_slice())])), + ], + ) + .unwrap(); + write_batch(&table, &batch).await; + + let builder = table.new_read_builder(); + let plan = builder + .new_incremental_scan(IncrementalScanMode::Delta, 0, 1) + .plan() + .await + .unwrap(); + let batches: Vec = builder + .new_read() + .unwrap() + .to_incremental_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 1); + let payloads = batches[0] + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(payloads.value(0), b"blob-data"); +} + async fn plan_incremental( table: &paimon::table::Table, mode: IncrementalScanMode, From ac9fe2c8337d3d5baae88f7ce2029390fc48d2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 16:48:58 +0800 Subject: [PATCH 19/21] fix(arrow): keep multiset map keys non-null --- crates/paimon/src/arrow/format/avro.rs | 6 +++--- crates/paimon/src/arrow/format/row.rs | 9 ++------- crates/paimon/src/arrow/mod.rs | 20 +++++++++++++++++++- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/crates/paimon/src/arrow/format/avro.rs b/crates/paimon/src/arrow/format/avro.rs index 546d963ee..ff4463a1b 100644 --- a/crates/paimon/src/arrow/format/avro.rs +++ b/crates/paimon/src/arrow/format/avro.rs @@ -394,14 +394,14 @@ fn build_column( DataType::Map(map_type) => build_map_column(records, name, map_type, num_rows)?, // Java encodes MULTISET as a map from the element to an INT count, // sharing the MAP path (`AvroSchemaConverter#extractValueTypeToAvroMap` - // returns IntType). Unlike MAP, `paimon_type_to_arrow` lets the key here - // follow the element's nullability and pins the count non-nullable. + // returns IntType). Arrow map keys are always non-null and the count is + // non-nullable too. DataType::Multiset(multiset_type) => build_map_like_column( records, name, multiset_type.element_type(), &DataType::Int(IntType::new()), - multiset_type.element_type().is_nullable(), + false, false, num_rows, )?, diff --git a/crates/paimon/src/arrow/format/row.rs b/crates/paimon/src/arrow/format/row.rs index fa00773a4..9338a1684 100644 --- a/crates/paimon/src/arrow/format/row.rs +++ b/crates/paimon/src/arrow/format/row.rs @@ -1266,12 +1266,7 @@ impl ColumnBuilder { DataType::Multiset(m) => { let count_type = DataType::Int(IntType::new()); Self::Map { - entries_field: map_entries_field( - m.element_type(), - &count_type, - m.element_type().is_nullable(), - false, - )?, + entries_field: map_entries_field(m.element_type(), &count_type, false, false)?, offsets: vec![0], validities: Vec::with_capacity(capacity), keys: Box::new(ColumnBuilder::new(m.element_type(), capacity)?), @@ -2992,7 +2987,7 @@ mod tests { let bag = test_map_array( vec![0, 2, 2, 3], vec![true, false, true], - true, + false, false, vec![Some("x"), Some("y"), Some("z")], vec![Some(2), Some(1), Some(4)], diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs index 2fe1a6e24..18858d3f7 100644 --- a/crates/paimon/src/arrow/mod.rs +++ b/crates/paimon/src/arrow/mod.rs @@ -104,7 +104,9 @@ pub fn paimon_type_to_arrow(dt: &PaimonDataType) -> crate::Result "entries", ArrowDataType::Struct( vec![ - ArrowField::new("key", element_type, m.element_type().is_nullable()), + // Arrow map keys are always non-null, including the + // element carrier used for a Paimon MULTISET. + ArrowField::new("key", element_type, false), ArrowField::new("value", ArrowDataType::Int32, false), ] .into(), @@ -490,6 +492,22 @@ mod tests { ); } + #[test] + fn test_multiset_arrow_key_is_non_nullable() { + let multiset = PaimonDataType::Multiset(MultisetType::new(PaimonDataType::VarChar( + VarCharType::new(VarCharType::MAX_LENGTH).unwrap(), + ))); + let ArrowDataType::Map(entries, false) = paimon_type_to_arrow(&multiset).unwrap() else { + panic!("expected multiset Arrow Map"); + }; + let ArrowDataType::Struct(fields) = entries.data_type() else { + panic!("expected multiset entries Struct"); + }; + assert!(!fields[0].is_nullable()); + assert_eq!(fields[1].data_type(), &ArrowDataType::Int32); + assert!(!fields[1].is_nullable()); + } + #[test] fn test_timestamp_roundtrip() { // millisecond precision From c966a55f3c492a9006c4e2c7eba415a290936dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 18:09:23 +0800 Subject: [PATCH 20/21] fix(stream): merge data evolution audit rows --- crates/paimon/src/table/table_read.rs | 33 ++++++++++++------- .../tests/incremental_batch_scan_test.rs | 30 ++++++++++++++++- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/crates/paimon/src/table/table_read.rs b/crates/paimon/src/table/table_read.rs index b49d81068..95d50ca4c 100644 --- a/crates/paimon/src/table/table_read.rs +++ b/crates/paimon/src/table/table_read.rs @@ -413,17 +413,28 @@ impl<'a> PaimonTableRead<'a> { )); } - let reader = DataFileReader::new( - self.table.file_io.clone(), - self.table.schema_manager().clone(), - self.table.schema().id(), - self.table.schema.fields().to_vec(), - read_type, - self.data_predicates.clone(), - ) - .with_batch_size(Some(self.table.schema().core_options().read_batch_size()?)) - .with_parquet_read_budget(Some(self.parquet_read_budget()?)); - let raw_stream = reader.read(&data_splits)?; + let core_options = self.table.schema().core_options(); + let raw_stream = if core_options.data_evolution_enabled() { + if has_value_kind || include_sequence { + return Err(crate::Error::Unsupported { + message: "Data-evolution audit reads with changelog or sequence-number fields are not supported" + .to_string(), + }); + } + self.read_with_evolution(&data_splits, &core_options)? + } else { + DataFileReader::new( + self.table.file_io.clone(), + self.table.schema_manager().clone(), + self.table.schema().id(), + self.table.schema.fields().to_vec(), + read_type, + self.data_predicates.clone(), + ) + .with_batch_size(Some(core_options.read_batch_size()?)) + .with_parquet_read_budget(Some(self.parquet_read_budget()?)) + .read(&data_splits)? + }; Ok(Box::pin(async_stream::try_stream! { futures::pin_mut!(raw_stream); diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index b37cdf3f5..29426338f 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -17,7 +17,7 @@ mod common; -use arrow_array::{Array, BinaryArray, Int32Array, RecordBatch}; +use arrow_array::{Array, BinaryArray, Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; use futures::TryStreamExt; use paimon::spec::{BlobType, DataType, IntType, Schema, TableSchema}; @@ -137,6 +137,34 @@ async fn delta_data_evolution_reads_blob_column_files() { .downcast_ref::() .unwrap(); assert_eq!(payloads.value(0), b"blob-data"); + + let audit_batches: Vec = builder + .new_read() + .unwrap() + .to_audit_log_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + audit_batches + .iter() + .map(RecordBatch::num_rows) + .sum::(), + 1 + ); + let rowkinds = audit_batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(rowkinds.value(0), "+I"); + let payloads = audit_batches[0] + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(payloads.value(0), b"blob-data"); } async fn plan_incremental( From 70017c609360a19e9f0388abc65c34a2cde5ed49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=9F=E5=BC=8B?= Date: Wed, 2 Sep 2026 18:09:24 +0800 Subject: [PATCH 21/21] revert(io): preserve filesystem commit compatibility --- bindings/cpp/README.md | 4 - crates/paimon/src/io/file_io.rs | 154 +------------------- crates/paimon/src/table/snapshot_manager.rs | 71 ++++----- docs/src/c-binding.md | 5 - 4 files changed, 31 insertions(+), 203 deletions(-) diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index ffff7d8f1..dd9d122bf 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -42,10 +42,6 @@ normal integrity/access controls. Before `abort_prepared`, fence every commit and abort for the same `(table, commit_user)` across processes. If snapshot history is too old to prove safety, abort fails closed and leaves cleanup to an orphan-file policy. -Filesystem-catalog commits require a backend with atomic publish-if-absent -(conditional rename/copy/write). Unsupported backends fail closed; use REST -commit or an external lock instead of relying on a racy existence check. - Continuous reading is a pull API. `StreamScan::poll` immediately returns data, waiting, or end; it never starts a callback thread and never waits for a future snapshot. A data result owns a `StreamPlan`, which can be read in data or audit diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index ccf0603b9..1257f6328 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -29,7 +29,7 @@ use chrono::{DateTime, Utc}; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use opendal::raw::normalize_root; -use opendal::{ErrorKind as OpendalErrorKind, Operator}; +use opendal::Operator; use snafu::ResultExt; use tokio_util::compat::FuturesAsyncWriteCompatExt; use url::Url; @@ -442,125 +442,6 @@ impl FileIO { Ok(()) } - - /// Publish a fully written temporary file without replacing an existing - /// destination. - /// - /// The operation uses only backend capabilities whose destination - /// precondition is atomic. It never falls back to `exists + write`, which - /// would allow two committers to overwrite the same snapshot. Backends - /// without a conditional rename, copy, or write must use REST commit or an - /// external lock. - pub(crate) async fn publish_if_not_exists( - &self, - src: &str, - dst: &str, - contents: Bytes, - ) -> Result { - let (op_src, relative_src) = self.create(src).await?; - let (op_dst, relative_dst) = self.create(dst).await?; - let src_cache_path = cache_object_path(&op_src, &relative_src); - let dst_cache_path = cache_object_path(&op_dst, &relative_dst); - let capability = op_src.info().capability(); - - if capability.rename_with_if_not_exists { - match op_src - .rename_with(&relative_src, &relative_dst) - .if_not_exists(true) - .await - { - Ok(_) => { - self.invalidate_publish_cache(&src_cache_path, &dst_cache_path) - .await; - return Ok(true); - } - Err(error) if destination_already_exists(&error) => { - let _ = op_src.delete(&relative_src).await; - return Ok(false); - } - Err(error) if error.kind() != OpendalErrorKind::Unsupported => { - let _ = op_src.delete(&relative_src).await; - return Err(error).context(IoUnexpectedSnafu { - message: format!("Failed to publish '{src}' as '{dst}'"), - }); - } - Err(_) => {} - } - } - - if capability.copy_with_if_not_exists { - match op_src - .copy_with(&relative_src, &relative_dst) - .if_not_exists(true) - .await - { - Ok(_) => { - let _ = op_src.delete(&relative_src).await; - self.invalidate_publish_cache(&src_cache_path, &dst_cache_path) - .await; - return Ok(true); - } - Err(error) if destination_already_exists(&error) => { - let _ = op_src.delete(&relative_src).await; - return Ok(false); - } - Err(error) if error.kind() != OpendalErrorKind::Unsupported => { - let _ = op_src.delete(&relative_src).await; - return Err(error).context(IoUnexpectedSnafu { - message: format!("Failed to publish '{src}' as '{dst}'"), - }); - } - Err(_) => {} - } - } - - let write_capability = op_dst.info().capability(); - if write_capability.write_with_if_not_exists { - match op_dst - .write_with(&relative_dst, contents) - .if_not_exists(true) - .await - { - Ok(_) => { - let _ = op_src.delete(&relative_src).await; - self.invalidate_publish_cache(&src_cache_path, &dst_cache_path) - .await; - return Ok(true); - } - Err(error) if destination_already_exists(&error) => { - let _ = op_src.delete(&relative_src).await; - return Ok(false); - } - Err(error) => { - let _ = op_src.delete(&relative_src).await; - return Err(error).context(IoUnexpectedSnafu { - message: format!("Failed to publish '{src}' as '{dst}'"), - }); - } - } - } - - let _ = op_src.delete(&relative_src).await; - Err(Error::Unsupported { - message: format!( - "Storage backend for '{dst}' has no atomic publish-if-absent operation; use REST commit or an external lock" - ), - }) - } - - async fn invalidate_publish_cache(&self, src: &str, dst: &str) { - if let Some(cache) = &self.cache { - cache.invalidate_prefix(src).await; - cache.invalidate_prefix(dst).await; - } - } -} - -fn destination_already_exists(error: &opendal::Error) -> bool { - matches!( - error.kind(), - OpendalErrorKind::ConditionNotMatch | OpendalErrorKind::AlreadyExists - ) } fn status_path(base_path: &str, entry_path: &str) -> String { @@ -1414,39 +1295,6 @@ mod file_action_test { common_test_list_status_paths(&file_io, "file:/tmp/test_list_status_paths_fs/").await; } - #[tokio::test] - async fn test_publish_if_not_exists_fs_has_one_winner_and_cleans_temps() { - let directory = tempdir().unwrap(); - let file_io = setup_fs_file_io(); - let first = local_file_path(&directory.path().join("first.tmp")); - let second = local_file_path(&directory.path().join("second.tmp")); - let target = local_file_path(&directory.path().join("snapshot-1")); - let first_bytes = Bytes::from_static(b"first"); - let second_bytes = Bytes::from_static(b"second"); - file_io - .new_output(&first) - .unwrap() - .write(first_bytes.clone()) - .await - .unwrap(); - file_io - .new_output(&second) - .unwrap() - .write(second_bytes.clone()) - .await - .unwrap(); - - let (first_result, second_result) = tokio::join!( - file_io.publish_if_not_exists(&first, &target, first_bytes), - file_io.publish_if_not_exists(&second, &target, second_bytes) - ); - assert_ne!(first_result.unwrap(), second_result.unwrap()); - let committed = file_io.new_input(&target).unwrap().read().await.unwrap(); - assert!(committed.as_ref() == b"first" || committed.as_ref() == b"second"); - assert!(!file_io.exists(&first).await.unwrap()); - assert!(!file_io.exists(&second).await.unwrap()); - } - #[test] fn test_from_path_detects_local_fs_path() { let dir = tempdir().unwrap(); diff --git a/crates/paimon/src/table/snapshot_manager.rs b/crates/paimon/src/table/snapshot_manager.rs index 1a4185413..a6edfbc51 100644 --- a/crates/paimon/src/table/snapshot_manager.rs +++ b/crates/paimon/src/table/snapshot_manager.rs @@ -248,10 +248,10 @@ impl SnapshotManager { /// Writes the snapshot JSON to the target path. Returns `false` if the /// target already exists (another writer won the race). /// - /// The snapshot is first written under a unique temporary name and then - /// published with a backend-enforced destination precondition. Backends - /// without an atomic publish-if-absent primitive fail closed instead of - /// using a racy `exists + write` fallback. + /// On file systems that support atomic rename, we write to a temp file + /// first then rename. On backends where rename is not supported (e.g. + /// memory, object stores), we fall back to a direct write after an + /// existence check. pub async fn commit_snapshot(&self, snapshot: &Snapshot) -> crate::Result { let target_path = self.snapshot_path(snapshot.id()); @@ -260,19 +260,37 @@ impl SnapshotManager { source: Some(Box::new(e)), })?; + // Try rename-based atomic commit first, fall back to check-and-write. + // + // TODO: opendal's rename uses POSIX semantics which silently overwrites the target. + // The exists() check below narrows the race window but does not eliminate it. + // Java Paimon uses `lock.runWithLock(() -> !fileIO.exists(newPath) && callable.call())` + // for full mutual exclusion. We need an external lock mechanism (like Java's Lock + // interface) for backends without atomic rename-no-replace support. let tmp_path = format!("{}.tmp-{}", target_path, uuid::Uuid::new_v4()); let output = self.file_io.new_output(&tmp_path)?; - let json = bytes::Bytes::from(json); - output.write(json.clone()).await?; - - if !self - .file_io - .publish_if_not_exists(&tmp_path, &target_path, json) - .await? - { + output.write(bytes::Bytes::from(json.clone())).await?; + + // Check before rename to avoid silent overwrite (opendal uses POSIX rename semantics) + if self.file_io.exists(&target_path).await? { + let _ = self.file_io.delete_file(&tmp_path).await; return Ok(false); } + match self.file_io.rename(&tmp_path, &target_path).await { + Ok(()) => {} + Err(_) => { + // Rename not supported (e.g. memory/object store). + // Clean up temp file, then check-and-write. + let _ = self.file_io.delete_file(&tmp_path).await; + if self.file_io.exists(&target_path).await? { + return Ok(false); + } + let output = self.file_io.new_output(&target_path)?; + output.write(bytes::Bytes::from(json)).await?; + } + } + // Update LATEST hint (best-effort) let _ = self.write_latest_hint(snapshot.id()).await; Ok(true) @@ -624,35 +642,6 @@ mod tests { assert!(!result); } - #[tokio::test] - async fn test_concurrent_snapshot_publish_has_one_winner() { - let (_, sm) = setup("memory:/test_commit_race").await; - let first = test_snapshot(1); - let second = Snapshot::builder() - .version(3) - .id(1) - .schema_id(0) - .base_manifest_list("other-base-list".to_string()) - .delta_manifest_list("other-delta-list".to_string()) - .commit_user("other-user".to_string()) - .commit_identifier(1) - .commit_kind(CommitKind::APPEND) - .time_millis(1001) - .build(); - - let (first_result, second_result) = - tokio::join!(sm.commit_snapshot(&first), sm.commit_snapshot(&second)); - let first_won = first_result.unwrap(); - let second_won = second_result.unwrap(); - assert_ne!(first_won, second_won); - - let committed = sm.get_snapshot(1).await.unwrap(); - assert!( - (first_won && committed.commit_user() == "test-user") - || (second_won && committed.commit_user() == "other-user") - ); - } - #[tokio::test] async fn test_commit_updates_latest_hint() { let (_, sm) = setup("memory:/test_commit_hint").await; diff --git a/docs/src/c-binding.md b/docs/src/c-binding.md index 71ab2608c..9d5343b58 100644 --- a/docs/src/c-binding.md +++ b/docs/src/c-binding.md @@ -436,11 +436,6 @@ Serialized plan/commit blobs are trusted checkpoint state and are not cryptographically authenticated, so persist them with appropriate integrity and access controls. -Filesystem-catalog snapshot publication requires an atomic -publish-if-not-exists capability. If the storage backend cannot provide a -conditional rename, copy, or write, commit returns `Unsupported`; use REST -commit or an external lock rather than a racy check-then-write fallback. - ## Error Handling and Resource Ownership Functions that can fail use one of two conventions: