diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 772d15b53..27b3c0f60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,34 @@ jobs: - name: Build run: cargo build --locked --features fulltext,vortex + cpp: + name: cpp (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + steps: + - uses: actions/checkout@v7 + + - name: Install cbindgen + uses: taiki-e/install-action@065d6a08a14e61e89fb0a4c10eecdbdef39c7d8e # v2.85.4 + with: + tool: cbindgen@0.29.4 + + - name: Configure C++ facade + run: > + cmake -S bindings/cpp -B target/cpp-ci + -DPAIMON_CPP_BUILD_EXAMPLES=ON + -DPAIMON_CPP_BUILD_TESTS=ON + + - name: Build C++ facade + run: cmake --build target/cpp-ci --parallel 4 + + - name: Test C++ facade + run: ctest --test-dir target/cpp-ci --output-on-failure + unit: runs-on: ${{ matrix.os }} strategy: diff --git a/.gitignore b/.gitignore index 2ed1d6f45..45d9e2707 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ # under the License. /target +/bindings/cpp/target/ .idea .vscode **/.DS_Store diff --git a/Cargo.lock b/Cargo.lock index 6af8279c3..3570b2938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4731,7 +4731,9 @@ dependencies = [ "futures", "paimon", "paimon-vindex-core", + "serde", "serde_json", + "sha2 0.10.9", "tempfile", "tokio", "url", diff --git a/bindings/c/Cargo.toml b/bindings/c/Cargo.toml index 949044a0b..944eaa00f 100644 --- a/bindings/c/Cargo.toml +++ b/bindings/c/Cargo.toml @@ -43,7 +43,9 @@ futures = "0.3" arrow = { workspace = true } arrow-array = { workspace = true } arrow-schema = { workspace = true } -serde_json = "1.0.120" +serde_json = { version = "1.0.120", features = ["raw_value"] } +serde = { version = "1.0", features = ["derive"] } +sha2 = "0.10" async-trait = "0.1.81" bytes = "1.7.1" diff --git a/bindings/c/cbindgen.toml b/bindings/c/cbindgen.toml new file mode 100644 index 000000000..d214b2137 --- /dev/null +++ b/bindings/c/cbindgen.toml @@ -0,0 +1,24 @@ +language = "C" +header = """ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +""" +include_guard = "PAIMON_C_H" +cpp_compat = true +documentation = true +usize_is_size_t = true +style = "both" +sort_by = "Name" diff --git a/bindings/c/src/catalog.rs b/bindings/c/src/catalog.rs index 9c5c683b7..a21dc078a 100644 --- a/bindings/c/src/catalog.rs +++ b/bindings/c/src/catalog.rs @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. -use std::ffi::c_void; +use std::ffi::{c_char, c_void}; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::Arc; use paimon::catalog::Identifier; +use paimon::spec::Schema; use paimon::{Catalog, CatalogFactory, Options}; use crate::error::{check_non_null, paimon_error, validate_cstr}; @@ -26,6 +28,38 @@ use crate::result::{paimon_result_catalog_new, paimon_result_get_table}; use crate::runtime; use crate::types::{paimon_catalog, paimon_option, paimon_table}; +fn catalog_panic_error(operation: &str) -> *mut paimon_error { + paimon_error::new( + crate::error::PaimonErrorCode::Unexpected, + format!("Rust panic while executing {operation}"), + ) +} + +fn validate_creation_schema_json(schema_json: &str) -> Result { + let parsed = serde_json::from_str::(schema_json).map_err(|error| { + paimon_error::new( + crate::error::PaimonErrorCode::InvalidInput, + format!("Failed to parse creation schema JSON: {error}"), + ) + })?; + + let mut builder = Schema::builder(); + for field in parsed.fields() { + builder = builder.column_with_description( + field.name(), + field.data_type().clone(), + field.description().map(str::to_string), + ); + } + builder + .partition_keys(parsed.partition_keys().iter().cloned()) + .primary_key(parsed.primary_keys().iter().cloned()) + .options(parsed.options().clone()) + .comment(parsed.comment().map(str::to_string)) + .build() + .map_err(paimon_error::from_paimon) +} + /// Create a catalog using CatalogFactory with the given options. /// /// # Safety @@ -136,3 +170,85 @@ pub unsafe extern "C" fn paimon_catalog_get_table( }, } } + +/// Create a table from a logical Paimon `Schema` JSON document. +/// +/// The input is normalized and validated through `SchemaBuilder` before it is +/// sent to the catalog. Field IDs in the JSON are therefore treated as input +/// ordering hints and reassigned canonically from zero. +/// +/// # Safety +/// `catalog` and `identifier` must be valid Paimon handles. `schema_json` must +/// point to a valid null-terminated UTF-8 string. +#[no_mangle] +pub unsafe extern "C" fn paimon_catalog_create_table_from_schema_json( + catalog: *const paimon_catalog, + identifier: *const crate::types::paimon_identifier, + schema_json: *const c_char, + ignore_if_exists: bool, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(catalog, "catalog") { + return error; + } + if let Err(error) = check_non_null(identifier, "identifier") { + return error; + } + let schema_json = match validate_cstr(schema_json, "schema_json") { + Ok(value) => value, + Err(error) => return error, + }; + let schema = match validate_creation_schema_json(&schema_json) { + Ok(value) => value, + Err(error) => return error, + }; + let catalog_ref = &*((*catalog).inner as *const Arc); + let identifier_ref = &*((*identifier).inner as *const Identifier); + match runtime().block_on(catalog_ref.create_table(identifier_ref, schema, ignore_if_exists)) + { + Ok(()) => std::ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| catalog_panic_error("paimon_catalog_create_table_from_schema_json")) +} + +/// Drop a table from the catalog. +/// +/// # Safety +/// `catalog` and `identifier` must be valid Paimon handles, or null (returns an +/// error). +#[no_mangle] +pub unsafe extern "C" fn paimon_catalog_drop_table( + catalog: *const paimon_catalog, + identifier: *const crate::types::paimon_identifier, + ignore_if_not_exists: bool, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(catalog, "catalog") { + return error; + } + if let Err(error) = check_non_null(identifier, "identifier") { + return error; + } + let catalog_ref = &*((*catalog).inner as *const Arc); + let identifier_ref = &*((*identifier).inner as *const Identifier); + match runtime().block_on(catalog_ref.drop_table(identifier_ref, ignore_if_not_exists)) { + Ok(()) => std::ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| catalog_panic_error("paimon_catalog_drop_table")) +} + +const _: unsafe extern "C" fn( + *const paimon_catalog, + *const crate::types::paimon_identifier, + *const c_char, + bool, +) -> *mut paimon_error = paimon_catalog_create_table_from_schema_json; +const _: unsafe extern "C" fn( + *const paimon_catalog, + *const crate::types::paimon_identifier, + bool, +) -> *mut paimon_error = paimon_catalog_drop_table; diff --git a/bindings/c/src/error.rs b/bindings/c/src/error.rs index 7b0a88fc3..1613c9e58 100644 --- a/bindings/c/src/error.rs +++ b/bindings/c/src/error.rs @@ -19,15 +19,25 @@ use std::ffi::{c_char, CStr}; use crate::types::paimon_bytes; +pub const PAIMON_ERROR_UNEXPECTED: i32 = 0; +pub const PAIMON_ERROR_UNSUPPORTED: i32 = 1; +pub const PAIMON_ERROR_NOT_FOUND: i32 = 2; +pub const PAIMON_ERROR_ALREADY_EXISTS: i32 = 3; +pub const PAIMON_ERROR_INVALID_INPUT: i32 = 4; +pub const PAIMON_ERROR_IO: i32 = 5; +pub const PAIMON_ERROR_OUT_OF_RANGE: i32 = 6; + /// Error codes for paimon C API. #[repr(i32)] pub enum PaimonErrorCode { - Unexpected = 0, - Unsupported = 1, - NotFound = 2, - AlreadyExists = 3, - InvalidInput = 4, - IoError = 5, + Unexpected = PAIMON_ERROR_UNEXPECTED, + Unsupported = PAIMON_ERROR_UNSUPPORTED, + NotFound = PAIMON_ERROR_NOT_FOUND, + AlreadyExists = PAIMON_ERROR_ALREADY_EXISTS, + InvalidInput = PAIMON_ERROR_INVALID_INPUT, + IoError = PAIMON_ERROR_IO, + /// A requested streaming checkpoint or snapshot is no longer readable. + OutOfRange = PAIMON_ERROR_OUT_OF_RANGE, } /// C-compatible error type. @@ -53,9 +63,18 @@ impl paimon_error { paimon::Error::TableNotExist { .. } | paimon::Error::DatabaseNotExist { .. } | paimon::Error::ColumnNotExist { .. } => PaimonErrorCode::NotFound, + paimon::Error::SnapshotNotExist { .. } => PaimonErrorCode::OutOfRange, paimon::Error::TableAlreadyExist { .. } | paimon::Error::DatabaseAlreadyExist { .. } | paimon::Error::ColumnAlreadyExist { .. } => PaimonErrorCode::AlreadyExists, + paimon::Error::DataInvalid { message, .. } + if message.contains("snapshot") + && (message.contains("expired") + || message.contains("out of range") + || message.contains("too large")) => + { + PaimonErrorCode::OutOfRange + } paimon::Error::ConfigInvalid { .. } | paimon::Error::DataTypeInvalid { .. } | paimon::Error::DataInvalid { .. } diff --git a/bindings/c/src/lib.rs b/bindings/c/src/lib.rs index 0a5710ccc..46908cd4b 100644 --- a/bindings/c/src/lib.rs +++ b/bindings/c/src/lib.rs @@ -25,6 +25,7 @@ mod error; mod file_io; mod identifier; mod result; +mod stream; mod table; #[cfg(test)] mod tests; diff --git a/bindings/c/src/result.rs b/bindings/c/src/result.rs index 94317572b..72e2cfe8c 100644 --- a/bindings/c/src/result.rs +++ b/bindings/c/src/result.rs @@ -146,6 +146,18 @@ pub struct paimon_result_prepare_commit { pub error: *mut paimon_error, } +#[repr(C)] +pub struct paimon_result_prepared_commit { + pub prepared: *mut paimon_prepared_commit, + pub error: *mut paimon_error, +} + +#[repr(C)] +pub struct paimon_result_bytes { + pub bytes: paimon_bytes, + pub error: *mut paimon_error, +} + #[repr(C)] pub struct paimon_result_postpone_fixed_bucket_write_builder { pub write_builder: *mut paimon_postpone_fixed_bucket_write_builder, diff --git a/bindings/c/src/stream.rs b/bindings/c/src/stream.rs new file mode 100644 index 000000000..6d0ab98dd --- /dev/null +++ b/bindings/c/src/stream.rs @@ -0,0 +1,1168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Stateful continuous-read C ABI. +//! +//! This is deliberately a pull API. It does not create a C++ callback thread, +//! so callers retain control of cancellation, backpressure and checkpoint +//! barriers. All handles in this module are single-thread-confined. + +use std::ffi::c_void; +use std::mem::size_of; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::ptr; + +use paimon::table::{ + ArrowRecordBatchStream, IncrementalPlan, IncrementalScanMode, IncrementalSplit, Plan, + StreamPlan, StreamScan, StreamScanFollowUpMode, StreamScanPoll, StreamScanStartupMode, + TableRead, +}; +use paimon::DataSplit; +use serde::{Deserialize, Serialize}; + +use crate::error::{check_non_null, paimon_error, PaimonErrorCode}; +use crate::result::{paimon_result_bytes, paimon_result_record_batch_reader}; +use crate::runtime; +use crate::types::{ + paimon_bytes, paimon_read_builder, paimon_record_batch_reader, paimon_table_read, + read_builder_fingerprint, ReadBuilderState, TableReadState, +}; + +pub const PAIMON_STREAM_STARTUP_LATEST_FULL: i32 = 0; +pub const PAIMON_STREAM_STARTUP_LATEST: i32 = 1; +pub const PAIMON_STREAM_STARTUP_FROM_SNAPSHOT: i32 = 2; +pub const PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL: i32 = 3; + +pub const PAIMON_STREAM_FOLLOW_UP_AUTO: i32 = 0; +pub const PAIMON_STREAM_FOLLOW_UP_DELTA: i32 = 1; +pub const PAIMON_STREAM_FOLLOW_UP_CHANGELOG: i32 = 2; + +pub const PAIMON_STREAM_POLL_DATA: i32 = 0; +pub const PAIMON_STREAM_POLL_WAITING: i32 = 1; +pub const PAIMON_STREAM_POLL_END: i32 = 2; + +pub const PAIMON_STREAM_READ_DATA: i32 = 0; +pub const PAIMON_STREAM_READ_AUDIT_LOG: i32 = 1; + +/// Extensible options for a continuous scan. +/// +/// Initialize this with `paimon_stream_scan_options_init`; future versions may +/// consume fields from `reserved` while preserving this prefix. +#[repr(C)] +pub struct paimon_stream_scan_options { + pub struct_size: u32, + pub startup_mode: i32, + pub follow_up_mode: i32, + pub snapshot_id: i64, + pub reserved: [u64; 4], +} + +#[repr(C)] +pub struct paimon_stream_scan { + pub inner: *mut c_void, +} + +#[repr(C)] +pub struct paimon_stream_plan { + pub inner: *mut c_void, +} + +#[repr(C)] +pub struct paimon_result_stream_scan { + pub scan: *mut paimon_stream_scan, + pub error: *mut paimon_error, +} + +#[repr(C)] +pub struct paimon_result_stream_poll { + pub status: i32, + pub plan: *mut paimon_stream_plan, + pub snapshot_id: i64, + pub next_snapshot_id: i64, + pub watermark: i64, + pub has_watermark: u8, + pub reserved: [u8; 7], + pub error: *mut paimon_error, +} + +const STREAM_PLAN_FORMAT: &str = "paimon-rust-stream-plan"; +// Version 3 also binds restored work to the table branch. Earlier versions are +// rejected because location + schema id alone cannot distinguish two branches. +const STREAM_PLAN_VERSION: u32 = 3; +const MAX_STREAM_PLAN_BYTES: usize = 64 * 1024 * 1024; +const MAX_STREAM_PLAN_SPLITS: usize = 100_000; +const MAX_STREAM_SPLIT_BYTES: usize = 16 * 1024 * 1024; +const MAX_STREAM_SPLIT_TOTAL_BYTES: usize = 64 * 1024 * 1024; +const MAX_STREAM_IDENTITY_BYTES: usize = 1024 * 1024; + +struct StreamScanState { + scan: StreamScan, + table_location: String, + table_branch: String, + schema_id: i64, + read_fingerprint: String, +} + +struct StreamPlanState { + plan: StreamPlan, + table_location: String, + table_branch: String, + schema_id: i64, + read_fingerprint: String, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct StreamPlanEnvelope { + format: String, + version: u32, + table_location: String, + #[serde(default)] + table_branch: String, + schema_id: i64, + read_fingerprint: String, + kind: i32, + incremental_mode: i32, + snapshot_id: i64, + next_snapshot_id: i64, + watermark: Option, + #[serde(with = "bounded_splits")] + splits: Vec>, +} + +mod bounded_splits { + use std::fmt; + + use serde::de::{DeserializeSeed, Error, IgnoredAny, SeqAccess, Visitor}; + use serde::{Deserializer, Serialize, Serializer}; + + use super::{MAX_STREAM_PLAN_SPLITS, MAX_STREAM_SPLIT_BYTES, MAX_STREAM_SPLIT_TOTAL_BYTES}; + + pub fn serialize(splits: &[Vec], serializer: S) -> Result + where + S: Serializer, + { + splits.serialize(serializer) + } + + struct SplitBytesSeed; + + impl<'de> DeserializeSeed<'de> for SplitBytesSeed { + type Value = Vec; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_seq(SplitBytesVisitor) + } + } + + struct SplitBytesVisitor; + + impl<'de> Visitor<'de> for SplitBytesVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a bounded stream split byte array") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut bytes = Vec::with_capacity( + sequence + .size_hint() + .unwrap_or_default() + .min(MAX_STREAM_SPLIT_BYTES), + ); + while bytes.len() < MAX_STREAM_SPLIT_BYTES { + let Some(value) = sequence.next_element::()? else { + return Ok(bytes); + }; + bytes.push(value); + } + if sequence.next_element::()?.is_some() { + return Err(A::Error::custom(format!( + "stream plan split exceeds {MAX_STREAM_SPLIT_BYTES} bytes" + ))); + } + Ok(bytes) + } + } + + struct SplitsVisitor; + + impl<'de> Visitor<'de> for SplitsVisitor { + type Value = Vec>; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a bounded list of stream split byte arrays") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut splits = Vec::with_capacity( + sequence + .size_hint() + .unwrap_or_default() + .min(MAX_STREAM_PLAN_SPLITS), + ); + let mut total_bytes = 0usize; + while splits.len() < MAX_STREAM_PLAN_SPLITS { + let Some(split) = sequence.next_element_seed(SplitBytesSeed)? else { + return Ok(splits); + }; + total_bytes = total_bytes + .checked_add(split.len()) + .ok_or_else(|| A::Error::custom("stream plan split byte count overflows"))?; + if total_bytes > MAX_STREAM_SPLIT_TOTAL_BYTES { + return Err(A::Error::custom(format!( + "stream plan split bytes exceed {MAX_STREAM_SPLIT_TOTAL_BYTES}" + ))); + } + splits.push(split); + } + if sequence.next_element::()?.is_some() { + return Err(A::Error::custom(format!( + "stream plan contains more than {MAX_STREAM_PLAN_SPLITS} splits" + ))); + } + Ok(splits) + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + deserializer.deserialize_seq(SplitsVisitor) + } +} + +fn validate_stream_plan_envelope(envelope: &StreamPlanEnvelope) -> Result<(), *mut paimon_error> { + if envelope.table_location.is_empty() + || envelope.table_location.len() > MAX_STREAM_IDENTITY_BYTES + || envelope.table_branch.is_empty() + || envelope.table_branch.len() > MAX_STREAM_IDENTITY_BYTES + || envelope.schema_id < 0 + || envelope.read_fingerprint.is_empty() + || envelope.read_fingerprint.len() > MAX_STREAM_IDENTITY_BYTES + { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan contains an invalid table or read identity".to_string(), + )); + } + if envelope.splits.len() > MAX_STREAM_PLAN_SPLITS { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!( + "stream plan contains {} splits; maximum is {}", + envelope.splits.len(), + MAX_STREAM_PLAN_SPLITS + ), + )); + } + if envelope + .splits + .iter() + .any(|split| split.len() > MAX_STREAM_SPLIT_BYTES) + { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("stream plan split exceeds {MAX_STREAM_SPLIT_BYTES} bytes"), + )); + } + let total_split_bytes = envelope + .splits + .iter() + .try_fold(0usize, |total, split| total.checked_add(split.len())); + if total_split_bytes.is_none_or(|total| total > MAX_STREAM_SPLIT_TOTAL_BYTES) { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("stream plan split bytes exceed {MAX_STREAM_SPLIT_TOTAL_BYTES}"), + )); + } + let Some(expected_next_snapshot_id) = envelope.snapshot_id.checked_add(1) else { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan snapshot cursor overflows".to_string(), + )); + }; + let valid_next_snapshot_id = envelope.next_snapshot_id == expected_next_snapshot_id + || (envelope.kind == 0 && envelope.next_snapshot_id == envelope.snapshot_id); + if envelope.snapshot_id < 1 || !valid_next_snapshot_id { + return Err(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan contains an invalid snapshot cursor".to_string(), + )); + } + Ok(()) +} + +fn validate_stream_plan_recovery_paths(state: &StreamPlanState) -> Result<(), *mut paimon_error> { + match &state.plan { + StreamPlan::Full { plan, .. } => { + for split in plan.splits() { + split + .validate_restored_containment(&state.table_location) + .map_err(paimon_error::from_paimon)?; + } + } + StreamPlan::Incremental { plan, .. } => { + for split in plan.splits() { + let IncrementalSplit::Data(split) = split else { + return Err(paimon_error::new( + PaimonErrorCode::Unsupported, + "DiffPair plans are not valid continuous stream plans".to_string(), + )); + }; + split + .validate_restored_containment(&state.table_location) + .map_err(paimon_error::from_paimon)?; + } + } + } + Ok(()) +} + +fn panic_error(operation: &str) -> *mut paimon_error { + paimon_error::new( + PaimonErrorCode::Unexpected, + format!("Rust panic while executing {operation}"), + ) +} + +fn invalid_mode(name: &str, value: i32) -> *mut paimon_error { + paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("invalid {name} value {value}"), + ) +} + +fn empty_bytes() -> paimon_bytes { + paimon_bytes { + data: ptr::null_mut(), + len: 0, + } +} + +fn empty_poll(status: i32, scan: Option<&StreamScan>) -> paimon_result_stream_poll { + paimon_result_stream_poll { + status, + plan: ptr::null_mut(), + snapshot_id: -1, + next_snapshot_id: scan.and_then(StreamScan::checkpoint).unwrap_or(-1), + watermark: scan.and_then(StreamScan::watermark).unwrap_or(0), + has_watermark: u8::from(scan.and_then(StreamScan::watermark).is_some()), + reserved: [0; 7], + error: ptr::null_mut(), + } +} + +fn error_poll(error: *mut paimon_error) -> paimon_result_stream_poll { + let mut result = empty_poll(PAIMON_STREAM_POLL_END, None); + result.error = error; + result +} + +fn configure_builder<'a>( + state: &'a ReadBuilderState, +) -> Result, *mut paimon_error> { + let mut builder = state.table.new_read_builder(); + builder.with_case_sensitive(state.case_sensitive); + if let Some(columns) = &state.projected_columns { + let columns: Vec<&str> = columns.iter().map(String::as_str).collect(); + builder + .with_projection(&columns) + .map_err(paimon_error::from_paimon)?; + } + if let Some(filter) = &state.filter { + builder.with_filter(filter.clone()); + } + Ok(builder) +} + +/// Fill stream options with forward-compatible defaults (`latest-full`, +/// automatic delta/changelog selection). +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_options_init( + options: *mut paimon_stream_scan_options, +) -> *mut paimon_error { + if let Err(error) = check_non_null(options, "options") { + return error; + } + ptr::write( + options, + paimon_stream_scan_options { + struct_size: size_of::() as u32, + startup_mode: PAIMON_STREAM_STARTUP_LATEST_FULL, + follow_up_mode: PAIMON_STREAM_FOLLOW_UP_AUTO, + snapshot_id: -1, + reserved: [0; 4], + }, + ); + ptr::null_mut() +} + +/// Create an owned stream scan from a read builder. +/// +/// The returned scan clones all required Rust state and remains valid after +/// the read builder and table handles are freed. A scan handle is +/// single-thread-confined: callers must serialize poll/checkpoint/restore/free. +#[no_mangle] +pub unsafe extern "C" fn paimon_read_builder_new_stream_scan( + read_builder: *const paimon_read_builder, + options: *const paimon_stream_scan_options, +) -> paimon_result_stream_scan { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(read_builder, "read_builder") { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error, + }; + } + if let Err(error) = check_non_null(options, "options") { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error, + }; + } + if (*options).struct_size < size_of::() as u32 { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::InvalidInput, + format!( + "stream options struct_size {} is smaller than required {}", + (*options).struct_size, + size_of::() + ), + ), + }; + } + if (*options).reserved.iter().any(|value| *value != 0) { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::Unsupported, + "stream options reserved fields must be zero for ABI version 1".to_string(), + ), + }; + } + let startup = match (*options).startup_mode { + PAIMON_STREAM_STARTUP_LATEST_FULL => StreamScanStartupMode::LatestFull, + PAIMON_STREAM_STARTUP_LATEST => StreamScanStartupMode::Latest, + PAIMON_STREAM_STARTUP_FROM_SNAPSHOT => { + StreamScanStartupMode::FromSnapshot((*options).snapshot_id) + } + PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL => { + StreamScanStartupMode::FromSnapshotFull((*options).snapshot_id) + } + value => { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error: invalid_mode("stream startup mode", value), + } + } + }; + let follow_up = match (*options).follow_up_mode { + PAIMON_STREAM_FOLLOW_UP_AUTO => StreamScanFollowUpMode::Auto, + PAIMON_STREAM_FOLLOW_UP_DELTA => StreamScanFollowUpMode::Delta, + PAIMON_STREAM_FOLLOW_UP_CHANGELOG => StreamScanFollowUpMode::Changelog, + value => { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error: invalid_mode("stream follow-up mode", value), + } + } + }; + let state = &*((*read_builder).inner as *const ReadBuilderState); + let table_location = state.table.location().to_string(); + let table_branch = state.table.branch().to_string(); + let schema_id = state.table.schema().id(); + let read_fingerprint = read_builder_fingerprint(state); + let builder = match configure_builder(state) { + Ok(builder) => builder, + Err(error) => { + return paimon_result_stream_scan { + scan: ptr::null_mut(), + error, + } + } + }; + match runtime().block_on(builder.new_stream_scan(startup, follow_up)) { + Ok(scan) => { + let inner = Box::into_raw(Box::new(StreamScanState { + scan, + table_location, + table_branch, + schema_id, + read_fingerprint, + })) as *mut c_void; + paimon_result_stream_scan { + scan: Box::into_raw(Box::new(paimon_stream_scan { inner })), + error: ptr::null_mut(), + } + } + Err(error) => paimon_result_stream_scan { + scan: ptr::null_mut(), + error: paimon_error::from_paimon(error), + }, + } + })); + outcome.unwrap_or_else(|_| paimon_result_stream_scan { + scan: ptr::null_mut(), + error: panic_error("paimon_read_builder_new_stream_scan"), + }) +} + +/// Poll once for a snapshot plan. This call never waits for a future snapshot. +/// Calls using the same scan handle must not overlap on different threads. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_poll( + scan: *mut paimon_stream_scan, +) -> paimon_result_stream_poll { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(scan, "scan") { + return error_poll(error); + } + let state = &mut *((*scan).inner as *mut StreamScanState); + match runtime().block_on(state.scan.poll_next()) { + Ok(StreamScanPoll::Data(plan)) => { + let snapshot_id = plan.snapshot_id(); + let next_snapshot_id = plan.next_snapshot_id(); + let watermark = plan.watermark(); + let inner = Box::into_raw(Box::new(StreamPlanState { + plan, + table_location: state.table_location.clone(), + table_branch: state.table_branch.clone(), + schema_id: state.schema_id, + read_fingerprint: state.read_fingerprint.clone(), + })) as *mut c_void; + paimon_result_stream_poll { + status: PAIMON_STREAM_POLL_DATA, + plan: Box::into_raw(Box::new(paimon_stream_plan { inner })), + snapshot_id, + next_snapshot_id, + watermark: watermark.unwrap_or(0), + has_watermark: u8::from(watermark.is_some()), + reserved: [0; 7], + error: ptr::null_mut(), + } + } + Ok(StreamScanPoll::Waiting) => { + empty_poll(PAIMON_STREAM_POLL_WAITING, Some(&state.scan)) + } + Ok(StreamScanPoll::End) => empty_poll(PAIMON_STREAM_POLL_END, Some(&state.scan)), + Err(error) => error_poll(paimon_error::from_paimon(error)), + } + })); + outcome.unwrap_or_else(|_| error_poll(panic_error("paimon_stream_scan_poll"))) +} + +/// Return the next-snapshot cursor, or -1 before a startup position exists. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_checkpoint(scan: *const paimon_stream_scan) -> i64 { + if scan.is_null() || (*scan).inner.is_null() { + return -1; + } + let state = &*((*scan).inner as *const StreamScanState); + state.scan.checkpoint().unwrap_or(-1) +} + +/// Restore a next-snapshot cursor. Pass -1 to reapply the configured startup +/// mode; non-negative values must name a valid Paimon snapshot position. +/// This call must not overlap poll/checkpoint/free on the same handle. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_restore( + scan: *mut paimon_stream_scan, + next_snapshot_id: i64, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(scan, "scan") { + return error; + } + if next_snapshot_id != -1 && next_snapshot_id < 1 { + return paimon_error::new( + PaimonErrorCode::InvalidInput, + "next_snapshot_id must be -1 or a positive snapshot id".to_string(), + ); + } + let state = &mut *((*scan).inner as *mut StreamScanState); + match state + .scan + .restore((next_snapshot_id >= 0).then_some(next_snapshot_id)) + { + Ok(()) => ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| panic_error("paimon_stream_scan_restore")) +} + +/// Free a stream scan. It is valid to pass null. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_scan_free(scan: *mut paimon_stream_scan) { + if !scan.is_null() { + let wrapper = Box::from_raw(scan); + if !wrapper.inner.is_null() { + drop(Box::from_raw(wrapper.inner as *mut StreamScanState)); + } + } +} + +/// Return whether a stream plan is an initial full-snapshot plan. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_is_full(plan: *const paimon_stream_plan) -> u8 { + if plan.is_null() || (*plan).inner.is_null() { + return 0; + } + let state = &*((*plan).inner as *const StreamPlanState); + u8::from(state.plan.full_plan().is_some()) +} + +/// Return the number of work splits in a stream plan. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_num_splits(plan: *const paimon_stream_plan) -> usize { + if plan.is_null() || (*plan).inner.is_null() { + return 0; + } + let state = &*((*plan).inner as *const StreamPlanState); + match &state.plan { + StreamPlan::Full { plan, .. } => plan.splits().len(), + StreamPlan::Incremental { plan, .. } => plan.splits().len(), + } +} + +/// Serialize planned-but-not-yet-consumed work for an external checkpoint. +/// +/// The current format checkpoints at plan boundaries. If rows from a plan have already +/// been exposed, callers must either replay the plan after recovery or persist +/// their own logical rows-to-skip position alongside this buffer. +/// Plans containing external data-file paths are rejected because version 1 +/// recovery cannot revalidate those paths against a trusted manifest. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_serialize( + plan: *const paimon_stream_plan, +) -> paimon_result_bytes { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(plan, "plan") { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + let state = &*((*plan).inner as *const StreamPlanState); + if let Err(error) = validate_stream_plan_recovery_paths(state) { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + let envelope = match &state.plan { + StreamPlan::Full { + snapshot_id, + watermark, + next_snapshot_id, + plan, + } => { + let splits = match plan + .splits() + .iter() + .map(DataSplit::serialize_split_v1) + .collect::>>() + { + Ok(splits) => splits, + Err(error) => { + return paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::from_paimon(error), + } + } + }; + StreamPlanEnvelope { + format: STREAM_PLAN_FORMAT.to_string(), + version: STREAM_PLAN_VERSION, + table_location: state.table_location.clone(), + table_branch: state.table_branch.clone(), + schema_id: state.schema_id, + read_fingerprint: state.read_fingerprint.clone(), + kind: 0, + incremental_mode: -1, + snapshot_id: *snapshot_id, + next_snapshot_id: *next_snapshot_id, + watermark: *watermark, + splits, + } + } + StreamPlan::Incremental { + snapshot_id, + watermark, + next_snapshot_id, + plan, + } => { + let mut splits = Vec::with_capacity(plan.splits().len()); + for split in plan.splits() { + let IncrementalSplit::Data(split) = split else { + return paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::Unsupported, + "DiffPair plans are not valid continuous stream plans".to_string(), + ), + }; + }; + match split.serialize_split_v1() { + Ok(bytes) => splits.push(bytes), + Err(error) => { + return paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::from_paimon(error), + } + } + } + } + let incremental_mode = match plan.mode() { + IncrementalScanMode::Delta => 0, + IncrementalScanMode::Changelog => 1, + IncrementalScanMode::Auto | IncrementalScanMode::Diff => { + return paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::Unsupported, + "unresolved Auto and Diff plans are not valid continuous stream plans" + .to_string(), + ), + }; + } + }; + StreamPlanEnvelope { + format: STREAM_PLAN_FORMAT.to_string(), + version: STREAM_PLAN_VERSION, + table_location: state.table_location.clone(), + table_branch: state.table_branch.clone(), + schema_id: state.schema_id, + read_fingerprint: state.read_fingerprint.clone(), + kind: 1, + incremental_mode, + snapshot_id: *snapshot_id, + next_snapshot_id: *next_snapshot_id, + watermark: *watermark, + splits, + } + } + }; + if let Err(error) = validate_stream_plan_envelope(&envelope) { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + match serde_json::to_vec(&envelope) { + Ok(bytes) if bytes.len() <= MAX_STREAM_PLAN_BYTES => paimon_result_bytes { + bytes: paimon_bytes::new(bytes), + error: ptr::null_mut(), + }, + Ok(bytes) => paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::InvalidInput, + format!( + "serialized stream plan is {} bytes; maximum is {}", + bytes.len(), + MAX_STREAM_PLAN_BYTES + ), + ), + }, + Err(error) => paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::Unexpected, + format!("failed to serialize stream plan: {error}"), + ), + }, + } + })); + outcome.unwrap_or_else(|_| paimon_result_bytes { + bytes: empty_bytes(), + error: panic_error("paimon_stream_plan_serialize"), + }) +} + +/// Restore a stream plan serialized by `paimon_stream_plan_serialize`. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_deserialize( + data: *const u8, + len: usize, +) -> paimon_result_stream_poll { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if data.is_null() || len == 0 { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan buffer must not be null or empty".to_string(), + )); + } + if len > MAX_STREAM_PLAN_BYTES { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("stream plan buffer exceeds {MAX_STREAM_PLAN_BYTES} bytes"), + )); + } + let envelope: StreamPlanEnvelope = + match serde_json::from_slice(std::slice::from_raw_parts(data, len)) { + Ok(envelope) => envelope, + Err(error) => { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + format!("invalid stream plan buffer: {error}"), + )) + } + }; + if envelope.format != STREAM_PLAN_FORMAT || envelope.version != STREAM_PLAN_VERSION { + return error_poll(paimon_error::new( + PaimonErrorCode::Unsupported, + format!( + "unsupported stream plan format '{}' version {}", + envelope.format, envelope.version + ), + )); + } + if let Err(error) = validate_stream_plan_envelope(&envelope) { + return error_poll(error); + } + let splits = match envelope + .splits + .iter() + .map(|split| DataSplit::deserialize_split_v1(split)) + .collect::>>() + { + Ok(splits) => splits, + Err(error) => return error_poll(paimon_error::from_paimon(error)), + }; + if splits + .iter() + .any(|split| split.snapshot_id() != envelope.snapshot_id) + { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan split snapshot does not match its envelope".to_string(), + )); + } + for split in &splits { + if let Err(error) = split.validate_restored_containment(&envelope.table_location) { + return error_poll(paimon_error::from_paimon(error)); + } + } + let plan = match (envelope.kind, envelope.incremental_mode) { + (0, -1) => StreamPlan::Full { + snapshot_id: envelope.snapshot_id, + watermark: envelope.watermark, + next_snapshot_id: envelope.next_snapshot_id, + plan: Plan::new(splits), + }, + (1, mode @ (0 | 1)) => { + let mode = if mode == 0 { + IncrementalScanMode::Delta + } else { + IncrementalScanMode::Changelog + }; + let splits = splits.into_iter().map(IncrementalSplit::Data).collect(); + let plan = match IncrementalPlan::try_new(mode, splits) { + Ok(plan) => plan, + Err(error) => return error_poll(paimon_error::from_paimon(error)), + }; + StreamPlan::Incremental { + snapshot_id: envelope.snapshot_id, + watermark: envelope.watermark, + next_snapshot_id: envelope.next_snapshot_id, + plan, + } + } + _ => { + return error_poll(paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan contains an invalid kind or mode".to_string(), + )) + } + }; + let snapshot_id = plan.snapshot_id(); + let next_snapshot_id = plan.next_snapshot_id(); + let watermark = plan.watermark(); + let inner = Box::into_raw(Box::new(StreamPlanState { + plan, + table_location: envelope.table_location, + table_branch: envelope.table_branch, + schema_id: envelope.schema_id, + read_fingerprint: envelope.read_fingerprint, + })) as *mut c_void; + paimon_result_stream_poll { + status: PAIMON_STREAM_POLL_DATA, + plan: Box::into_raw(Box::new(paimon_stream_plan { inner })), + snapshot_id, + next_snapshot_id, + watermark: watermark.unwrap_or(0), + has_watermark: u8::from(watermark.is_some()), + reserved: [0; 7], + error: ptr::null_mut(), + } + })); + outcome.unwrap_or_else(|_| error_poll(panic_error("paimon_stream_plan_deserialize"))) +} + +/// Read a contiguous split range from a stream plan. +/// +/// `read_mode=PAIMON_STREAM_READ_AUDIT_LOG` exposes a stable UTF-8 `rowkind` +/// column for incremental plans. Full startup plans currently support data +/// mode only; callers requiring one fixed audit schema should start at +/// `latest` or `from-snapshot`. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_read_to_arrow( + read: *const paimon_table_read, + plan: *const paimon_stream_plan, + offset: usize, + length: usize, + read_mode: i32, +) -> paimon_result_record_batch_reader { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(read, "read") { + return paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error, + }; + } + if let Err(error) = check_non_null(plan, "plan") { + return paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error, + }; + } + if read_mode != PAIMON_STREAM_READ_DATA && read_mode != PAIMON_STREAM_READ_AUDIT_LOG { + return paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error: invalid_mode("stream read mode", read_mode), + }; + } + let state = &*((*read).inner as *const TableReadState); + let plan_state = &*((*plan).inner as *const StreamPlanState); + if state.table_location != plan_state.table_location + || state.table_branch != plan_state.table_branch + || state.schema_id != plan_state.schema_id + || state.read_fingerprint != plan_state.read_fingerprint + { + return paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::InvalidInput, + "stream plan was created for a different table, branch, schema, or read builder" + .to_string(), + ), + }; + } + let stream_plan = &plan_state.plan; + let table_read = TableRead::new( + &state.table, + state.read_type.clone(), + state.data_predicates.clone(), + ); + let stream_result: paimon::Result = match stream_plan { + StreamPlan::Full { plan, .. } => { + if read_mode == PAIMON_STREAM_READ_AUDIT_LOG { + Err(paimon::Error::Unsupported { + message: "Audit-log mode for a full stream startup plan is not implemented; use latest/from-snapshot startup or data mode".to_string(), + }) + } else { + let splits = plan.splits(); + let start = offset.min(splits.len()); + let end = offset.saturating_add(length).min(splits.len()); + table_read.to_arrow(&splits[start..end]) + } + } + StreamPlan::Incremental { plan, .. } => { + let splits = plan.splits(); + let start = offset.min(splits.len()); + let end = offset.saturating_add(length).min(splits.len()); + let selected = paimon::table::IncrementalPlan::try_new( + plan.mode(), + splits[start..end].to_vec(), + ); + match selected { + Ok(selected) if read_mode == PAIMON_STREAM_READ_AUDIT_LOG => { + table_read.to_audit_log_arrow(&selected) + } + Ok(selected) => table_read.to_incremental_arrow(&selected), + Err(error) => Err(error), + } + } + }; + match stream_result { + Ok(stream) => { + let inner = Box::into_raw(Box::new(stream)) as *mut c_void; + paimon_result_record_batch_reader { + reader: Box::into_raw(Box::new(paimon_record_batch_reader { inner })), + error: ptr::null_mut(), + } + } + Err(error) => paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error: paimon_error::from_paimon(error), + }, + } + })); + outcome.unwrap_or_else(|_| paimon_result_record_batch_reader { + reader: ptr::null_mut(), + error: panic_error("paimon_stream_plan_read_to_arrow"), + }) +} + +/// Free a stream plan. It is valid to pass null. +#[no_mangle] +pub unsafe extern "C" fn paimon_stream_plan_free(plan: *mut paimon_stream_plan) { + if !plan.is_null() { + let wrapper = Box::from_raw(plan); + if !wrapper.inner.is_null() { + drop(Box::from_raw(wrapper.inner as *mut StreamPlanState)); + } + } +} + +// C ABI signature guards. +const _: unsafe extern "C" fn( + *const paimon_read_builder, + *const paimon_stream_scan_options, +) -> paimon_result_stream_scan = paimon_read_builder_new_stream_scan; +const _: unsafe extern "C" fn(*mut paimon_stream_scan) -> paimon_result_stream_poll = + paimon_stream_scan_poll; +const _: unsafe extern "C" fn( + *const paimon_table_read, + *const paimon_stream_plan, + usize, + usize, + i32, +) -> paimon_result_record_batch_reader = paimon_stream_plan_read_to_arrow; +const _: unsafe extern "C" fn(*const paimon_stream_plan) -> paimon_result_bytes = + paimon_stream_plan_serialize; +const _: unsafe extern "C" fn(*const u8, usize) -> paimon_result_stream_poll = + paimon_stream_plan_deserialize; + +#[cfg(test)] +mod tests { + use paimon::spec::BinaryRow; + use paimon::table::DataSplitBuilder; + + use super::*; + + fn serialized_plan(bucket_path: &str, next_snapshot_id: i64) -> Vec { + let split = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(bucket_path.to_string()) + .with_total_buckets(1) + .with_data_files(Vec::new()) + .build() + .unwrap(); + serde_json::to_vec(&StreamPlanEnvelope { + format: STREAM_PLAN_FORMAT.to_string(), + version: STREAM_PLAN_VERSION, + table_location: "memory:/table".to_string(), + table_branch: "main".to_string(), + schema_id: 0, + read_fingerprint: "fingerprint".to_string(), + kind: 0, + incremental_mode: -1, + snapshot_id: 1, + next_snapshot_id, + watermark: None, + splits: vec![split.serialize_split_v1().unwrap()], + }) + .unwrap() + } + + fn full_plan_state(bucket_path: &str) -> StreamPlanState { + let split = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(bucket_path.to_string()) + .with_total_buckets(1) + .with_data_files(Vec::new()) + .build() + .unwrap(); + StreamPlanState { + plan: StreamPlan::Full { + snapshot_id: 1, + watermark: None, + next_snapshot_id: 2, + plan: Plan::new(vec![split]), + }, + table_location: "memory:/table".to_string(), + table_branch: "main".to_string(), + schema_id: 0, + read_fingerprint: "fingerprint".to_string(), + } + } + + #[test] + fn serialization_rejects_plan_which_cannot_be_restored() { + assert!( + validate_stream_plan_recovery_paths(&full_plan_state("memory:/table/bucket-0")).is_ok() + ); + let error = + validate_stream_plan_recovery_paths(&full_plan_state("memory:/table-evil/bucket-0")) + .unwrap_err(); + unsafe { crate::error::paimon_error_free(error) }; + } + + #[test] + fn restored_plan_rejects_bucket_path_outside_table() { + let bytes = serialized_plan("memory:/table-evil/bucket-0", 2); + let result = unsafe { paimon_stream_plan_deserialize(bytes.as_ptr(), bytes.len()) }; + assert!(result.plan.is_null()); + assert!(!result.error.is_null()); + unsafe { crate::error::paimon_error_free(result.error) }; + } + + #[test] + fn full_plan_allows_same_snapshot_follow_up_cursor() { + let bytes = serialized_plan("memory:/table/bucket-0", 1); + let result = unsafe { paimon_stream_plan_deserialize(bytes.as_ptr(), bytes.len()) }; + assert!(result.error.is_null()); + assert_eq!(result.next_snapshot_id, 1); + unsafe { paimon_stream_plan_free(result.plan) }; + } + + #[test] + fn older_plan_version_is_reported_as_unsupported() { + let bytes = serialized_plan("memory:/table/bucket-0", 2); + let mut value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + value["version"] = serde_json::json!(2); + value.as_object_mut().unwrap().remove("table_branch"); + let bytes = serde_json::to_vec(&value).unwrap(); + let result = unsafe { paimon_stream_plan_deserialize(bytes.as_ptr(), bytes.len()) }; + assert!(result.plan.is_null()); + assert!(!result.error.is_null()); + assert_eq!( + unsafe { (*result.error).code }, + PaimonErrorCode::Unsupported as i32 + ); + unsafe { crate::error::paimon_error_free(result.error) }; + } +} diff --git a/bindings/c/src/table.rs b/bindings/c/src/table.rs index 0b18de703..08782fc15 100644 --- a/bindings/c/src/table.rs +++ b/bindings/c/src/table.rs @@ -675,6 +675,10 @@ pub unsafe extern "C" fn paimon_read_builder_new_read( table: state.table.clone(), read_type: table_read.read_type().to_vec(), data_predicates: table_read.data_predicates().to_vec(), + table_location: state.table.location().to_string(), + table_branch: state.table.branch().to_string(), + schema_id: state.table.schema().id(), + read_fingerprint: read_builder_fingerprint(state), }; paimon_result_new_read { read: box_table_read_state(read_state), diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs index 31b62de27..39eace0c2 100644 --- a/bindings/c/src/tests.rs +++ b/bindings/c/src/tests.rs @@ -46,13 +46,109 @@ use paimon::spec::{ use paimon::table::{SnapshotManager, Table}; use crate::blob_reader::*; +use crate::catalog::*; use crate::error::*; use crate::file_io::*; +use crate::identifier::*; +use crate::stream::*; use crate::table::*; use crate::types::*; use crate::vector_search::*; use crate::write::*; +#[test] +fn test_catalog_create_and_drop_table_from_schema_json() { + let directory = tempfile::tempdir().unwrap(); + let warehouse = CString::new(directory.path().to_string_lossy().as_bytes()).unwrap(); + let warehouse_key = CString::new("warehouse").unwrap(); + let options = [paimon_option { + key: warehouse_key.as_ptr(), + value: warehouse.as_ptr(), + }]; + let catalog_result = unsafe { paimon_catalog_create(options.as_ptr(), options.len()) }; + assert!(catalog_result.error.is_null()); + assert!(!catalog_result.catalog.is_null()); + + let catalog = unsafe { &*((*catalog_result.catalog).inner as *const Arc) }; + crate::runtime() + .block_on(catalog.create_database("default", true, HashMap::new())) + .unwrap(); + + let database = CString::new("default").unwrap(); + let table_name = CString::new("ffi_ddl").unwrap(); + let identifier_result = + unsafe { paimon_identifier_new(database.as_ptr(), table_name.as_ptr()) }; + assert!(identifier_result.error.is_null()); + assert!(!identifier_result.identifier.is_null()); + + let schema = Schema::builder() + .column("id", DataType::Int(IntType::with_nullable(false))) + .option("bucket", "1") + .option("bucket-key", "id") + .build() + .unwrap(); + let schema_json = CString::new(serde_json::to_string(&schema).unwrap()).unwrap(); + + let create_error = unsafe { + paimon_catalog_create_table_from_schema_json( + catalog_result.catalog, + identifier_result.identifier, + schema_json.as_ptr(), + false, + ) + }; + assert!(create_error.is_null()); + + let table_result = + unsafe { paimon_catalog_get_table(catalog_result.catalog, identifier_result.identifier) }; + assert!(table_result.error.is_null()); + assert!(!table_result.table.is_null()); + unsafe { paimon_table_free(table_result.table) }; + + let duplicate_error = unsafe { + paimon_catalog_create_table_from_schema_json( + catalog_result.catalog, + identifier_result.identifier, + schema_json.as_ptr(), + false, + ) + }; + assert!(!duplicate_error.is_null()); + assert_eq!( + unsafe { (*duplicate_error).code }, + PAIMON_ERROR_ALREADY_EXISTS + ); + unsafe { paimon_error_free(duplicate_error) }; + assert!(unsafe { + paimon_catalog_create_table_from_schema_json( + catalog_result.catalog, + identifier_result.identifier, + schema_json.as_ptr(), + true, + ) + } + .is_null()); + + assert!(unsafe { + paimon_catalog_drop_table(catalog_result.catalog, identifier_result.identifier, false) + } + .is_null()); + let missing = + unsafe { paimon_catalog_get_table(catalog_result.catalog, identifier_result.identifier) }; + assert!(missing.table.is_null()); + assert!(!missing.error.is_null()); + unsafe { paimon_error_free(missing.error) }; + assert!(unsafe { + paimon_catalog_drop_table(catalog_result.catalog, identifier_result.identifier, true) + } + .is_null()); + + unsafe { + paimon_identifier_free(identifier_result.identifier); + paimon_catalog_free(catalog_result.catalog); + } +} + // ========================================================================= // Helpers // ========================================================================= @@ -314,6 +410,42 @@ unsafe fn collect_rows(reader: *mut paimon_record_batch_reader) -> Vec<(i32, Str rows } +/// Collect the audit-log row-kind strings while exercising Arrow ownership. +unsafe fn collect_rowkinds(reader: *mut paimon_record_batch_reader) -> Vec { + let mut kinds = Vec::new(); + loop { + let result = paimon_record_batch_reader_next(reader); + assert!(result.error.is_null(), "reader_next should not error"); + if result.batch.array.is_null() { + break; + } + let ffi_array = ptr::read(result.batch.array as *const FFI_ArrowArray); + let ffi_schema = ptr::read(result.batch.schema as *const FFI_ArrowSchema); + let data = arrow_array::ffi::from_ffi(ffi_array, &ffi_schema).unwrap(); + ptr::write( + result.batch.array as *mut FFI_ArrowArray, + FFI_ArrowArray::empty(), + ); + ptr::write( + result.batch.schema as *mut FFI_ArrowSchema, + FFI_ArrowSchema::empty(), + ); + paimon_arrow_batch_free(result.batch); + + let batch = RecordBatch::from(StructArray::from(data)); + let rowkind = batch + .column_by_name("rowkind") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..batch.num_rows() { + kinds.push(rowkind.value(index).to_string()); + } + } + kinds +} + /// Full read via C FFI: read_builder -> scan -> plan -> read -> stream -> rows. /// Called OUTSIDE of any block_on — the C FFI functions use block_on internally. unsafe fn read_rows_ffi(table: *const paimon_table) -> Vec<(i32, String)> { @@ -348,6 +480,234 @@ unsafe fn read_rows_ffi(table: *const paimon_table) -> Vec<(i32, String)> { rows } +#[test] +fn test_stream_scan_tails_snapshots_and_restores_cursor() { + let path = "memory:/test_stream_scan_tails_snapshots"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io, + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + let handle = unsafe { wrap_table(table.clone()) }; + + unsafe { + let rb_result = paimon_table_new_read_builder(handle); + assert!(rb_result.error.is_null()); + let rb = rb_result.read_builder; + let read_result = paimon_read_builder_new_read(rb); + assert!(read_result.error.is_null()); + let read = read_result.read; + + let mut options = std::mem::MaybeUninit::::uninit(); + assert!(paimon_stream_scan_options_init(options.as_mut_ptr()).is_null()); + let mut options = options.assume_init(); + options.startup_mode = PAIMON_STREAM_STARTUP_LATEST; + options.follow_up_mode = PAIMON_STREAM_FOLLOW_UP_DELTA; + + let scan_result = paimon_read_builder_new_stream_scan(rb, &options); + assert!(scan_result.error.is_null()); + let scan = scan_result.scan; + assert_eq!(paimon_stream_scan_checkpoint(scan), 1); + + // Commit before the first poll. Eager initialization at scan creation + // must retain snapshot 1 instead of treating it as pre-existing data. + write_data_rust(&table, &[make_batch(vec![1], vec!["first"])]); + let first = paimon_stream_scan_poll(scan); + assert!(first.error.is_null()); + assert_eq!(first.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(first.snapshot_id, 1); + assert_eq!(first.next_snapshot_id, 2); + assert_eq!(paimon_stream_scan_checkpoint(scan), 2); + assert_eq!(paimon_stream_plan_is_full(first.plan), 0); + + let serialized = paimon_stream_plan_serialize(first.plan); + assert!(serialized.error.is_null()); + let checkpoint_bytes = + std::slice::from_raw_parts(serialized.bytes.data, serialized.bytes.len).to_vec(); + paimon_bytes_free(serialized.bytes); + let restored_plan = + paimon_stream_plan_deserialize(checkpoint_bytes.as_ptr(), checkpoint_bytes.len()); + assert!(restored_plan.error.is_null()); + assert_eq!(restored_plan.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(restored_plan.snapshot_id, 1); + assert_eq!(restored_plan.next_snapshot_id, 2); + let restored_reader = paimon_stream_plan_read_to_arrow( + read, + restored_plan.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(restored_reader.error.is_null()); + assert_eq!( + collect_rows(restored_reader.reader), + vec![(1, "first".into())] + ); + paimon_record_batch_reader_free(restored_reader.reader); + + let mismatched_builder = paimon_table_new_read_builder(handle); + assert!(mismatched_builder.error.is_null()); + assert!( + paimon_read_builder_with_case_sensitive(mismatched_builder.read_builder, false,) + .is_null() + ); + let mismatched_read = paimon_read_builder_new_read(mismatched_builder.read_builder); + assert!(mismatched_read.error.is_null()); + let mismatched_result = paimon_stream_plan_read_to_arrow( + mismatched_read.read, + restored_plan.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(mismatched_result.reader.is_null()); + assert!(!mismatched_result.error.is_null()); + assert_eq!( + (*mismatched_result.error).code, + PaimonErrorCode::InvalidInput as i32 + ); + paimon_error_free(mismatched_result.error); + paimon_table_read_free(mismatched_read.read); + paimon_read_builder_free(mismatched_builder.read_builder); + + let branch_table = Table::from_resolved_schema( + table.file_io().clone(), + Identifier::new("default", "test"), + path.to_string(), + table.schema().clone(), + "branch-review", + ) + .unwrap(); + let branch_handle = wrap_table(branch_table); + let branch_builder = paimon_table_new_read_builder(branch_handle); + assert!(branch_builder.error.is_null()); + let branch_read = paimon_read_builder_new_read(branch_builder.read_builder); + assert!(branch_read.error.is_null()); + let branch_result = paimon_stream_plan_read_to_arrow( + branch_read.read, + restored_plan.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(branch_result.reader.is_null()); + assert!(!branch_result.error.is_null()); + assert_eq!( + (*branch_result.error).code, + PaimonErrorCode::InvalidInput as i32 + ); + paimon_error_free(branch_result.error); + paimon_table_read_free(branch_read.read); + paimon_read_builder_free(branch_builder.read_builder); + unwrap_table(branch_handle); + + paimon_stream_plan_free(restored_plan.plan); + + let first_reader = paimon_stream_plan_read_to_arrow( + read, + first.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(first_reader.error.is_null()); + assert_eq!(collect_rows(first_reader.reader), vec![(1, "first".into())]); + paimon_record_batch_reader_free(first_reader.reader); + + let audit_reader = paimon_stream_plan_read_to_arrow( + read, + first.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_AUDIT_LOG, + ); + assert!(audit_reader.error.is_null()); + assert_eq!(collect_rowkinds(audit_reader.reader), vec!["+I"]); + paimon_record_batch_reader_free(audit_reader.reader); + paimon_stream_plan_free(first.plan); + + write_data_rust(&table, &[make_batch(vec![2], vec!["second"])]); + let second = paimon_stream_scan_poll(scan); + assert!(second.error.is_null()); + assert_eq!(second.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(second.snapshot_id, 2); + assert_eq!(second.next_snapshot_id, 3); + let second_reader = paimon_stream_plan_read_to_arrow( + read, + second.plan, + 0, + usize::MAX, + PAIMON_STREAM_READ_DATA, + ); + assert!(second_reader.error.is_null()); + assert_eq!( + collect_rows(second_reader.reader), + vec![(2, "second".into())] + ); + paimon_record_batch_reader_free(second_reader.reader); + paimon_stream_plan_free(second.plan); + + // Restoring nextSnapshotId=2 replays snapshot 2 rather than losing it. + assert!(paimon_stream_scan_restore(scan, 2).is_null()); + let replay = paimon_stream_scan_poll(scan); + assert!(replay.error.is_null()); + assert_eq!(replay.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(replay.snapshot_id, 2); + paimon_stream_plan_free(replay.plan); + + paimon_stream_scan_free(scan); + paimon_table_read_free(read); + paimon_read_builder_free(rb); + unwrap_table(handle); + } +} + +#[test] +fn test_stream_scan_latest_full_then_waits_for_follow_up() { + let path = "memory:/test_stream_scan_latest_full"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io, + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + write_data_rust(&table, &[make_batch(vec![7], vec!["existing"])]); + let handle = unsafe { wrap_table(table) }; + + unsafe { + let rb_result = paimon_table_new_read_builder(handle); + assert!(rb_result.error.is_null()); + let mut options = std::mem::MaybeUninit::::uninit(); + assert!(paimon_stream_scan_options_init(options.as_mut_ptr()).is_null()); + let options = options.assume_init(); + let scan_result = paimon_read_builder_new_stream_scan(rb_result.read_builder, &options); + assert!(scan_result.error.is_null()); + + let full = paimon_stream_scan_poll(scan_result.scan); + assert!(full.error.is_null()); + assert_eq!(full.status, PAIMON_STREAM_POLL_DATA); + assert_eq!(full.snapshot_id, 1); + assert_eq!(paimon_stream_plan_is_full(full.plan), 1); + assert_eq!(paimon_stream_scan_checkpoint(scan_result.scan), 2); + paimon_stream_plan_free(full.plan); + + let waiting = paimon_stream_scan_poll(scan_result.scan); + assert!(waiting.error.is_null()); + assert_eq!(waiting.status, PAIMON_STREAM_POLL_WAITING); + + paimon_stream_scan_free(scan_result.scan); + paimon_read_builder_free(rb_result.read_builder); + unwrap_table(handle); + } +} + // ========================================================================= // Catalog-free table construction tests // ========================================================================= @@ -1803,6 +2163,298 @@ fn test_caller_supplied_commit_identity_is_shared_and_persisted() { assert_eq!(snapshot.id(), 1, "retry must not create another snapshot"); } +#[test] +fn test_stream_write_v1_reuses_writer_across_monotonic_checkpoints() { + let path = "memory:/test_stream_write_v1_reuses_writer"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io.clone(), + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + let handle = unsafe { wrap_table(table) }; + let commit_user = CString::new("stream-write-job-9").unwrap(); + + unsafe { + let wb_result = + paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()); + assert!(wb_result.error.is_null()); + let wb = wb_result.write_builder; + + let tw_result = paimon_write_builder_new_write(wb); + assert!(tw_result.error.is_null()); + let tw = tw_result.write; + + let commit_result = paimon_write_builder_new_commit(wb); + assert!(commit_result.error.is_null()); + let commit = commit_result.commit; + + // Checkpoint 100: retain the prepared messages until a successful + // filter-and-commit confirms an intentionally lost commit ACK. + let (array, schema) = export_batch_to_ffi(make_batch(vec![1], vec!["first"])); + let error = paimon_table_write_write_arrow_batch( + tw, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + + let prepared_100 = paimon_table_write_prepare_commit(tw); + assert!(prepared_100.error.is_null()); + let error = paimon_table_commit_commit_with_identifier(commit, prepared_100.messages, 100); + assert!(error.is_null()); + + let error = paimon_table_commit_filter_and_commit_with_identifier( + commit, + prepared_100.messages, + 100, + ); + assert!( + error.is_null(), + "a retry after a lost commit ACK must be idempotent" + ); + paimon_commit_messages_free(prepared_100.messages); + + // Checkpoint 101 deliberately reuses both the writer and committer. + // prepare_commit must drain only the data written since checkpoint 100. + let (array, schema) = export_batch_to_ffi(make_batch(vec![2], vec!["second"])); + let error = paimon_table_write_write_arrow_batch( + tw, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + + let prepared_101 = paimon_table_write_prepare_commit(tw); + assert!(prepared_101.error.is_null()); + let error = paimon_table_commit_commit_with_identifier(commit, prepared_101.messages, 101); + assert!(error.is_null()); + paimon_commit_messages_free(prepared_101.messages); + + assert_eq!( + read_rows_ffi(handle), + vec![(1, "first".into()), (2, "second".into())] + ); + + // A later prepared checkpoint can be abandoned without publishing a + // snapshot or making its rows visible. + let (array, schema) = export_batch_to_ffi(make_batch(vec![3], vec!["aborted"])); + let error = paimon_table_write_write_arrow_batch( + tw, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + + let abandoned = paimon_table_write_prepare_commit(tw); + assert!(abandoned.error.is_null()); + let abandoned_prepared = paimon_commit_messages_prepare(abandoned.messages, 102); + assert!(abandoned_prepared.error.is_null()); + paimon_commit_messages_free(abandoned.messages); + let error = paimon_table_commit_abort_prepared(commit, abandoned_prepared.prepared); + assert!(error.is_null()); + paimon_prepared_commit_free(abandoned_prepared.prepared); + + assert_eq!( + read_rows_ffi(handle), + vec![(1, "first".into()), (2, "second".into())] + ); + + paimon_table_commit_free(commit); + paimon_table_write_free(tw); + paimon_write_builder_free(wb); + unwrap_table(handle); + } + + let snapshots = crate::runtime().block_on(async { + let manager = SnapshotManager::new(file_io, path.to_string()); + ( + manager.get_snapshot(1).await.unwrap(), + manager.get_snapshot(2).await.unwrap(), + manager.get_latest_snapshot_id().await.unwrap(), + ) + }); + assert_eq!(snapshots.0.commit_user(), "stream-write-job-9"); + assert_eq!(snapshots.0.commit_identifier(), 100); + assert_eq!(snapshots.1.commit_user(), "stream-write-job-9"); + assert_eq!(snapshots.1.commit_identifier(), 101); + assert_eq!( + snapshots.2, + Some(2), + "the retry and abort must not publish snapshots" + ); +} + +#[test] +fn test_prepared_commit_roundtrip_and_lost_ack_retry() { + let path = "memory:/test_prepared_commit_roundtrip"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io.clone(), + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + let handle = unsafe { wrap_table(table) }; + let commit_user = CString::new("durable-stream-job-5").unwrap(); + + unsafe { + let writer_builder = + paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()); + assert!(writer_builder.error.is_null()); + let writer_builder = writer_builder.write_builder; + + let writer = paimon_write_builder_new_write(writer_builder); + assert!(writer.error.is_null()); + let writer = writer.write; + + let (array, schema) = export_batch_to_ffi(make_batch(vec![5], vec!["durable"])); + let error = paimon_table_write_write_arrow_batch( + writer, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + + let messages = paimon_table_write_prepare_commit(writer); + assert!(messages.error.is_null()); + let reserved = paimon_commit_messages_prepare(messages.messages, i64::MAX); + assert!(reserved.prepared.is_null()); + assert!(!reserved.error.is_null()); + assert_eq!((*reserved.error).code, PaimonErrorCode::InvalidInput as i32); + paimon_error_free(reserved.error); + let prepared = paimon_commit_messages_prepare(messages.messages, 500); + assert!(prepared.error.is_null()); + assert_eq!(paimon_prepared_commit_identifier(prepared.prepared), 500); + + let serialized = paimon_prepared_commit_serialize(prepared.prepared); + assert!(serialized.error.is_null()); + assert!(!serialized.bytes.data.is_null()); + assert!(serialized.bytes.len > 0); + + let mut unsafe_checkpoint: serde_json::Value = serde_json::from_slice( + std::slice::from_raw_parts(serialized.bytes.data, serialized.bytes.len), + ) + .unwrap(); + unsafe_checkpoint["messages"][0]["new_files"][0]["_EXTERNAL_PATH"] = + serde_json::json!("file:/tmp/not-owned-by-the-prepared-commit"); + let unsafe_checkpoint = serde_json::to_vec(&unsafe_checkpoint).unwrap(); + let rejected = + paimon_prepared_commit_deserialize(unsafe_checkpoint.as_ptr(), unsafe_checkpoint.len()); + assert!(rejected.prepared.is_null()); + assert!(!rejected.error.is_null()); + assert_eq!((*rejected.error).code, PaimonErrorCode::InvalidInput as i32); + paimon_error_free(rejected.error); + + let mut duplicated_checkpoint: serde_json::Value = serde_json::from_slice( + std::slice::from_raw_parts(serialized.bytes.data, serialized.bytes.len), + ) + .unwrap(); + let duplicate_message = duplicated_checkpoint["messages"][0].clone(); + duplicated_checkpoint["messages"] + .as_array_mut() + .unwrap() + .push(duplicate_message); + let duplicated_checkpoint = serde_json::to_vec(&duplicated_checkpoint).unwrap(); + + let mut conflicting_checkpoint: serde_json::Value = serde_json::from_slice( + std::slice::from_raw_parts(serialized.bytes.data, serialized.bytes.len), + ) + .unwrap(); + let mut conflicting_message = conflicting_checkpoint["messages"][0].clone(); + conflicting_message["new_files"][0]["_FILE_SIZE"] = serde_json::json!(123456789); + conflicting_checkpoint["messages"] + .as_array_mut() + .unwrap() + .push(conflicting_message); + let conflicting_checkpoint = serde_json::to_vec(&conflicting_checkpoint).unwrap(); + let rejected = paimon_prepared_commit_deserialize( + conflicting_checkpoint.as_ptr(), + conflicting_checkpoint.len(), + ); + assert!(rejected.prepared.is_null()); + assert!(!rejected.error.is_null()); + assert!(error_message(rejected.error).contains("same file identity")); + paimon_error_free(rejected.error); + + // The serialized bytes, rather than either in-process source handle, + // are the durable checkpoint boundary. + paimon_commit_messages_free(messages.messages); + paimon_prepared_commit_free(prepared.prepared); + + let restored = paimon_prepared_commit_deserialize( + duplicated_checkpoint.as_ptr(), + duplicated_checkpoint.len(), + ); + assert!(restored.error.is_null()); + assert_eq!(paimon_prepared_commit_identifier(restored.prepared), 500); + + let first_committer_builder = + paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()); + assert!(first_committer_builder.error.is_null()); + let first_committer_builder = first_committer_builder.write_builder; + let first_committer = paimon_write_builder_new_commit(first_committer_builder); + assert!(first_committer.error.is_null()); + let error = paimon_table_commit_commit_prepared(first_committer.commit, restored.prepared); + assert!(error.is_null()); + + // A stale abort request after a successful commit (including a lost + // acknowledgement recovered by identifier) must not delete files now + // referenced by the committed snapshot. + let error = paimon_table_commit_abort_prepared(first_committer.commit, restored.prepared); + assert!(error.is_null()); + assert_eq!(read_rows_ffi(handle), vec![(5, "durable".into())]); + + // Treat the successful return above as a lost ACK. Discard all + // in-memory commit state, recover from the same durable bytes, and + // retry through the identifier-filtering commit path. + paimon_prepared_commit_free(restored.prepared); + paimon_table_commit_free(first_committer.commit); + paimon_write_builder_free(first_committer_builder); + + let retry = paimon_prepared_commit_deserialize( + serialized.bytes.data.cast_const(), + serialized.bytes.len, + ); + assert!(retry.error.is_null()); + let retry_committer_builder = + paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()); + assert!(retry_committer_builder.error.is_null()); + let retry_committer_builder = retry_committer_builder.write_builder; + let retry_committer = paimon_write_builder_new_commit(retry_committer_builder); + assert!(retry_committer.error.is_null()); + let error = paimon_table_commit_commit_prepared(retry_committer.commit, retry.prepared); + assert!( + error.is_null(), + "recovered commit_prepared must filter a previously committed identifier" + ); + + paimon_prepared_commit_free(retry.prepared); + paimon_bytes_free(serialized.bytes); + paimon_table_commit_free(retry_committer.commit); + paimon_write_builder_free(retry_committer_builder); + + assert_eq!(read_rows_ffi(handle), vec![(5, "durable".into())]); + + paimon_table_write_free(writer); + paimon_write_builder_free(writer_builder); + unwrap_table(handle); + } + + let snapshot = crate::runtime() + .block_on(SnapshotManager::new(file_io, path.to_string()).get_latest_snapshot()) + .unwrap() + .unwrap(); + assert_eq!(snapshot.id(), 1, "the lost-ACK retry must be a no-op"); + assert_eq!(snapshot.commit_user(), "durable-stream-job-5"); + assert_eq!(snapshot.commit_identifier(), 500); +} + #[test] fn test_commit_messages_merge_preserves_all_writer_files() { let path = "memory:/test_commit_messages_merge"; @@ -1840,6 +2492,11 @@ fn test_commit_messages_merge_preserves_all_writer_files() { let messages2 = paimon_table_write_prepare_commit(tw2).messages; let err = paimon_commit_messages_merge(messages1, messages2); assert!(err.is_null()); + let err = paimon_commit_messages_merge(messages1, messages2); + assert!( + err.is_null(), + "re-merging the same fragment must be a no-op" + ); let commit = paimon_write_builder_new_commit(wb1).commit; let err = paimon_table_commit_commit_with_identifier(commit, messages1, 7); @@ -1860,6 +2517,81 @@ fn test_commit_messages_merge_preserves_all_writer_files() { } } +#[test] +fn test_prepared_commit_merge_preserves_parallel_writer_files() { + let path = "memory:/test_prepared_commit_merge"; + let file_io = memory_file_io(); + setup_table_dirs(&file_io, path); + let table = Table::new( + file_io, + Identifier::new("default", "test"), + path.to_string(), + simple_table_schema(), + None, + ); + let handle = unsafe { wrap_table(table) }; + let commit_user = CString::new("durable-distributed-job-700").unwrap(); + + unsafe { + let wb1 = paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()) + .write_builder; + let wb2 = paimon_table_new_write_builder_with_commit_user(handle, commit_user.as_ptr()) + .write_builder; + let tw1 = paimon_write_builder_new_write(wb1).write; + let tw2 = paimon_write_builder_new_write(wb2).write; + + for (writer, ids, names) in [ + (tw1, vec![10], vec!["left"]), + (tw2, vec![20], vec!["right"]), + ] { + let (array, schema) = export_batch_to_ffi(make_batch(ids, names)); + let error = paimon_table_write_write_arrow_batch( + writer, + (&**array) as *const FFI_ArrowArray as *mut c_void, + (&**schema) as *const FFI_ArrowSchema as *mut c_void, + ); + assert!(error.is_null()); + } + + let messages1 = paimon_table_write_prepare_commit(tw1); + assert!(messages1.error.is_null()); + let messages2 = paimon_table_write_prepare_commit(tw2); + assert!(messages2.error.is_null()); + let prepared1 = paimon_commit_messages_prepare(messages1.messages, 700); + assert!(prepared1.error.is_null()); + let prepared2 = paimon_commit_messages_prepare(messages2.messages, 700); + assert!(prepared2.error.is_null()); + paimon_commit_messages_free(messages2.messages); + paimon_commit_messages_free(messages1.messages); + + let error = paimon_prepared_commit_merge(prepared1.prepared, prepared2.prepared); + assert!(error.is_null()); + let error = paimon_prepared_commit_merge(prepared1.prepared, prepared2.prepared); + assert!( + error.is_null(), + "re-merging the same durable fragment must be a no-op" + ); + let commit = paimon_write_builder_new_commit(wb1); + assert!(commit.error.is_null()); + let error = paimon_table_commit_commit_prepared(commit.commit, prepared1.prepared); + assert!(error.is_null()); + + assert_eq!( + read_rows_ffi(handle), + vec![(10, "left".into()), (20, "right".into())] + ); + + paimon_table_commit_free(commit.commit); + paimon_prepared_commit_free(prepared2.prepared); + paimon_prepared_commit_free(prepared1.prepared); + paimon_table_write_free(tw2); + paimon_table_write_free(tw1); + paimon_write_builder_free(wb2); + paimon_write_builder_free(wb1); + unwrap_table(handle); + } +} + #[test] fn test_postpone_bucket_plan_arrow_ownership_on_errors() { let path = "memory:/test_postpone_bucket_plan_arrow_ownership"; @@ -2125,6 +2857,17 @@ fn test_write_multiple_batches() { let tc_result = paimon_write_builder_new_commit(wb); let tc = tc_result.commit; + for invalid in [-1, i64::MAX] { + let err = paimon_table_commit_commit_with_identifier(tc, pc_result.messages, invalid); + assert!(!err.is_null()); + assert_eq!((*err).code, PaimonErrorCode::InvalidInput as i32); + paimon_error_free(err); + let err = paimon_table_commit_truncate_table_with_identifier(tc, invalid); + assert!(!err.is_null()); + assert_eq!((*err).code, PaimonErrorCode::InvalidInput as i32); + paimon_error_free(err); + } + let err = paimon_table_commit_commit(tc, pc_result.messages); assert!(err.is_null()); paimon_commit_messages_free(pc_result.messages); diff --git a/bindings/c/src/types.rs b/bindings/c/src/types.rs index 4e6d2709f..5bc4d5b66 100644 --- a/bindings/c/src/types.rs +++ b/bindings/c/src/types.rs @@ -24,6 +24,7 @@ use paimon::table::{ CommitMessage, PostponeBucketPlan, PostponeFixedBucketTableCommit, PostponeFixedBucketTableWrite, Table, TableCommit, TableWrite, }; +use sha2::{Digest, Sha256}; /// C-compatible key-value pair for options. #[repr(C)] @@ -201,6 +202,68 @@ pub(crate) struct ReadBuilderState { pub case_sensitive: bool, } +fn digest_read_builder_canonical(canonical: &str) -> String { + let digest = Sha256::digest(canonical.as_bytes()); + format!("paimon-c-read-builder-sha256-v1:{digest:x}") +} + +/// Build the stable identity component shared by a stream plan and the +/// `TableRead` which consumes it. +/// +/// The canonical input uses length-prefixed components and the persisted value +/// is a SHA-256 digest, so predicate literals are not exposed in a checkpoint. +/// Predicate `Debug` formatting is versioned by the fingerprint prefix and +/// must be bumped if it changes incompatibly. +pub(crate) fn read_builder_fingerprint(state: &ReadBuilderState) -> String { + fn push_component(target: &mut String, value: &str) { + target.push_str(&value.len().to_string()); + target.push(':'); + target.push_str(value); + target.push(';'); + } + + let mut canonical = String::from("paimon-c-read-builder-canonical-v1;"); + canonical.push_str(if state.case_sensitive { + "case=1;" + } else { + "case=0;" + }); + match &state.projected_columns { + None => canonical.push_str("projection=none;"), + Some(columns) => { + canonical.push_str("projection=some;"); + canonical.push_str(&columns.len().to_string()); + canonical.push(';'); + for column in columns { + push_component(&mut canonical, column); + } + } + } + match &state.filter { + None => canonical.push_str("filter=none;"), + Some(filter) => { + canonical.push_str("filter=some;"); + push_component(&mut canonical, &format!("{filter:?}")); + } + } + digest_read_builder_canonical(&canonical) +} + +#[cfg(test)] +mod fingerprint_tests { + use super::digest_read_builder_canonical; + + #[test] + fn digest_does_not_expose_predicate_literals() { + let canonical = "filter=some;24:secret-customer-id=42;"; + let digest = digest_read_builder_canonical(canonical); + assert!(digest.starts_with("paimon-c-read-builder-sha256-v1:")); + assert!(!digest.contains("secret-customer-id")); + assert_eq!(digest.len(), "paimon-c-read-builder-sha256-v1:".len() + 64); + assert_eq!(digest, digest_read_builder_canonical(canonical)); + } +} + /// Internal state for TableScan that stores table and filter. pub(crate) struct TableScanState { pub table: Table, @@ -222,6 +285,10 @@ pub(crate) struct TableReadState { pub table: Table, pub read_type: Vec, pub data_predicates: Vec, + pub table_location: String, + pub table_branch: String, + pub schema_id: i64, + pub read_fingerprint: String, } #[repr(C)] @@ -368,6 +435,15 @@ pub(crate) struct CommitMessagesState { pub commit_user: String, } +/// Durable, versioned representation of one standard streaming checkpoint. +/// +/// Unlike `paimon_commit_messages`, this state also carries the monotonically +/// increasing commit identifier and can be serialized across process restarts. +pub(crate) struct PreparedCommitState { + pub commit_identifier: i64, + pub messages: CommitMessagesState, +} + pub(crate) struct PostponeFixedBucketCommitMessagesState { pub messages: Vec, pub overwrite: bool, @@ -396,6 +472,12 @@ pub struct paimon_commit_messages { pub inner: *mut c_void, } +/// Opaque durable prepared-commit handle for a standard table write. +#[repr(C)] +pub struct paimon_prepared_commit { + pub inner: *mut c_void, +} + #[repr(C)] pub struct paimon_postpone_fixed_bucket_write_builder { pub inner: *mut c_void, diff --git a/bindings/c/src/write.rs b/bindings/c/src/write.rs index c6e9afa7d..474156942 100644 --- a/bindings/c/src/write.rs +++ b/bindings/c/src/write.rs @@ -15,22 +15,28 @@ // specific language governing permissions and limitations // under the License. +use std::collections::{HashMap, HashSet}; use std::ffi::{c_char, c_void}; +use std::fmt; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::ptr; use std::sync::Arc; use arrow_array::ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}; use arrow_array::{Array, RecordBatch, RecordBatchOptions, StructArray}; use arrow_schema::{DataType as ArrowDataType, Schema as ArrowSchema}; -use paimon::table::{PostponeBucketPlan, Table}; +use paimon::table::{CommitMessage, PostponeBucketPlan, Table}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; use crate::error::{check_non_null, paimon_error, validate_cstr, PaimonErrorCode}; use crate::result::{ - paimon_result_postpone_fixed_bucket_prepare_commit, + paimon_result_bytes, paimon_result_postpone_fixed_bucket_prepare_commit, paimon_result_postpone_fixed_bucket_table_commit, paimon_result_postpone_fixed_bucket_table_write, paimon_result_postpone_fixed_bucket_write_builder, paimon_result_prepare_commit, - paimon_result_table_commit, paimon_result_table_write, paimon_result_write_builder, + paimon_result_prepared_commit, paimon_result_table_commit, paimon_result_table_write, + paimon_result_write_builder, }; use crate::runtime; use crate::types::*; @@ -817,6 +823,490 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_commit_messages_free( } } +const PREPARED_COMMIT_FORMAT: &str = "paimon-rust-prepared-commit"; +// Version 2 adds strict resource and path validation. Version 1 is rejected: +// accepting its unconstrained internal CommitMessage representation would +// reintroduce unsafe file references after recovery. +const PREPARED_COMMIT_VERSION: u32 = 2; +const MAX_PREPARED_COMMIT_BYTES: usize = 64 * 1024 * 1024; +const MAX_PREPARED_MESSAGES: usize = 100_000; +const MAX_PREPARED_MESSAGE_BYTES: usize = 16 * 1024 * 1024; +const MAX_FILES_PER_MESSAGE: usize = 100_000; +const MAX_TOTAL_FILE_REFERENCES: usize = 1_000_000; +const MAX_EXTRA_FILES_PER_DATA_FILE: usize = 10_000; +const MAX_PARTITION_BYTES: usize = 16 * 1024 * 1024; +const MAX_IDENTITY_BYTES: usize = 1024 * 1024; +const MAX_FILE_NAME_BYTES: usize = 4 * 1024; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PreparedCommitEnvelope { + format: String, + version: u32, + commit_identifier: i64, + table_location: String, + commit_user: String, + overwrite: bool, + messages: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawPreparedCommitEnvelope<'a> { + format: String, + version: u32, + commit_identifier: i64, + table_location: String, + commit_user: String, + overwrite: bool, + #[serde(borrow, deserialize_with = "deserialize_bounded_raw_messages")] + messages: Vec<&'a RawValue>, +} + +fn deserialize_bounded_raw_messages<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + use serde::de::{Error, IgnoredAny, SeqAccess, Visitor}; + + struct RawMessagesVisitor; + + impl<'de> Visitor<'de> for RawMessagesVisitor { + type Value = Vec<&'de RawValue>; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a bounded list of prepared commit messages") + } + + fn visit_seq(self, mut sequence: A) -> Result + where + A: SeqAccess<'de>, + { + let mut messages = Vec::with_capacity( + sequence + .size_hint() + .unwrap_or_default() + .min(MAX_PREPARED_MESSAGES), + ); + while messages.len() < MAX_PREPARED_MESSAGES { + let Some(message) = sequence.next_element::<&'de RawValue>()? else { + return Ok(messages); + }; + messages.push(message); + } + if sequence.next_element::()?.is_some() { + return Err(A::Error::custom(format!( + "prepared commit contains more than {MAX_PREPARED_MESSAGES} messages" + ))); + } + Ok(messages) + } + } + + deserializer.deserialize_seq(RawMessagesVisitor) +} + +fn prepared_panic_error(operation: &str) -> *mut paimon_error { + paimon_error::new( + PaimonErrorCode::Unexpected, + format!("Rust panic while executing {operation}"), + ) +} + +fn validate_file_component(kind: &str, name: &str) -> Result<(), *mut paimon_error> { + if name.is_empty() + || name.len() > MAX_FILE_NAME_BYTES + || name == "." + || name == ".." + || name.contains('/') + || name.contains('\\') + || name.contains('\0') + { + return Err(invalid_input(format!( + "prepared commit contains unsafe {kind} '{name}'" + ))); + } + Ok(()) +} + +fn validate_data_file(file: &paimon::spec::DataFileMeta) -> Result { + if file.external_path.is_some() { + return Err(invalid_input( + "prepared commits with external data-file paths are not supported", + )); + } + validate_file_component("data file name", &file.file_name)?; + if file.extra_files.len() > MAX_EXTRA_FILES_PER_DATA_FILE { + return Err(invalid_input(format!( + "data file contains {} extra files; maximum is {}", + file.extra_files.len(), + MAX_EXTRA_FILES_PER_DATA_FILE + ))); + } + for extra in &file.extra_files { + validate_file_component("extra file name", extra)?; + } + Ok(1 + file.extra_files.len()) +} + +fn validate_index_file(file: &paimon::spec::IndexFileMeta) -> Result { + validate_file_component("index file name", &file.file_name)?; + if let Some(ranges) = &file.deletion_vectors_ranges { + for data_file_name in ranges.keys() { + validate_file_component("deletion-vector data file name", data_file_name)?; + } + } + Ok(1) +} + +fn validate_prepared_commit_envelope( + envelope: &PreparedCommitEnvelope, +) -> Result<(), *mut paimon_error> { + if envelope.commit_identifier < 0 + || envelope.commit_identifier == i64::MAX + || envelope.table_location.is_empty() + || envelope.table_location.len() > MAX_IDENTITY_BYTES + || envelope.commit_user.is_empty() + || envelope.commit_user.len() > MAX_IDENTITY_BYTES + { + return Err(invalid_input( + "prepared commit contains an invalid identity", + )); + } + if envelope.messages.len() > MAX_PREPARED_MESSAGES { + return Err(invalid_input(format!( + "prepared commit contains {} messages; maximum is {}", + envelope.messages.len(), + MAX_PREPARED_MESSAGES + ))); + } + + let mut total_file_references = 0usize; + for message in &envelope.messages { + if message.partition.len() > MAX_PARTITION_BYTES { + return Err(invalid_input(format!( + "prepared commit partition exceeds {MAX_PARTITION_BYTES} bytes" + ))); + } + let message_file_count = message + .new_files + .len() + .checked_add(message.new_changelog_files.len()) + .and_then(|count| count.checked_add(message.deleted_files.len())) + .and_then(|count| count.checked_add(message.new_index_files.len())) + .and_then(|count| count.checked_add(message.deleted_index_files.len())) + .ok_or_else(|| invalid_input("prepared commit file count overflows"))?; + if message_file_count > MAX_FILES_PER_MESSAGE { + return Err(invalid_input(format!( + "prepared commit message contains {message_file_count} files; maximum is {MAX_FILES_PER_MESSAGE}" + ))); + } + for file in message + .new_files + .iter() + .chain(message.new_changelog_files.iter()) + .chain(message.deleted_files.iter()) + { + total_file_references = total_file_references + .checked_add(validate_data_file(file)?) + .ok_or_else(|| invalid_input("prepared commit file count overflows"))?; + } + for file in message + .new_index_files + .iter() + .chain(message.deleted_index_files.iter()) + { + total_file_references = total_file_references + .checked_add(validate_index_file(file)?) + .ok_or_else(|| invalid_input("prepared commit file count overflows"))?; + } + if total_file_references > MAX_TOTAL_FILE_REFERENCES { + return Err(invalid_input(format!( + "prepared commit contains more than {MAX_TOTAL_FILE_REFERENCES} file references" + ))); + } + } + Ok(()) +} + +fn empty_bytes() -> paimon_bytes { + paimon_bytes { + data: ptr::null_mut(), + len: 0, + } +} + +/// Bind standard commit messages to a monotonically increasing streaming +/// commit identifier. The returned prepared commit owns a clone of the +/// messages, so the source handle remains valid. Valid identifiers are in +/// `[0, INT64_MAX)`; `INT64_MAX` is reserved for unidentified batch commits. +#[no_mangle] +pub unsafe extern "C" fn paimon_commit_messages_prepare( + msgs: *const paimon_commit_messages, + commit_identifier: i64, +) -> paimon_result_prepared_commit { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(msgs, "msgs") { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error, + }; + } + if commit_identifier < 0 || commit_identifier == i64::MAX { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ), + }; + } + let source = &*((*msgs).inner as *const CommitMessagesState); + let mut messages = Vec::new(); + if let Err(error) = merge_messages_idempotently(&mut messages, &source.messages) { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error, + }; + } + let state = PreparedCommitState { + commit_identifier, + messages: CommitMessagesState { + messages, + overwrite: source.overwrite, + table_location: source.table_location.clone(), + commit_user: source.commit_user.clone(), + }, + }; + let inner = Box::into_raw(Box::new(state)) as *mut c_void; + paimon_result_prepared_commit { + prepared: Box::into_raw(Box::new(paimon_prepared_commit { inner })), + error: ptr::null_mut(), + } + })); + outcome.unwrap_or_else(|_| paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: prepared_panic_error("paimon_commit_messages_prepare"), + }) +} + +/// Serialize a prepared commit into a process-independent, versioned buffer. +/// The bytes must be released with `paimon_bytes_free`. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_serialize( + prepared: *const paimon_prepared_commit, +) -> paimon_result_bytes { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(prepared, "prepared") { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + let state = &*((*prepared).inner as *const PreparedCommitState); + let envelope = PreparedCommitEnvelope { + format: PREPARED_COMMIT_FORMAT.to_string(), + version: PREPARED_COMMIT_VERSION, + commit_identifier: state.commit_identifier, + table_location: state.messages.table_location.clone(), + commit_user: state.messages.commit_user.clone(), + overwrite: state.messages.overwrite, + messages: state.messages.messages.clone(), + }; + if let Err(error) = validate_prepared_commit_envelope(&envelope) { + return paimon_result_bytes { + bytes: empty_bytes(), + error, + }; + } + match serde_json::to_vec(&envelope) { + Ok(bytes) if bytes.len() <= MAX_PREPARED_COMMIT_BYTES => paimon_result_bytes { + bytes: paimon_bytes::new(bytes), + error: ptr::null_mut(), + }, + Ok(bytes) => paimon_result_bytes { + bytes: empty_bytes(), + error: invalid_input(format!( + "serialized prepared commit is {} bytes; maximum is {}", + bytes.len(), + MAX_PREPARED_COMMIT_BYTES + )), + }, + Err(error) => paimon_result_bytes { + bytes: empty_bytes(), + error: paimon_error::new( + PaimonErrorCode::Unexpected, + format!("failed to serialize prepared commit: {error}"), + ), + }, + } + })); + outcome.unwrap_or_else(|_| paimon_result_bytes { + bytes: empty_bytes(), + error: prepared_panic_error("paimon_prepared_commit_serialize"), + }) +} + +/// Restore a prepared commit serialized by `paimon_prepared_commit_serialize`. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_deserialize( + data: *const u8, + len: usize, +) -> paimon_result_prepared_commit { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if data.is_null() || len == 0 { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input("prepared commit buffer must not be null or empty"), + }; + } + if len > MAX_PREPARED_COMMIT_BYTES { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!( + "prepared commit buffer exceeds {MAX_PREPARED_COMMIT_BYTES} bytes" + )), + }; + } + let bytes = std::slice::from_raw_parts(data, len); + let raw: RawPreparedCommitEnvelope<'_> = match serde_json::from_slice(bytes) { + Ok(envelope) => envelope, + Err(error) => { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!("invalid prepared commit buffer: {error}")), + }; + } + }; + if raw.format != PREPARED_COMMIT_FORMAT || raw.version != PREPARED_COMMIT_VERSION { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::Unsupported, + format!( + "unsupported prepared commit format '{}' version {}", + raw.format, raw.version + ), + ), + }; + } + if raw.commit_identifier < 0 + || raw.commit_identifier == i64::MAX + || raw.table_location.is_empty() + || raw.table_location.len() > MAX_IDENTITY_BYTES + || raw.commit_user.is_empty() + || raw.commit_user.len() > MAX_IDENTITY_BYTES + { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input("prepared commit contains an invalid identity"), + }; + } + if raw.messages.len() > MAX_PREPARED_MESSAGES { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!( + "prepared commit contains {} messages; maximum is {}", + raw.messages.len(), + MAX_PREPARED_MESSAGES + )), + }; + } + let mut messages = Vec::with_capacity(raw.messages.len()); + for (index, raw_message) in raw.messages.into_iter().enumerate() { + if raw_message.get().len() > MAX_PREPARED_MESSAGE_BYTES { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!( + "prepared commit message {index} exceeds {MAX_PREPARED_MESSAGE_BYTES} bytes" + )), + }; + } + match serde_json::from_str::(raw_message.get()) { + Ok(message) => messages.push(message), + Err(error) => { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: invalid_input(format!( + "invalid prepared commit message {index}: {error}" + )), + }; + } + } + } + let mut envelope = PreparedCommitEnvelope { + format: raw.format, + version: raw.version, + commit_identifier: raw.commit_identifier, + table_location: raw.table_location, + commit_user: raw.commit_user, + overwrite: raw.overwrite, + messages, + }; + if let Err(error) = validate_prepared_commit_envelope(&envelope) { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error, + }; + } + let mut normalized_messages = Vec::new(); + if let Err(error) = + merge_messages_idempotently(&mut normalized_messages, &envelope.messages) + { + return paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error, + }; + } + envelope.messages = normalized_messages; + let state = PreparedCommitState { + commit_identifier: envelope.commit_identifier, + messages: CommitMessagesState { + messages: envelope.messages, + overwrite: envelope.overwrite, + table_location: envelope.table_location, + commit_user: envelope.commit_user, + }, + }; + let inner = Box::into_raw(Box::new(state)) as *mut c_void; + paimon_result_prepared_commit { + prepared: Box::into_raw(Box::new(paimon_prepared_commit { inner })), + error: ptr::null_mut(), + } + })); + outcome.unwrap_or_else(|_| paimon_result_prepared_commit { + prepared: ptr::null_mut(), + error: prepared_panic_error("paimon_prepared_commit_deserialize"), + }) +} + +/// Return the commit identifier carried by a prepared commit, or -1 for null. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_identifier( + prepared: *const paimon_prepared_commit, +) -> i64 { + catch_unwind(AssertUnwindSafe(|| { + if prepared.is_null() || (*prepared).inner.is_null() { + return -1; + } + let state = &*((*prepared).inner as *const PreparedCommitState); + state.commit_identifier + })) + .unwrap_or(-1) +} + +/// Free a prepared commit. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_free(prepared: *mut paimon_prepared_commit) { + let _ = catch_unwind(AssertUnwindSafe(|| { + if !prepared.is_null() { + let wrapper = Box::from_raw(prepared); + if !wrapper.inner.is_null() { + drop(Box::from_raw(wrapper.inner as *mut PreparedCommitState)); + } + } + })); +} + fn validate_message_context( target_table: &str, target_user: &str, @@ -838,6 +1328,90 @@ fn validate_message_context( Ok(()) } +type CommitMessageGroupKey = (Vec, i32); +type CommitFileKey = (u8, String); + +fn commit_message_file_keys(message: &CommitMessage) -> Vec { + let mut keys = Vec::new(); + let mut add_data_files = |category: u8, files: &[paimon::spec::DataFileMeta]| { + for file in files { + keys.push((category, file.file_name.clone())); + for extra in &file.extra_files { + keys.push((category + 1, extra.clone())); + } + } + }; + add_data_files(0, &message.new_files); + add_data_files(2, &message.new_changelog_files); + add_data_files(4, &message.deleted_files); + for (category, files) in [ + (6u8, &message.new_index_files), + (7u8, &message.deleted_index_files), + ] { + for file in files { + keys.push((category, file.file_name.clone())); + } + } + keys +} + +fn merge_messages_idempotently( + target: &mut Vec, + source: &[CommitMessage], +) -> Result<(), *mut paimon_error> { + let capacity = target + .len() + .checked_add(source.len()) + .unwrap_or(MAX_PREPARED_MESSAGES) + .min(MAX_PREPARED_MESSAGES); + let mut merged = Vec::with_capacity(capacity); + let mut key_owners: HashMap> = + HashMap::new(); + + for message in target.iter().chain(source) { + let message_keys = commit_message_file_keys(message); + // Empty writer fragments do not publish any metadata and can be + // removed without changing commit semantics. + if message_keys.is_empty() { + continue; + } + let unique_keys = message_keys.iter().cloned().collect::>(); + if unique_keys.len() != message_keys.len() { + return Err(invalid_input( + "commit message contains a duplicate file identity", + )); + } + + // Hash/copy a partition only once per message. Putting it in every + // file key makes merge CPU and memory proportional to + // partition_bytes * file_count. + let group = (message.partition.clone(), message.bucket); + let group_owners = key_owners.entry(group).or_default(); + let owners = unique_keys + .iter() + .filter_map(|key| group_owners.get(key).copied()) + .collect::>(); + if !owners.is_empty() { + if owners.iter().any(|index| merged[*index] == *message) { + continue; + } + return Err(invalid_input( + "commit message merge found the same file identity with different fragment metadata", + )); + } + if merged.len() >= MAX_PREPARED_MESSAGES { + return Err(invalid_input(format!( + "merged commit contains more than {MAX_PREPARED_MESSAGES} messages" + ))); + } + let owner = merged.len(); + group_owners.extend(unique_keys.into_iter().map(|key| (key, owner))); + merged.push(message.clone()); + } + *target = merged; + Ok(()) +} + /// Merge standard commit messages for one logical commit. #[no_mangle] pub unsafe extern "C" fn paimon_commit_messages_merge( @@ -865,8 +1439,51 @@ pub unsafe extern "C" fn paimon_commit_messages_merge( ) { return error; } - target.messages.extend(source.messages.clone()); - ptr::null_mut() + match merge_messages_idempotently(&mut target.messages, &source.messages) { + Ok(()) => ptr::null_mut(), + Err(error) => error, + } +} + +/// Merge two durable prepared commits produced by parallel writers for the +/// same table, commit user, mode and identifier. +#[no_mangle] +pub unsafe extern "C" fn paimon_prepared_commit_merge( + target: *mut paimon_prepared_commit, + source: *const paimon_prepared_commit, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(target, "target") { + return error; + } + if let Err(error) = check_non_null(source, "source") { + return error; + } + if ptr::eq(target, source.cast_mut()) { + return invalid_input("target and source prepared commits must be distinct handles"); + } + let target = &mut *((*target).inner as *mut PreparedCommitState); + let source = &*((*source).inner as *const PreparedCommitState); + if target.commit_identifier != source.commit_identifier { + return invalid_input("prepared commits must have the same commit_identifier"); + } + if let Err(error) = validate_message_context( + &target.messages.table_location, + &target.messages.commit_user, + target.messages.overwrite, + &source.messages.table_location, + &source.messages.commit_user, + source.messages.overwrite, + ) { + return error; + } + match merge_messages_idempotently(&mut target.messages.messages, &source.messages.messages) + { + Ok(()) => ptr::null_mut(), + Err(error) => error, + } + })); + outcome.unwrap_or_else(|_| prepared_panic_error("paimon_prepared_commit_merge")) } /// Merge postpone fixed-bucket messages for one logical commit. @@ -896,8 +1513,10 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_commit_messages_merge( ) { return error; } - target.messages.extend(source.messages.clone()); - ptr::null_mut() + match merge_messages_idempotently(&mut target.messages, &source.messages) { + Ok(()) => ptr::null_mut(), + Err(error) => error, + } } // ======================= Commit operations =============================== @@ -929,11 +1548,107 @@ fn validate_commit_context( Ok(()) } +/// Commit a durable prepared commit using the retry-safe identifier path. +/// +/// This is the correct operation after restoring a prepared commit or after a +/// previous commit returned an indeterminate transport/IO error. A successful +/// earlier commit with the same `(commit_user, commit_identifier)` is filtered. +#[no_mangle] +pub unsafe extern "C" fn paimon_table_commit_commit_prepared( + tc: *const paimon_table_commit, + prepared: *const paimon_prepared_commit, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(tc, "tc") { + return error; + } + if let Err(error) = check_non_null(prepared, "prepared") { + return error; + } + let table_commit = &*((*tc).inner as *const TableCommitState); + let prepared = &*((*prepared).inner as *const PreparedCommitState); + let messages = &prepared.messages; + if let Err(error) = validate_commit_context( + &table_commit.table_location, + &table_commit.commit_user, + table_commit.overwrite, + &messages.table_location, + &messages.commit_user, + messages.overwrite, + ) { + return error; + } + let result = if messages.overwrite { + runtime().block_on(table_commit.commit.overwrite_with_identifier( + messages.messages.clone(), + None, + prepared.commit_identifier, + )) + } else { + runtime().block_on(table_commit.commit.filter_and_commit_with_identifier( + messages.messages.clone(), + prepared.commit_identifier, + )) + }; + match result { + Ok(()) => ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| prepared_panic_error("paimon_table_commit_commit_prepared")) +} + +/// Abort files referenced by a durable prepared commit. +/// +/// Do not call this after an indeterminate commit response: retry +/// `paimon_table_commit_commit_prepared` first so a successful commit is not +/// followed by deletion of its files. The caller must also fence/serialize all +/// commit and abort operations for the same `(table, commit_user)` across +/// processes. If retained snapshot history cannot prove that abort is safe, +/// this function fails closed and deletes nothing. +#[no_mangle] +pub unsafe extern "C" fn paimon_table_commit_abort_prepared( + tc: *const paimon_table_commit, + prepared: *const paimon_prepared_commit, +) -> *mut paimon_error { + let outcome = catch_unwind(AssertUnwindSafe(|| { + if let Err(error) = check_non_null(tc, "tc") { + return error; + } + if let Err(error) = check_non_null(prepared, "prepared") { + return error; + } + let table_commit = &*((*tc).inner as *const TableCommitState); + let prepared = &*((*prepared).inner as *const PreparedCommitState); + let messages = &prepared.messages; + if let Err(error) = validate_commit_context( + &table_commit.table_location, + &table_commit.commit_user, + table_commit.overwrite, + &messages.table_location, + &messages.commit_user, + messages.overwrite, + ) { + return error; + } + match runtime().block_on( + table_commit + .commit + .abort_if_uncommitted(&messages.messages, prepared.commit_identifier), + ) { + Ok(()) => ptr::null_mut(), + Err(error) => paimon_error::from_paimon(error), + } + })); + outcome.unwrap_or_else(|_| prepared_panic_error("paimon_table_commit_abort_prepared")) +} + unsafe fn standard_commit_with_identifier_impl( tc: *const paimon_table_commit, msgs: *mut paimon_commit_messages, commit_identifier: i64, filter_committed: bool, + batch_commit: bool, ) -> *mut paimon_error { if let Err(error) = check_non_null(tc, "tc") { return error; @@ -941,6 +1656,11 @@ unsafe fn standard_commit_with_identifier_impl( if let Err(error) = check_non_null(msgs, "msgs") { return error; } + if commit_identifier < 0 || (!batch_commit && commit_identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const TableCommitState); let messages = &*((*msgs).inner as *const CommitMessagesState); if let Err(error) = validate_commit_context( @@ -959,7 +1679,9 @@ unsafe fn standard_commit_with_identifier_impl( ); } let messages = messages.messages.clone(); - let result = if filter_committed { + let result = if batch_commit { + runtime().block_on(table_commit.commit.commit(messages)) + } else if filter_committed { runtime().block_on( table_commit .commit @@ -984,7 +1706,7 @@ pub unsafe extern "C" fn paimon_table_commit_commit( tc: *const paimon_table_commit, msgs: *mut paimon_commit_messages, ) -> *mut paimon_error { - paimon_table_commit_commit_with_identifier(tc, msgs, i64::MAX) + standard_commit_with_identifier_impl(tc, msgs, i64::MAX, false, true) } /// Commit standard append messages with an identifier. @@ -994,7 +1716,7 @@ pub unsafe extern "C" fn paimon_table_commit_commit_with_identifier( msgs: *mut paimon_commit_messages, commit_identifier: i64, ) -> *mut paimon_error { - standard_commit_with_identifier_impl(tc, msgs, commit_identifier, false) + standard_commit_with_identifier_impl(tc, msgs, commit_identifier, false, false) } /// Filter a committed identifier before committing standard append messages. @@ -1004,7 +1726,7 @@ pub unsafe extern "C" fn paimon_table_commit_filter_and_commit_with_identifier( msgs: *mut paimon_commit_messages, commit_identifier: i64, ) -> *mut paimon_error { - standard_commit_with_identifier_impl(tc, msgs, commit_identifier, true) + standard_commit_with_identifier_impl(tc, msgs, commit_identifier, true, false) } /// Commit standard overwrite messages. @@ -1037,6 +1759,11 @@ unsafe fn standard_overwrite_impl( if let Err(error) = check_non_null(msgs, "msgs") { return error; } + if commit_identifier.is_some_and(|identifier| identifier < 0 || identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const TableCommitState); let messages = &*((*msgs).inner as *const CommitMessagesState); if let Err(error) = validate_commit_context( @@ -1093,6 +1820,11 @@ unsafe fn paimon_table_commit_truncate_table_impl( if let Err(error) = check_non_null(tc, "tc") { return error; } + if commit_identifier.is_some_and(|identifier| identifier < 0 || identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const TableCommitState); let result = match commit_identifier { Some(commit_identifier) => runtime().block_on( @@ -1143,6 +1875,7 @@ unsafe fn fixed_commit_with_identifier_impl( msgs: *mut paimon_postpone_fixed_bucket_commit_messages, commit_identifier: i64, filter_committed: bool, + batch_commit: bool, ) -> *mut paimon_error { if let Err(error) = check_non_null(tc, "tc") { return error; @@ -1150,6 +1883,11 @@ unsafe fn fixed_commit_with_identifier_impl( if let Err(error) = check_non_null(msgs, "msgs") { return error; } + if commit_identifier < 0 || (!batch_commit && commit_identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const PostponeFixedBucketTableCommitState); let messages = &*((*msgs).inner as *const PostponeFixedBucketCommitMessagesState); if let Err(error) = validate_commit_context( @@ -1163,7 +1901,9 @@ unsafe fn fixed_commit_with_identifier_impl( return error; } let messages = messages.messages.clone(); - let result = if filter_committed { + let result = if batch_commit { + runtime().block_on(table_commit.commit.commit(messages)) + } else if filter_committed { runtime().block_on( table_commit .commit @@ -1188,7 +1928,7 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_table_commit_commit( tc: *const paimon_postpone_fixed_bucket_table_commit, msgs: *mut paimon_postpone_fixed_bucket_commit_messages, ) -> *mut paimon_error { - paimon_postpone_fixed_bucket_table_commit_commit_with_identifier(tc, msgs, i64::MAX) + fixed_commit_with_identifier_impl(tc, msgs, i64::MAX, false, true) } /// Commit postpone fixed-bucket messages with an identifier. @@ -1198,7 +1938,7 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_table_commit_commit_with_i msgs: *mut paimon_postpone_fixed_bucket_commit_messages, commit_identifier: i64, ) -> *mut paimon_error { - fixed_commit_with_identifier_impl(tc, msgs, commit_identifier, false) + fixed_commit_with_identifier_impl(tc, msgs, commit_identifier, false, false) } /// Filter a committed identifier before committing fixed-bucket messages. @@ -1208,7 +1948,7 @@ pub unsafe extern "C" fn paimon_postpone_fixed_bucket_table_commit_filter_and_co msgs: *mut paimon_postpone_fixed_bucket_commit_messages, commit_identifier: i64, ) -> *mut paimon_error { - fixed_commit_with_identifier_impl(tc, msgs, commit_identifier, true) + fixed_commit_with_identifier_impl(tc, msgs, commit_identifier, true, false) } /// Truncate a table with a postpone fixed-bucket TableCommit. @@ -1235,6 +1975,11 @@ unsafe fn fixed_truncate_table_impl( if let Err(error) = check_non_null(tc, "tc") { return error; } + if commit_identifier.is_some_and(|identifier| identifier < 0 || identifier == i64::MAX) { + return invalid_input( + "streaming commit_identifier must be non-negative and less than i64::MAX", + ); + } let table_commit = &*((*tc).inner as *const PostponeFixedBucketTableCommitState); let result = match commit_identifier { Some(commit_identifier) => runtime().block_on( @@ -1322,6 +2067,16 @@ const _: unsafe extern "C" fn( *mut paimon_commit_messages, *const paimon_commit_messages, ) -> *mut paimon_error = paimon_commit_messages_merge; +const _: unsafe extern "C" fn(*const paimon_commit_messages, i64) -> paimon_result_prepared_commit = + paimon_commit_messages_prepare; +const _: unsafe extern "C" fn(*const paimon_prepared_commit) -> paimon_result_bytes = + paimon_prepared_commit_serialize; +const _: unsafe extern "C" fn(*const u8, usize) -> paimon_result_prepared_commit = + paimon_prepared_commit_deserialize; +const _: unsafe extern "C" fn( + *mut paimon_prepared_commit, + *const paimon_prepared_commit, +) -> *mut paimon_error = paimon_prepared_commit_merge; const _: unsafe extern "C" fn( *mut paimon_postpone_fixed_bucket_commit_messages, *const paimon_postpone_fixed_bucket_commit_messages, @@ -1382,7 +2137,39 @@ const _: unsafe extern "C" fn( *const paimon_table_commit, *mut paimon_commit_messages, ) -> *mut paimon_error = paimon_table_commit_abort; +const _: unsafe extern "C" fn( + *const paimon_table_commit, + *const paimon_prepared_commit, +) -> *mut paimon_error = paimon_table_commit_commit_prepared; +const _: unsafe extern "C" fn( + *const paimon_table_commit, + *const paimon_prepared_commit, +) -> *mut paimon_error = paimon_table_commit_abort_prepared; const _: unsafe extern "C" fn( *const paimon_postpone_fixed_bucket_table_commit, *mut paimon_postpone_fixed_bucket_commit_messages, ) -> *mut paimon_error = paimon_postpone_fixed_bucket_table_commit_abort; + +#[cfg(test)] +mod raw_message_limit_tests { + use super::{RawPreparedCommitEnvelope, MAX_PREPARED_MESSAGES}; + + #[test] + fn raw_message_count_is_rejected_during_deserialization() { + let messages = (0..=MAX_PREPARED_MESSAGES) + .map(|_| "{}") + .collect::>() + .join(","); + let json = format!( + r#"{{"format":"paimon-rust-prepared-commit","version":2,"commit_identifier":1,"table_location":"memory:/table","commit_user":"job","overwrite":false,"messages":[{messages}]}}"# + ); + let error = match serde_json::from_str::>(&json) { + Ok(_) => panic!("oversized raw message list must be rejected"), + Err(error) => error, + }; + assert!(error.to_string().contains("more than")); + assert!(error + .to_string() + .contains(&MAX_PREPARED_MESSAGES.to_string())); + } +} diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt new file mode 100644 index 000000000..5c20ece08 --- /dev/null +++ b/bindings/cpp/CMakeLists.txt @@ -0,0 +1,331 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 3.15) +project(PaimonCpp VERSION 0.1.0 LANGUAGES CXX) + +include(CMakePackageConfigHelpers) +include(GNUInstallDirs) + +option(PAIMON_CPP_BUILD_EXAMPLES "Build the C++ facade examples" OFF) +option(PAIMON_CPP_BUILD_TESTS "Build the header compile smoke test" OFF) +get_filename_component( + paimon_rust_root "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE) +get_filename_component( + paimon_c_binding_dir "${CMAKE_CURRENT_SOURCE_DIR}/../c" ABSOLUTE) +set(paimon_c_generated_include_dir + "${CMAKE_CURRENT_BINARY_DIR}/generated/include") +set(paimon_c_generated_header + "${paimon_c_generated_include_dir}/paimon.h") + +if(TARGET Paimon::c) + message( + FATAL_ERROR + "bindings/cpp owns Paimon::c and does not accept an external C ABI target") +endif() + +if(DEFINED ENV{CARGO} AND EXISTS "$ENV{CARGO}") + set(paimon_cargo_executable "$ENV{CARGO}") +elseif(DEFINED ENV{HOME} AND EXISTS "$ENV{HOME}/.cargo/bin/cargo") + set(paimon_cargo_executable "$ENV{HOME}/.cargo/bin/cargo") +else() + set(paimon_find_appbundle "${CMAKE_FIND_APPBUNDLE}") + set(CMAKE_FIND_APPBUNDLE NEVER) + find_program(paimon_cargo_executable NAMES cargo) + set(CMAKE_FIND_APPBUNDLE "${paimon_find_appbundle}") +endif() +if(NOT paimon_cargo_executable) + message(FATAL_ERROR "Rust Cargo was not found") +endif() +if(DEFINED ENV{CBINDGEN} AND EXISTS "$ENV{CBINDGEN}") + set(paimon_cbindgen_executable "$ENV{CBINDGEN}") +else() + set(paimon_find_appbundle "${CMAKE_FIND_APPBUNDLE}") + set(CMAKE_FIND_APPBUNDLE NEVER) + find_program(paimon_cbindgen_executable NAMES cbindgen) + set(CMAKE_FIND_APPBUNDLE "${paimon_find_appbundle}") +endif() +if(NOT paimon_cbindgen_executable) + message(FATAL_ERROR "cbindgen was not found") +endif() +execute_process( + COMMAND "${paimon_cargo_executable}" -vV + RESULT_VARIABLE paimon_cargo_version_result + OUTPUT_VARIABLE paimon_cargo_version + OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT paimon_cargo_version_result EQUAL 0 OR + NOT paimon_cargo_version MATCHES "^cargo [0-9]") + message(FATAL_ERROR "Not a Rust Cargo executable: ${paimon_cargo_executable}") +endif() +string(REGEX MATCH "host: ([^\n\r]+)" paimon_cargo_host_match + "${paimon_cargo_version}") +if(NOT paimon_cargo_host_match) + message(FATAL_ERROR "Cargo did not report its host target") +endif() +set(paimon_rust_host "${CMAKE_MATCH_1}") +set(paimon_cargo_target_dir "${paimon_rust_root}/target") +set(paimon_c_release_dir + "${paimon_cargo_target_dir}/${paimon_rust_host}/release") +if(APPLE) + set(paimon_c_library "${paimon_c_release_dir}/libpaimon_c.dylib") +elseif(UNIX) + set(paimon_c_library "${paimon_c_release_dir}/libpaimon_c.so") +elseif(WIN32) + set(paimon_c_library "${paimon_c_release_dir}/paimon_c.dll") +else() + message(FATAL_ERROR "Unsupported platform for automatic paimon-c build") +endif() +set(paimon_c_build_command + "${CMAKE_COMMAND}" -E env "MAKEFLAGS=" + "CARGO_TARGET_DIR=${paimon_cargo_target_dir}" + "${paimon_cargo_executable}" build --locked --release -p paimon-c + --target "${paimon_rust_host}") + +add_custom_target( + paimon_c_cargo_build ALL + COMMAND ${paimon_c_build_command} + WORKING_DIRECTORY "${paimon_rust_root}" + BYPRODUCTS "${paimon_c_library}" + COMMENT "Building the in-tree Rust paimon-c library" + VERBATIM + USES_TERMINAL) + +file( + GLOB_RECURSE paimon_c_header_sources CONFIGURE_DEPENDS + "${paimon_c_binding_dir}/src/*.rs") +add_custom_command( + OUTPUT "${paimon_c_generated_header}" + COMMAND + "${CMAKE_COMMAND}" -E make_directory + "${paimon_c_generated_include_dir}" + COMMAND + "${paimon_cbindgen_executable}" --quiet + --config "${paimon_c_binding_dir}/cbindgen.toml" + "${paimon_c_binding_dir}" + --output "${paimon_c_generated_header}" + DEPENDS + ${paimon_c_header_sources} + "${paimon_c_binding_dir}/Cargo.toml" + "${paimon_c_binding_dir}/cbindgen.toml" + COMMENT "Generating paimon.h from the Rust C ABI" + VERBATIM) +add_custom_target( + paimon_c_header ALL DEPENDS "${paimon_c_generated_header}") +add_dependencies(paimon_c_header paimon_c_cargo_build) + +get_filename_component( + PAIMON_C_INSTALL_FILENAME "${paimon_c_library}" NAME) + +# Keep the C ABI dependency as a target in both the build and install trees. +# PaimonCppConfig.cmake recreates this imported target for installed consumers. +add_library(Paimon::c SHARED IMPORTED GLOBAL) +set_target_properties( + Paimon::c + PROPERTIES + IMPORTED_LOCATION "${paimon_c_library}" + IMPORTED_NO_SONAME TRUE + INTERFACE_INCLUDE_DIRECTORIES "${paimon_c_generated_include_dir}") +add_dependencies(Paimon::c paimon_c_cargo_build paimon_c_header) + +add_library(paimon_cpp INTERFACE) +add_library(Paimon::cpp ALIAS paimon_cpp) +add_dependencies(paimon_cpp paimon_c_cargo_build paimon_c_header) +set_target_properties(paimon_cpp PROPERTIES EXPORT_NAME cpp) +target_compile_features(paimon_cpp INTERFACE cxx_std_17) +target_include_directories( + paimon_cpp + INTERFACE + "$" + "$") + +target_include_directories( + paimon_cpp INTERFACE "$") + +target_link_libraries(paimon_cpp INTERFACE Paimon::c) + +if(PAIMON_CPP_BUILD_EXAMPLES) + add_executable(paimon_cpp_batch_read examples/batch_read.cpp) + target_link_libraries(paimon_cpp_batch_read PRIVATE Paimon::cpp) + add_executable(paimon_cpp_streaming_write examples/streaming_write.cpp) + target_link_libraries(paimon_cpp_streaming_write PRIVATE Paimon::cpp) + add_executable(paimon_cpp_stream_read examples/stream_read.cpp) + target_link_libraries(paimon_cpp_stream_read PRIVATE Paimon::cpp) +endif() + +if(PAIMON_CPP_BUILD_TESTS) + enable_testing() + add_library(paimon_cpp_header_smoke OBJECT tests/header_smoke.cpp) + target_compile_definitions( + paimon_cpp_header_smoke + PRIVATE PAIMON_C_HEADER="paimon_test_stub.h") + target_include_directories( + paimon_cpp_header_smoke PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/tests") + target_link_libraries(paimon_cpp_header_smoke PRIVATE Paimon::cpp) + add_test( + NAME paimon_cpp_header_compile_smoke + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} + --target paimon_cpp_header_smoke) + + set(paimon_cpp_install_test_root + "${CMAKE_CURRENT_BINARY_DIR}/install-tree-consumer-test") + add_test( + NAME paimon_cpp_install_tree_consumer + COMMAND + "${CMAKE_COMMAND}" + "-DMAIN_BUILD_DIR=${CMAKE_BINARY_DIR}" + "-DCONSUMER_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/tests/install_tree_consumer" + "-DTEST_ROOT=${paimon_cpp_install_test_root}" + "-DCXX_COMPILER=${CMAKE_CXX_COMPILER}" + "-DINSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/tests/run_install_tree_consumer.cmake") +endif() + +set(paimon_cpp_install_component PaimonCppSdk) +install( + DIRECTORY include/ + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT ${paimon_cpp_install_component}) +install( + FILES "${paimon_c_generated_header}" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT ${paimon_cpp_install_component}) +install( + PROGRAMS "${paimon_c_library}" + DESTINATION "${CMAKE_INSTALL_LIBDIR}" + COMPONENT ${paimon_cpp_install_component}) +if(APPLE) + find_program(paimon_install_name_tool NAMES install_name_tool) + find_program(paimon_codesign NAMES codesign) + if(NOT paimon_install_name_tool OR NOT paimon_codesign) + message(FATAL_ERROR "install_name_tool and codesign are required on macOS") + endif() + install( + CODE + "set(paimon_installed_library \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/${PAIMON_C_INSTALL_FILENAME}\") + execute_process( + COMMAND \"${paimon_install_name_tool}\" -id \"@rpath/${PAIMON_C_INSTALL_FILENAME}\" \"\${paimon_installed_library}\" + RESULT_VARIABLE paimon_install_name_result) + if(NOT paimon_install_name_result EQUAL 0) + message(FATAL_ERROR \"failed to set the paimon-c install name\") + endif() + execute_process( + COMMAND \"${paimon_codesign}\" --force --sign - \"\${paimon_installed_library}\" + RESULT_VARIABLE paimon_codesign_result) + if(NOT paimon_codesign_result EQUAL 0) + message(FATAL_ERROR \"failed to sign the installed paimon-c library\") + endif()" + COMPONENT ${paimon_cpp_install_component}) +endif() +install( + TARGETS paimon_cpp + EXPORT PaimonCppTargets + INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +install( + EXPORT PaimonCppTargets + FILE PaimonCppTargets.cmake + NAMESPACE Paimon:: + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp" + COMPONENT ${paimon_cpp_install_component}) + +configure_package_config_file( + cmake/PaimonCppConfig.cmake.in + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfig.cmake" + INSTALL_DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp") +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfigVersion.cmake" + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion) +install( + FILES + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/PaimonCpp" + COMPONENT ${paimon_cpp_install_component}) +install( + FILES "${paimon_rust_root}/LICENSE" "${paimon_rust_root}/NOTICE" + DESTINATION "${CMAKE_INSTALL_DATADIR}/doc/paimon-cpp" + COMPONENT ${paimon_cpp_install_component}) + +# Keep the normal CMake install target, and also materialize the ready-to-use +# headers, library, and package files directly in the CMake build directory. +add_custom_target( + paimon_cpp_artifacts ALL + COMMAND + "${CMAKE_COMMAND}" -E env "DESTDIR=" + "${CMAKE_COMMAND}" --install "${CMAKE_BINARY_DIR}" + --prefix "${CMAKE_CURRENT_BINARY_DIR}" + --component ${paimon_cpp_install_component} + DEPENDS paimon_c_cargo_build paimon_c_header + COMMENT "Staging the Paimon C++ artifacts in ${CMAKE_CURRENT_BINARY_DIR}" + VERBATIM + USES_TERMINAL) + +# A Linux package build emits the native package formats plus a portable +# archive in one CPack invocation. All three contain the same binary, so build +# on the oldest Linux/OpenSSL ABI baseline that the resulting packages support. +set(CPACK_PACKAGE_NAME "paimon-cpp-sdk") +set(CPACK_PACKAGE_VENDOR "Apache Software Foundation") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY + "Apache Paimon C and header-only C++ SDK") +set(CPACK_PACKAGE_HOMEPAGE_URL "https://paimon.apache.org/") +set(CPACK_PACKAGE_CONTACT "dev@paimon.apache.org") +set(CPACK_PACKAGE_VERSION "${PROJECT_VERSION}") +set(CPACK_RESOURCE_FILE_LICENSE "${paimon_rust_root}/LICENSE") +set(CPACK_PACKAGE_DIRECTORY "${CMAKE_BINARY_DIR}/packages") +set(CPACK_PACKAGE_CHECKSUM SHA256) +set(CPACK_MONOLITHIC_INSTALL ON) +set(CPACK_PACKAGE_RELOCATABLE FALSE) + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(CPACK_GENERATOR "DEB;RPM;TGZ") + set(CPACK_PACKAGING_INSTALL_PREFIX "/usr") + + set(CPACK_DEBIAN_PACKAGE_NAME "paimon-cpp-dev") + set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Apache Paimon Developers") + set(CPACK_DEBIAN_PACKAGE_SECTION "libdevel") + set(CPACK_DEBIAN_PACKAGE_RELEASE "1") + set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) + if(paimon_rust_host MATCHES "^x86_64-") + set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE amd64) + elseif(paimon_rust_host MATCHES "^aarch64-") + set(CPACK_DEBIAN_PACKAGE_ARCHITECTURE arm64) + else() + message( + FATAL_ERROR + "No Debian architecture mapping for Rust host ${paimon_rust_host}") + endif() + + set(CPACK_RPM_PACKAGE_NAME "paimon-cpp-devel") + set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") + set(CPACK_RPM_PACKAGE_GROUP "Development/Libraries") + set(CPACK_RPM_PACKAGE_RELEASE "1") + set(CPACK_RPM_PACKAGE_RELEASE_DIST ON) + set(CPACK_RPM_PACKAGE_AUTOREQ ON) + set(CPACK_RPM_PACKAGE_RELOCATABLE FALSE) + set(CPACK_RPM_FILE_NAME RPM-DEFAULT) + + configure_file( + cmake/PaimonCppCPackOptions.cmake.in + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppCPackOptions.cmake" + @ONLY) + set(CPACK_PROJECT_CONFIG_FILE + "${CMAKE_CURRENT_BINARY_DIR}/PaimonCppCPackOptions.cmake") +else() + set(CPACK_GENERATOR TGZ) +endif() + +include(CPack) diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md new file mode 100644 index 000000000..dd9d122bf --- /dev/null +++ b/bindings/cpp/README.md @@ -0,0 +1,160 @@ + + +# Paimon C++ facade + +This directory provides a header-only C++17 RAII facade over the stable Paimon +C ABI. It deliberately builds no C++ shared library: the only Paimon binary is +`libpaimon_c`, produced by Rust, and every symbol called by the facade has C +linkage. The facade does not depend on Arrow C++; Arrow batches cross the API as +raw Arrow C Data `array` and `schema` pointers. + +All native handles are move-only. Their destructors are `noexcept` and only +release resources. In particular, destroying `PreparedMessages` or +`PreparedCommit` never commits or aborts them. A streaming writer binds messages +to a checkpoint with `PreparedMessages::prepare`, persists the bytes from +`PreparedCommit::serialize`, then calls `TableCommit::commit_prepared`. After an +uncertain result or process restart, deserialize the same bytes and retry with +the same stable `commit_user`; the identifier path filters duplicate commits. +Exactly-once filtering is recorded in retained snapshot metadata. Keep snapshot +history for at least the maximum writer-recovery horizon, never retry a +checkpoint older than that horizon, and give each fresh job a new globally +unique `commit_user`. Checkpoint identifiers must be in `[0, INT64_MAX)`; +`INT64_MAX` is reserved for unidentified batch commits. +Checkpoint blobs are trusted state, not a security token: store them behind +normal integrity/access controls. Before `abort_prepared`, fence every commit +and abort for the same `(table, commit_user)` across processes. If snapshot +history is too old to prove safety, abort fails closed and leaves cleanup to an +orphan-file policy. +Continuous reading is a pull API. `StreamScan::poll` immediately returns data, +waiting, or end; it never starts a callback thread and never waits for a future +snapshot. A data result owns a `StreamPlan`, which can be read in data or audit +log mode using the same Arrow C Data `RecordBatchReader` as bounded reads. +Each `StreamScan` is single-thread-confined; serialize poll, checkpoint, +restore, and destruction. Decoupled changelog fallback and consumer-retention +registration are not implemented yet, so snapshot retention must cover the +maximum expected reader lag. +Persisted stream plans currently reject external data-file paths. The failure +is reported by `StreamPlan::serialize` before a checkpoint can be acknowledged, +instead of producing a checkpoint that cannot be restored. + +Catalog DDL is available directly from the facade. Creation accepts the JSON +form of Paimon's logical `Schema`; it validates and canonically reassigns field +IDs before calling the catalog. Both operations return `Status`, so callers can +choose strict or idempotent create/drop semantics without a Java helper: + +```cpp +auto identifier = paimon::Identifier::create("default", "events"); +auto created = catalog.create_table_from_schema_json( + identifier.value(), schema_json, /*ignore_if_exists=*/false); +auto dropped = catalog.drop_table( + identifier.value(), /*ignore_if_not_exists=*/true); +``` + +## Build + +Configure and build the C++ facade directly. The build always compiles the +in-tree `bindings/c` crate first, so the C and C++ layers come from the same +source revision: + +```bash +cmake -S bindings/cpp -B target/cpp-build \ + -DPAIMON_CPP_BUILD_EXAMPLES=ON +cmake --build target/cpp-build +``` + +The CMake build directory itself is a complete, directly consumable artifact +tree: + +```text +target/cpp-build/ +├── include/paimon.h +├── include/paimon/paimon.hpp +├── /libpaimon_c.so # Linux +└── /cmake/PaimonCpp/ +``` + +`` follows GNUInstallDirs and is normally `lib` or `lib64`. macOS uses +`libpaimon_c.dylib` in the same location. The facade is header-only, so there is +intentionally no separate `libpaimon_cpp` shared library. + +All platforms use `cargo build --locked --release -p paimon-c`. Build Linux +release artifacts on the oldest glibc version that must be supported; glibc is +backward compatible with binaries built against older symbol versions. +External prebuilt paimon-c libraries and parent-provided `Paimon::c` targets +are deliberately unsupported. + +Linux builds use the OpenSSL selected by the locked `openssl-sys` dependency +and link it dynamically. OpenSSL 1.0.2 is not supported by the current lock; +an old-glibc build host must provide a parallel OpenSSL 1.1 or newer development +installation. The resulting package requires that exact OpenSSL SONAME at +runtime. + +Source builds require `cbindgen`. CMake regenerates `paimon.h` from the Rust C +ABI in `target/cpp-build/generated/include`; the generated header is not stored +in Git. The staged build tree and every installed package still contain it at +`include/paimon.h`. + +Install the CMake interface target elsewhere when needed: + +```bash +cmake --install target/cpp-build --prefix /your/prefix +``` + +Installation always bundles the just-built `libpaimon_c` and exports +`Paimon::c` plus the header-only `Paimon::cpp` target. A consumer only needs: + +```cmake +find_package(PaimonCpp CONFIG REQUIRED) +add_executable(my_paimon_app main.cpp) +target_link_libraries(my_paimon_app PRIVATE Paimon::cpp) +``` + +## Linux packages + +The CPack `package` target builds all supported Linux package formats in one +run after compiling the in-tree Rust library: + +```bash +cmake -S bindings/cpp -B target/cpp-build +cmake --build target/cpp-build --target package +ls target/cpp-build/packages +``` + +It produces a Debian/Ubuntu `paimon-cpp-dev` DEB, an RPM-family +`paimon-cpp-devel` RPM, and a `paimon-cpp-sdk` TGZ, plus a SHA-256 checksum for +each package. Building the RPM requires the distribution's `rpmbuild` tool. +Install the native package with, for example: + +```bash +sudo apt install ./paimon-cpp-dev_*.deb +sudo dnf install ./paimon-cpp-devel-*.rpm +``` + +All formats from one run contain the same `libpaimon_c.so`. Package format does +not change its glibc or OpenSSL ABI: build on each binary compatibility baseline +that customers need. Publish a DEB from a Debian/Ubuntu baseline and an RPM from +an RPM-family baseline so native library-directory and dependency conventions +match the target distribution. The TGZ is the format-neutral fallback and +contains the same `/usr` installation tree. + +`Scan::plan()` remains a bounded scan. Use `StreamScanOptions` and +`ReadBuilder::new_stream_scan` for a stateful continuous scan. Persist +`StreamScan::checkpoint()` only after every split in the returned plan has been +durably accounted for by the surrounding checkpoint barrier. diff --git a/bindings/cpp/cmake/PaimonCppCPackOptions.cmake.in b/bindings/cpp/cmake/PaimonCppCPackOptions.cmake.in new file mode 100644 index 000000000..2de9a4b30 --- /dev/null +++ b/bindings/cpp/cmake/PaimonCppCPackOptions.cmake.in @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# CPack loads this file once per generator. The RPM generator derives ELF +# requirements itself; for a DEB built on a non-Debian host, derive the Ubuntu +# OpenSSL package from the actual SONAME linked by libpaimon_c. +if(CPACK_GENERATOR STREQUAL "DEB") + execute_process( + COMMAND "@CMAKE_READELF@" --dynamic "@paimon_c_library@" + RESULT_VARIABLE paimon_readelf_result + OUTPUT_VARIABLE paimon_dynamic_section + ERROR_VARIABLE paimon_readelf_error) + if(NOT paimon_readelf_result EQUAL 0) + message( + FATAL_ERROR + "Cannot inspect libpaimon_c dependencies: ${paimon_readelf_error}") + endif() + + execute_process( + COMMAND "@CMAKE_READELF@" --version-info "@paimon_c_library@" + RESULT_VARIABLE paimon_version_info_result + OUTPUT_VARIABLE paimon_version_info + ERROR_VARIABLE paimon_version_info_error) + if(NOT paimon_version_info_result EQUAL 0) + message( + FATAL_ERROR + "Cannot inspect libpaimon_c symbol versions: ${paimon_version_info_error}") + endif() + string( + REGEX MATCHALL "GLIBC_[0-9]+\\.[0-9]+(\\.[0-9]+)?" + paimon_glibc_symbols "${paimon_version_info}") + set(paimon_minimum_glibc 0) + foreach(paimon_glibc_symbol IN LISTS paimon_glibc_symbols) + string(REPLACE "GLIBC_" "" paimon_glibc_version "${paimon_glibc_symbol}") + if(paimon_glibc_version VERSION_GREATER paimon_minimum_glibc) + set(paimon_minimum_glibc "${paimon_glibc_version}") + endif() + endforeach() + if(paimon_minimum_glibc STREQUAL 0) + message(FATAL_ERROR "No GLIBC symbol versions found in libpaimon_c") + endif() + + set(paimon_debian_dependencies "libc6 (>= ${paimon_minimum_glibc})") + if(paimon_dynamic_section MATCHES "libssl\\.so\\.3") + list(APPEND paimon_debian_dependencies libssl3) + elseif(paimon_dynamic_section MATCHES "libssl\\.so\\.1\\.1") + list(APPEND paimon_debian_dependencies libssl1.1) + elseif(paimon_dynamic_section MATCHES "libssl\\.so") + message( + FATAL_ERROR + "Unsupported OpenSSL SONAME in libpaimon_c; set an explicit DEB mapping") + endif() + if(paimon_dynamic_section MATCHES "libgcc_s\\.so") + list(APPEND paimon_debian_dependencies "libgcc-s1 | libgcc1") + endif() + list(JOIN paimon_debian_dependencies ", " CPACK_DEBIAN_PACKAGE_DEPENDS) +endif() diff --git a/bindings/cpp/cmake/PaimonCppConfig.cmake.in b/bindings/cpp/cmake/PaimonCppConfig.cmake.in new file mode 100644 index 000000000..a9e30c487 --- /dev/null +++ b/bindings/cpp/cmake/PaimonCppConfig.cmake.in @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +@PACKAGE_INIT@ + +# CMake may discover /usr/lib64 packages through the /lib64 -> /usr/lib64 +# symlink. Resolve the config directory before deriving the install prefix so +# imported include and library paths still point at /usr. +get_filename_component( + _paimon_cpp_config_dir "${CMAKE_CURRENT_LIST_DIR}" REALPATH) +get_filename_component( + PACKAGE_PREFIX_DIR "${_paimon_cpp_config_dir}/../../.." ABSOLUTE) +unset(_paimon_cpp_config_dir) + +if(TARGET Paimon::cpp) + if(NOT TARGET Paimon::c) + set(PaimonCpp_FOUND FALSE) + set(PaimonCpp_NOT_FOUND_MESSAGE + "Paimon::cpp exists without its bundled Paimon::c target") + return() + endif() + check_required_components(PaimonCpp) + return() +endif() + +if(TARGET Paimon::c) + set(PaimonCpp_FOUND FALSE) + set(PaimonCpp_NOT_FOUND_MESSAGE + "Paimon::c already exists; PaimonCpp requires its bundled paimon-c library") + return() +endif() + +set(_paimon_c_library + "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_LIBDIR@/@PAIMON_C_INSTALL_FILENAME@") +if(NOT EXISTS "${_paimon_c_library}") + set(PaimonCpp_FOUND FALSE) + set(PaimonCpp_NOT_FOUND_MESSAGE + "the bundled paimon-c library is missing: ${_paimon_c_library}") + return() +endif() + +add_library(Paimon::c SHARED IMPORTED) +set_target_properties( + Paimon::c + PROPERTIES + IMPORTED_LOCATION "${_paimon_c_library}" + IMPORTED_NO_SONAME TRUE + INTERFACE_INCLUDE_DIRECTORIES "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") + +include("${CMAKE_CURRENT_LIST_DIR}/PaimonCppTargets.cmake") +set_target_properties( + Paimon::cpp + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES + "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") +check_required_components(PaimonCpp) diff --git a/bindings/cpp/examples/batch_read.cpp b/bindings/cpp/examples/batch_read.cpp new file mode 100644 index 000000000..d8654179d --- /dev/null +++ b/bindings/cpp/examples/batch_read.cpp @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +namespace { + +void print_error(const paimon::Error& error) { + const auto message = error.message(); + std::fprintf(stderr, "Paimon error %d: %.*s\n", + static_cast(error.code()), + static_cast(message.size()), message.data()); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 4) { + std::fprintf(stderr, "usage: %s WAREHOUSE DATABASE TABLE\n", argv[0]); + return EXIT_FAILURE; + } + + const paimon::Option options[] = {{"warehouse", argv[1]}}; + auto catalog_result = paimon::Catalog::create(options); + if (!catalog_result) { + print_error(catalog_result.error()); + return EXIT_FAILURE; + } + auto catalog = std::move(catalog_result).value(); + + auto identifier_result = paimon::Identifier::create(argv[2], argv[3]); + if (!identifier_result) { + print_error(identifier_result.error()); + return EXIT_FAILURE; + } + auto identifier = std::move(identifier_result).value(); + + auto table_result = catalog.get_table(identifier); + if (!table_result) { + print_error(table_result.error()); + return EXIT_FAILURE; + } + auto table = std::move(table_result).value(); + + auto builder_result = table.new_read_builder(); + if (!builder_result) { + print_error(builder_result.error()); + return EXIT_FAILURE; + } + auto builder = std::move(builder_result).value(); + + auto scan_result = builder.new_scan(); + auto read_result = builder.new_read(); + if (!scan_result || !read_result) { + print_error(!scan_result ? scan_result.error() : read_result.error()); + return EXIT_FAILURE; + } + auto scan = std::move(scan_result).value(); + auto read = std::move(read_result).value(); + + auto plan_result = scan.plan(); + if (!plan_result) { + print_error(plan_result.error()); + return EXIT_FAILURE; + } + auto plan = std::move(plan_result).value(); + + auto reader_result = read.to_arrow(plan); + if (!reader_result) { + print_error(reader_result.error()); + return EXIT_FAILURE; + } + auto reader = std::move(reader_result).value(); + + std::size_t batch_count = 0; + for (;;) { + auto next = reader.next(); + if (!next) { + print_error(next.error()); + return EXIT_FAILURE; + } + auto batch = std::move(next).value(); + if (!batch) { + break; + } + // Import batch.array()/batch.schema() with any Arrow C Data consumer here. + // ArrowBatch releases both native containers when it leaves this scope. + ++batch_count; + } + + std::printf("splits=%zu batches=%zu\n", plan.num_splits(), batch_count); + return EXIT_SUCCESS; +} diff --git a/bindings/cpp/examples/stream_read.cpp b/bindings/cpp/examples/stream_read.cpp new file mode 100644 index 000000000..ffdae6a02 --- /dev/null +++ b/bindings/cpp/examples/stream_read.cpp @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +namespace { + +void print_error(const paimon::Error& error) { + const auto message = error.message(); + std::fprintf(stderr, "Paimon error %d: %.*s\n", + static_cast(error.code()), + static_cast(message.size()), message.data()); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 4) { + std::fprintf(stderr, "usage: %s WAREHOUSE DATABASE TABLE\n", argv[0]); + return EXIT_FAILURE; + } + + const paimon::Option catalog_options[] = {{"warehouse", argv[1]}}; + auto catalog_result = paimon::Catalog::create(catalog_options); + auto identifier_result = paimon::Identifier::create(argv[2], argv[3]); + if (!catalog_result || !identifier_result) { + print_error(!catalog_result ? catalog_result.error() + : identifier_result.error()); + return EXIT_FAILURE; + } + auto catalog = std::move(catalog_result).value(); + auto identifier = std::move(identifier_result).value(); + + auto table_result = catalog.get_table(identifier); + if (!table_result) { + print_error(table_result.error()); + return EXIT_FAILURE; + } + auto table = std::move(table_result).value(); + + auto builder_result = table.new_read_builder(); + auto options_result = paimon::StreamScanOptions::defaults(); + if (!builder_result || !options_result) { + print_error(!builder_result ? builder_result.error() + : options_result.error()); + return EXIT_FAILURE; + } + auto builder = std::move(builder_result).value(); + auto options = std::move(options_result).value(); + options.with_startup(paimon::StreamStartupMode::latest_full) + .with_follow_up(paimon::StreamFollowUpMode::automatic); + + auto read_result = builder.new_read(); + auto scan_result = builder.new_stream_scan(options); + if (!read_result || !scan_result) { + print_error(!read_result ? read_result.error() : scan_result.error()); + return EXIT_FAILURE; + } + auto read = std::move(read_result).value(); + auto scan = std::move(scan_result).value(); + + // poll() is a pull operation and never waits. A scheduler should call it + // again later after Waiting; this standalone example exits instead. + auto poll_result = scan.poll(); + if (!poll_result) { + print_error(poll_result.error()); + return EXIT_FAILURE; + } + auto poll = std::move(poll_result).value(); + if (poll.waiting()) { + std::printf("waiting next_snapshot_id=%lld\n", + static_cast(poll.next_snapshot_id())); + return EXIT_SUCCESS; + } + if (poll.end()) { + std::puts("end"); + return EXIT_SUCCESS; + } + + auto pending_plan_result = poll.plan().serialize(); + if (!pending_plan_result) { + print_error(pending_plan_result.error()); + return EXIT_FAILURE; + } + auto pending_plan = std::move(pending_plan_result).value(); + // Persist pending_plan together with the cursor before exposing rows. On + // recovery, StreamPlan::deserialize recreates this PollResult for replay. + std::printf("pending-plan-bytes=%zu\n", pending_plan.size()); + + auto reader_result = poll.plan().read_to_arrow( + read, paimon::StreamReadMode::data); + if (!reader_result) { + print_error(reader_result.error()); + return EXIT_FAILURE; + } + auto reader = std::move(reader_result).value(); + std::size_t batches = 0; + for (;;) { + auto next = reader.next(); + if (!next) { + print_error(next.error()); + return EXIT_FAILURE; + } + auto batch = std::move(next).value(); + if (!batch) { + break; + } + // Import through any Arrow C Data consumer before batch is destroyed. + ++batches; + } + + // Persist this cursor only after the plan's split progress is durably part of + // the surrounding checkpoint barrier. + std::printf("snapshot=%lld splits=%zu batches=%zu checkpoint=%lld\n", + static_cast(poll.snapshot_id()), + poll.plan().num_splits(), batches, + static_cast(scan.checkpoint())); + return EXIT_SUCCESS; +} diff --git a/bindings/cpp/examples/streaming_write.cpp b/bindings/cpp/examples/streaming_write.cpp new file mode 100644 index 000000000..5e8b2fe2b --- /dev/null +++ b/bindings/cpp/examples/streaming_write.cpp @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +// Data and checkpoint barriers are separate events so an idle stream can still +// checkpoint. Paimon consumes Arrow contents in place; the producer continues +// to own the ArrowArray and ArrowSchema container memory. +enum class StreamEventKind : std::uint8_t { batch, checkpoint, end }; + +struct StreamEvent { + StreamEventKind kind; + void* array; + void* schema; +}; + +using Producer = paimon::Status (*)(void* context, StreamEvent* output); +using Persist = paimon::Status (*)(void* context, std::int64_t checkpoint_id, + const std::uint8_t* data, std::size_t size); + +paimon::Status recover_checkpoint(const paimon::TableCommit& committer, + const std::uint8_t* data, + std::size_t size) { + // A zero-length checkpoint records source progress but has no Paimon data to + // commit. The surrounding engine owns that source-state representation. + if (size == 0) { + return paimon::Status::success(); + } + auto restored_result = paimon::PreparedCommit::deserialize(data, size); + if (!restored_result) { + return paimon::Status::failure(std::move(restored_result).error()); + } + auto restored = std::move(restored_result).value(); + return committer.commit_prepared(restored); +} + +paimon::Status complete_checkpoint(paimon::TableWrite& writer, + const paimon::TableCommit& committer, + Persist persist, void* context, + std::int64_t checkpoint_id) { + auto prepared_result = writer.prepare_commit(); + if (!prepared_result) { + return paimon::Status::failure(std::move(prepared_result).error()); + } + auto prepared = std::move(prepared_result).value(); + + auto durable_result = prepared.prepare(checkpoint_id); + if (!durable_result) { + return paimon::Status::failure(std::move(durable_result).error()); + } + auto durable = std::move(durable_result).value(); + auto bytes_result = durable.serialize(); + if (!bytes_result) { + return paimon::Status::failure(std::move(bytes_result).error()); + } + auto bytes = std::move(bytes_result).value(); + + // persist must not report success until the checkpoint blob and the engine's + // source state are durable in the same checkpoint protocol. + auto persist_status = + persist(context, checkpoint_id, bytes.data(), bytes.size()); + if (!persist_status) { + return persist_status; + } + + // commit_prepared is retry-safe. After a crash, deserialize the persisted + // blob and call this again with the same stable commit_user. + return committer.commit_prepared(durable); +} + +paimon::Status run_stream(paimon::TableWrite& writer, + const paimon::TableCommit& committer, + Producer producer, void* context, + Persist persist, + std::int64_t first_checkpoint_id) { + auto checkpoint_id = first_checkpoint_id; + bool dirty = false; + for (;;) { + StreamEvent event{StreamEventKind::end, nullptr, nullptr}; + auto producer_status = producer(context, &event); + if (!producer_status) { + return producer_status; + } + + switch (event.kind) { + case StreamEventKind::batch: { + auto write_status = writer.write_arrow(event.array, event.schema); + if (!write_status) { + return write_status; + } + dirty = true; + break; + } + case StreamEventKind::checkpoint: { + auto checkpoint_status = + dirty ? complete_checkpoint(writer, committer, persist, context, + checkpoint_id) + : persist(context, checkpoint_id, nullptr, 0); + if (!checkpoint_status) { + return checkpoint_status; + } + dirty = false; + ++checkpoint_id; + break; + } + case StreamEventKind::end: + // Never discard a tail batch merely because the producer ended before + // emitting its next periodic checkpoint barrier. + return dirty ? complete_checkpoint(writer, committer, persist, context, + checkpoint_id) + : paimon::Status::success(); + } + } +} + +namespace { + +void print_error(const paimon::Error& error) { + const auto message = error.message(); + std::fprintf(stderr, "Paimon error %d: %.*s\n", + static_cast(error.code()), + static_cast(message.size()), message.data()); +} + +paimon::Status no_input(void*, StreamEvent* event) { + *event = {StreamEventKind::end, nullptr, nullptr}; + return paimon::Status::success(); +} + +paimon::Status no_op_persist(void*, std::int64_t, const std::uint8_t*, + std::size_t) { + // Replace this with fsync/rename or the surrounding engine's durable state. + return paimon::Status::success(); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 5) { + std::fprintf(stderr, + "usage: %s WAREHOUSE DATABASE TABLE STABLE_COMMIT_USER\n", + argv[0]); + return EXIT_FAILURE; + } + + const paimon::Option options[] = {{"warehouse", argv[1]}}; + auto catalog_result = paimon::Catalog::create(options); + auto identifier_result = paimon::Identifier::create(argv[2], argv[3]); + if (!catalog_result || !identifier_result) { + print_error(!catalog_result ? catalog_result.error() + : identifier_result.error()); + return EXIT_FAILURE; + } + auto catalog = std::move(catalog_result).value(); + auto identifier = std::move(identifier_result).value(); + + auto table_result = catalog.get_table(identifier); + if (!table_result) { + print_error(table_result.error()); + return EXIT_FAILURE; + } + auto table = std::move(table_result).value(); + + auto builder_result = table.new_write_builder(argv[4]); + if (!builder_result) { + print_error(builder_result.error()); + return EXIT_FAILURE; + } + auto builder = std::move(builder_result).value(); + + auto writer_result = builder.new_write(); + auto committer_result = builder.new_commit(); + if (!writer_result || !committer_result) { + print_error(!writer_result ? writer_result.error() + : committer_result.error()); + return EXIT_FAILURE; + } + auto writer = std::move(writer_result).value(); + auto committer = std::move(committer_result).value(); + + // Replace no_input and no_op_persist with the application's Arrow producer + // and durable checkpoint store. + auto status = run_stream(writer, committer, no_input, nullptr, + no_op_persist, 1); + if (!status) { + print_error(status.error()); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/bindings/cpp/include/paimon/paimon.hpp b/bindings/cpp/include/paimon/paimon.hpp new file mode 100644 index 000000000..e225037ee --- /dev/null +++ b/bindings/cpp/include/paimon/paimon.hpp @@ -0,0 +1,1491 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef PAIMON_CPP_PAIMON_HPP +#define PAIMON_CPP_PAIMON_HPP + +#include +#include +#include +#include +#include +#include +#include + +// Tests and embedders may override this with a quoted header name. Normal +// consumers use the cbindgen-generated paimon.h shipped with libpaimon_c. +#ifndef PAIMON_C_HEADER +#define PAIMON_C_HEADER +#endif + +// Keep overridden test/embedding headers under C linkage too. Nesting is +// harmless for the generated paimon.h, which has its own C++ compatibility +// guard. +extern "C" { +#include PAIMON_C_HEADER +} + +namespace paimon { + +struct adopt_handle_t { + explicit constexpr adopt_handle_t() noexcept = default; +}; + +inline constexpr adopt_handle_t adopt_handle{}; + +enum class ErrorCode : std::int32_t { + unexpected = PAIMON_ERROR_UNEXPECTED, + unsupported = PAIMON_ERROR_UNSUPPORTED, + not_found = PAIMON_ERROR_NOT_FOUND, + already_exists = PAIMON_ERROR_ALREADY_EXISTS, + invalid_input = PAIMON_ERROR_INVALID_INPUT, + io_error = PAIMON_ERROR_IO, + out_of_range = PAIMON_ERROR_OUT_OF_RANGE, +}; + +// Owns one paimon_error. Error is also used as the storage for Status: a null +// native handle means success. The message view remains valid until this Error +// is moved, reset, or destroyed. +class Error final { + public: + constexpr Error() noexcept = default; + + explicit Error(adopt_handle_t, ::paimon_error* error) noexcept + : error_(error) {} + + Error(const Error&) = delete; + Error& operator=(const Error&) = delete; + + Error(Error&& other) noexcept : error_(other.release()) {} + + Error& operator=(Error&& other) noexcept { + if (this != &other) { + reset(other.release()); + } + return *this; + } + + ~Error() noexcept { reset(); } + + [[nodiscard]] bool ok() const noexcept { return error_ == nullptr; } + [[nodiscard]] explicit operator bool() const noexcept { return !ok(); } + + [[nodiscard]] ErrorCode code() const noexcept { + return error_ == nullptr + ? ErrorCode::unexpected + : static_cast(error_->code); + } + + [[nodiscard]] std::string_view message() const noexcept { + if (error_ == nullptr || error_->message.data == nullptr) { + return {}; + } + return {reinterpret_cast(error_->message.data), + error_->message.len}; + } + + [[nodiscard]] ::paimon_error* native_handle() const noexcept { + return error_; + } + + [[nodiscard]] ::paimon_error* release() noexcept { + auto* result = error_; + error_ = nullptr; + return result; + } + + void reset(::paimon_error* error = nullptr) noexcept { + if (error_ == error) { + return; + } + if (error_ != nullptr) { + ::paimon_error_free(error_); + } + error_ = error; + } + + private: + ::paimon_error* error_ = nullptr; +}; + +// Owns a byte buffer allocated by libpaimon_c. This is used by durable stream +// plan and prepared-commit APIs and never allocates through a C++ runtime. +class Bytes final { + public: + constexpr Bytes() noexcept : bytes_{nullptr, 0} {} + explicit constexpr Bytes(adopt_handle_t, ::paimon_bytes bytes) noexcept + : bytes_(bytes) {} + + Bytes(const Bytes&) = delete; + Bytes& operator=(const Bytes&) = delete; + + Bytes(Bytes&& other) noexcept : bytes_(other.release()) {} + + Bytes& operator=(Bytes&& other) noexcept { + if (this != &other) { + reset(); + bytes_ = other.release(); + } + return *this; + } + + ~Bytes() noexcept { reset(); } + + [[nodiscard]] const std::uint8_t* data() const noexcept { + return bytes_.data; + } + + [[nodiscard]] std::size_t size() const noexcept { return bytes_.len; } + [[nodiscard]] bool empty() const noexcept { return bytes_.len == 0; } + + [[nodiscard]] std::string_view string_view() const noexcept { + if (bytes_.data == nullptr) { + return {}; + } + return {reinterpret_cast(bytes_.data), bytes_.len}; + } + + [[nodiscard]] ::paimon_bytes native_handle() const noexcept { return bytes_; } + + [[nodiscard]] ::paimon_bytes release() noexcept { + const auto result = bytes_; + bytes_ = {nullptr, 0}; + return result; + } + + void reset() noexcept { + if (bytes_.data != nullptr) { + ::paimon_bytes_free(bytes_); + bytes_ = {nullptr, 0}; + } + } + + private: + ::paimon_bytes bytes_; +}; + +// A small C++17 expected-like result. It deliberately does not throw and does +// not allocate. Accessing the wrong alternative is a programming error. +template +class [[nodiscard]] Result final { + public: + Result(const Result&) = delete; + Result& operator=(const Result&) = delete; + + Result(Result&& other) noexcept( + std::is_nothrow_move_constructible::value) + : has_value_(other.has_value_) { + if (has_value_) { + new (&storage_.value) T(std::move(other.storage_.value)); + } else { + new (&storage_.error) Error(std::move(other.storage_.error)); + } + } + + Result& operator=(Result&& other) noexcept( + std::is_nothrow_move_constructible::value) { + if (this != &other) { + destroy(); + has_value_ = other.has_value_; + if (has_value_) { + new (&storage_.value) T(std::move(other.storage_.value)); + } else { + new (&storage_.error) Error(std::move(other.storage_.error)); + } + } + return *this; + } + + ~Result() noexcept { destroy(); } + + static Result success(T value) noexcept( + std::is_nothrow_move_constructible::value) { + return Result(value_tag{}, std::move(value)); + } + + static Result failure(Error error) noexcept { + return Result(error_tag{}, std::move(error)); + } + + [[nodiscard]] bool ok() const noexcept { return has_value_; } + [[nodiscard]] explicit operator bool() const noexcept { return ok(); } + + [[nodiscard]] T& value() & noexcept { + assert(has_value_); + return storage_.value; + } + + [[nodiscard]] const T& value() const& noexcept { + assert(has_value_); + return storage_.value; + } + + [[nodiscard]] T&& value() && noexcept { + assert(has_value_); + return std::move(storage_.value); + } + + [[nodiscard]] Error& error() & noexcept { + assert(!has_value_); + return storage_.error; + } + + [[nodiscard]] const Error& error() const& noexcept { + assert(!has_value_); + return storage_.error; + } + + [[nodiscard]] Error&& error() && noexcept { + assert(!has_value_); + return std::move(storage_.error); + } + + private: + struct value_tag {}; + struct error_tag {}; + + union Storage { + T value; + Error error; + + Storage() noexcept {} + ~Storage() noexcept {} + } storage_; + + explicit Result(value_tag, T&& value) noexcept( + std::is_nothrow_move_constructible::value) + : has_value_(true) { + new (&storage_.value) T(std::move(value)); + } + + explicit Result(error_tag, Error&& error) noexcept : has_value_(false) { + new (&storage_.error) Error(std::move(error)); + } + + void destroy() noexcept { + if (has_value_) { + storage_.value.~T(); + } else { + storage_.error.~Error(); + } + } + + bool has_value_; +}; + +template <> +class [[nodiscard]] Result final { + public: + Result(const Result&) = delete; + Result& operator=(const Result&) = delete; + Result(Result&&) noexcept = default; + Result& operator=(Result&&) noexcept = default; + ~Result() noexcept = default; + + static Result success() noexcept { return Result(Error{}); } + static Result failure(Error error) noexcept { + return Result(std::move(error)); + } + + [[nodiscard]] bool ok() const noexcept { return error_.ok(); } + [[nodiscard]] explicit operator bool() const noexcept { return ok(); } + + [[nodiscard]] Error& error() & noexcept { + assert(!ok()); + return error_; + } + + [[nodiscard]] const Error& error() const& noexcept { + assert(!ok()); + return error_; + } + + [[nodiscard]] Error&& error() && noexcept { + assert(!ok()); + return std::move(error_); + } + + private: + explicit Result(Error error) noexcept : error_(std::move(error)) {} + Error error_; +}; + +using Status = Result; +using Option = ::paimon_option; + +namespace detail { + +inline Status status_from(::paimon_error* error) noexcept { + if (error == nullptr) { + return Status::success(); + } + return Status::failure(Error(adopt_handle, error)); +} + +template +class UniqueHandle final { + public: + constexpr UniqueHandle() noexcept = default; + explicit UniqueHandle(adopt_handle_t, Raw* raw) noexcept : raw_(raw) {} + + UniqueHandle(const UniqueHandle&) = delete; + UniqueHandle& operator=(const UniqueHandle&) = delete; + + UniqueHandle(UniqueHandle&& other) noexcept : raw_(other.release()) {} + + UniqueHandle& operator=(UniqueHandle&& other) noexcept { + if (this != &other) { + reset(other.release()); + } + return *this; + } + + ~UniqueHandle() noexcept { reset(); } + + [[nodiscard]] Raw* get() const noexcept { return raw_; } + [[nodiscard]] explicit operator bool() const noexcept { + return raw_ != nullptr; + } + + [[nodiscard]] Raw* release() noexcept { + Raw* result = raw_; + raw_ = nullptr; + return result; + } + + void reset(Raw* raw = nullptr) noexcept { + if (raw_ != nullptr) { + Free(raw_); + } + raw_ = raw; + } + + private: + Raw* raw_ = nullptr; +}; + +} // namespace detail + +class Identifier; +class Table; +class ReadBuilder; +class Scan; +class Plan; +class TableRead; +class RecordBatchReader; +class WriteBuilder; +class TableWrite; +class PreparedMessages; +class TableCommit; +class PreparedCommit; +class StreamScan; +class StreamPlan; +class PollResult; + +enum class StreamStartupMode : std::int32_t { + latest_full = PAIMON_STREAM_STARTUP_LATEST_FULL, + latest = PAIMON_STREAM_STARTUP_LATEST, + from_snapshot = PAIMON_STREAM_STARTUP_FROM_SNAPSHOT, + from_snapshot_full = PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL, +}; + +enum class StreamFollowUpMode : std::int32_t { + automatic = PAIMON_STREAM_FOLLOW_UP_AUTO, + delta = PAIMON_STREAM_FOLLOW_UP_DELTA, + changelog = PAIMON_STREAM_FOLLOW_UP_CHANGELOG, +}; + +enum class StreamPollStatus : std::int32_t { + data = PAIMON_STREAM_POLL_DATA, + waiting = PAIMON_STREAM_POLL_WAITING, + end = PAIMON_STREAM_POLL_END, +}; + +enum class StreamReadMode : std::int32_t { + data = PAIMON_STREAM_READ_DATA, + audit_log = PAIMON_STREAM_READ_AUDIT_LOG, +}; + +class StreamScanOptions final { + public: + StreamScanOptions(const StreamScanOptions&) noexcept = default; + StreamScanOptions& operator=(const StreamScanOptions&) noexcept = default; + StreamScanOptions(StreamScanOptions&&) noexcept = default; + StreamScanOptions& operator=(StreamScanOptions&&) noexcept = default; + ~StreamScanOptions() noexcept = default; + + [[nodiscard]] static Result defaults() noexcept; + + StreamScanOptions& with_startup(StreamStartupMode mode, + std::int64_t snapshot_id = -1) noexcept { + options_.startup_mode = static_cast(mode); + options_.snapshot_id = snapshot_id; + return *this; + } + + StreamScanOptions& with_follow_up(StreamFollowUpMode mode) noexcept { + options_.follow_up_mode = static_cast(mode); + return *this; + } + + [[nodiscard]] const ::paimon_stream_scan_options* native_handle() + const noexcept { + return &options_; + } + + [[nodiscard]] ::paimon_stream_scan_options* native_handle() noexcept { + return &options_; + } + + private: + explicit StreamScanOptions(::paimon_stream_scan_options options) noexcept + : options_(options) {} + + ::paimon_stream_scan_options options_{}; +}; + +class Catalog final { + public: + Catalog() noexcept = default; + explicit Catalog(adopt_handle_t tag, ::paimon_catalog* raw) noexcept + : handle_(tag, raw) {} + + Catalog(const Catalog&) = delete; + Catalog& operator=(const Catalog&) = delete; + Catalog(Catalog&&) noexcept = default; + Catalog& operator=(Catalog&&) noexcept = default; + ~Catalog() noexcept = default; + + static Result create(const Option* options = nullptr, + std::size_t options_len = 0) noexcept; + + template + static Result create(const Option (&options)[N]) noexcept { + return create(options, N); + } + + [[nodiscard]] Result get_table( + const Identifier& identifier) const noexcept; + + [[nodiscard]] Status create_table_from_schema_json( + const Identifier& identifier, const char* schema_json, + bool ignore_if_exists = false) const noexcept; + + [[nodiscard]] Status drop_table( + const Identifier& identifier, + bool ignore_if_not_exists = false) const noexcept; + + [[nodiscard]] ::paimon_catalog* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_catalog, ::paimon_catalog_free> handle_; +}; + +class Identifier final { + public: + Identifier() noexcept = default; + explicit Identifier(adopt_handle_t tag, ::paimon_identifier* raw) noexcept + : handle_(tag, raw) {} + + Identifier(const Identifier&) = delete; + Identifier& operator=(const Identifier&) = delete; + Identifier(Identifier&&) noexcept = default; + Identifier& operator=(Identifier&&) noexcept = default; + ~Identifier() noexcept = default; + + static Result create(const char* database, + const char* object) noexcept; + + [[nodiscard]] ::paimon_identifier* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_identifier, ::paimon_identifier_free> handle_; +}; + +class Table final { + public: + Table() noexcept = default; + explicit Table(adopt_handle_t tag, ::paimon_table* raw) noexcept + : handle_(tag, raw) {} + + Table(const Table&) = delete; + Table& operator=(const Table&) = delete; + Table(Table&&) noexcept = default; + Table& operator=(Table&&) noexcept = default; + ~Table() noexcept = default; + + static Result
from_schema_json( + const char* table_path, const char* table_schema_json, + const char* database, const char* table_name, const char* branch = nullptr, + const Option* storage_options = nullptr, + std::size_t storage_options_len = 0) noexcept; + + [[nodiscard]] Result new_read_builder() const noexcept; + [[nodiscard]] Result new_read_builder( + const Option* options, std::size_t options_len) const noexcept; + + template + [[nodiscard]] Result new_read_builder( + const Option (&options)[N]) const noexcept; + + [[nodiscard]] Result new_write_builder() const noexcept; + [[nodiscard]] Result new_write_builder( + const char* stable_commit_user) const noexcept; + + [[nodiscard]] ::paimon_table* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table, ::paimon_table_free> handle_; +}; + +class ReadBuilder final { + public: + ReadBuilder() noexcept = default; + explicit ReadBuilder(adopt_handle_t tag, ::paimon_read_builder* raw) noexcept + : handle_(tag, raw) {} + + ReadBuilder(const ReadBuilder&) = delete; + ReadBuilder& operator=(const ReadBuilder&) = delete; + ReadBuilder(ReadBuilder&&) noexcept = default; + ReadBuilder& operator=(ReadBuilder&&) noexcept = default; + ~ReadBuilder() noexcept = default; + + // columns must be a null-terminated array. Passing nullptr clears projection. + [[nodiscard]] Status with_projection( + const char* const* columns) noexcept { + return detail::status_from( + ::paimon_read_builder_with_projection(handle_.get(), columns)); + } + + [[nodiscard]] Status with_case_sensitive(bool case_sensitive) noexcept { + return detail::status_from(::paimon_read_builder_with_case_sensitive( + handle_.get(), case_sensitive)); + } + + [[nodiscard]] Result new_scan() const noexcept; + [[nodiscard]] Result new_read() const noexcept; + [[nodiscard]] Result new_stream_scan( + const StreamScanOptions& options) const noexcept; + + [[nodiscard]] ::paimon_read_builder* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_read_builder, ::paimon_read_builder_free> + handle_; +}; + +class Scan final { + public: + Scan() noexcept = default; + explicit Scan(adopt_handle_t tag, ::paimon_table_scan* raw) noexcept + : handle_(tag, raw) {} + + Scan(const Scan&) = delete; + Scan& operator=(const Scan&) = delete; + Scan(Scan&&) noexcept = default; + Scan& operator=(Scan&&) noexcept = default; + ~Scan() noexcept = default; + + [[nodiscard]] Result plan() const noexcept; + + [[nodiscard]] ::paimon_table_scan* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table_scan, ::paimon_table_scan_free> handle_; +}; + +class Plan final { + public: + Plan() noexcept = default; + explicit Plan(adopt_handle_t tag, ::paimon_plan* raw) noexcept + : handle_(tag, raw) {} + + Plan(const Plan&) = delete; + Plan& operator=(const Plan&) = delete; + Plan(Plan&&) noexcept = default; + Plan& operator=(Plan&&) noexcept = default; + ~Plan() noexcept = default; + + static Result from_split_bytes(const std::uint8_t* data, + std::size_t size) noexcept; + + [[nodiscard]] std::size_t num_splits() const noexcept { + return ::paimon_plan_num_splits(handle_.get()); + } + + [[nodiscard]] ::paimon_plan* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_plan, ::paimon_plan_free> handle_; +}; + +// Owns the two heap-allocated Arrow C Data container structs returned by +// libpaimon_c. This type intentionally has no dependency on Arrow C++. +class ArrowBatch final { + public: + constexpr ArrowBatch() noexcept : batch_{nullptr, nullptr} {} + explicit constexpr ArrowBatch(adopt_handle_t, + ::paimon_arrow_batch batch) noexcept + : batch_(batch) {} + + ArrowBatch(const ArrowBatch&) = delete; + ArrowBatch& operator=(const ArrowBatch&) = delete; + + ArrowBatch(ArrowBatch&& other) noexcept : batch_(other.release()) {} + + ArrowBatch& operator=(ArrowBatch&& other) noexcept { + if (this != &other) { + reset(); + batch_ = other.release(); + } + return *this; + } + + ~ArrowBatch() noexcept { reset(); } + + [[nodiscard]] bool empty() const noexcept { + return batch_.array == nullptr && batch_.schema == nullptr; + } + + [[nodiscard]] explicit operator bool() const noexcept { return !empty(); } + [[nodiscard]] void* array() const noexcept { return batch_.array; } + [[nodiscard]] void* schema() const noexcept { return batch_.schema; } + + [[nodiscard]] ::paimon_arrow_batch native_handle() const noexcept { + return batch_; + } + + // The caller becomes responsible for paimon_arrow_batch_free(raw). + [[nodiscard]] ::paimon_arrow_batch release() noexcept { + const auto result = batch_; + batch_ = {nullptr, nullptr}; + return result; + } + + void reset() noexcept { + if (!empty()) { + ::paimon_arrow_batch_free(batch_); + batch_ = {nullptr, nullptr}; + } + } + + private: + ::paimon_arrow_batch batch_; +}; + +class RecordBatchReader final { + public: + RecordBatchReader() noexcept = default; + explicit RecordBatchReader(adopt_handle_t tag, + ::paimon_record_batch_reader* raw) noexcept + : handle_(tag, raw) {} + + RecordBatchReader(const RecordBatchReader&) = delete; + RecordBatchReader& operator=(const RecordBatchReader&) = delete; + RecordBatchReader(RecordBatchReader&&) noexcept = default; + RecordBatchReader& operator=(RecordBatchReader&&) noexcept = default; + ~RecordBatchReader() noexcept = default; + + // A successful empty ArrowBatch is end-of-stream. + [[nodiscard]] Result next() noexcept; + + [[nodiscard]] ::paimon_record_batch_reader* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_record_batch_reader, + ::paimon_record_batch_reader_free> + handle_; +}; + +class TableRead final { + public: + TableRead() noexcept = default; + explicit TableRead(adopt_handle_t tag, ::paimon_table_read* raw) noexcept + : handle_(tag, raw) {} + + TableRead(const TableRead&) = delete; + TableRead& operator=(const TableRead&) = delete; + TableRead(TableRead&&) noexcept = default; + TableRead& operator=(TableRead&&) noexcept = default; + ~TableRead() noexcept = default; + + [[nodiscard]] Result to_arrow( + const Plan& plan, std::size_t offset = 0, + std::size_t length = static_cast(-1)) const noexcept; + + [[nodiscard]] ::paimon_table_read* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table_read, ::paimon_table_read_free> handle_; +}; + +class StreamPlan final { + public: + StreamPlan() noexcept = default; + explicit StreamPlan(adopt_handle_t tag, ::paimon_stream_plan* raw) noexcept + : handle_(tag, raw) {} + + StreamPlan(const StreamPlan&) = delete; + StreamPlan& operator=(const StreamPlan&) = delete; + StreamPlan(StreamPlan&&) noexcept = default; + StreamPlan& operator=(StreamPlan&&) noexcept = default; + ~StreamPlan() noexcept = default; + + [[nodiscard]] bool is_full() const noexcept { + return ::paimon_stream_plan_is_full(handle_.get()) != 0; + } + + [[nodiscard]] std::size_t num_splits() const noexcept { + return ::paimon_stream_plan_num_splits(handle_.get()); + } + + [[nodiscard]] Result serialize() const noexcept; + + [[nodiscard]] static Result deserialize( + const std::uint8_t* data, std::size_t size) noexcept; + + [[nodiscard]] Result read_to_arrow( + const TableRead& read, StreamReadMode mode = StreamReadMode::data, + std::size_t offset = 0, + std::size_t length = static_cast(-1)) const noexcept; + + [[nodiscard]] ::paimon_stream_plan* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_stream_plan, ::paimon_stream_plan_free> handle_; +}; + +class PollResult final { + public: + PollResult(StreamPollStatus status, StreamPlan plan, + std::int64_t snapshot_id, std::int64_t next_snapshot_id, + std::int64_t watermark, bool has_watermark) noexcept + : status_(status), + plan_(std::move(plan)), + snapshot_id_(snapshot_id), + next_snapshot_id_(next_snapshot_id), + watermark_(watermark), + has_watermark_(has_watermark) {} + + PollResult(const PollResult&) = delete; + PollResult& operator=(const PollResult&) = delete; + PollResult(PollResult&&) noexcept = default; + PollResult& operator=(PollResult&&) noexcept = default; + ~PollResult() noexcept = default; + + [[nodiscard]] StreamPollStatus status() const noexcept { return status_; } + [[nodiscard]] bool has_data() const noexcept { + return status_ == StreamPollStatus::data; + } + [[nodiscard]] bool waiting() const noexcept { + return status_ == StreamPollStatus::waiting; + } + [[nodiscard]] bool end() const noexcept { + return status_ == StreamPollStatus::end; + } + + [[nodiscard]] StreamPlan& plan() & noexcept { + assert(has_data()); + return plan_; + } + + [[nodiscard]] const StreamPlan& plan() const& noexcept { + assert(has_data()); + return plan_; + } + + [[nodiscard]] StreamPlan&& plan() && noexcept { + assert(has_data()); + return std::move(plan_); + } + + [[nodiscard]] std::int64_t snapshot_id() const noexcept { + return snapshot_id_; + } + + [[nodiscard]] std::int64_t next_snapshot_id() const noexcept { + return next_snapshot_id_; + } + + [[nodiscard]] bool has_watermark() const noexcept { return has_watermark_; } + + [[nodiscard]] std::int64_t watermark() const noexcept { + assert(has_watermark_); + return watermark_; + } + + private: + StreamPollStatus status_; + StreamPlan plan_; + std::int64_t snapshot_id_; + std::int64_t next_snapshot_id_; + std::int64_t watermark_; + bool has_watermark_; +}; + +class StreamScan final { + public: + StreamScan() noexcept = default; + explicit StreamScan(adopt_handle_t tag, ::paimon_stream_scan* raw) noexcept + : handle_(tag, raw) {} + + StreamScan(const StreamScan&) = delete; + StreamScan& operator=(const StreamScan&) = delete; + StreamScan(StreamScan&&) noexcept = default; + StreamScan& operator=(StreamScan&&) noexcept = default; + ~StreamScan() noexcept = default; + + // poll() never waits for a future snapshot. Waiting is a normal result, not + // an error, so the caller controls scheduling, cancellation and backpressure. + // One StreamScan is single-thread-confined; poll/checkpoint/restore/free must + // be externally serialized. + [[nodiscard]] Result poll() noexcept; + + // This cursor is safe to persist only after every split in the returned plan + // has been durably accounted for by the caller's checkpoint barrier. + [[nodiscard]] std::int64_t checkpoint() const noexcept { + return ::paimon_stream_scan_checkpoint(handle_.get()); + } + + [[nodiscard]] Status restore(std::int64_t next_snapshot_id) noexcept { + return detail::status_from( + ::paimon_stream_scan_restore(handle_.get(), next_snapshot_id)); + } + + [[nodiscard]] ::paimon_stream_scan* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_stream_scan, ::paimon_stream_scan_free> handle_; +}; + +class PreparedMessages final { + public: + PreparedMessages() noexcept = default; + explicit PreparedMessages(adopt_handle_t tag, + ::paimon_commit_messages* raw) noexcept + : handle_(tag, raw) {} + + PreparedMessages(const PreparedMessages&) = delete; + PreparedMessages& operator=(const PreparedMessages&) = delete; + PreparedMessages(PreparedMessages&&) noexcept = default; + PreparedMessages& operator=(PreparedMessages&&) noexcept = default; + + // Destruction only frees the messages. It never commits or aborts files. + ~PreparedMessages() noexcept = default; + + [[nodiscard]] Status merge(const PreparedMessages& source) noexcept { + return detail::status_from(::paimon_commit_messages_merge( + handle_.get(), source.handle_.get())); + } + + [[nodiscard]] Result prepare( + std::int64_t checkpoint_id) const noexcept; + + [[nodiscard]] ::paimon_commit_messages* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_commit_messages, + ::paimon_commit_messages_free> + handle_; +}; + +class PreparedCommit final { + public: + PreparedCommit() noexcept = default; + explicit PreparedCommit(adopt_handle_t tag, + ::paimon_prepared_commit* raw) noexcept + : handle_(tag, raw) {} + + PreparedCommit(const PreparedCommit&) = delete; + PreparedCommit& operator=(const PreparedCommit&) = delete; + PreparedCommit(PreparedCommit&&) noexcept = default; + PreparedCommit& operator=(PreparedCommit&&) noexcept = default; + + // Destruction only releases the durable in-memory envelope. It never commits + // or aborts the referenced data files. + ~PreparedCommit() noexcept = default; + + [[nodiscard]] static Result deserialize( + const std::uint8_t* data, std::size_t size) noexcept; + + [[nodiscard]] std::int64_t identifier() const noexcept { + return ::paimon_prepared_commit_identifier(handle_.get()); + } + + [[nodiscard]] Result serialize() const noexcept; + + [[nodiscard]] Status merge(const PreparedCommit& source) noexcept { + return detail::status_from(::paimon_prepared_commit_merge( + handle_.get(), source.handle_.get())); + } + + [[nodiscard]] ::paimon_prepared_commit* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_prepared_commit, + ::paimon_prepared_commit_free> + handle_; +}; + +class TableWrite final { + public: + TableWrite() noexcept = default; + explicit TableWrite(adopt_handle_t tag, ::paimon_table_write* raw) noexcept + : handle_(tag, raw) {} + + TableWrite(const TableWrite&) = delete; + TableWrite& operator=(const TableWrite&) = delete; + TableWrite(TableWrite&&) noexcept = default; + TableWrite& operator=(TableWrite&&) noexcept = default; + ~TableWrite() noexcept = default; + + // The Arrow C Data contents are consumed in place once import begins. The + // caller continues to own the ArrowArray/ArrowSchema container memory. + [[nodiscard]] Status write_arrow(void* array, void* schema) noexcept { + return detail::status_from(::paimon_table_write_write_arrow_batch( + handle_.get(), array, schema)); + } + + // Convenient bridge for a Rust-allocated batch. Its heap container structs + // remain owned by batch and are released before this call returns. + [[nodiscard]] Status write_arrow(ArrowBatch&& batch) noexcept { + auto status = write_arrow(batch.array(), batch.schema()); + batch.reset(); + return status; + } + + [[nodiscard]] Result prepare_commit() noexcept; + + [[nodiscard]] ::paimon_table_write* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table_write, ::paimon_table_write_free> handle_; +}; + +class TableCommit final { + public: + TableCommit() noexcept = default; + explicit TableCommit(adopt_handle_t tag, ::paimon_table_commit* raw) noexcept + : handle_(tag, raw) {} + + TableCommit(const TableCommit&) = delete; + TableCommit& operator=(const TableCommit&) = delete; + TableCommit(TableCommit&&) noexcept = default; + TableCommit& operator=(TableCommit&&) noexcept = default; + ~TableCommit() noexcept = default; + + // Commit calls never consume messages. Keep them until the outcome is known; + // retry an uncertain outcome with filter_and_commit(checkpoint_id). + [[nodiscard]] Status commit(PreparedMessages& messages) const noexcept { + return detail::status_from(::paimon_table_commit_commit( + handle_.get(), messages.native_handle())); + } + + [[nodiscard]] Status commit(PreparedMessages& messages, + std::int64_t checkpoint_id) const noexcept { + return detail::status_from(::paimon_table_commit_commit_with_identifier( + handle_.get(), messages.native_handle(), checkpoint_id)); + } + + [[nodiscard]] Status filter_and_commit( + PreparedMessages& messages, std::int64_t checkpoint_id) const noexcept { + return detail::status_from( + ::paimon_table_commit_filter_and_commit_with_identifier( + handle_.get(), messages.native_handle(), checkpoint_id)); + } + + // Retry-safe commit for a serialized/restored checkpoint. The PreparedCommit + // remains owned by the caller and can be retried after an uncertain result. + [[nodiscard]] Status commit_prepared( + const PreparedCommit& prepared) const noexcept { + return detail::status_from(::paimon_table_commit_commit_prepared( + handle_.get(), prepared.native_handle())); + } + + [[nodiscard]] Status overwrite(PreparedMessages& messages) const noexcept { + return detail::status_from(::paimon_table_commit_overwrite( + handle_.get(), messages.native_handle())); + } + + [[nodiscard]] Status overwrite(PreparedMessages& messages, + std::int64_t checkpoint_id) const noexcept { + return detail::status_from( + ::paimon_table_commit_overwrite_with_identifier( + handle_.get(), messages.native_handle(), checkpoint_id)); + } + + [[nodiscard]] Status truncate_table() const noexcept { + return detail::status_from( + ::paimon_table_commit_truncate_table(handle_.get())); + } + + [[nodiscard]] Status truncate_table( + std::int64_t checkpoint_id) const noexcept { + return detail::status_from( + ::paimon_table_commit_truncate_table_with_identifier( + handle_.get(), checkpoint_id)); + } + + // Abort is always explicit. PreparedMessages destruction does not call it. + [[nodiscard]] Status abort(PreparedMessages& messages) const noexcept { + return detail::status_from(::paimon_table_commit_abort( + handle_.get(), messages.native_handle())); + } + + // Fence all commit/abort calls for the same table and commit_user across + // processes. Truncated snapshot history is reported as an error and no file + // is removed. + [[nodiscard]] Status abort_prepared( + const PreparedCommit& prepared) const noexcept { + return detail::status_from(::paimon_table_commit_abort_prepared( + handle_.get(), prepared.native_handle())); + } + + [[nodiscard]] ::paimon_table_commit* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_table_commit, ::paimon_table_commit_free> + handle_; +}; + +class WriteBuilder final { + public: + WriteBuilder() noexcept = default; + explicit WriteBuilder(adopt_handle_t tag, ::paimon_write_builder* raw) noexcept + : handle_(tag, raw) {} + + WriteBuilder(const WriteBuilder&) = delete; + WriteBuilder& operator=(const WriteBuilder&) = delete; + WriteBuilder(WriteBuilder&&) noexcept = default; + WriteBuilder& operator=(WriteBuilder&&) noexcept = default; + ~WriteBuilder() noexcept = default; + + [[nodiscard]] Status with_overwrite() noexcept { + return detail::status_from( + ::paimon_write_builder_with_overwrite(handle_.get())); + } + + [[nodiscard]] Result new_write() const noexcept; + [[nodiscard]] Result new_commit() const noexcept; + + [[nodiscard]] ::paimon_write_builder* native_handle() const noexcept { + return handle_.get(); + } + + private: + detail::UniqueHandle<::paimon_write_builder, ::paimon_write_builder_free> + handle_; +}; + +inline Result Catalog::create(const Option* options, + std::size_t options_len) noexcept { + const auto result = ::paimon_catalog_create(options, options_len); + if (result.error != nullptr) { + if (result.catalog != nullptr) { + ::paimon_catalog_free(result.catalog); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + Catalog(adopt_handle, result.catalog)); +} + +inline Result StreamScanOptions::defaults() noexcept { + ::paimon_stream_scan_options options{}; + auto* error = ::paimon_stream_scan_options_init(&options); + if (error != nullptr) { + return Result::failure(Error(adopt_handle, error)); + } + return Result::success(StreamScanOptions(options)); +} + +inline Result Identifier::create(const char* database, + const char* object) noexcept { + const auto result = ::paimon_identifier_new(database, object); + if (result.error != nullptr) { + if (result.identifier != nullptr) { + ::paimon_identifier_free(result.identifier); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + Identifier(adopt_handle, result.identifier)); +} + +inline Result
Catalog::get_table( + const Identifier& identifier) const noexcept { + const auto result = + ::paimon_catalog_get_table(handle_.get(), identifier.native_handle()); + if (result.error != nullptr) { + if (result.table != nullptr) { + ::paimon_table_free(result.table); + } + return Result
::failure(Error(adopt_handle, result.error)); + } + return Result
::success(Table(adopt_handle, result.table)); +} + +inline Status Catalog::create_table_from_schema_json( + const Identifier& identifier, const char* schema_json, + bool ignore_if_exists) const noexcept { + return detail::status_from(::paimon_catalog_create_table_from_schema_json( + handle_.get(), identifier.native_handle(), schema_json, + ignore_if_exists)); +} + +inline Status Catalog::drop_table(const Identifier& identifier, + bool ignore_if_not_exists) const noexcept { + return detail::status_from(::paimon_catalog_drop_table( + handle_.get(), identifier.native_handle(), ignore_if_not_exists)); +} + +inline Result
Table::from_schema_json( + const char* table_path, const char* table_schema_json, const char* database, + const char* table_name, const char* branch, const Option* storage_options, + std::size_t storage_options_len) noexcept { + const auto result = ::paimon_table_from_schema_json( + table_path, table_schema_json, database, table_name, branch, + storage_options, storage_options_len); + if (result.error != nullptr) { + if (result.table != nullptr) { + ::paimon_table_free(result.table); + } + return Result
::failure(Error(adopt_handle, result.error)); + } + return Result
::success(Table(adopt_handle, result.table)); +} + +inline Result Table::new_read_builder() const noexcept { + const auto result = ::paimon_table_new_read_builder(handle_.get()); + if (result.error != nullptr) { + if (result.read_builder != nullptr) { + ::paimon_read_builder_free(result.read_builder); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + ReadBuilder(adopt_handle, result.read_builder)); +} + +inline Result Table::new_read_builder( + const Option* options, std::size_t options_len) const noexcept { + const auto result = ::paimon_table_new_read_builder_with_options( + handle_.get(), options, options_len); + if (result.error != nullptr) { + if (result.read_builder != nullptr) { + ::paimon_read_builder_free(result.read_builder); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + ReadBuilder(adopt_handle, result.read_builder)); +} + +template +inline Result Table::new_read_builder( + const Option (&options)[N]) const noexcept { + return new_read_builder(options, N); +} + +inline Result ReadBuilder::new_scan() const noexcept { + const auto result = ::paimon_read_builder_new_scan(handle_.get()); + if (result.error != nullptr) { + if (result.scan != nullptr) { + ::paimon_table_scan_free(result.scan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Scan(adopt_handle, result.scan)); +} + +inline Result ReadBuilder::new_read() const noexcept { + const auto result = ::paimon_read_builder_new_read(handle_.get()); + if (result.error != nullptr) { + if (result.read != nullptr) { + ::paimon_table_read_free(result.read); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(TableRead(adopt_handle, result.read)); +} + +inline Result ReadBuilder::new_stream_scan( + const StreamScanOptions& options) const noexcept { + const auto result = ::paimon_read_builder_new_stream_scan( + handle_.get(), options.native_handle()); + if (result.error != nullptr) { + if (result.scan != nullptr) { + ::paimon_stream_scan_free(result.scan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(StreamScan(adopt_handle, result.scan)); +} + +inline Result Scan::plan() const noexcept { + const auto result = ::paimon_table_scan_plan(handle_.get()); + if (result.error != nullptr) { + if (result.plan != nullptr) { + ::paimon_plan_free(result.plan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Plan(adopt_handle, result.plan)); +} + +inline Result Plan::from_split_bytes(const std::uint8_t* data, + std::size_t size) noexcept { + const auto result = ::paimon_plan_from_split_bytes(data, size); + if (result.error != nullptr) { + if (result.plan != nullptr) { + ::paimon_plan_free(result.plan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Plan(adopt_handle, result.plan)); +} + +inline Result TableRead::to_arrow( + const Plan& plan, std::size_t offset, std::size_t length) const noexcept { + const auto result = ::paimon_table_read_to_arrow( + handle_.get(), plan.native_handle(), offset, length); + if (result.error != nullptr) { + if (result.reader != nullptr) { + ::paimon_record_batch_reader_free(result.reader); + } + return Result::failure( + Error(adopt_handle, result.error)); + } + return Result::success( + RecordBatchReader(adopt_handle, result.reader)); +} + +inline Result RecordBatchReader::next() noexcept { + auto result = ::paimon_record_batch_reader_next(handle_.get()); + if (result.error != nullptr) { + if (result.batch.array != nullptr || result.batch.schema != nullptr) { + ::paimon_arrow_batch_free(result.batch); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + ArrowBatch(adopt_handle, result.batch)); +} + +inline Result StreamPlan::read_to_arrow( + const TableRead& read, StreamReadMode mode, std::size_t offset, + std::size_t length) const noexcept { + const auto result = ::paimon_stream_plan_read_to_arrow( + read.native_handle(), handle_.get(), offset, length, + static_cast(mode)); + if (result.error != nullptr) { + if (result.reader != nullptr) { + ::paimon_record_batch_reader_free(result.reader); + } + return Result::failure( + Error(adopt_handle, result.error)); + } + return Result::success( + RecordBatchReader(adopt_handle, result.reader)); +} + +inline Result StreamPlan::serialize() const noexcept { + auto result = ::paimon_stream_plan_serialize(handle_.get()); + if (result.error != nullptr) { + if (result.bytes.data != nullptr) { + ::paimon_bytes_free(result.bytes); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Bytes(adopt_handle, result.bytes)); +} + +inline Result StreamPlan::deserialize( + const std::uint8_t* data, std::size_t size) noexcept { + auto result = ::paimon_stream_plan_deserialize(data, size); + if (result.error != nullptr) { + if (result.plan != nullptr) { + ::paimon_stream_plan_free(result.plan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(PollResult( + static_cast(result.status), + StreamPlan(adopt_handle, result.plan), result.snapshot_id, + result.next_snapshot_id, result.watermark, result.has_watermark != 0)); +} + +inline Result StreamScan::poll() noexcept { + auto result = ::paimon_stream_scan_poll(handle_.get()); + if (result.error != nullptr) { + if (result.plan != nullptr) { + ::paimon_stream_plan_free(result.plan); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(PollResult( + static_cast(result.status), + StreamPlan(adopt_handle, result.plan), result.snapshot_id, + result.next_snapshot_id, result.watermark, result.has_watermark != 0)); +} + +inline Result Table::new_write_builder() const noexcept { + const auto result = ::paimon_table_new_write_builder(handle_.get()); + if (result.error != nullptr) { + if (result.write_builder != nullptr) { + ::paimon_write_builder_free(result.write_builder); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + WriteBuilder(adopt_handle, result.write_builder)); +} + +inline Result Table::new_write_builder( + const char* stable_commit_user) const noexcept { + const auto result = ::paimon_table_new_write_builder_with_commit_user( + handle_.get(), stable_commit_user); + if (result.error != nullptr) { + if (result.write_builder != nullptr) { + ::paimon_write_builder_free(result.write_builder); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + WriteBuilder(adopt_handle, result.write_builder)); +} + +inline Result WriteBuilder::new_write() const noexcept { + const auto result = ::paimon_write_builder_new_write(handle_.get()); + if (result.error != nullptr) { + if (result.write != nullptr) { + ::paimon_table_write_free(result.write); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(TableWrite(adopt_handle, result.write)); +} + +inline Result WriteBuilder::new_commit() const noexcept { + const auto result = ::paimon_write_builder_new_commit(handle_.get()); + if (result.error != nullptr) { + if (result.commit != nullptr) { + ::paimon_table_commit_free(result.commit); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + TableCommit(adopt_handle, result.commit)); +} + +inline Result TableWrite::prepare_commit() noexcept { + const auto result = ::paimon_table_write_prepare_commit(handle_.get()); + if (result.error != nullptr) { + if (result.messages != nullptr) { + ::paimon_commit_messages_free(result.messages); + } + return Result::failure( + Error(adopt_handle, result.error)); + } + return Result::success( + PreparedMessages(adopt_handle, result.messages)); +} + +inline Result PreparedMessages::prepare( + std::int64_t checkpoint_id) const noexcept { + const auto result = + ::paimon_commit_messages_prepare(handle_.get(), checkpoint_id); + if (result.error != nullptr) { + if (result.prepared != nullptr) { + ::paimon_prepared_commit_free(result.prepared); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + PreparedCommit(adopt_handle, result.prepared)); +} + +inline Result PreparedCommit::serialize() const noexcept { + auto result = ::paimon_prepared_commit_serialize(handle_.get()); + if (result.error != nullptr) { + if (result.bytes.data != nullptr) { + ::paimon_bytes_free(result.bytes); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success(Bytes(adopt_handle, result.bytes)); +} + +inline Result PreparedCommit::deserialize( + const std::uint8_t* data, std::size_t size) noexcept { + const auto result = ::paimon_prepared_commit_deserialize(data, size); + if (result.error != nullptr) { + if (result.prepared != nullptr) { + ::paimon_prepared_commit_free(result.prepared); + } + return Result::failure(Error(adopt_handle, result.error)); + } + return Result::success( + PreparedCommit(adopt_handle, result.prepared)); +} + +static_assert(!std::is_copy_constructible::value, + "native handles must stay move-only"); +static_assert(std::is_nothrow_destructible::value, + "native handle destructors must be noexcept"); +static_assert(!std::is_copy_constructible::value, + "prepared messages must stay move-only"); +static_assert(std::is_nothrow_destructible::value, + "prepared-message destruction must be noexcept"); +static_assert(!std::is_copy_constructible::value, + "stream scans must stay move-only"); +static_assert(std::is_nothrow_destructible::value, + "stream plan destruction must be noexcept"); +static_assert(!std::is_copy_constructible::value, + "durable prepared commits must stay move-only"); + +} // namespace paimon + +#endif // PAIMON_CPP_PAIMON_HPP diff --git a/bindings/cpp/tests/header_smoke.cpp b/bindings/cpp/tests/header_smoke.cpp new file mode 100644 index 000000000..9797ed590 --- /dev/null +++ b/bindings/cpp/tests/header_smoke.cpp @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include + +static_assert(std::is_move_constructible::value, "moveable"); +static_assert(!std::is_copy_constructible::value, + "not copyable"); +static_assert(std::is_nothrow_destructible::value, + "noexcept Arrow ownership"); +static_assert(std::is_nothrow_destructible::value, + "noexcept committer ownership"); + +void paimon_cpp_header_smoke(const paimon::Option* options, + std::size_t option_count, + void* arrow_array, void* arrow_schema) { + auto catalog = paimon::Catalog::create(options, option_count); + auto identifier = paimon::Identifier::create("default", "table"); + auto direct_table = paimon::Table::from_schema_json( + "/tmp/table", "{}", "default", "table"); + auto split_plan = paimon::Plan::from_split_bytes(nullptr, 0); + (void)direct_table; + (void)split_plan; + if (!catalog || !identifier) { + return; + } + + auto table = catalog.value().get_table(identifier.value()); + auto create_table_status = catalog.value().create_table_from_schema_json( + identifier.value(), "{}", true); + auto drop_table_status = catalog.value().drop_table(identifier.value(), true); + (void)create_table_status; + (void)drop_table_status; + if (!table) { + return; + } + + auto read_builder = table.value().new_read_builder(); + if (read_builder) { + const char* projection[] = {"id", nullptr}; + auto projection_status = read_builder.value().with_projection(projection); + auto case_status = read_builder.value().with_case_sensitive(true); + auto scan = read_builder.value().new_scan(); + auto read = read_builder.value().new_read(); + auto stream_options = paimon::StreamScanOptions::defaults(); + (void)projection_status; + (void)case_status; + if (scan && read) { + auto plan = scan.value().plan(); + if (plan) { + auto reader = read.value().to_arrow(plan.value()); + if (reader) { + auto batch = reader.value().next(); + (void)batch; + } + } + if (stream_options) { + stream_options.value().with_startup( + paimon::StreamStartupMode::latest); + stream_options.value().with_follow_up( + paimon::StreamFollowUpMode::automatic); + auto stream_scan = read_builder.value().new_stream_scan( + stream_options.value()); + if (stream_scan) { + const auto checkpoint = stream_scan.value().checkpoint(); + auto restore = stream_scan.value().restore(checkpoint); + auto poll = stream_scan.value().poll(); + (void)restore; + if (poll && poll.value().has_data()) { + auto plan_bytes = poll.value().plan().serialize(); + if (plan_bytes) { + auto restored_plan = paimon::StreamPlan::deserialize( + plan_bytes.value().data(), plan_bytes.value().size()); + (void)restored_plan; + } + auto stream_reader = poll.value().plan().read_to_arrow( + read.value(), paimon::StreamReadMode::data); + (void)stream_reader; + } + } + } + } + } + + auto write_builder = table.value().new_write_builder("stable-writer"); + if (!write_builder) { + return; + } + auto overwrite_status = write_builder.value().with_overwrite(); + auto writer = write_builder.value().new_write(); + auto committer = write_builder.value().new_commit(); + (void)overwrite_status; + if (!writer || !committer) { + return; + } + auto write_status = writer.value().write_arrow(arrow_array, arrow_schema); + auto prepared = writer.value().prepare_commit(); + (void)write_status; + if (prepared) { + auto durable = prepared.value().prepare(1); + if (durable) { + auto serialized = durable.value().serialize(); + if (serialized) { + auto restored = paimon::PreparedCommit::deserialize( + serialized.value().data(), serialized.value().size()); + if (restored) { + auto merge_status = durable.value().merge(restored.value()); + auto commit_status = committer.value().commit_prepared(durable.value()); + (void)merge_status; + (void)commit_status; + } + } + } + } +} diff --git a/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt new file mode 100644 index 000000000..ad6ed378c --- /dev/null +++ b/bindings/cpp/tests/install_tree_consumer/CMakeLists.txt @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 3.15) +project(PaimonCppInstallTreeConsumer LANGUAGES CXX) + +find_package(PaimonCpp CONFIG REQUIRED) +# Package discovery may happen through more than one dependency. A repeated +# lookup must retain the same bundled Paimon::c target instead of treating it +# as an external override. +find_package(PaimonCpp CONFIG REQUIRED) +add_executable(paimon_install_tree_consumer main.cpp) +target_link_libraries(paimon_install_tree_consumer PRIVATE Paimon::cpp) diff --git a/bindings/cpp/tests/install_tree_consumer/main.cpp b/bindings/cpp/tests/install_tree_consumer/main.cpp new file mode 100644 index 000000000..a8f6f5ab2 --- /dev/null +++ b/bindings/cpp/tests/install_tree_consumer/main.cpp @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +int main() { + auto options = paimon::StreamScanOptions::defaults(); + return options ? 0 : 1; +} diff --git a/bindings/cpp/tests/paimon_test_stub.h b/bindings/cpp/tests/paimon_test_stub.h new file mode 100644 index 000000000..1a7303443 --- /dev/null +++ b/bindings/cpp/tests/paimon_test_stub.h @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef PAIMON_CPP_TEST_PAIMON_STUB_H +#define PAIMON_CPP_TEST_PAIMON_STUB_H + +#include +#include +#include + +#define PAIMON_ERROR_UNEXPECTED 0 +#define PAIMON_ERROR_UNSUPPORTED 1 +#define PAIMON_ERROR_NOT_FOUND 2 +#define PAIMON_ERROR_ALREADY_EXISTS 3 +#define PAIMON_ERROR_INVALID_INPUT 4 +#define PAIMON_ERROR_IO 5 +#define PAIMON_ERROR_OUT_OF_RANGE 6 + +#define PAIMON_STREAM_STARTUP_LATEST_FULL 0 +#define PAIMON_STREAM_STARTUP_LATEST 1 +#define PAIMON_STREAM_STARTUP_FROM_SNAPSHOT 2 +#define PAIMON_STREAM_STARTUP_FROM_SNAPSHOT_FULL 3 +#define PAIMON_STREAM_FOLLOW_UP_AUTO 0 +#define PAIMON_STREAM_FOLLOW_UP_DELTA 1 +#define PAIMON_STREAM_FOLLOW_UP_CHANGELOG 2 +#define PAIMON_STREAM_POLL_DATA 0 +#define PAIMON_STREAM_POLL_WAITING 1 +#define PAIMON_STREAM_POLL_END 2 +#define PAIMON_STREAM_READ_DATA 0 +#define PAIMON_STREAM_READ_AUDIT_LOG 1 + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct paimon_bytes { + uint8_t* data; + size_t len; +} paimon_bytes; + +typedef struct paimon_error { + int32_t code; + paimon_bytes message; +} paimon_error; + +typedef struct paimon_option { + const char* key; + const char* value; +} paimon_option; + +typedef struct paimon_catalog paimon_catalog; +typedef struct paimon_identifier paimon_identifier; +typedef struct paimon_table paimon_table; +typedef struct paimon_read_builder paimon_read_builder; +typedef struct paimon_table_scan paimon_table_scan; +typedef struct paimon_plan paimon_plan; +typedef struct paimon_table_read paimon_table_read; +typedef struct paimon_record_batch_reader paimon_record_batch_reader; +typedef struct paimon_write_builder paimon_write_builder; +typedef struct paimon_table_write paimon_table_write; +typedef struct paimon_commit_messages paimon_commit_messages; +typedef struct paimon_table_commit paimon_table_commit; +typedef struct paimon_prepared_commit paimon_prepared_commit; +typedef struct paimon_stream_scan paimon_stream_scan; +typedef struct paimon_stream_plan paimon_stream_plan; + +typedef struct paimon_stream_scan_options { + uint32_t struct_size; + int32_t startup_mode; + int32_t follow_up_mode; + int64_t snapshot_id; + uint64_t reserved[4]; +} paimon_stream_scan_options; + +typedef struct paimon_arrow_batch { + void* array; + void* schema; +} paimon_arrow_batch; + +typedef struct paimon_result_catalog_new { + paimon_catalog* catalog; + paimon_error* error; +} paimon_result_catalog_new; + +typedef struct paimon_result_identifier_new { + paimon_identifier* identifier; + paimon_error* error; +} paimon_result_identifier_new; + +typedef struct paimon_result_get_table { + paimon_table* table; + paimon_error* error; +} paimon_result_get_table; + +typedef struct paimon_result_read_builder { + paimon_read_builder* read_builder; + paimon_error* error; +} paimon_result_read_builder; + +typedef struct paimon_result_table_scan { + paimon_table_scan* scan; + paimon_error* error; +} paimon_result_table_scan; + +typedef struct paimon_result_new_read { + paimon_table_read* read; + paimon_error* error; +} paimon_result_new_read; + +typedef struct paimon_result_plan { + paimon_plan* plan; + paimon_error* error; +} paimon_result_plan; + +typedef struct paimon_result_record_batch_reader { + paimon_record_batch_reader* reader; + paimon_error* error; +} paimon_result_record_batch_reader; + +typedef struct paimon_result_next_batch { + paimon_arrow_batch batch; + paimon_error* error; +} paimon_result_next_batch; + +typedef struct paimon_result_write_builder { + paimon_write_builder* write_builder; + paimon_error* error; +} paimon_result_write_builder; + +typedef struct paimon_result_table_write { + paimon_table_write* write; + paimon_error* error; +} paimon_result_table_write; + +typedef struct paimon_result_table_commit { + paimon_table_commit* commit; + paimon_error* error; +} paimon_result_table_commit; + +typedef struct paimon_result_prepare_commit { + paimon_commit_messages* messages; + paimon_error* error; +} paimon_result_prepare_commit; + +typedef struct paimon_result_prepared_commit { + paimon_prepared_commit* prepared; + paimon_error* error; +} paimon_result_prepared_commit; + +typedef struct paimon_result_bytes { + paimon_bytes bytes; + paimon_error* error; +} paimon_result_bytes; + +typedef struct paimon_result_stream_scan { + paimon_stream_scan* scan; + paimon_error* error; +} paimon_result_stream_scan; + +typedef struct paimon_result_stream_poll { + int32_t status; + paimon_stream_plan* plan; + int64_t snapshot_id; + int64_t next_snapshot_id; + int64_t watermark; + uint8_t has_watermark; + uint8_t reserved[7]; + paimon_error* error; +} paimon_result_stream_poll; + +void paimon_error_free(paimon_error* error); +void paimon_bytes_free(paimon_bytes bytes); +paimon_result_catalog_new paimon_catalog_create(const paimon_option* options, + size_t options_len); +void paimon_catalog_free(paimon_catalog* catalog); +paimon_result_get_table paimon_catalog_get_table( + const paimon_catalog* catalog, const paimon_identifier* identifier); +paimon_error* paimon_catalog_create_table_from_schema_json( + const paimon_catalog* catalog, const paimon_identifier* identifier, + const char* schema_json, bool ignore_if_exists); +paimon_error* paimon_catalog_drop_table( + const paimon_catalog* catalog, const paimon_identifier* identifier, + bool ignore_if_not_exists); +paimon_result_identifier_new paimon_identifier_new(const char* database, + const char* object); +void paimon_identifier_free(paimon_identifier* identifier); + +paimon_result_get_table paimon_table_from_schema_json( + const char* table_path, const char* table_schema_json, + const char* database, const char* table_name, const char* branch, + const paimon_option* storage_options, size_t storage_options_len); +void paimon_table_free(paimon_table* table); +paimon_result_read_builder paimon_table_new_read_builder( + const paimon_table* table); +paimon_result_read_builder paimon_table_new_read_builder_with_options( + const paimon_table* table, const paimon_option* options, + size_t options_len); +void paimon_read_builder_free(paimon_read_builder* builder); +paimon_error* paimon_read_builder_with_projection( + paimon_read_builder* builder, const char* const* columns); +paimon_error* paimon_read_builder_with_case_sensitive( + paimon_read_builder* builder, bool case_sensitive); +paimon_result_table_scan paimon_read_builder_new_scan( + const paimon_read_builder* builder); +paimon_result_new_read paimon_read_builder_new_read( + const paimon_read_builder* builder); +paimon_error* paimon_stream_scan_options_init( + paimon_stream_scan_options* options); +paimon_result_stream_scan paimon_read_builder_new_stream_scan( + const paimon_read_builder* builder, + const paimon_stream_scan_options* options); +paimon_result_stream_poll paimon_stream_scan_poll(paimon_stream_scan* scan); +int64_t paimon_stream_scan_checkpoint(const paimon_stream_scan* scan); +paimon_error* paimon_stream_scan_restore(paimon_stream_scan* scan, + int64_t next_snapshot_id); +void paimon_stream_scan_free(paimon_stream_scan* scan); +uint8_t paimon_stream_plan_is_full(const paimon_stream_plan* plan); +size_t paimon_stream_plan_num_splits(const paimon_stream_plan* plan); +paimon_result_bytes paimon_stream_plan_serialize( + const paimon_stream_plan* plan); +paimon_result_stream_poll paimon_stream_plan_deserialize( + const uint8_t* data, size_t size); +paimon_result_record_batch_reader paimon_stream_plan_read_to_arrow( + const paimon_table_read* read, const paimon_stream_plan* plan, + size_t offset, size_t length, int32_t read_mode); +void paimon_stream_plan_free(paimon_stream_plan* plan); +void paimon_table_scan_free(paimon_table_scan* scan); +paimon_result_plan paimon_table_scan_plan(const paimon_table_scan* scan); +paimon_result_plan paimon_plan_from_split_bytes(const uint8_t* data, + size_t size); +void paimon_plan_free(paimon_plan* plan); +size_t paimon_plan_num_splits(const paimon_plan* plan); +void paimon_table_read_free(paimon_table_read* read); +paimon_result_record_batch_reader paimon_table_read_to_arrow( + const paimon_table_read* read, const paimon_plan* plan, size_t offset, + size_t length); +paimon_result_next_batch paimon_record_batch_reader_next( + paimon_record_batch_reader* reader); +void paimon_record_batch_reader_free(paimon_record_batch_reader* reader); +void paimon_arrow_batch_free(paimon_arrow_batch batch); + +paimon_result_write_builder paimon_table_new_write_builder( + const paimon_table* table); +paimon_result_write_builder paimon_table_new_write_builder_with_commit_user( + const paimon_table* table, const char* commit_user); +void paimon_write_builder_free(paimon_write_builder* builder); +paimon_error* paimon_write_builder_with_overwrite( + paimon_write_builder* builder); +paimon_result_table_write paimon_write_builder_new_write( + const paimon_write_builder* builder); +paimon_result_table_commit paimon_write_builder_new_commit( + const paimon_write_builder* builder); +void paimon_table_write_free(paimon_table_write* writer); +paimon_error* paimon_table_write_write_arrow_batch(paimon_table_write* writer, + void* array, + void* schema); +paimon_result_prepare_commit paimon_table_write_prepare_commit( + paimon_table_write* writer); +void paimon_commit_messages_free(paimon_commit_messages* messages); +paimon_result_prepared_commit paimon_commit_messages_prepare( + const paimon_commit_messages* messages, int64_t checkpoint_id); +paimon_result_bytes paimon_prepared_commit_serialize( + const paimon_prepared_commit* prepared); +paimon_result_prepared_commit paimon_prepared_commit_deserialize( + const uint8_t* data, size_t size); +int64_t paimon_prepared_commit_identifier( + const paimon_prepared_commit* prepared); +void paimon_prepared_commit_free(paimon_prepared_commit* prepared); +paimon_error* paimon_commit_messages_merge( + paimon_commit_messages* target, const paimon_commit_messages* source); +paimon_error* paimon_prepared_commit_merge( + paimon_prepared_commit* target, const paimon_prepared_commit* source); +void paimon_table_commit_free(paimon_table_commit* committer); +paimon_error* paimon_table_commit_commit( + const paimon_table_commit* committer, paimon_commit_messages* messages); +paimon_error* paimon_table_commit_commit_with_identifier( + const paimon_table_commit* committer, paimon_commit_messages* messages, + int64_t checkpoint_id); +paimon_error* paimon_table_commit_filter_and_commit_with_identifier( + const paimon_table_commit* committer, paimon_commit_messages* messages, + int64_t checkpoint_id); +paimon_error* paimon_table_commit_commit_prepared( + const paimon_table_commit* committer, + const paimon_prepared_commit* prepared); +paimon_error* paimon_table_commit_overwrite( + const paimon_table_commit* committer, paimon_commit_messages* messages); +paimon_error* paimon_table_commit_overwrite_with_identifier( + const paimon_table_commit* committer, paimon_commit_messages* messages, + int64_t checkpoint_id); +paimon_error* paimon_table_commit_truncate_table( + const paimon_table_commit* committer); +paimon_error* paimon_table_commit_truncate_table_with_identifier( + const paimon_table_commit* committer, int64_t checkpoint_id); +paimon_error* paimon_table_commit_abort( + const paimon_table_commit* committer, paimon_commit_messages* messages); +paimon_error* paimon_table_commit_abort_prepared( + const paimon_table_commit* committer, + const paimon_prepared_commit* prepared); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // PAIMON_CPP_TEST_PAIMON_STUB_H diff --git a/bindings/cpp/tests/run_install_tree_consumer.cmake b/bindings/cpp/tests/run_install_tree_consumer.cmake new file mode 100644 index 000000000..a99c0319a --- /dev/null +++ b/bindings/cpp/tests/run_install_tree_consumer.cmake @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +foreach(required IN ITEMS MAIN_BUILD_DIR CONSUMER_SOURCE_DIR TEST_ROOT + CXX_COMPILER INSTALL_LIBDIR) + if(NOT DEFINED ${required}) + message(FATAL_ERROR "missing -D${required}=...") + endif() +endforeach() + +set(test_prefix "${TEST_ROOT}/prefix") +set(consumer_build "${TEST_ROOT}/build") +file(REMOVE_RECURSE "${TEST_ROOT}") + +execute_process( + COMMAND "${CMAKE_COMMAND}" --install "${MAIN_BUILD_DIR}" + --prefix "${test_prefix}" + RESULT_VARIABLE install_result + OUTPUT_VARIABLE install_stdout + ERROR_VARIABLE install_stderr) +if(NOT install_result EQUAL 0) + message(FATAL_ERROR + "install-tree setup failed:\n${install_stdout}\n${install_stderr}") +endif() + +set(config_libdir_alias "${TEST_ROOT}/libdir-alias") +execute_process( + COMMAND "${CMAKE_COMMAND}" -E create_symlink + "${test_prefix}/${INSTALL_LIBDIR}" "${config_libdir_alias}" + RESULT_VARIABLE alias_result + ERROR_VARIABLE alias_stderr) +if(NOT alias_result EQUAL 0) + message(FATAL_ERROR "package config alias setup failed:\n${alias_stderr}") +endif() + +execute_process( + COMMAND "${CMAKE_COMMAND}" + -S "${CONSUMER_SOURCE_DIR}" + -B "${consumer_build}" + "-DPaimonCpp_DIR=${config_libdir_alias}/cmake/PaimonCpp" + "-DCMAKE_CXX_COMPILER=${CXX_COMPILER}" + RESULT_VARIABLE configure_result + OUTPUT_VARIABLE configure_stdout + ERROR_VARIABLE configure_stderr) +if(NOT configure_result EQUAL 0) + message(FATAL_ERROR + "install-tree consumer configure failed:\n${configure_stdout}\n${configure_stderr}") +endif() + +execute_process( + COMMAND "${CMAKE_COMMAND}" --build "${consumer_build}" + RESULT_VARIABLE build_result + OUTPUT_VARIABLE build_stdout + ERROR_VARIABLE build_stderr) +if(NOT build_result EQUAL 0) + message(FATAL_ERROR + "install-tree consumer build failed:\n${build_stdout}\n${build_stderr}") +endif() + +execute_process( + COMMAND "${consumer_build}/paimon_install_tree_consumer${CMAKE_EXECUTABLE_SUFFIX}" + RESULT_VARIABLE run_result + OUTPUT_VARIABLE run_stdout + ERROR_VARIABLE run_stderr) +if(NOT run_result EQUAL 0) + message(FATAL_ERROR + "install-tree consumer run failed:\n${run_stdout}\n${run_stderr}") +endif() diff --git a/crates/paimon/src/arrow/format/avro.rs b/crates/paimon/src/arrow/format/avro.rs index 546d963ee..ff4463a1b 100644 --- a/crates/paimon/src/arrow/format/avro.rs +++ b/crates/paimon/src/arrow/format/avro.rs @@ -394,14 +394,14 @@ fn build_column( DataType::Map(map_type) => build_map_column(records, name, map_type, num_rows)?, // Java encodes MULTISET as a map from the element to an INT count, // sharing the MAP path (`AvroSchemaConverter#extractValueTypeToAvroMap` - // returns IntType). Unlike MAP, `paimon_type_to_arrow` lets the key here - // follow the element's nullability and pins the count non-nullable. + // returns IntType). Arrow map keys are always non-null and the count is + // non-nullable too. DataType::Multiset(multiset_type) => build_map_like_column( records, name, multiset_type.element_type(), &DataType::Int(IntType::new()), - multiset_type.element_type().is_nullable(), + false, false, num_rows, )?, diff --git a/crates/paimon/src/arrow/format/row.rs b/crates/paimon/src/arrow/format/row.rs index fa00773a4..9338a1684 100644 --- a/crates/paimon/src/arrow/format/row.rs +++ b/crates/paimon/src/arrow/format/row.rs @@ -1266,12 +1266,7 @@ impl ColumnBuilder { DataType::Multiset(m) => { let count_type = DataType::Int(IntType::new()); Self::Map { - entries_field: map_entries_field( - m.element_type(), - &count_type, - m.element_type().is_nullable(), - false, - )?, + entries_field: map_entries_field(m.element_type(), &count_type, false, false)?, offsets: vec![0], validities: Vec::with_capacity(capacity), keys: Box::new(ColumnBuilder::new(m.element_type(), capacity)?), @@ -2992,7 +2987,7 @@ mod tests { let bag = test_map_array( vec![0, 2, 2, 3], vec![true, false, true], - true, + false, false, vec![Some("x"), Some("y"), Some("z")], vec![Some(2), Some(1), Some(4)], diff --git a/crates/paimon/src/arrow/mod.rs b/crates/paimon/src/arrow/mod.rs index 2fe1a6e24..18858d3f7 100644 --- a/crates/paimon/src/arrow/mod.rs +++ b/crates/paimon/src/arrow/mod.rs @@ -104,7 +104,9 @@ pub fn paimon_type_to_arrow(dt: &PaimonDataType) -> crate::Result "entries", ArrowDataType::Struct( vec![ - ArrowField::new("key", element_type, m.element_type().is_nullable()), + // Arrow map keys are always non-null, including the + // element carrier used for a Paimon MULTISET. + ArrowField::new("key", element_type, false), ArrowField::new("value", ArrowDataType::Int32, false), ] .into(), @@ -490,6 +492,22 @@ mod tests { ); } + #[test] + fn test_multiset_arrow_key_is_non_nullable() { + let multiset = PaimonDataType::Multiset(MultisetType::new(PaimonDataType::VarChar( + VarCharType::new(VarCharType::MAX_LENGTH).unwrap(), + ))); + let ArrowDataType::Map(entries, false) = paimon_type_to_arrow(&multiset).unwrap() else { + panic!("expected multiset Arrow Map"); + }; + let ArrowDataType::Struct(fields) = entries.data_type() else { + panic!("expected multiset entries Struct"); + }; + assert!(!fields[0].is_nullable()); + assert_eq!(fields[1].data_type(), &ArrowDataType::Int32); + assert!(!fields[1].is_nullable()); + } + #[test] fn test_timestamp_roundtrip() { // millisecond precision diff --git a/crates/paimon/src/lib.rs b/crates/paimon/src/lib.rs index 9e6b13400..9c407cf0f 100644 --- a/crates/paimon/src/lib.rs +++ b/crates/paimon/src/lib.rs @@ -54,7 +54,8 @@ pub use table::{ IncrementalScanMode, IncrementalSplit, PartitionBucket, Plan, PostponeBucketPlan, PostponeFixedBucketTableCommit, PostponeFixedBucketTableWrite, RESTEnv, RESTSnapshotCommit, ReadBuilder, RenamingSnapshotCommit, RowRange, ScanTrace, SnapshotCommit, SnapshotManager, - Table, TableCommit, TableRead, TableScan, TableUpdate, TableWrite, TagManager, WriteBuilder, + StreamPlan, StreamScan, StreamScanFollowUpMode, StreamScanPoll, StreamScanStartupMode, Table, + TableCommit, TableRead, TableScan, TableUpdate, TableWrite, TagManager, WriteBuilder, }; pub use table::{ diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index 7afff6f67..5b37a3c64 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -95,6 +95,9 @@ pub(crate) const DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL_OPTION: &str = "alter-column-null-to-not-null.disabled"; const MERGE_ENGINE_OPTION: &str = "merge-engine"; pub(crate) const CHANGELOG_PRODUCER_OPTION: &str = "changelog-producer"; +const NUM_LEVELS_OPTION: &str = "num-levels"; +const NUM_SORTED_RUN_COMPACTION_TRIGGER_OPTION: &str = "num-sorted-run.compaction-trigger"; +const DEFAULT_NUM_SORTED_RUN_COMPACTION_TRIGGER: i32 = 5; const ROWKIND_FIELD_OPTION: &str = "rowkind.field"; const IGNORE_DELETE_OPTION: &str = "ignore-delete"; const IGNORE_UPDATE_BEFORE_OPTION: &str = "ignore-update-before"; @@ -614,6 +617,44 @@ impl<'a> CoreOptions<'a> { } } + /// Total number of merge-tree levels. + /// + /// Java defaults this to `num-sorted-run.compaction-trigger + 1` so a + /// compaction always has at least one non-zero target level. + pub fn num_levels(&self) -> crate::Result { + fn positive_i32(raw: &str, option: &str) -> crate::Result { + let value = raw + .parse::() + .map_err(|error| crate::Error::DataInvalid { + message: format!("Option '{option}' must be a positive integer, got: {raw}"), + source: Some(Box::new(error)), + })?; + if value <= 0 { + return Err(crate::Error::DataInvalid { + message: format!("Option '{option}' must be greater than 0, got: {value}"), + source: None, + }); + } + Ok(value) + } + + if let Some(raw) = self.options.get(NUM_LEVELS_OPTION) { + return positive_i32(raw, NUM_LEVELS_OPTION); + } + let trigger = match self.options.get(NUM_SORTED_RUN_COMPACTION_TRIGGER_OPTION) { + Some(raw) => positive_i32(raw, NUM_SORTED_RUN_COMPACTION_TRIGGER_OPTION)?, + None => DEFAULT_NUM_SORTED_RUN_COMPACTION_TRIGGER, + }; + trigger + .checked_add(1) + .ok_or_else(|| crate::Error::DataInvalid { + message: format!( + "Option '{NUM_SORTED_RUN_COMPACTION_TRIGGER_OPTION}' cannot be incremented: {trigger}" + ), + source: None, + }) + } + /// The `rowkind.field` option: a user column whose value encodes the row kind. pub fn rowkind_field(&self) -> Option<&str> { self.options.get(ROWKIND_FIELD_OPTION).map(String::as_str) diff --git a/crates/paimon/src/table/commit_message.rs b/crates/paimon/src/table/commit_message.rs index 55afbf643..d338ddfae 100644 --- a/crates/paimon/src/table/commit_message.rs +++ b/crates/paimon/src/table/commit_message.rs @@ -17,11 +17,12 @@ use crate::spec::DataFileMeta; use crate::spec::IndexFileMeta; +use serde::{Deserialize, Serialize}; /// A commit message representing new files to be committed for a specific partition and bucket. /// /// Reference: [org.apache.paimon.table.sink.CommitMessage](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/sink/CommitMessageImpl.java) -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct CommitMessage { /// Binary row bytes for the partition. pub partition: Vec, diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 92c73a6fb..db3276f02 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -96,6 +96,7 @@ mod sorted_global_index_build_builder; mod sorted_global_index_options; mod source; mod stats_filter; +mod stream_scan; pub(crate) mod table_commit; mod table_read; mod table_scan; @@ -149,6 +150,9 @@ pub use sorted_global_index_build_builder::{ pub use source::{ merge_row_ranges, DataSplit, DataSplitBuilder, DeletionFile, PartitionBucket, Plan, RowRange, }; +pub use stream_scan::{ + StreamPlan, StreamScan, StreamScanFollowUpMode, StreamScanPoll, StreamScanStartupMode, +}; pub use table_commit::TableCommit; pub use table_read::TableRead; pub use table_scan::TableScan; diff --git a/crates/paimon/src/table/read_builder.rs b/crates/paimon/src/table/read_builder.rs index ec8ef966e..51b4f6923 100644 --- a/crates/paimon/src/table/read_builder.rs +++ b/crates/paimon/src/table/read_builder.rs @@ -24,6 +24,7 @@ use super::bucket_filter::{extract_predicate_for_keys, split_partition_and_data_ use super::format_read_builder::FormatReadBuilder; use super::incremental_scan::{IncrementalScan, IncrementalScanMode}; use super::partition_filter::PartitionFilter; +use super::stream_scan::{StreamScan, StreamScanFollowUpMode, StreamScanStartupMode}; use super::table_read::{configured_parquet_read_budget, TableRead}; use super::{Table, TableScan}; use crate::spec::{CoreOptions, DataField, Predicate}; @@ -288,6 +289,33 @@ impl<'a> ReadBuilder<'a> { } } + /// Create an owned, stateful continuous snapshot scanner. + /// + /// The returned scanner clones the table and scan configuration. It remains + /// valid after this builder and its originating table handle are dropped. + /// Filters, projection-driven data-evolution pruning, and row ranges are + /// preserved. Limit pushdown is rejected because advancing a stream cursor + /// past a partially planned snapshot would lose data. + pub async fn new_stream_scan( + &self, + startup_mode: StreamScanStartupMode, + follow_up_mode: StreamScanFollowUpMode, + ) -> Result { + let mut scan = match &self.0 { + ReadBuilderKind::Paimon(builder) => { + builder.new_stream_scan(startup_mode, follow_up_mode) + } + ReadBuilderKind::Format(_) => Err(Error::Unsupported { + message: "Continuous stream scan is not supported for format tables".to_string(), + }), + }?; + // Freeze the `Latest` boundary before returning. If initialization is + // deferred until the first poll, a concurrently committed snapshot can + // be mistaken for pre-existing data and skipped. + scan.initialize().await?; + Ok(scan) + } + /// Create a table read for consuming splits (e.g. from a scan plan). pub fn new_read(&self) -> Result> { match &self.0 { @@ -332,6 +360,37 @@ impl<'a> PaimonReadBuilder<'a> { } } + fn new_stream_scan( + &self, + startup_mode: StreamScanStartupMode, + follow_up_mode: StreamScanFollowUpMode, + ) -> Result { + if self.limit.is_some() { + return Err(Error::Unsupported { + message: "Continuous stream scan does not support limit pushdown".to_string(), + }); + } + let partition_filter = self.filter.partition_predicate.clone().map(|pred| { + PartitionFilter::from_predicate(pred, &self.table.schema().partition_fields()) + }); + let read_type = self.resolve_read_type()?; + let projected_read_field_ids = projected_read_field_ids_with_predicates( + &read_type, + &self.filter.data_predicates, + self.table.schema().fields(), + ); + StreamScan::try_new( + self.table.clone(), + partition_filter, + self.filter.data_predicates.clone(), + self.filter.bucket_predicate.clone(), + self.effective_row_ranges(), + projected_read_field_ids, + startup_mode, + follow_up_mode, + ) + } + /// Set column projection by name. Output order follows the caller-specified order. /// An empty list is a valid zero-column projection. /// diff --git a/crates/paimon/src/table/snapshot_manager.rs b/crates/paimon/src/table/snapshot_manager.rs index 1de8baa9e..a6edfbc51 100644 --- a/crates/paimon/src/table/snapshot_manager.rs +++ b/crates/paimon/src/table/snapshot_manager.rs @@ -145,9 +145,13 @@ impl SnapshotManager { let hint_path = self.latest_hint_path(); if let Some(hint_id) = self.read_hint(&hint_path).await { if hint_id > 0 { - let next_path = self.snapshot_path(hint_id + 1); - let next_input = self.file_io.new_input(&next_path)?; - if !next_input.exists().await? { + if let Some(next_id) = hint_id.checked_add(1) { + let next_path = self.snapshot_path(next_id); + let next_input = self.file_io.new_input(&next_path)?; + if !next_input.exists().await? { + return Ok(Some(hint_id)); + } + } else { return Ok(Some(hint_id)); } } @@ -656,6 +660,13 @@ mod tests { assert_eq!(hint, Some(42)); } + #[tokio::test] + async fn test_latest_hint_at_max_id_does_not_overflow() { + let (_, sm) = setup("memory:/test_latest_hint_max").await; + sm.write_latest_hint(i64::MAX).await.unwrap(); + assert_eq!(sm.get_latest_snapshot_id().await.unwrap(), Some(i64::MAX)); + } + #[tokio::test] async fn test_list_all_ids_empty() { let (_, sm) = setup("memory:/test_list_empty").await; diff --git a/crates/paimon/src/table/source.rs b/crates/paimon/src/table/source.rs index aaaf66cfc..65697ec75 100644 --- a/crates/paimon/src/table/source.rs +++ b/crates/paimon/src/table/source.rs @@ -23,6 +23,71 @@ use crate::spec::{BinaryRow, DataFileMeta, DataFileMetaRowLayout}; use crate::table::stats_filter::group_by_overlapping_row_id; use serde::{Deserialize, Serialize}; use std::sync::Arc; +use url::Url; + +const MAX_RESTORED_FILE_NAME_BYTES: usize = 4 * 1024; + +fn safe_restored_file_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= MAX_RESTORED_FILE_NAME_BYTES + && name != "." + && name != ".." + && !name.contains('/') + && !name.contains('\\') + && !name.contains('\0') +} + +fn url_authority_matches(left: &Url, right: &Url) -> bool { + left.scheme() == right.scheme() + && left.username() == right.username() + && left.password() == right.password() + && left.host_str() == right.host_str() + && left.port_or_known_default() == right.port_or_known_default() +} + +fn path_has_root(root: &str, candidate: &str) -> bool { + match (Url::parse(root), Url::parse(candidate)) { + (Ok(root), Ok(candidate)) => { + if !url_authority_matches(&root, &candidate) + || root.query().is_some() + || root.fragment().is_some() + || candidate.query().is_some() + || candidate.fragment().is_some() + { + return false; + } + path_text_has_root(root.path(), candidate.path()) + } + (Err(_), Err(_)) => path_text_has_root(root, candidate), + _ => false, + } +} + +fn path_text_has_root(root: &str, candidate: &str) -> bool { + fn components(path: &str) -> Option> { + let mut result = Vec::new(); + for component in path.split(['/', '\\']) { + match component { + "" | "." => {} + ".." => return None, + value => result.push(value), + } + } + Some(result) + } + + if root.is_empty() + || candidate.is_empty() + || root.starts_with('/') != candidate.starts_with('/') + || root.starts_with('\\') != candidate.starts_with('\\') + { + return false; + } + let (Some(root), Some(candidate)) = (components(root), components(candidate)) else { + return false; + }; + candidate.len() >= root.len() && candidate[..root.len()] == root +} fn is_vector_store_file_name(file_name: &str) -> bool { file_name.to_ascii_lowercase().contains(".vector.") @@ -574,6 +639,65 @@ impl DataSplit { file.data_file_path(&self.bucket_path) } + /// Validate that a deserialized split cannot escape its table root. + /// + /// Planned splits are trusted Rust objects, but persisted split bytes can + /// cross an FFI/process boundary. Version 1 recovery therefore rejects + /// external data paths and requires every referenced path to remain under + /// the table location. + pub fn validate_restored_containment(&self, table_location: &str) -> crate::Result<()> { + if !path_has_root(table_location, self.bucket_path()) { + return Err(crate::Error::DataInvalid { + message: format!( + "Restored split bucket path '{}' is outside table root '{table_location}'", + self.bucket_path() + ), + source: None, + }); + } + for file in self.data_files() { + if file.external_path.is_some() { + return Err(crate::Error::Unsupported { + message: + "Restored stream plans with external data-file paths are not supported" + .to_string(), + }); + } + if !safe_restored_file_name(&file.file_name) + || file + .extra_files + .iter() + .any(|name| !safe_restored_file_name(name)) + { + return Err(crate::Error::DataInvalid { + message: "Restored stream plan contains an unsafe data-file name".to_string(), + source: None, + }); + } + } + if let Some(deletion_files) = self.data_deletion_files() { + for deletion_file in deletion_files.iter().flatten() { + if deletion_file.offset() < 0 + || deletion_file.length() < 0 + || deletion_file + .offset() + .checked_add(deletion_file.length()) + .is_none() + || !path_has_root(table_location, deletion_file.path()) + { + return Err(crate::Error::DataInvalid { + message: format!( + "Restored deletion file '{}' is invalid or outside table root '{table_location}'", + deletion_file.path() + ), + source: None, + }); + } + } + } + Ok(()) + } + /// Sum of the physical row counts this split knows about. /// /// Files whose count is [`DataFileMeta::ROW_COUNT_UNKNOWN`] contribute @@ -1376,6 +1500,61 @@ mod tests { .unwrap() } + #[test] + fn restored_split_paths_are_confined_to_table() { + let safe = split(vec![file("data.orc", 1, None)], true); + assert!(safe.validate_restored_containment("file:/tmp").is_ok()); + assert!(safe + .validate_restored_containment("file:/tmp-other") + .is_err()); + + let outside_bucket = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path("s3://warehouse/table-evil/bucket-0".to_string()) + .with_total_buckets(1) + .with_data_files(vec![file("data.orc", 1, None)]) + .build() + .unwrap(); + assert!(outside_bucket + .validate_restored_containment("s3://warehouse/table") + .is_err()); + } + + #[test] + fn restored_split_rejects_external_and_unsafe_files() { + let mut external = file("data.orc", 1, None); + external.external_path = Some("file:/etc/passwd".to_string()); + assert!(split(vec![external], true) + .validate_restored_containment("file:/tmp") + .is_err()); + + let unsafe_name = file("../data.orc", 1, None); + assert!(split(vec![unsafe_name], true) + .validate_restored_containment("file:/tmp") + .is_err()); + + let with_outside_deletion = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path("file:/tmp/bucket-0".to_string()) + .with_total_buckets(1) + .with_data_files(vec![file("data.orc", 1, None)]) + .with_data_deletion_files(vec![Some(DeletionFile::new( + "file:/elsewhere/dv.idx".to_string(), + 0, + 8, + Some(1), + ))]) + .build() + .unwrap(); + assert!(with_outside_deletion + .validate_restored_containment("file:/tmp") + .is_err()); + } + #[test] fn data_split_clone_shares_planned_metadata() { let split = DataSplitBuilder::new() diff --git a/crates/paimon/src/table/stream_scan.rs b/crates/paimon/src/table/stream_scan.rs new file mode 100644 index 000000000..d1b8016da --- /dev/null +++ b/crates/paimon/src/table/stream_scan.rs @@ -0,0 +1,685 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Stateful continuous snapshot scan. +//! +//! The cursor has the same meaning as Java `DataTableStreamScan`: planning a +//! snapshot advances `next_snapshot_id` immediately. Callers which hand plans +//! to asynchronous workers must therefore persist their own safe checkpoint +//! only after the planned work has completed. + +use std::collections::HashSet; + +use super::incremental_scan::{IncrementalPlan, IncrementalScanMode, IncrementalSplit}; +use super::partition_filter::PartitionFilter; +use super::table_scan::SnapshotLevelFilter; +use super::{Plan, RowRange, SnapshotManager, Table, TableScan}; +use crate::spec::{ChangelogProducer, CommitKind, Predicate, Snapshot}; + +const FIRST_SNAPSHOT_ID: i64 = 1; +const RANGE_READ_ATTEMPTS: usize = 2; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AvailableRange { + Empty, + Range { earliest: i64, latest: i64 }, + Transient, +} + +/// How a continuous scan chooses its first snapshot. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamScanStartupMode { + /// Read the latest table state in full, then follow later snapshots. + LatestFull, + /// Ignore snapshots which already exist when the scan starts and follow + /// snapshots committed afterwards. + /// + /// When the table has no snapshot at startup, snapshot 1 is consumed as an + /// incremental snapshot once it appears, matching Java Paimon. + Latest, + /// Read `snapshot_id` inclusively as the first incremental snapshot. + FromSnapshot(i64), + /// Read `snapshot_id` as a full table state, then follow later snapshots. + FromSnapshotFull(i64), +} + +/// How snapshots after the startup phase are planned. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamScanFollowUpMode { + /// Use delta manifests for `changelog-producer=none`; otherwise use + /// changelog manifests. + Auto, + /// Read APPEND snapshot delta manifests. + Delta, + /// Read changelog manifests. + Changelog, +} + +/// A plan emitted by a continuous scan. +#[derive(Debug)] +pub enum StreamPlan { + /// A complete table state at one snapshot. + Full { + snapshot_id: i64, + watermark: Option, + next_snapshot_id: i64, + plan: Plan, + }, + /// Delta or changelog work for one snapshot. + Incremental { + snapshot_id: i64, + watermark: Option, + next_snapshot_id: i64, + plan: IncrementalPlan, + }, +} + +impl StreamPlan { + /// Snapshot represented by this plan. + pub fn snapshot_id(&self) -> i64 { + match self { + Self::Full { snapshot_id, .. } | Self::Incremental { snapshot_id, .. } => *snapshot_id, + } + } + + /// Snapshot watermark, when one was committed. + pub fn watermark(&self) -> Option { + match self { + Self::Full { watermark, .. } | Self::Incremental { watermark, .. } => *watermark, + } + } + + /// Cursor immediately after this plan was produced. + pub fn next_snapshot_id(&self) -> i64 { + match self { + Self::Full { + next_snapshot_id, .. + } + | Self::Incremental { + next_snapshot_id, .. + } => *next_snapshot_id, + } + } + + pub fn full_plan(&self) -> Option<&Plan> { + match self { + Self::Full { plan, .. } => Some(plan), + Self::Incremental { .. } => None, + } + } + + pub fn incremental_plan(&self) -> Option<&IncrementalPlan> { + match self { + Self::Incremental { plan, .. } => Some(plan), + Self::Full { .. } => None, + } + } + + pub fn into_full_plan(self) -> Option { + match self { + Self::Full { plan, .. } => Some(plan), + Self::Incremental { .. } => None, + } + } + + pub fn into_incremental_plan(self) -> Option { + match self { + Self::Incremental { plan, .. } => Some(plan), + Self::Full { .. } => None, + } + } +} + +/// Result of one non-blocking continuous-scan poll. +#[derive(Debug)] +pub enum StreamScanPoll { + /// Work is available. + Data(StreamPlan), + /// The next expected snapshot has not been committed yet. + Waiting, + /// The configured bounded scan is complete. + /// + /// Bounded-watermark configuration is not implemented in the first + /// version, so this variant is reserved for forward-compatible consumers. + End, +} + +/// An owned, stateful continuous scanner. +/// +/// The scanner clones the table and all scan-time predicates at construction, +/// so it remains valid after the originating [`Table`] or read builder is +/// dropped. It does not spawn a background task; callers control polling and +/// backpressure. +#[derive(Debug)] +pub struct StreamScan { + table: Table, + snapshot_manager: SnapshotManager, + partition_filter: Option, + data_predicates: Vec, + bucket_predicate: Option, + row_ranges: Option>, + projected_read_field_ids: Option>, + startup_mode: StreamScanStartupMode, + follow_up_mode: IncrementalScanMode, + startup_complete: bool, + next_snapshot_id: Option, + current_watermark: Option, +} + +impl StreamScan { + #[allow(clippy::too_many_arguments)] + pub(crate) fn try_new( + table: Table, + partition_filter: Option, + data_predicates: Vec, + bucket_predicate: Option, + row_ranges: Option>, + projected_read_field_ids: Option>, + startup_mode: StreamScanStartupMode, + follow_up_mode: StreamScanFollowUpMode, + ) -> crate::Result { + match startup_mode { + StreamScanStartupMode::FromSnapshot(snapshot_id) + | StreamScanStartupMode::FromSnapshotFull(snapshot_id) + if snapshot_id < FIRST_SNAPSHOT_ID => + { + return Err(crate::Error::DataInvalid { + message: format!( + "Stream scan starting snapshot id must be at least {FIRST_SNAPSHOT_ID}, got {snapshot_id}" + ), + source: None, + }); + } + _ => {} + } + + if table.is_format_table() { + return Err(crate::Error::Unsupported { + message: "Continuous stream scan is not supported for format tables".to_string(), + }); + } + + let core_options = table.schema().core_options(); + let changelog_producer = core_options.try_changelog_producer()?; + let deletion_vectors_enabled = core_options.deletion_vectors_enabled(); + let follow_up_mode = match (follow_up_mode, changelog_producer) { + (StreamScanFollowUpMode::Delta, ChangelogProducer::Lookup) + if deletion_vectors_enabled + && matches!( + startup_mode, + StreamScanStartupMode::LatestFull + | StreamScanStartupMode::FromSnapshotFull(_) + ) => + { + return Err(crate::Error::Unsupported { + message: "Deletion-vector lookup tables require changelog follow-up" + .to_string(), + }); + } + (StreamScanFollowUpMode::Delta, _) => IncrementalScanMode::Delta, + (StreamScanFollowUpMode::Changelog, ChangelogProducer::None) => { + return Err(crate::Error::Unsupported { + message: "Changelog stream follow-up requires a changelog producer".to_string(), + }); + } + (StreamScanFollowUpMode::Changelog, _) => IncrementalScanMode::Changelog, + (StreamScanFollowUpMode::Auto, ChangelogProducer::None) => IncrementalScanMode::Delta, + (StreamScanFollowUpMode::Auto, _) => IncrementalScanMode::Changelog, + }; + let snapshot_manager = table.snapshot_manager(); + + Ok(Self { + table, + snapshot_manager, + partition_filter, + data_predicates, + bucket_predicate, + row_ranges, + projected_read_field_ids, + startup_mode, + follow_up_mode, + startup_complete: false, + next_snapshot_id: None, + current_watermark: None, + }) + } + + /// Resolved follow-up mode. `Auto` is collapsed during construction. + pub fn follow_up_mode(&self) -> IncrementalScanMode { + self.follow_up_mode + } + + /// The next snapshot which will be considered, suitable for checkpointing. + /// + /// This cursor advances when a plan is produced, not when its splits finish. + pub fn checkpoint(&self) -> Option { + self.next_snapshot_id + } + + /// Restore a previously checkpointed next snapshot id. + /// + /// `Some(id)` bypasses startup selection. `None` resets the scanner and + /// applies its configured startup mode again. + pub fn restore(&mut self, next_snapshot_id: Option) -> crate::Result<()> { + if next_snapshot_id.is_some_and(|id| id < FIRST_SNAPSHOT_ID) { + return Err(crate::Error::DataInvalid { + message: format!( + "Stream scan checkpoint must be at least {FIRST_SNAPSHOT_ID}, got {}", + next_snapshot_id.unwrap() + ), + source: None, + }); + } + self.next_snapshot_id = next_snapshot_id; + self.startup_complete = next_snapshot_id.is_some(); + self.current_watermark = None; + Ok(()) + } + + /// Most recent watermark observed on a planned (including empty) snapshot. + pub fn watermark(&self) -> Option { + self.current_watermark + } + + /// Freeze startup state which is observable at scanner creation time. + /// + /// `Latest` must remember whether the table was empty when the source was + /// created. Without this explicit async initialization, a snapshot + /// committed between construction and the first poll could be mistaken for + /// pre-existing data and skipped. Other startup modes resolve their first + /// plan during polling and need no eager IO. + pub async fn initialize(&mut self) -> crate::Result<()> { + if self.startup_complete || self.startup_mode != StreamScanStartupMode::Latest { + return Ok(()); + } + self.table + .schema() + .core_options() + .ensure_read_authorized()?; + let next_snapshot_id = match self.snapshot_manager.get_latest_snapshot_id().await? { + Some(latest) => next_id(latest)?, + None => FIRST_SNAPSHOT_ID, + }; + self.next_snapshot_id = Some(next_snapshot_id); + self.startup_complete = true; + Ok(()) + } + + /// Poll once without waiting for future snapshots. + pub async fn poll_next(&mut self) -> crate::Result { + self.table + .schema() + .core_options() + .ensure_read_authorized()?; + self.initialize().await?; + if !self.startup_complete { + return self.poll_startup().await; + } + self.poll_follow_up().await + } + + fn table_scan(&self) -> TableScan<'_> { + TableScan::new( + &self.table, + self.partition_filter.clone(), + self.data_predicates.clone(), + self.bucket_predicate.clone(), + None, + self.row_ranges.clone(), + ) + .with_projected_read_field_ids(self.projected_read_field_ids.clone()) + } + + async fn poll_startup(&mut self) -> crate::Result { + match self.startup_mode { + StreamScanStartupMode::LatestFull => { + let Some(snapshot) = self.snapshot_manager.get_latest_snapshot().await? else { + return Ok(StreamScanPoll::Waiting); + }; + self.plan_full_startup(snapshot).await + } + StreamScanStartupMode::Latest => { + unreachable!("Latest startup is resolved by initialize") + } + StreamScanStartupMode::FromSnapshot(snapshot_id) => { + let (earliest, latest) = match self.available_range().await? { + AvailableRange::Empty | AvailableRange::Transient => { + return Ok(StreamScanPoll::Waiting) + } + AvailableRange::Range { earliest, latest } => (earliest, latest), + }; + validate_incremental_start(snapshot_id, earliest, latest)?; + self.next_snapshot_id = Some(snapshot_id); + self.startup_complete = true; + Ok(StreamScanPoll::Waiting) + } + StreamScanStartupMode::FromSnapshotFull(snapshot_id) => { + let (earliest, latest) = match self.available_range().await? { + AvailableRange::Empty | AvailableRange::Transient => { + return Ok(StreamScanPoll::Waiting) + } + AvailableRange::Range { earliest, latest } => (earliest, latest), + }; + validate_full_start(snapshot_id, earliest, latest)?; + let Some(snapshot) = self.try_get_snapshot(snapshot_id).await? else { + return Ok(StreamScanPoll::Waiting); + }; + self.plan_full_startup(snapshot).await + } + } + } + + async fn plan_full_startup(&mut self, snapshot: Snapshot) -> crate::Result { + // Reading a full state at an overwrite snapshot is well-defined. The + // overwrite restriction applies only to follow-up change plans. + let snapshot_id = snapshot.id(); + let watermark = snapshot.watermark(); + let next_snapshot_id = self.full_start_next_snapshot_id(snapshot_id)?; + let level_filter = self.full_start_level_filter()?; + let plan = self + .table_scan() + .plan_snapshot_full(&snapshot, level_filter) + .await?; + self.next_snapshot_id = Some(next_snapshot_id); + self.current_watermark = watermark; + self.startup_complete = true; + Ok(StreamScanPoll::Data(StreamPlan::Full { + snapshot_id, + watermark, + next_snapshot_id, + plan, + })) + } + + fn full_start_level_filter(&self) -> crate::Result> { + let options = self.table.schema().core_options(); + // Lookup-style tables expose their stable materialized state above + // level 0. Deletion-vector-only tables replay the starting snapshot in + // the incremental phase so its un-compacted level-0 changes are not + // lost. + if options.deletion_vectors_enabled() { + return Ok(Some(SnapshotLevelFilter::GreaterThan(0))); + } + if self.follow_up_mode != IncrementalScanMode::Changelog { + return Ok(None); + } + match options.try_changelog_producer()? { + // Lookup compaction will emit level-0 input through a later + // changelog. Reading it in the full phase would emit it twice. + ChangelogProducer::Lookup => Ok(Some(SnapshotLevelFilter::GreaterThan(0))), + // Full-compaction changelog covers all changes since the previous + // last-level state. Start from that materialized state only. + ChangelogProducer::FullCompaction => { + Ok(Some(SnapshotLevelFilter::Equal(options.num_levels()? - 1))) + } + ChangelogProducer::None | ChangelogProducer::Input => Ok(None), + } + } + + fn full_start_next_snapshot_id(&self, snapshot_id: i64) -> crate::Result { + let options = self.table.schema().core_options(); + if options.deletion_vectors_enabled() + && options.try_changelog_producer()? != ChangelogProducer::Lookup + { + // The full plan deliberately excludes level 0. Revisit this same + // snapshot once through Delta/Changelog to emit those changes. + Ok(snapshot_id) + } else { + next_id(snapshot_id) + } + } + + async fn poll_follow_up(&mut self) -> crate::Result { + loop { + let snapshot_id = + self.next_snapshot_id + .ok_or_else(|| crate::Error::UnexpectedError { + message: "Stream scan startup completed without a next snapshot id" + .to_string(), + source: None, + })?; + let Some(snapshot) = self.next_snapshot(snapshot_id).await? else { + return Ok(StreamScanPoll::Waiting); + }; + + if snapshot.commit_kind() == &CommitKind::OVERWRITE { + return Err(crate::Error::Unsupported { + message: format!( + "Streaming follow-up scan cannot safely consume OVERWRITE snapshot {snapshot_id}" + ), + }); + } + + let should_scan = match self.follow_up_mode { + IncrementalScanMode::Delta => snapshot.commit_kind() == &CommitKind::APPEND, + IncrementalScanMode::Changelog => snapshot.changelog_manifest_list().is_some(), + IncrementalScanMode::Auto | IncrementalScanMode::Diff => { + unreachable!("stream follow-up mode must resolve to Delta or Changelog") + } + }; + + let following_snapshot_id = next_id(snapshot_id)?; + if !should_scan { + self.next_snapshot_id = Some(following_snapshot_id); + continue; + } + + let raw_plan = match self.follow_up_mode { + IncrementalScanMode::Delta => { + self.table_scan() + .plan_snapshot_delta_streaming(&snapshot) + .await? + } + IncrementalScanMode::Changelog => { + self.table_scan() + .plan_snapshot_changelog_streaming(&snapshot) + .await? + } + IncrementalScanMode::Auto | IncrementalScanMode::Diff => unreachable!(), + }; + let splits = raw_plan + .into_splits() + .into_iter() + .map(IncrementalSplit::Data) + .collect(); + let plan = IncrementalPlan::try_new(self.follow_up_mode, splits)?; + + // Match Java DataTableStreamScan: the checkpoint cursor advances as + // soon as planning succeeds. Empty snapshots also advance and the + // same poll keeps looking for useful work. + self.next_snapshot_id = Some(following_snapshot_id); + self.current_watermark = snapshot.watermark(); + if plan.splits().is_empty() { + continue; + } + + return Ok(StreamScanPoll::Data(StreamPlan::Incremental { + snapshot_id, + watermark: snapshot.watermark(), + next_snapshot_id: following_snapshot_id, + plan, + })); + } + } + + async fn next_snapshot(&mut self, snapshot_id: i64) -> crate::Result> { + if let Some(snapshot) = self.try_get_snapshot(snapshot_id).await? { + return Ok(Some(snapshot)); + } + + // The snapshot may be committed after the first lookup but before the + // range observation. Re-read it before classifying the miss as a gap. + let range = self.available_range().await?; + if let Some(snapshot) = self.try_get_snapshot(snapshot_id).await? { + return Ok(Some(snapshot)); + } + + match range { + AvailableRange::Transient => Ok(None), + AvailableRange::Empty if snapshot_id == FIRST_SNAPSHOT_ID => Ok(None), + AvailableRange::Empty => Err(crate::Error::DataInvalid { + message: format!( + "Next expected snapshot {snapshot_id} is out of range because the table currently has no snapshots" + ), + source: None, + }), + AvailableRange::Range { earliest, latest } if snapshot_id < earliest => { + Err(crate::Error::DataInvalid { + message: format!( + "Next expected snapshot {snapshot_id} has expired; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }) + } + // A range hint/listing can become visible before the snapshot + // object itself on an eventually consistent backend. Polling + // frequency must never turn that transient state into data loss. + AvailableRange::Range { latest, .. } if snapshot_id <= latest => Ok(None), + AvailableRange::Range { latest, .. } + if latest.checked_add(1) == Some(snapshot_id) => + { + Ok(None) + } + AvailableRange::Range { earliest, latest } => Err(crate::Error::DataInvalid { + message: format!( + "Next expected snapshot {snapshot_id} is too large; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }), + } + } + + async fn try_get_snapshot(&self, snapshot_id: i64) -> crate::Result> { + match self.snapshot_manager.get_snapshot(snapshot_id).await { + Ok(snapshot) => Ok(Some(snapshot)), + Err(crate::Error::SnapshotNotExist { + snapshot_id: missing, + }) if missing == snapshot_id => Ok(None), + Err(error) => Err(error), + } + } + + async fn available_range(&mut self) -> crate::Result { + let mut last_observation = (None, None); + for _ in 0..RANGE_READ_ATTEMPTS { + let earliest = self.snapshot_manager.earliest_snapshot_id().await?; + let latest = self.snapshot_manager.get_latest_snapshot_id().await?; + match (earliest, latest) { + (None, None) => { + return Ok(AvailableRange::Empty); + } + (Some(earliest), Some(latest)) if earliest <= latest => { + return Ok(AvailableRange::Range { earliest, latest }); + } + observation => last_observation = observation, + } + } + + let _ = last_observation; + Ok(AvailableRange::Transient) + } +} + +fn next_id(snapshot_id: i64) -> crate::Result { + snapshot_id + .checked_add(1) + .ok_or_else(|| crate::Error::DataInvalid { + message: format!("Snapshot id {snapshot_id} cannot be advanced"), + source: None, + }) +} + +fn validate_incremental_start(snapshot_id: i64, earliest: i64, latest: i64) -> crate::Result<()> { + if snapshot_id < earliest { + return Err(crate::Error::DataInvalid { + message: format!( + "Stream starting snapshot {snapshot_id} has expired; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }); + } + if snapshot_id > latest.saturating_add(1) { + return Err(crate::Error::DataInvalid { + message: format!( + "Stream starting snapshot {snapshot_id} is too large; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }); + } + Ok(()) +} + +fn validate_full_start(snapshot_id: i64, earliest: i64, latest: i64) -> crate::Result<()> { + if snapshot_id < earliest { + return Err(crate::Error::DataInvalid { + message: format!( + "Full stream starting snapshot {snapshot_id} has expired; available snapshot range is [{earliest}, {latest}]" + ), + source: None, + }); + } + if snapshot_id > latest { + return Err(crate::Error::SnapshotNotExist { snapshot_id }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + validate_full_start, validate_incremental_start, StreamPlan, StreamScanFollowUpMode, + StreamScanPoll, StreamScanStartupMode, + }; + + #[test] + fn start_range_validation_is_explicit() { + assert!(validate_incremental_start(4, 4, 6).is_ok()); + assert!(validate_incremental_start(7, 4, 6).is_ok()); + assert!(matches!( + validate_incremental_start(3, 4, 6), + Err(crate::Error::DataInvalid { .. }) + )); + assert!(matches!( + validate_incremental_start(8, 4, 6), + Err(crate::Error::DataInvalid { .. }) + )); + assert!(matches!( + validate_full_start(7, 4, 6), + Err(crate::Error::SnapshotNotExist { snapshot_id: 7 }) + )); + } + + #[test] + fn public_modes_and_poll_are_matchable() { + let _ = [ + StreamScanStartupMode::LatestFull, + StreamScanStartupMode::Latest, + StreamScanStartupMode::FromSnapshot(1), + StreamScanStartupMode::FromSnapshotFull(1), + ]; + let _ = [ + StreamScanFollowUpMode::Auto, + StreamScanFollowUpMode::Delta, + StreamScanFollowUpMode::Changelog, + ]; + let waiting = StreamScanPoll::Waiting; + assert!(matches!(waiting, StreamScanPoll::Waiting)); + let end = StreamScanPoll::End; + assert!(matches!(end, StreamScanPoll::End)); + let _: Option = None; + } +} diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index 9f3f707d1..72ca64840 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -40,7 +40,7 @@ use crate::Result; use apache_avro::{to_value, Schema}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; /// Batch commit identifier (i64::MAX), same as Python's BATCH_COMMIT_IDENTIFIER. const BATCH_COMMIT_IDENTIFIER: i64 = i64::MAX; @@ -48,10 +48,50 @@ const BATCH_COMMIT_IDENTIFIER: i64 = i64::MAX; const CHECK_ROLLING_RECORD_COUNT: usize = 1000; const DELETION_VECTORS_INDEX_TYPE: &str = "DELETION_VECTORS"; +fn checked_next_snapshot_id(snapshot_id: i64) -> Result { + snapshot_id + .checked_add(1) + .ok_or_else(|| crate::Error::DataInvalid { + message: format!("Snapshot id {snapshot_id} cannot be incremented"), + source: None, + }) +} + +fn validate_commit_identifier(commit_identifier: i64) -> Result<()> { + if commit_identifier < 0 { + return Err(crate::Error::DataInvalid { + message: format!( + "Streaming commit identifier must be non-negative, got {commit_identifier}" + ), + source: None, + }); + } + Ok(()) +} + +fn validate_streaming_commit_identifier(commit_identifier: i64) -> Result<()> { + validate_commit_identifier(commit_identifier)?; + if commit_identifier == BATCH_COMMIT_IDENTIFIER { + return Err(crate::Error::DataInvalid { + message: format!( + "Streaming commit identifier {BATCH_COMMIT_IDENTIFIER} is reserved for batch commits" + ), + source: None, + }); + } + Ok(()) +} + type PartitionBucketKey = (Vec, i32); type RowIdRange = (i64, i64); type ExistingRowIdRanges = HashMap>; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum IdentifierCommitStatus { + Known(bool), + HistoryTruncated { earliest_snapshot_id: i64 }, +} + fn validate_bucket_ownership(messages: &[CommitMessage]) -> Result<()> { let mut owners = HashSet::new(); for message in messages { @@ -164,13 +204,14 @@ impl TableCommit { /// Commit new files in APPEND mode. pub async fn commit(&self, commit_messages: Vec) -> Result<()> { - self.commit_with_identifier(commit_messages, BATCH_COMMIT_IDENTIFIER) + self.commit_with_identifier_impl(commit_messages, BATCH_COMMIT_IDENTIFIER, false) .await } /// Commit new files with a caller-provided commit identifier. /// /// Identifiers must increase monotonically for a given `commit_user`. + /// `i64::MAX` is reserved for unidentified batch commits. /// All messages for one identifier must be submitted in a single call. /// This method does not filter previously committed identifiers. Use /// [`Self::filter_and_commit_with_identifier`] when retrying an uncertain @@ -180,20 +221,23 @@ impl TableCommit { commit_messages: Vec, commit_identifier: i64, ) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.commit_with_identifier_impl(commit_messages, commit_identifier, false) .await } /// Filter a previously committed identifier, then commit if it is new. /// - /// Identifiers must increase monotonically for a given `commit_user`. This - /// method is intended for retrying the same uncertain result; regular + /// Identifiers must increase monotonically for a given `commit_user`. + /// `i64::MAX` is reserved for unidentified batch commits. + /// This method is intended for retrying the same uncertain result; regular /// commits should use [`Self::commit_with_identifier`]. pub async fn filter_and_commit_with_identifier( &self, commit_messages: Vec, commit_identifier: i64, ) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.commit_with_identifier_impl(commit_messages, commit_identifier, true) .await } @@ -204,6 +248,7 @@ impl TableCommit { commit_identifier: i64, filter_committed: bool, ) -> Result<()> { + validate_commit_identifier(commit_identifier)?; // A commit validates against the existing snapshot. CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; @@ -325,6 +370,7 @@ impl TableCommit { static_partitions: Option>>, commit_identifier: i64, ) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.overwrite_impl(commit_messages, static_partitions, commit_identifier, true) .await } @@ -336,6 +382,7 @@ impl TableCommit { commit_identifier: i64, filter_committed: bool, ) -> Result<()> { + validate_commit_identifier(commit_identifier)?; // A commit validates against the existing snapshot. CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; @@ -582,6 +629,7 @@ impl TableCommit { partitions: Vec>>, commit_identifier: i64, ) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.truncate_partitions_impl(partitions, commit_identifier, true) .await } @@ -592,6 +640,7 @@ impl TableCommit { commit_identifier: i64, filter_committed: bool, ) -> Result<()> { + validate_commit_identifier(commit_identifier)?; // A commit validates against the existing snapshot. CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; @@ -665,6 +714,7 @@ impl TableCommit { /// A previously committed identifier is filtered so retrying an uncertain /// result cannot delete data committed in between. pub async fn truncate_table_with_identifier(&self, commit_identifier: i64) -> Result<()> { + validate_streaming_commit_identifier(commit_identifier)?; self.truncate_table_impl(commit_identifier, true).await } @@ -673,6 +723,7 @@ impl TableCommit { commit_identifier: i64, filter_committed: bool, ) -> Result<()> { + validate_commit_identifier(commit_identifier)?; // A commit validates against the existing snapshot. CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; self.table.ensure_not_branch_reference_for_write()?; @@ -695,6 +746,50 @@ impl TableCommit { .await } + /// Return whether this commit user has already published `commit_identifier` + /// or a later identifier. + /// + /// Streaming identifiers are monotonically increasing, so a later snapshot + /// also proves that an older checkpoint must not be committed or aborted. + pub async fn is_identifier_committed(&self, commit_identifier: i64) -> Result { + validate_streaming_commit_identifier(commit_identifier)?; + CoreOptions::new(self.table.schema().options()).ensure_read_authorized()?; + self.table.ensure_not_branch_reference_for_write()?; + let latest_snapshot = self.snapshot_manager.get_latest_snapshot().await?; + match self + .commit_identifier_status(&latest_snapshot, commit_identifier) + .await? + { + IdentifierCommitStatus::Known(committed) => Ok(committed), + IdentifierCommitStatus::HistoryTruncated { + earliest_snapshot_id, + } => Err(crate::Error::DataInvalid { + message: format!( + "Commit identifier {commit_identifier} is indeterminate because retained snapshot history starts at {earliest_snapshot_id}; the required snapshot history is out of range" + ), + source: None, + }), + } + } + + /// Abort prepared files only if their streaming identifier has not already + /// been published. + /// + /// This makes recovery after a lost commit acknowledgement safe: retrying + /// or accidentally aborting a committed prepared checkpoint is a no-op. + /// The caller must still serialize commit and abort operations for one + /// `commit_user` so a new commit cannot race this check. + pub async fn abort_if_uncommitted( + &self, + commit_messages: &[CommitMessage], + commit_identifier: i64, + ) -> Result<()> { + if self.is_identifier_committed(commit_identifier).await? { + return Ok(()); + } + self.abort(commit_messages).await + } + /// Abort a prepared commit by deleting newly written data, changelog and index files. /// /// Deletion is best-effort and mirrors Python `FileStoreCommit.abort`: missing @@ -778,12 +873,14 @@ impl TableCommit { let mut retry_count = 0u32; let mut duplicate_check_start_snapshot_id: Option = None; let mut retry_state: Option> = None; - let start_time_ms = current_time_millis(); + let start_time = Instant::now(); // An identified destructive no-op must still record its identifier. // Otherwise a retry after an intervening write can execute the operation // for the first time and delete data which was not present originally. let commit_empty_overwrite = filter_committed && plan.commit_kind_hint() == CommitKind::OVERWRITE; + let enforce_monotonic_identifier = + !filter_committed && commit_identifier != BATCH_COMMIT_IDENTIFIER; let mut filter_committed = filter_committed; loop { @@ -810,6 +907,19 @@ impl TableCommit { break; } } + if enforce_monotonic_identifier + && self + .is_committed_identifier(&latest_snapshot, commit_identifier) + .await? + { + return Err(crate::Error::DataInvalid { + message: format!( + "Commit identifier {commit_identifier} is not greater than the latest identifier retained for commit_user '{}'", + self.commit_user + ), + source: None, + }); + } validate_expected_latest_snapshot(expected_snapshot_id, &latest_snapshot)?; let resolved = self .resolve_commit(&mut plan, &latest_snapshot, retry_state.as_deref()) @@ -830,14 +940,17 @@ impl TableCommit { match result { CommitAttemptResult::Success => break, CommitAttemptResult::Retry(state) => { - duplicate_check_start_snapshot_id.get_or_insert_with(|| { - latest_snapshot.as_ref().map(|s| s.id() + 1).unwrap_or(1) - }); + if duplicate_check_start_snapshot_id.is_none() { + duplicate_check_start_snapshot_id = Some(match &latest_snapshot { + Some(snapshot) => checked_next_snapshot_id(snapshot.id())?, + None => 1, + }); + } retry_state = Some(state); } } - let elapsed_ms = current_time_millis() - start_time_ms; + let elapsed_ms = u64::try_from(start_time.elapsed().as_millis()).unwrap_or(u64::MAX); if elapsed_ms > self.commit_timeout_ms || retry_count >= self.commit_max_retries { let snap_id = duplicate_check_start_snapshot_id.unwrap_or(1); return Err(crate::Error::DataInvalid { @@ -864,7 +977,10 @@ impl TableCommit { latest_snapshot: &Option, commit_identifier: i64, ) -> Result { - let new_snapshot_id = latest_snapshot.as_ref().map(|s| s.id() + 1).unwrap_or(1); + let new_snapshot_id = match latest_snapshot { + Some(snapshot) => checked_next_snapshot_id(snapshot.id())?, + None => 1, + }; // Row tracking let mut next_row_id: Option = None; @@ -1274,8 +1390,26 @@ impl TableCommit { latest_snapshot: &Option, commit_identifier: i64, ) -> Result { + match self + .commit_identifier_status(latest_snapshot, commit_identifier) + .await? + { + IdentifierCommitStatus::Known(committed) => Ok(committed), + // A fresh, globally unique commit_user must be able to begin on a + // table whose early snapshots have expired. Retry safety across + // that retention boundary cannot be proven; abort uses the public + // fail-closed path above instead. + IdentifierCommitStatus::HistoryTruncated { .. } => Ok(false), + } + } + + async fn commit_identifier_status( + &self, + latest_snapshot: &Option, + commit_identifier: i64, + ) -> Result { let Some(latest) = latest_snapshot else { - return Ok(false); + return Ok(IdentifierCommitStatus::Known(false)); }; let earliest_snapshot_id = self .snapshot_manager @@ -1288,11 +1422,19 @@ impl TableCommit { } else { self.snapshot_manager.get_snapshot(snapshot_id).await? }; - if snapshot.commit_user() == self.commit_user { - return Ok(commit_identifier <= snapshot.commit_identifier()); + if snapshot.commit_user() == self.commit_user + && snapshot.commit_identifier() != BATCH_COMMIT_IDENTIFIER + && commit_identifier <= snapshot.commit_identifier() + { + return Ok(IdentifierCommitStatus::Known(true)); } } - Ok(false) + if earliest_snapshot_id > 1 { + return Ok(IdentifierCommitStatus::HistoryTruncated { + earliest_snapshot_id, + }); + } + Ok(IdentifierCommitStatus::Known(false)) } /// Check if this commit was already completed during an in-process retry. @@ -1792,7 +1934,13 @@ impl TableCommit { return Ok(false); }; - for snapshot_id in cached_snapshot.id() + 1..=latest_snapshot.id() { + if cached_snapshot.id() > latest_snapshot.id() { + return Ok(false); + } + if cached_snapshot.id() == latest_snapshot.id() { + return Ok(true); + } + for snapshot_id in checked_next_snapshot_id(cached_snapshot.id())?..=latest_snapshot.id() { *delta_probe_count += 1; let snapshot = match self.snapshot_manager.get_snapshot(snapshot_id).await { Ok(snapshot) => snapshot, @@ -1897,7 +2045,10 @@ impl TableCommit { let entry_refs = commit_entries.iter().collect::>(); let partition_filter = self.build_entries_partition_filter(&entry_refs)?; let mut entries = Vec::new(); - for snapshot_id in from_snapshot.id() + 1..=to_snapshot.id() { + if from_snapshot.id() >= to_snapshot.id() { + return Ok(Some(entries)); + } + for snapshot_id in checked_next_snapshot_id(from_snapshot.id())?..=to_snapshot.id() { let snapshot = match self.snapshot_manager.get_snapshot(snapshot_id).await { Ok(snapshot) => snapshot, Err(_) => return Ok(None), @@ -2037,7 +2188,11 @@ impl TableCommit { .collect::>(); let partition_filter = self.build_entries_partition_filter(&fixed_entries)?; - for snapshot_id in check_from_snapshot.max(0) + 1..=latest_snapshot.id() { + let check_from_snapshot = check_from_snapshot.max(0); + if check_from_snapshot >= latest_snapshot.id() { + return Ok(()); + } + for snapshot_id in checked_next_snapshot_id(check_from_snapshot)?..=latest_snapshot.id() { let snapshot = self.snapshot_manager.get_snapshot(snapshot_id).await?; let concurrent_entries = self .read_delta_entries(partition_filter.as_ref(), &snapshot) @@ -2351,7 +2506,10 @@ impl TableCommit { let delta_entry_refs = delta_entries.iter().collect::>(); let partition_filter = self.build_entries_partition_filter(&delta_entry_refs)?; - for snapshot_id in check_from_snapshot + 1..=latest_snapshot.id() { + if check_from_snapshot >= latest_snapshot.id() { + return Ok(()); + } + for snapshot_id in checked_next_snapshot_id(check_from_snapshot)?..=latest_snapshot.id() { let snapshot = self.snapshot_manager.get_snapshot(snapshot_id).await?; if snapshot.commit_kind() == &CommitKind::COMPACT { continue; @@ -3159,6 +3317,44 @@ fn rand_f64() -> f64 { mod tests { use super::*; + #[test] + fn test_snapshot_successor_rejects_overflow() { + assert_eq!(checked_next_snapshot_id(1).unwrap(), 2); + assert!(checked_next_snapshot_id(i64::MAX).is_err()); + } + + #[tokio::test] + async fn test_invalid_streaming_identifiers_are_rejected_even_for_noops() { + let file_io = test_file_io(); + let table_path = "memory:/test_negative_commit_identifier"; + setup_dirs(&file_io, table_path).await; + let commit = setup_commit(&file_io, table_path); + + for invalid in [-1, BATCH_COMMIT_IDENTIFIER] { + assert!(commit + .commit_with_identifier(Vec::new(), invalid) + .await + .is_err()); + assert!(commit + .filter_and_commit_with_identifier(Vec::new(), invalid) + .await + .is_err()); + assert!(commit + .overwrite_with_identifier(Vec::new(), None, invalid) + .await + .is_err()); + assert!(commit + .truncate_partitions_with_identifier(Vec::new(), invalid) + .await + .is_err()); + assert!(commit + .truncate_table_with_identifier(invalid) + .await + .is_err()); + } + assert!(latest_snapshot(&file_io, table_path).await.is_none()); + } + #[tokio::test] async fn abort_still_cleans_up_for_a_query_auth_table() { let table = crate::table::query_auth_table(); @@ -3617,6 +3813,84 @@ mod tests { ); } + #[tokio::test] + async fn test_non_filtering_identifiers_must_increase() { + let file_io = test_file_io(); + let table_path = "memory:/test_monotonic_commit_identifier"; + setup_dirs(&file_io, table_path).await; + let commit = setup_commit(&file_io, table_path); + + commit + .commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file("data-7.parquet", 100)], + )], + 7, + ) + .await + .unwrap(); + for identifier in [7, 6] { + let error = commit + .commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file( + &format!("invalid-{identifier}.parquet"), + 100, + )], + )], + identifier, + ) + .await + .expect_err("non-filtering identifiers must increase"); + assert!(error.to_string().contains("not greater")); + } + commit + .commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file("data-8.parquet", 100)], + )], + 8, + ) + .await + .unwrap(); + + let latest = latest_snapshot(&file_io, table_path).await.unwrap(); + assert_eq!(latest.id(), 2); + assert_eq!(latest.commit_identifier(), 8); + } + + #[tokio::test] + async fn test_identifier_lookup_checks_all_retained_snapshots() { + let file_io = test_file_io(); + let table_path = "memory:/test_legacy_out_of_order_identifiers"; + setup_dirs(&file_io, table_path).await; + let snapshot_manager = SnapshotManager::new(file_io.clone(), table_path.to_string()); + for (snapshot_id, commit_identifier) in [(1, 10), (2, 5)] { + let snapshot = Snapshot::builder() + .version(3) + .id(snapshot_id) + .schema_id(0) + .base_manifest_list("base-list".to_string()) + .delta_manifest_list("delta-list".to_string()) + .commit_user("test-user".to_string()) + .commit_identifier(commit_identifier) + .commit_kind(CommitKind::APPEND) + .time_millis(snapshot_id as u64) + .build(); + assert!(snapshot_manager.commit_snapshot(&snapshot).await.unwrap()); + } + + let commit = setup_commit(&file_io, table_path); + assert!(commit.is_identifier_committed(7).await.unwrap()); + assert!(!commit.is_identifier_committed(11).await.unwrap()); + } + #[tokio::test] async fn test_filter_and_commit_rejects_expired_older_identifier() { let file_io = test_file_io(); @@ -3650,6 +3924,61 @@ mod tests { ); } + #[tokio::test] + async fn test_abort_fails_closed_when_commit_history_is_truncated() { + let file_io = test_file_io(); + let table_path = "memory:/test_abort_truncated_identifier_history"; + setup_dirs(&file_io, table_path).await; + + let commit = setup_commit(&file_io, table_path); + let first = CommitMessage::new( + vec![], + 0, + vec![test_data_file("possibly-live.parquet", 100)], + ); + commit + .commit_with_identifier(vec![first.clone()], 1) + .await + .unwrap(); + let other_commit = TableCommit::new(test_table(&file_io, table_path), "other-user".into()); + other_commit + .commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file("other.parquet", 100)], + )], + 1, + ) + .await + .unwrap(); + + let snapshot_manager = SnapshotManager::new(file_io.clone(), table_path.to_string()); + snapshot_manager.delete_snapshot(1).await.unwrap(); + let error = commit + .abort_if_uncommitted(&[first], 1) + .await + .expect_err("truncated history cannot prove that abort is safe"); + assert!(error.to_string().contains("indeterminate")); + assert!(error.to_string().contains("out of range")); + + let new_job = TableCommit::new(test_table(&file_io, table_path), "brand-new-user".into()); + new_job + .filter_and_commit_with_identifier( + vec![CommitMessage::new( + vec![], + 0, + vec![test_data_file("new-job.parquet", 100)], + )], + 1, + ) + .await + .expect("a new commit_user must be able to start after history truncation"); + let latest = latest_snapshot(&file_io, table_path).await.unwrap(); + assert_eq!(latest.id(), 3); + assert_eq!(latest.commit_user(), "brand-new-user"); + } + #[tokio::test] async fn test_overwrite_retry_preserves_intervening_commit() { let file_io = test_file_io(); diff --git a/crates/paimon/src/table/table_read.rs b/crates/paimon/src/table/table_read.rs index 99c01f7d9..95d50ca4c 100644 --- a/crates/paimon/src/table/table_read.rs +++ b/crates/paimon/src/table/table_read.rs @@ -319,8 +319,15 @@ impl<'a> PaimonTableRead<'a> { } } // Delta / Changelog rows are read as-is from planned files (no full-table - // merge against historical base versions). - self.new_data_file_reader()?.read(&data_splits) + // merge against historical base versions). Data-evolution tables still + // need their column files merged with the main data file; reading only + // the latter would silently return NULL for BLOB/vector columns. + let core_options = self.table.schema.core_options(); + if core_options.data_evolution_enabled() { + self.read_with_evolution(&data_splits, &core_options) + } else { + self.new_data_file_reader()?.read(&data_splits) + } } fn to_incremental_diff_arrow( @@ -406,17 +413,28 @@ impl<'a> PaimonTableRead<'a> { )); } - let reader = DataFileReader::new( - self.table.file_io.clone(), - self.table.schema_manager().clone(), - self.table.schema().id(), - self.table.schema.fields().to_vec(), - read_type, - self.data_predicates.clone(), - ) - .with_batch_size(Some(self.table.schema().core_options().read_batch_size()?)) - .with_parquet_read_budget(Some(self.parquet_read_budget()?)); - let raw_stream = reader.read(&data_splits)?; + let core_options = self.table.schema().core_options(); + let raw_stream = if core_options.data_evolution_enabled() { + if has_value_kind || include_sequence { + return Err(crate::Error::Unsupported { + message: "Data-evolution audit reads with changelog or sequence-number fields are not supported" + .to_string(), + }); + } + self.read_with_evolution(&data_splits, &core_options)? + } else { + DataFileReader::new( + self.table.file_io.clone(), + self.table.schema_manager().clone(), + self.table.schema().id(), + self.table.schema.fields().to_vec(), + read_type, + self.data_predicates.clone(), + ) + .with_batch_size(Some(core_options.read_batch_size()?)) + .with_parquet_read_budget(Some(self.parquet_read_budget()?)) + .read(&data_splits)? + }; Ok(Box::pin(async_stream::try_stream! { futures::pin_mut!(raw_stream); diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index d5de9048a..10230e266 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -61,6 +61,22 @@ const MANIFEST_DIR: &str = "manifest"; /// Path segment for index directory under table. const DELETION_VECTORS_INDEX_TYPE: &str = "DELETION_VECTORS"; +/// Additional file-level restriction used by streaming full-start scans. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SnapshotLevelFilter { + GreaterThan(i32), + Equal(i32), +} + +impl SnapshotLevelFilter { + fn matches(self, level: i32) -> bool { + match self { + Self::GreaterThan(bound) => level > bound, + Self::Equal(expected) => level == expected, + } + } +} + #[derive(Debug, Default)] struct ManifestReadCounters { entries_read: usize, @@ -118,6 +134,7 @@ async fn read_all_manifest_entries( table_path: &str, snapshot: &Snapshot, skip_level_zero: bool, + level_filter: Option, scan_all_files: bool, has_primary_keys: bool, partition_filter: Option<&PartitionFilter>, @@ -237,7 +254,9 @@ async fn read_all_manifest_entries( // Post-filter: level-0 and data predicates (need DataFileMeta) let mut filtered = Vec::with_capacity(entries.len()); for entry in entries { - if skip_level_zero && has_primary_keys && entry.file().level == 0 { + if (skip_level_zero && has_primary_keys && entry.file().level == 0) + || level_filter.is_some_and(|filter| !filter.matches(entry.file().level)) + { counters.pruned_by_level += 1; continue; } @@ -538,6 +557,14 @@ fn merge_manifest_entries(mut entries: Vec) -> Vec entries } +/// Keep change files without netting DELETE against ADD. A stream/batch +/// incremental manifest describes events, so a rewrite's ADD must survive even +/// when the same identity also appears as DELETE in that manifest. +fn retain_incremental_add_entries(mut entries: Vec) -> Vec { + entries.retain(|entry| *entry.kind() == FileKind::Add); + entries +} + /// Whether scan-owned pruning still preserves `merged_row_count()` as a safe /// row-count hint. /// @@ -968,6 +995,19 @@ impl<'a> TableScan<'a> { } } + /// Plan stream changes without attaching post-commit index/DV state. + pub(crate) async fn plan_snapshot_delta_streaming( + &self, + snapshot: &Snapshot, + ) -> crate::Result { + match &self.0 { + TableScanKind::Paimon(scan) => scan.plan_snapshot_delta_streaming(snapshot).await, + TableScanKind::Format(_) => Err(crate::Error::Unsupported { + message: "Format tables do not support incremental delta scan".to_string(), + }), + } + } + /// Plan data splits from a snapshot's changelog manifest list only. pub(crate) async fn plan_snapshot_changelog(&self, snapshot: &Snapshot) -> crate::Result { match &self.0 { @@ -978,6 +1018,33 @@ impl<'a> TableScan<'a> { } } + /// Plan stream changelog work without current-state index/DV pruning. + pub(crate) async fn plan_snapshot_changelog_streaming( + &self, + snapshot: &Snapshot, + ) -> crate::Result { + match &self.0 { + TableScanKind::Paimon(scan) => scan.plan_snapshot_changelog_streaming(snapshot).await, + TableScanKind::Format(_) => Err(crate::Error::Unsupported { + message: "Format tables do not support incremental changelog scan".to_string(), + }), + } + } + + /// Plan the complete table state at an already resolved snapshot. + pub(crate) async fn plan_snapshot_full( + &self, + snapshot: &Snapshot, + level_filter: Option, + ) -> crate::Result { + match &self.0 { + TableScanKind::Paimon(scan) => scan.plan_snapshot_full(snapshot, level_filter).await, + TableScanKind::Format(_) => Err(crate::Error::Unsupported { + message: "Format tables do not support Paimon snapshot stream scan".to_string(), + }), + } + } + /// Plan before/after full-snapshot splits for batch incremental Diff. pub(crate) async fn plan_snapshot_diff( &self, @@ -1103,7 +1170,7 @@ impl<'a> PaimonTableScan<'a> { Some(snapshot) => snapshot, None => return Ok(Plan::new(Vec::new())), }; - self.plan_snapshot(snapshot, data_evolution_read_field_ids.as_ref(), None) + self.plan_snapshot(snapshot, data_evolution_read_field_ids.as_ref(), None, None) .await } @@ -1124,6 +1191,7 @@ impl<'a> PaimonTableScan<'a> { .plan_snapshot( snapshot, data_evolution_read_field_ids.as_ref(), + None, Some(&mut trace), ) .await?; @@ -1177,7 +1245,7 @@ impl<'a> PaimonTableScan<'a> { &self, snapshot: &Snapshot, ) -> crate::Result> { - self.plan_manifest_entries_with_trace(snapshot, None, None) + self.plan_manifest_entries_with_trace(snapshot, None, None, None) .await } @@ -1185,6 +1253,7 @@ impl<'a> PaimonTableScan<'a> { &self, snapshot: &Snapshot, row_range_index: Option<&RowRangeIndex>, + level_filter: Option, trace: Option<&mut ScanTrace>, ) -> crate::Result> { let file_io = self.table.file_io(); @@ -1251,6 +1320,7 @@ impl<'a> PaimonTableScan<'a> { table_path, snapshot, skip_level_zero, + level_filter, self.scan_all_files, has_primary_keys, self.partition_filter.as_ref(), @@ -1474,6 +1544,22 @@ impl<'a> PaimonTableScan<'a> { snapshot, snapshot.delta_manifest_list(), data_evolution_read_field_ids.as_ref(), + false, + ) + .await + } + + pub(crate) async fn plan_snapshot_delta_streaming( + &self, + snapshot: &Snapshot, + ) -> crate::Result { + self.ensure_query_auth_allowed()?; + let data_evolution_read_field_ids = self.projected_read_field_ids()?; + self.plan_snapshot_manifest_list( + snapshot, + snapshot.delta_manifest_list(), + data_evolution_read_field_ids.as_ref(), + true, ) .await } @@ -1493,6 +1579,42 @@ impl<'a> PaimonTableScan<'a> { snapshot, list_name, data_evolution_read_field_ids.as_ref(), + false, + ) + .await + } + + pub(crate) async fn plan_snapshot_changelog_streaming( + &self, + snapshot: &Snapshot, + ) -> crate::Result { + self.ensure_query_auth_allowed()?; + let Some(list_name) = snapshot.changelog_manifest_list() else { + return Ok(Plan::new(Vec::new())); + }; + let data_evolution_read_field_ids = self.projected_read_field_ids()?; + self.plan_snapshot_manifest_list( + snapshot, + list_name, + data_evolution_read_field_ids.as_ref(), + true, + ) + .await + } + + /// Plan the complete table state at an already resolved snapshot. + pub(crate) async fn plan_snapshot_full( + &self, + snapshot: &Snapshot, + level_filter: Option, + ) -> crate::Result { + self.ensure_query_auth_allowed()?; + let data_evolution_read_field_ids = self.projected_read_field_ids()?; + self.plan_snapshot( + snapshot.clone(), + data_evolution_read_field_ids.as_ref(), + level_filter, + None, ) .await } @@ -1502,24 +1624,34 @@ impl<'a> PaimonTableScan<'a> { snapshot: &Snapshot, manifest_list_name: &str, data_evolution_read_field_ids: Option<&HashSet>, + streaming_changes: bool, ) -> crate::Result { if matches!(self.limit, Some(0)) { return Ok(Plan::new(Vec::new())); } let core_options = CoreOptions::new(self.table.schema().options()); let data_evolution_enabled = core_options.data_evolution_enabled(); - let global_index_settings = - self.global_index_scan_settings(&core_options, data_evolution_enabled)?; - let index_entries = self - .read_index_manifest_entries( - snapshot, - global_index_settings.is_some(), - core_options.deletion_vectors_enabled(), - ) - .await?; - let manifest_row_ranges = self - .manifest_row_ranges(snapshot, index_entries.as_deref(), global_index_settings) - .await?; + let (index_entries, global_index_settings, manifest_row_ranges) = if streaming_changes { + // Stream plans represent changes, not the post-commit table state. + // A current-state global index can prune a required retract or + // UPDATE_BEFORE event, and a current deletion vector can mask the + // very row the stream must emit. Keep only explicit row ranges. + (None, None, self.row_ranges.clone()) + } else { + let settings = + self.global_index_scan_settings(&core_options, data_evolution_enabled)?; + let entries = self + .read_index_manifest_entries( + snapshot, + settings.is_some(), + core_options.deletion_vectors_enabled(), + ) + .await?; + let ranges = self + .manifest_row_ranges(snapshot, entries.as_deref(), settings) + .await?; + (entries, settings, ranges) + }; if manifest_row_ranges.as_ref().is_some_and(Vec::is_empty) { return Ok(Plan::new(Vec::new())); } @@ -1703,7 +1835,13 @@ impl<'a> PaimonTableScan<'a> { let manifest_entries = crate::spec::avro::from_manifest_bytes_filtered_shared( &bytes, &shared_cache, - &mut |_kind, partition_bytes, bucket, total_buckets| { + &mut |kind, partition_bytes, bucket, total_buckets| { + // Java's incremental reader uses readAndNoMergeFileEntries + // and then selects ADD. Merging a DELETE+ADD rewrite here + // would cancel the new change before it can be emitted. + if kind != FileKind::Add { + return false; + } if has_primary_keys && !scan_all_files && bucket < 0 { return false; } @@ -1734,7 +1872,7 @@ impl<'a> PaimonTableScan<'a> { )?; entries.extend(manifest_entries); } - let entries = merge_manifest_entries(entries); + let entries = retain_incremental_add_entries(entries); let entries = if let Some(index) = row_range_index { retain_manifest_entry_row_ranges(entries, index) } else { @@ -1747,6 +1885,7 @@ impl<'a> PaimonTableScan<'a> { &self, snapshot: Snapshot, data_evolution_read_field_ids: Option<&HashSet>, + level_filter: Option, mut trace: Option<&mut ScanTrace>, ) -> crate::Result { if matches!(self.limit, Some(0)) { @@ -1784,6 +1923,7 @@ impl<'a> PaimonTableScan<'a> { .plan_manifest_entries_with_trace( &snapshot, row_range_index.as_ref(), + level_filter, trace.as_deref_mut(), ) .await?; @@ -2164,11 +2304,11 @@ mod tests { use super::{ data_evolution_row_range_groups, data_file_overlaps_row_range_index, group_data_files_by_partition_bucket, manifest_file_overlaps_row_range_index, - prune_data_evolution_group_by_read_fields, retain_index_manifest_entry, - retain_index_manifest_entry_for_scan, retain_manifest_entry_row_ranges, - retain_manifest_row_ranges, scan_predicate_field_ids, should_skip_level_zero_for_scan, - split_row_ranges_for_files, LimitPushdownAccumulator, PaimonTableScan, RowRangeIndex, - TableScan, + prune_data_evolution_group_by_read_fields, retain_incremental_add_entries, + retain_index_manifest_entry, retain_index_manifest_entry_for_scan, + retain_manifest_entry_row_ranges, retain_manifest_row_ranges, scan_predicate_field_ids, + should_skip_level_zero_for_scan, split_row_ranges_for_files, LimitPushdownAccumulator, + PaimonTableScan, RowRangeIndex, TableScan, }; use crate::catalog::Identifier; use crate::io::FileIOBuilder; @@ -2530,6 +2670,25 @@ mod tests { ); } + #[test] + fn test_incremental_entries_do_not_net_delete_against_add() { + let entry = |kind: FileKind| { + ManifestEntry::new( + kind, + Vec::new(), + 0, + 1, + make_evo_file("same.parquet", 1, 1, 1, None), + 2, + ) + }; + let changes = + retain_incremental_add_entries(vec![entry(FileKind::Delete), entry(FileKind::Add)]); + assert_eq!(changes.len(), 1); + assert_eq!(*changes[0].kind(), FileKind::Add); + assert_eq!(changes[0].file().file_name, "same.parquet"); + } + fn file_names(groups: &[Vec]) -> Vec> { groups .iter() diff --git a/crates/paimon/tests/incremental_batch_scan_test.rs b/crates/paimon/tests/incremental_batch_scan_test.rs index 30217e7be..29426338f 100644 --- a/crates/paimon/tests/incremental_batch_scan_test.rs +++ b/crates/paimon/tests/incremental_batch_scan_test.rs @@ -17,9 +17,12 @@ mod common; -use arrow_array::{Array, Int32Array, RecordBatch}; +use arrow_array::{Array, BinaryArray, Int32Array, RecordBatch, StringArray}; +use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; use futures::TryStreamExt; +use paimon::spec::{BlobType, DataType, IntType, Schema, TableSchema}; use paimon::table::IncrementalScanMode; +use std::sync::Arc; use common::incremental_helpers::{ make_batch, make_batch_with_kinds, make_partitioned_batch, memory_table, partitioned_pk_schema, @@ -82,6 +85,88 @@ async fn read_current_pairs(table: &paimon::table::Table) -> Vec<(i32, i32)> { collect_pairs(&batches) } +#[tokio::test] +async fn delta_data_evolution_reads_blob_column_files() { + let table_path = "memory:/incremental_batch/data_evolution_blob"; + let schema = TableSchema::new( + 0, + &Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("payload", DataType::Blob(BlobType::new())) + .option("bucket", "-1") + .option("row-tracking.enabled", "true") + .option("data-evolution.enabled", "true") + .build() + .unwrap(), + ); + let (file_io, table) = memory_table(table_path, schema); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int32, false), + ArrowField::new("payload", ArrowDataType::Binary, true), + ])), + vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(BinaryArray::from(vec![Some(b"blob-data".as_slice())])), + ], + ) + .unwrap(); + write_batch(&table, &batch).await; + + let builder = table.new_read_builder(); + let plan = builder + .new_incremental_scan(IncrementalScanMode::Delta, 0, 1) + .plan() + .await + .unwrap(); + let batches: Vec = builder + .new_read() + .unwrap() + .to_incremental_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 1); + let payloads = batches[0] + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(payloads.value(0), b"blob-data"); + + let audit_batches: Vec = builder + .new_read() + .unwrap() + .to_audit_log_arrow(&plan) + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!( + audit_batches + .iter() + .map(RecordBatch::num_rows) + .sum::(), + 1 + ); + let rowkinds = audit_batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(rowkinds.value(0), "+I"); + let payloads = audit_batches[0] + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(payloads.value(0), b"blob-data"); +} + async fn plan_incremental( table: &paimon::table::Table, mode: IncrementalScanMode, diff --git a/crates/paimon/tests/stream_scan_test.rs b/crates/paimon/tests/stream_scan_test.rs new file mode 100644 index 000000000..e66ccf466 --- /dev/null +++ b/crates/paimon/tests/stream_scan_test.rs @@ -0,0 +1,524 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod common; + +use common::incremental_helpers::{ + make_batch, make_partitioned_batch, memory_table, partitioned_pk_schema, persist_table_schema, + pk_schema, setup_dirs, write_batch, write_partitioned, +}; +use paimon::spec::{CommitKind, Datum, PredicateBuilder, Snapshot}; +use paimon::table::{ + IncrementalScanMode, IncrementalSplit, StreamPlan, StreamScanFollowUpMode, StreamScanPoll, + StreamScanStartupMode, +}; + +async fn commit_metadata_snapshot(table: &paimon::Table, snapshot_id: i64, kind: CommitKind) { + let snapshot = Snapshot::builder() + .version(3) + .id(snapshot_id) + .schema_id(table.schema().id()) + .base_manifest_list(String::new()) + .delta_manifest_list(String::new()) + .commit_user("stream-test".to_string()) + .commit_identifier(snapshot_id) + .commit_kind(kind) + .time_millis(snapshot_id as u64) + .watermark(Some(snapshot_id * 100)) + .build(); + assert!(table + .snapshot_manager() + .commit_snapshot(&snapshot) + .await + .unwrap()); +} + +async fn write_batch_at_level(table: &paimon::Table, ids: Vec, values: Vec, level: i32) { + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + writer + .write_arrow_batch(&make_batch(ids, values)) + .await + .unwrap(); + let mut messages = writer.prepare_commit().await.unwrap(); + for message in &mut messages { + for file in &mut message.new_files { + file.level = level; + } + } + builder.new_commit().commit(messages).await.unwrap(); +} + +async fn full_start_levels(table: &paimon::Table) -> Vec { + let mut scan = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::LatestFull, + StreamScanFollowUpMode::Auto, + ) + .await + .unwrap(); + let plan = expect_data(scan.poll_next().await.unwrap()); + let mut levels = plan + .full_plan() + .unwrap() + .splits() + .iter() + .flat_map(|split| split.data_files()) + .map(|file| file.level) + .collect::>(); + levels.sort_unstable(); + levels +} + +fn expect_data(poll: StreamScanPoll) -> StreamPlan { + match poll { + StreamScanPoll::Data(plan) => plan, + other => panic!("expected stream data, got {other:?}"), + } +} + +#[tokio::test] +async fn latest_full_is_owned_and_then_follows_new_delta_snapshots() { + let table_path = "memory:/stream_scan/latest_full"; + let (file_io, table) = memory_table(table_path, partitioned_pk_schema("1")); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_partitioned( + &table, + make_partitioned_batch(vec!["a", "b"], vec![1, 2], vec![10, 20]), + ) + .await; + + let writer_table = table.clone(); + let mut builder = table.new_read_builder(); + let filter = PredicateBuilder::new(table.schema().fields()) + .equal("pt", Datum::String("a".to_string())) + .unwrap(); + builder.with_filter(filter); + builder.with_projection(&["pt", "id"]).unwrap(); + let mut scan = builder + .new_stream_scan( + StreamScanStartupMode::LatestFull, + StreamScanFollowUpMode::Auto, + ) + .await + .unwrap(); + drop(builder); + drop(table); + + let first = expect_data(scan.poll_next().await.unwrap()); + assert_eq!(first.snapshot_id(), 1); + assert_eq!(first.next_snapshot_id(), 2); + assert_eq!(scan.checkpoint(), Some(2)); + assert!(matches!(first, StreamPlan::Full { .. })); + assert_eq!(first.full_plan().unwrap().splits().len(), 1); + assert_eq!(scan.follow_up_mode(), IncrementalScanMode::Delta); + + write_partitioned( + &writer_table, + make_partitioned_batch(vec!["a", "b"], vec![3, 4], vec![30, 40]), + ) + .await; + let second = expect_data(scan.poll_next().await.unwrap()); + assert_eq!(second.snapshot_id(), 2); + assert_eq!(second.next_snapshot_id(), 3); + assert_eq!(scan.checkpoint(), Some(3)); + let incremental = second.incremental_plan().unwrap(); + assert_eq!(incremental.mode(), IncrementalScanMode::Delta); + assert_eq!(incremental.splits().len(), 1); + + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); +} + +#[tokio::test] +async fn changelog_full_start_uses_java_level_filters() { + let lookup_path = "memory:/stream_scan/lookup_full_levels"; + let (lookup_io, lookup_table) = + memory_table(lookup_path, pk_schema(&[("changelog-producer", "lookup")])); + setup_dirs(&lookup_io, lookup_path).await; + persist_table_schema(&lookup_io, lookup_path, lookup_table.schema()).await; + write_batch_at_level(&lookup_table, vec![1], vec![10], 0).await; + write_batch_at_level(&lookup_table, vec![2], vec![20], 1).await; + assert_eq!(full_start_levels(&lookup_table).await, vec![1]); + + let full_compaction_path = "memory:/stream_scan/full_compaction_levels"; + let (full_compaction_io, full_compaction_table) = memory_table( + full_compaction_path, + pk_schema(&[ + ("changelog-producer", "full-compaction"), + ("num-levels", "3"), + ]), + ); + setup_dirs(&full_compaction_io, full_compaction_path).await; + persist_table_schema( + &full_compaction_io, + full_compaction_path, + full_compaction_table.schema(), + ) + .await; + write_batch_at_level(&full_compaction_table, vec![1], vec![10], 0).await; + write_batch_at_level(&full_compaction_table, vec![2], vec![20], 1).await; + write_batch_at_level(&full_compaction_table, vec![3], vec![30], 2).await; + assert_eq!(full_start_levels(&full_compaction_table).await, vec![2]); +} + +#[tokio::test] +async fn explicit_follow_up_validation_rejects_only_unsafe_combinations() { + let input_path = "memory:/stream_scan/input_explicit_delta"; + let (input_io, input_table) = memory_table( + input_path, + pk_schema(&[("changelog-producer", "input"), ("bucket", "1")]), + ); + setup_dirs(&input_io, input_path).await; + persist_table_schema(&input_io, input_path, input_table.schema()).await; + let scan = input_table + .new_read_builder() + .new_stream_scan(StreamScanStartupMode::Latest, StreamScanFollowUpMode::Delta) + .await + .expect("input changelog tables may be consumed explicitly as delta"); + assert_eq!(scan.follow_up_mode(), IncrementalScanMode::Delta); + + let dv_lookup_path = "memory:/stream_scan/dv_lookup_explicit_delta"; + let (dv_lookup_io, dv_lookup_table) = memory_table( + dv_lookup_path, + pk_schema(&[ + ("changelog-producer", "lookup"), + ("deletion-vectors.enabled", "true"), + ("bucket", "1"), + ]), + ); + setup_dirs(&dv_lookup_io, dv_lookup_path).await; + persist_table_schema(&dv_lookup_io, dv_lookup_path, dv_lookup_table.schema()).await; + let error = dv_lookup_table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::LatestFull, + StreamScanFollowUpMode::Delta, + ) + .await + .expect_err("DV lookup requires future compaction changelog"); + assert!(matches!(error, paimon::Error::Unsupported { .. })); + let scan = dv_lookup_table + .new_read_builder() + .new_stream_scan(StreamScanStartupMode::Latest, StreamScanFollowUpMode::Delta) + .await + .expect("non-full startup may explicitly consume future lookup deltas"); + assert_eq!(scan.follow_up_mode(), IncrementalScanMode::Delta); + + let none_path = "memory:/stream_scan/none_explicit_changelog"; + let (none_io, none_table) = memory_table( + none_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&none_io, none_path).await; + persist_table_schema(&none_io, none_path, none_table.schema()).await; + let error = none_table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::Latest, + StreamScanFollowUpMode::Changelog, + ) + .await + .expect_err("a table without changelog files cannot use changelog follow-up"); + assert!(matches!(error, paimon::Error::Unsupported { .. })); +} + +#[tokio::test] +async fn deletion_vector_full_start_replays_starting_level_zero() { + let table_path = "memory:/stream_scan/dv_full_start"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[ + ("changelog-producer", "none"), + ("deletion-vectors.enabled", "true"), + ("bucket", "1"), + ]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + + for startup in [ + StreamScanStartupMode::LatestFull, + StreamScanStartupMode::FromSnapshotFull(1), + ] { + let mut scan = table + .new_read_builder() + .new_stream_scan(startup, StreamScanFollowUpMode::Auto) + .await + .unwrap(); + let full = expect_data(scan.poll_next().await.unwrap()); + assert!(full.full_plan().unwrap().splits().is_empty()); + assert_eq!(full.next_snapshot_id(), 1); + assert_eq!(scan.checkpoint(), Some(1)); + + let incremental = expect_data(scan.poll_next().await.unwrap()); + assert_eq!(incremental.snapshot_id(), 1); + assert_eq!(incremental.next_snapshot_id(), 2); + let splits = incremental.incremental_plan().unwrap().splits(); + assert!(!splits.is_empty()); + for split in splits { + let IncrementalSplit::Data(split) = split else { + panic!("stream delta must contain data splits"); + }; + assert!(split.data_files().iter().all(|file| file.level == 0)); + assert!(split.data_deletion_files().is_none()); + } + } +} + +#[tokio::test] +async fn latest_on_initially_empty_table_includes_first_snapshot() { + let table_path = "memory:/stream_scan/latest_empty"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + + let mut scan = table + .new_read_builder() + .new_stream_scan(StreamScanStartupMode::Latest, StreamScanFollowUpMode::Delta) + .await + .unwrap(); + assert_eq!(scan.checkpoint(), Some(1)); + + // Commit after construction but before the first poll. The async + // constructor freezes the empty-table boundary, so snapshot 1 is retained. + write_batch(&table, &make_batch(vec![1], vec![10])).await; + let first = expect_data(scan.poll_next().await.unwrap()); + assert_eq!(first.snapshot_id(), 1); + assert_eq!(scan.checkpoint(), Some(2)); +} + +#[tokio::test] +async fn inconsistent_range_observation_recovers_across_polls() { + let table_path = "memory:/stream_scan/transient_range"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + + // Model a commit between earliest/latest observations: the latest hint is + // visible while the corresponding snapshot is not yet visible. + table.snapshot_manager().write_latest_hint(1).await.unwrap(); + let mut scan = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(1), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + + commit_metadata_snapshot(&table, 1, CommitKind::APPEND).await; + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert_eq!(scan.checkpoint(), Some(2)); +} + +#[tokio::test] +async fn empty_append_snapshot_advances_cursor_and_restore_replays_from_checkpoint() { + let table_path = "memory:/stream_scan/empty_snapshot"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + commit_metadata_snapshot(&table, 2, CommitKind::APPEND).await; + + let mut scan = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(1), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert_eq!(scan.checkpoint(), Some(1)); + assert_eq!( + expect_data(scan.poll_next().await.unwrap()).snapshot_id(), + 1 + ); + + // Snapshot 2 is an APPEND with an empty delta manifest. One poll consumes + // it and waits for snapshot 3 instead of returning the same empty plan. + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert_eq!(scan.checkpoint(), Some(3)); + assert_eq!(scan.watermark(), Some(200)); + + scan.restore(Some(1)).unwrap(); + assert_eq!( + expect_data(scan.poll_next().await.unwrap()).snapshot_id(), + 1 + ); + assert_eq!(scan.checkpoint(), Some(2)); +} + +#[tokio::test] +async fn transient_missing_snapshot_waits_but_range_errors_are_explicit() { + let table_path = "memory:/stream_scan/range_errors"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + commit_metadata_snapshot(&table, 3, CommitKind::APPEND).await; + + let mut missing = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(2), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + // Poll frequency must not turn an eventually consistent in-range miss into + // a permanent gap. It remains Waiting until the object becomes visible. + for _ in 0..16 { + assert!(matches!( + missing.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + } + + table.snapshot_manager().delete_snapshot(1).await.unwrap(); + let mut expired = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(1), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + let expired_error = expired.poll_next().await.unwrap_err(); + assert!(matches!(expired_error, paimon::Error::DataInvalid { .. })); + assert!(expired_error.to_string().contains("expired")); + + let mut too_large = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(5), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + let too_large_error = too_large.poll_next().await.unwrap_err(); + assert!(matches!(too_large_error, paimon::Error::DataInvalid { .. })); + assert!(too_large_error.to_string().contains("too large")); +} + +#[tokio::test] +async fn overwrite_follow_up_is_not_silently_skipped() { + let table_path = "memory:/stream_scan/overwrite"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + + let mut scan = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshot(1), + StreamScanFollowUpMode::Auto, + ) + .await + .unwrap(); + assert!(matches!( + scan.poll_next().await.unwrap(), + StreamScanPoll::Waiting + )); + assert_eq!( + expect_data(scan.poll_next().await.unwrap()).snapshot_id(), + 1 + ); + commit_metadata_snapshot(&table, 2, CommitKind::OVERWRITE).await; + + let error = scan.poll_next().await.unwrap_err(); + assert!(matches!(error, paimon::Error::Unsupported { .. })); + assert!(error.to_string().contains("OVERWRITE snapshot 2")); + assert_eq!(scan.checkpoint(), Some(2)); +} + +#[tokio::test] +async fn from_snapshot_full_reads_exact_snapshot_and_reports_missing_target() { + let table_path = "memory:/stream_scan/from_full"; + let (file_io, table) = memory_table( + table_path, + pk_schema(&[("changelog-producer", "none"), ("bucket", "1")]), + ); + setup_dirs(&file_io, table_path).await; + persist_table_schema(&file_io, table_path, table.schema()).await; + write_batch(&table, &make_batch(vec![1], vec![10])).await; + + let mut full = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshotFull(1), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + let plan = expect_data(full.poll_next().await.unwrap()); + assert!(matches!(plan, StreamPlan::Full { .. })); + assert_eq!(plan.snapshot_id(), 1); + assert_eq!(full.checkpoint(), Some(2)); + + let mut missing = table + .new_read_builder() + .new_stream_scan( + StreamScanStartupMode::FromSnapshotFull(2), + StreamScanFollowUpMode::Delta, + ) + .await + .unwrap(); + assert!(matches!( + missing.poll_next().await.unwrap_err(), + paimon::Error::SnapshotNotExist { snapshot_id: 2 } + )); +} diff --git a/docs/src/c-binding.md b/docs/src/c-binding.md index 1cd25ad94..9d5343b58 100644 --- a/docs/src/c-binding.md +++ b/docs/src/c-binding.md @@ -25,7 +25,7 @@ writes and commits, and vector search. Record batches cross the ABI through the [Arrow C Data Interface](https://arrow.apache.org/docs/format/CDataInterface.html). The C binding is currently built from source. The repository does not check in -a generated header or publish pre-built C packages. +its generated public header. ## Prerequisites @@ -46,7 +46,8 @@ Run the following commands from the repository root: ```bash cargo build --release -p paimon-c -cbindgen bindings/c --lang c --output target/release/paimon.h +cbindgen --config bindings/c/cbindgen.toml bindings/c \ + --output target/release/paimon.h ``` The build produces a dynamic library and a static library under @@ -78,6 +79,11 @@ LD_LIBRARY_PATH=target/release ./example /path/to/warehouse DYLD_LIBRARY_PATH=target/release ./example /path/to/warehouse ``` +The header-only C++17 facade under `bindings/cpp` adds move-only RAII handles +without creating a C++ shared library. Its CMake build compiles `libpaimon_c` +and generates `paimon.h` automatically; install and CPack targets include the +generated header. + ## Opening and Scanning a Table The following program opens `default.my_table` from a filesystem catalog and @@ -275,6 +281,60 @@ range is clamped to the number of available splits. reversed: `paimon_table_write_write_arrow_batch` consumes the exported Arrow structures, so the caller must not release them again. +## Continuous Stream Reading + +`paimon_read_builder_new_stream_scan` creates an owned pull-based scanner. It +does not start a callback thread. Each call to `paimon_stream_scan_poll` returns +one of `PAIMON_STREAM_POLL_DATA`, `PAIMON_STREAM_POLL_WAITING`, or +`PAIMON_STREAM_POLL_END`: + +```c +paimon_stream_scan_options options; +paimon_error *error = paimon_stream_scan_options_init(&options); +options.startup_mode = PAIMON_STREAM_STARTUP_LATEST; +options.follow_up_mode = PAIMON_STREAM_FOLLOW_UP_AUTO; + +paimon_result_stream_scan created = + paimon_read_builder_new_stream_scan(read_builder, &options); + +for (;;) { + paimon_result_stream_poll poll = paimon_stream_scan_poll(created.scan); + if (poll.error != NULL) { + /* Inspect and free poll.error. */ + break; + } + if (poll.status == PAIMON_STREAM_POLL_WAITING) { + /* Schedule the next poll with application-controlled backoff. */ + continue; + } + if (poll.status == PAIMON_STREAM_POLL_END) { + break; + } + + paimon_result_record_batch_reader batches = + paimon_stream_plan_read_to_arrow( + read, poll.plan, 0, SIZE_MAX, PAIMON_STREAM_READ_DATA); + /* Drain batches before checkpointing poll.next_snapshot_id. */ + paimon_stream_plan_free(poll.plan); +} +``` + +The scan checkpoint is the next snapshot ID and advances when planning +succeeds. Persist it only after all returned work is durably accounted for. +`paimon_stream_plan_serialize` preserves a pending plan across restart. The +current format recovers at plan boundaries, so a partially consumed plan may be +replayed. A stream-scan handle is single-thread-confined; poll, checkpoint, +restore, and free calls for one handle must be externally serialized. +Persisted stream plans with external data-file paths are not supported; plan +serialization fails before the checkpoint is persisted. +Audit-log mode is available for incremental plans and prepends the UTF-8 +`rowkind` column (`+I`, `-U`, `+U`, `-D`). Follow-up `OVERWRITE` snapshots are +reported as unsupported in version 1 rather than silently skipped. + +Decoupled changelog fallback and consumer snapshot-retention registration are +not implemented yet. Configure snapshot retention to exceed the maximum reader +lag; an expired cursor fails explicitly instead of skipping data. + ## Projection and Predicates Projection uses a null-terminated array of column names: @@ -355,6 +415,27 @@ values. `paimon_commit_messages_free` even after a successful commit. The caller retains message ownership and may retry a failed commit. +For a recoverable streaming checkpoint, bind the messages to a non-negative, +monotonically increasing identifier with `paimon_commit_messages_prepare`. +`INT64_MAX` is reserved for unidentified batch commits and is not a valid +streaming checkpoint identifier. +Persist the bytes returned by `paimon_prepared_commit_serialize` before +committing. After a crash or an indeterminate commit response, deserialize the +same bytes and call `paimon_table_commit_commit_prepared`; this retry-safe path +filters an identifier which was already committed. Parallel writers may merge +prepared commits only when table, `commit_user`, overwrite mode, and identifier +all match. Do not call `paimon_table_commit_abort_prepared` after an +indeterminate response: retry first so files from a successful commit are not +deleted. Commit and abort for the same `(table, commit_user)` must also be +fenced across processes; truncated snapshot history makes abort fail closed. +Duplicate filtering is stored in retained snapshots, so snapshot retention +must cover the maximum writer-recovery horizon. Do not retry a prepared commit +older than that horizon, and use a new globally unique `commit_user` for each +fresh job. +Serialized plan/commit blobs are trusted checkpoint state and are not +cryptographically authenticated, so persist them with appropriate integrity +and access controls. + ## Error Handling and Resource Ownership Functions that can fail use one of two conventions: