diff --git a/Cargo.lock b/Cargo.lock index 8bf6fd19a5..8e2b99d718 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3893,6 +3893,10 @@ name = "iceberg-catalog-rest" version = "0.11.0" dependencies = [ "async-trait", + "aws-config", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-runtime-api", "bytes", "chrono", "futures", diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index 8dc9a86d7f..3ad1381c73 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -31,6 +31,13 @@ repository = { workspace = true } [dependencies] async-trait = { workspace = true } +# AWS SigV4 request signing for AWS-native Iceberg REST endpoints +# (S3 Tables, Glue). Enabled per-catalog via the `rest.sigv4-enabled` +# property; otherwise these are inert. +aws-config = "1.8" +aws-credential-types = "1.2" +aws-sigv4 = { version = "1.2", features = ["http1"] } +aws-smithy-runtime-api = { version = "1", features = ["client"] } chrono = { workspace = true } http = { workspace = true } iceberg = { workspace = true } diff --git a/crates/catalog/rest/src/auth/mod.rs b/crates/catalog/rest/src/auth/mod.rs index b15dcdb480..66588b6882 100644 --- a/crates/catalog/rest/src/auth/mod.rs +++ b/crates/catalog/rest/src/auth/mod.rs @@ -19,6 +19,7 @@ //! `AuthManager`/`AuthSession` API. mod oauth2; +mod sigv4; use std::collections::HashMap; use std::fmt::Debug; @@ -27,12 +28,16 @@ use std::sync::Arc; use async_trait::async_trait; use iceberg::Result; pub use oauth2::OAuth2Manager; +pub(crate) use sigv4::sigv4_enabled; +pub use sigv4::{REST_SIGNING_NAME, REST_SIGNING_REGION, REST_SIGV4_ENABLED, SigV4AuthManager}; use crate::client::HttpClient; use crate::request::HttpRequest; /// `rest.auth.type` value disabling authentication. pub const AUTH_TYPE_NONE: &str = "none"; +/// `rest.auth.type` value selecting AWS SigV4 request signing. +pub const AUTH_TYPE_SIGV4: &str = "sigv4"; /// `rest.auth.type` value selecting OAuth2 token authentication. pub const AUTH_TYPE_OAUTH2: &str = "oauth2"; diff --git a/crates/catalog/rest/src/auth/sigv4.rs b/crates/catalog/rest/src/auth/sigv4.rs new file mode 100644 index 0000000000..bb6153782a --- /dev/null +++ b/crates/catalog/rest/src/auth/sigv4.rs @@ -0,0 +1,343 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS SigV4 request signing for AWS-native Iceberg REST endpoints (S3 +//! Tables, Glue), as Iceberg Java's `SigV4AuthManager`: selected by +//! `rest.auth.type=sigv4`, or by the older `rest.sigv4-enabled=true`. + +use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; + +use async_trait::async_trait; +use aws_credential_types::provider::SharedCredentialsProvider; +use iceberg::{Error, ErrorKind, Result}; +use tokio::sync::OnceCell; + +use super::{AuthManager, AuthSession, HttpRequest}; +use crate::client::HttpClient; + +/// The legacy property enabling SigV4 signing, without `rest.auth.type`. +pub const REST_SIGV4_ENABLED: &str = "rest.sigv4-enabled"; +/// The AWS region requests are signed for; falls back to `s3.region`, then +/// `us-east-1`. +pub const REST_SIGNING_REGION: &str = "rest.signing-region"; +/// The AWS service requests are signed for (`s3tables` for S3 Tables, +/// `glue` for Glue); defaults to `execute-api`. +pub const REST_SIGNING_NAME: &str = "rest.signing-name"; + +/// Whether `props` turn on SigV4 through the legacy `rest.sigv4-enabled`. +pub(crate) fn sigv4_enabled(props: &HashMap) -> bool { + props + .get(REST_SIGV4_ENABLED) + .is_some_and(|v| v.eq_ignore_ascii_case("true")) +} + +/// [`AuthManager`] signing every request with AWS SigV4. Credentials come +/// from the standard AWS chain (environment, ECS/EKS task role, EC2 IMDS, +/// shared config), loaded on first use and kept for the catalog's lifetime. +pub struct SigV4AuthManager { + region: String, + name: String, + credentials: Arc>, +} + +impl Debug for SigV4AuthManager { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SigV4AuthManager") + .field("region", &self.region) + .field("name", &self.name) + .finish_non_exhaustive() + } +} + +impl SigV4AuthManager { + /// A manager signing for the region and service `props` name. + pub fn from_props(props: &HashMap) -> Self { + let (region, name) = signing(props, None); + Self { + region, + name, + credentials: Arc::new(OnceCell::new()), + } + } + + /// The same, signing with `provider`'s credentials instead of the + /// default chain's. + pub fn with_credentials_provider(self, provider: SharedCredentialsProvider) -> Self { + Self { + credentials: Arc::new(OnceCell::new_with(Some(provider))), + ..self + } + } + + fn session(&self, region: String, name: String) -> SigV4Session { + SigV4Session { + region, + name, + credentials: self.credentials.clone(), + } + } +} + +/// The region and service `props` sign for, or `fallback`'s. +fn signing(props: &HashMap, fallback: Option<(&str, &str)>) -> (String, String) { + let region = props + .get(REST_SIGNING_REGION) + .or_else(|| props.get("s3.region")) + .cloned() + .or_else(|| fallback.map(|(region, _)| region.to_string())) + .unwrap_or_else(|| "us-east-1".to_string()); + let name = props + .get(REST_SIGNING_NAME) + .cloned() + .or_else(|| fallback.map(|(_, name)| name.to_string())) + .unwrap_or_else(|| "execute-api".to_string()); + (region, name) +} + +#[async_trait] +impl AuthManager for SigV4AuthManager { + async fn init_session( + &self, + _client: &HttpClient, + _props: &HashMap, + ) -> Result> { + Ok(Box::new( + self.session(self.region.clone(), self.name.clone()), + )) + } + + async fn catalog_session( + &self, + _client: &HttpClient, + props: &HashMap, + ) -> Result> { + // The server's config may name the region or service. + let (region, name) = signing(props, Some((&self.region, &self.name))); + Ok(Arc::new(self.session(region, name))) + } +} + +/// [`AuthSession`] signing every request with AWS SigV4. +struct SigV4Session { + region: String, + name: String, + credentials: Arc>, +} + +impl Debug for SigV4Session { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SigV4Session") + .field("region", &self.region) + .field("name", &self.name) + .finish_non_exhaustive() + } +} + +impl SigV4Session { + async fn provider(&self) -> Result<&SharedCredentialsProvider> { + self.credentials + .get_or_try_init(|| async { + let cfg = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + cfg.credentials_provider().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "no AWS credentials provider for SigV4 signing", + ) + }) + }) + .await + } +} + +#[async_trait] +impl AuthSession for SigV4Session { + /// Adds `authorization`, `x-amz-date` and `x-amz-content-sha256`, + /// computed over the request's method, URL, headers and body as they + /// stand: the client sets every other header first. + async fn authenticate(&self, request: &mut HttpRequest) -> Result<()> { + use aws_credential_types::provider::ProvideCredentials; + use aws_sigv4::http_request::{ + PayloadChecksumKind, SignableBody, SignableRequest, SigningSettings, sign, + }; + use aws_sigv4::sign::v4; + + let creds = self + .provider() + .await? + .provide_credentials() + .await + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "load AWS credentials").with_source(e) + })?; + let identity: aws_smithy_runtime_api::client::identity::Identity = creds.into(); + + let mut settings = SigningSettings::default(); + // AWS-native catalogs (S3 Tables, Glue) expect the payload hash. + settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256; + let params = v4::SigningParams::builder() + .identity(&identity) + .region(&self.region) + .name(&self.name) + .time(std::time::SystemTime::now()) + .settings(settings) + .build() + .map_err(|e| Error::new(ErrorKind::Unexpected, "build SigV4 params").with_source(e))? + .into(); + + let body = request.body().as_bytes().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "SigV4 can't sign a request with a streaming body", + ) + })?; + let method = request.method().as_str().to_string(); + let uri = request.url_str().to_string(); + let headers: Vec<(String, String)> = request + .headers() + .iter() + .map(|(k, v)| (k.as_str().to_string(), v.to_str().unwrap_or("").to_string())) + .collect(); + let signable = SignableRequest::new( + &method, + &uri, + headers.iter().map(|(k, v)| (k.as_str(), v.as_str())), + SignableBody::Bytes(body), + ) + .map_err(|e| Error::new(ErrorKind::Unexpected, "build signable request").with_source(e))?; + let (instructions, _signature) = sign(signable, ¶ms) + .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 sign").with_source(e))? + .into_parts(); + + // Apply the signing headers to a throwaway request mirroring this + // one, then copy them over. + let mut mirror = http::Request::builder() + .method(request.method().clone()) + .uri(uri.as_str()); + for (k, v) in request.headers().iter() { + mirror = mirror.header(k, v); + } + let mut mirror = mirror.body(()).map_err(|e| { + Error::new(ErrorKind::Unexpected, "build http request for signing").with_source(e) + })?; + instructions.apply_to_request_http1x(&mut mirror); + for (k, v) in mirror.headers() { + request.headers_mut().insert(k.clone(), v.clone()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use aws_credential_types::Credentials; + use reqwest::{Client, Method}; + + use super::*; + + fn manager(props: &[(&str, &str)]) -> SigV4AuthManager { + let props = props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + SigV4AuthManager::from_props(&props).with_credentials_provider( + SharedCredentialsProvider::new(Credentials::new("AKID", "SECRET", None, None, "test")), + ) + } + + async fn signed(manager: &SigV4AuthManager, catalog_props: &[(&str, &str)]) -> HttpRequest { + let config = crate::RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .build(); + let client = HttpClient::new(&config).unwrap(); + let props = catalog_props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + let session = manager.catalog_session(&client, &props).await.unwrap(); + let mut request = HttpRequest::new( + Client::new() + .request( + Method::POST, + "https://s3tables.eu-west-1.amazonaws.com/iceberg/v1/config", + ) + .body("{\"a\":1}") + .build() + .unwrap(), + ); + session.authenticate(&mut request).await.unwrap(); + request + } + + fn header(request: &HttpRequest, name: &str) -> String { + request + .headers() + .get(name) + .map(|v| v.to_str().unwrap().to_string()) + .unwrap_or_default() + } + + #[tokio::test] + async fn test_signs_for_the_configured_region_and_service() { + let manager = manager(&[ + (REST_SIGV4_ENABLED, "true"), + (REST_SIGNING_REGION, "eu-west-1"), + (REST_SIGNING_NAME, "s3tables"), + ]); + let request = signed(&manager, &[]).await; + let authorization = header(&request, "authorization"); + assert!( + authorization.starts_with("AWS4-HMAC-SHA256 Credential=AKID/"), + "{authorization}" + ); + assert!( + authorization.contains("/eu-west-1/s3tables/aws4_request"), + "{authorization}" + ); + assert!(!header(&request, "x-amz-date").is_empty()); + assert_eq!(header(&request, "x-amz-content-sha256").len(), 64); + } + + #[tokio::test] + async fn test_server_config_names_the_region_and_service() { + let manager = manager(&[(REST_SIGNING_REGION, "eu-west-1")]); + let request = signed(&manager, &[(REST_SIGNING_NAME, "glue")]).await; + let authorization = header(&request, "authorization"); + assert!( + authorization.contains("/eu-west-1/glue/aws4_request"), + "{authorization}" + ); + } + + #[tokio::test] + async fn test_region_falls_back_to_s3_region_then_us_east_1() { + let request = signed(&manager(&[("s3.region", "ap-southeast-1")]), &[]).await; + assert!(header(&request, "authorization").contains("/ap-southeast-1/execute-api/")); + let request = signed(&manager(&[]), &[]).await; + assert!(header(&request, "authorization").contains("/us-east-1/execute-api/")); + } + + #[test] + fn test_legacy_property_enables_signing() { + let props = |v: &str| HashMap::from([(REST_SIGV4_ENABLED.to_string(), v.to_string())]); + assert!(sigv4_enabled(&props("true"))); + assert!(sigv4_enabled(&props("TRUE"))); + assert!(!sigv4_enabled(&props("false"))); + assert!(!sigv4_enabled(&HashMap::new())); + } +} diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index aa36a074e7..a059799cae 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -39,7 +39,10 @@ use reqwest::{Client, Method, StatusCode, Url}; use tokio::sync::OnceCell; use typed_builder::TypedBuilder; -use crate::auth::{AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, AuthManager, NoopAuthManager, OAuth2Manager}; +use crate::auth::{ + AUTH_TYPE_NONE, AUTH_TYPE_OAUTH2, AUTH_TYPE_SIGV4, AuthManager, NoopAuthManager, OAuth2Manager, + SigV4AuthManager, sigv4_enabled, +}; use crate::client::{ HttpClient, deserialize_catalog_response, deserialize_unexpected_catalog_error, }; @@ -769,7 +772,10 @@ impl RestSessionCatalog { // Matched case-insensitively, as the other flag properties are. .map(|auth_type| auth_type.to_ascii_lowercase()) .unwrap_or_else(|| { - if config.token().is_some() + // The property Java's SigV4 signing was first enabled by. + if sigv4_enabled(&config.props) { + AUTH_TYPE_SIGV4.to_string() + } else if config.token().is_some() || config.credential().is_some() || config.explicit_oauth2_server_uri().is_some() { @@ -800,6 +806,7 @@ impl RestSessionCatalog { match auth_type.as_str() { AUTH_TYPE_NONE => Ok(Arc::new(NoopAuthManager)), AUTH_TYPE_OAUTH2 => Ok(Arc::new(OAuth2Manager::from_config(config)?)), + AUTH_TYPE_SIGV4 => Ok(Arc::new(SigV4AuthManager::from_props(&config.props))), other => Err(Error::new( ErrorKind::DataInvalid, format!( @@ -1245,6 +1252,12 @@ impl SessionCatalog for RestSessionCatalog { } }; + // Stash the response-side config map BEFORE merging it with + // the catalog-level user_config — `Table::properties()` + // surfaces *only* the per-table response props so downstream + // consumers can spot vended credentials without re-issuing + // the request. + let response_config = response.config.clone(); let config = response .config .into_iter() @@ -1259,7 +1272,8 @@ impl SessionCatalog for RestSessionCatalog { .identifier(table_ident.clone()) .file_io(file_io) .metadata(response.metadata) - .runtime(self.runtime.clone()); + .runtime(self.runtime.clone()) + .properties(response_config); if let Some(kms_client) = self.kms_client.clone() { table_builder = table_builder.kms_client(kms_client); } @@ -2877,6 +2891,38 @@ mod tests { ); } + #[tokio::test] + async fn test_auth_type_sigv4() { + let manager = |props: &[(&str, &str)]| { + let config = RestCatalogConfig::builder() + .uri("http://localhost".to_string()) + .props( + props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) + .build(); + format!("{:?}", test_catalog(config).resolve_auth_manager().unwrap()) + }; + // Explicitly, or through the property that first enabled it. + assert!(manager(&[(REST_CATALOG_PROP_AUTH_TYPE, "sigv4")]).contains("SigV4AuthManager")); + assert!(manager(&[("rest.sigv4-enabled", "true")]).contains("SigV4AuthManager")); + assert!( + manager(&[("rest.sigv4-enabled", "true"), ("token", "tok")]) + .contains("SigV4AuthManager") + ); + assert!(manager(&[("rest.sigv4-enabled", "false")]).contains("NoopAuthManager")); + // An explicit type wins. + assert!( + manager(&[ + ("rest.sigv4-enabled", "true"), + (REST_CATALOG_PROP_AUTH_TYPE, "none") + ]) + .contains("NoopAuthManager") + ); + } + #[tokio::test] async fn test_unknown_auth_type_is_rejected() { let props = HashMap::from([( diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index c9fd022724..4c9e8a3189 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -334,9 +334,14 @@ pub(crate) fn update_snapshot_summaries( previous_summary: Option<&Summary>, truncate_full_table: bool, ) -> Result { - // Validate that the operation is supported + // Validate that the operation is supported. `Replace` is treated like + // `Overwrite` for summary-update purposes — both swap files in/out + // without changing logical contents — but we keep them distinct in + // the snapshot summary so MoR tooling can distinguish compaction + // (`replace`) from logical overwrites (`overwrite`). if summary.operation != Operation::Append && summary.operation != Operation::Overwrite + && summary.operation != Operation::Replace && summary.operation != Operation::Delete { return Err(Error::new( diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 21cd8681a7..d8a39b6625 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -17,6 +17,7 @@ //! Table API for Apache Iceberg +use std::collections::HashMap; use std::sync::Arc; use crate::arrow::ArrowReaderBuilder; @@ -43,6 +44,7 @@ pub struct TableBuilder { disable_cache: bool, cache_size_bytes: Option, runtime: Option, + properties: HashMap, } impl TableBuilder { @@ -57,6 +59,7 @@ impl TableBuilder { disable_cache: false, cache_size_bytes: None, runtime: None, + properties: HashMap::new(), } } @@ -120,6 +123,15 @@ impl TableBuilder { self } + /// optional - per-table properties returned by the catalog (e.g. the + /// `config` map from a REST `loadTable` response, which carries + /// vended S3 credentials when the catalog is asked for delegated + /// access). + pub fn properties(mut self, properties: HashMap) -> Self { + self.properties = properties; + self + } + /// build the Table pub fn build(self) -> Result { let Self { @@ -132,6 +144,7 @@ impl TableBuilder { disable_cache, cache_size_bytes, runtime, + properties, } = self; let Some(file_io) = file_io else { @@ -192,6 +205,7 @@ impl TableBuilder { object_cache, runtime, encryption_manager, + properties, }) } } @@ -207,6 +221,13 @@ pub struct Table { object_cache: Arc, runtime: Runtime, encryption_manager: Option>, + /// Per-table properties returned by the catalog. For REST catalogs, + /// this is the `config` map in the `loadTable` response — empty for + /// catalogs that don't surface table-scoped config (file, glue, + /// memory). Useful for downstream consumers that need to extract + /// vended credentials (`s3.access-key-id`, `s3.session-token`, + /// etc.) without re-issuing the REST request. + properties: HashMap, } impl Table { @@ -262,6 +283,17 @@ impl Table { &self.file_io } + /// Returns per-table properties returned by the catalog. For REST + /// catalogs, this is the `config` map from `loadTable`. Empty for + /// catalogs that don't surface table-scoped config. + /// + /// Polynya patch: lets downstream consumers (e.g. pg2iceberg's + /// vended-S3 router) extract per-table S3 credentials without + /// duplicating the REST request. + pub fn properties(&self) -> &HashMap { + &self.properties + } + /// Returns this table's object cache pub(crate) fn object_cache(&self) -> Arc { self.object_cache.clone() diff --git a/crates/iceberg/src/transaction/action.rs b/crates/iceberg/src/transaction/action.rs index aa0a05d0d9..7b50f3cfa6 100644 --- a/crates/iceberg/src/transaction/action.rs +++ b/crates/iceberg/src/transaction/action.rs @@ -26,7 +26,7 @@ use crate::transaction::Transaction; use crate::{Result, TableRequirement, TableUpdate}; /// A boxed, thread-safe reference to a `TransactionAction`. -pub(crate) type BoxedTransactionAction = Arc; +pub type BoxedTransactionAction = Arc; /// A trait representing an atomic action that can be part of a transaction. /// @@ -34,7 +34,7 @@ pub(crate) type BoxedTransactionAction = Arc; /// Each action is responsible for generating the updates and requirements needed /// to modify the table metadata. #[async_trait] -pub(crate) trait TransactionAction: AsAny + Sync + Send { +pub trait TransactionAction: AsAny + Sync + Send { /// Commits this action against the provided table and returns the resulting updates. /// NOTE: This function is intended for internal use only and should not be called directly by users. /// diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 3260f9ffa2..a15c09fc6c 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -25,17 +25,24 @@ use crate::error::Result; use crate::spec::{DataFile, ManifestEntry, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::snapshot::{ - DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, + DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, StagedSnapshot, }; use crate::transaction::{ActionCommit, TransactionAction}; -/// FastAppendAction is a transaction action for fast append data files to the table. +/// FastAppendAction is a transaction action that appends data files and/or +/// equality-/position-delete files to the table in a single snapshot. +/// +/// Files passed via [`Self::add_data_files`] are routed by `content_type()` +/// into separate manifests at commit time — `Data` files into a data +/// manifest, `EqualityDeletes`/`PositionDeletes` into a delete manifest. The +/// Iceberg spec forbids mixing content types in one manifest. pub struct FastAppendAction { check_duplicate: bool, // below are properties used to create SnapshotProducer when commit commit_uuid: Option, snapshot_properties: HashMap, added_data_files: Vec, + added_delete_files: Vec, } impl FastAppendAction { @@ -45,6 +52,7 @@ impl FastAppendAction { commit_uuid: None, snapshot_properties: HashMap::default(), added_data_files: vec![], + added_delete_files: vec![], } } @@ -54,9 +62,19 @@ impl FastAppendAction { self } - /// Add data files to the snapshot. + /// Add files to the snapshot. Files are routed by `content_type()` + /// into the data manifest (for `Data`) or the delete manifest (for + /// `EqualityDeletes` / `PositionDeletes`). pub fn add_data_files(mut self, data_files: impl IntoIterator) -> Self { - self.added_data_files.extend(data_files); + for file in data_files { + match file.content_type() { + crate::spec::DataContentType::Data => self.added_data_files.push(file), + crate::spec::DataContentType::EqualityDeletes + | crate::spec::DataContentType::PositionDeletes => { + self.added_delete_files.push(file) + } + } + } self } @@ -75,9 +93,9 @@ impl FastAppendAction { /// Collapse files sharing a path to their first occurrence, so a single /// manifest never references the same file twice. Always runs (unlike the /// `check_duplicate`-gated cross-snapshot check) since it is in-memory only. - fn dedupe_added_files(&self) -> Vec { - let mut seen = HashSet::with_capacity(self.added_data_files.len()); - self.added_data_files + fn dedupe_added_files(files: &[DataFile]) -> Vec { + let mut seen = HashSet::with_capacity(files.len()); + files .iter() .filter(|data_file| seen.insert(data_file.file_path.as_str())) .cloned() @@ -85,6 +103,27 @@ impl FastAppendAction { } } +impl FastAppendAction { + /// Plan this append's snapshot against `table` without writing it: the + /// commit that adds it, and its files to write. For a chain of appends + /// one commit lands, whose files are then written together — their + /// manifests, then their manifest lists, each naming its parent's + /// manifests and its own. Doesn't check for duplicate files. V1 and v2 + /// tables only. + pub async fn stage(&self, table: &Table) -> Result<(ActionCommit, StagedSnapshot)> { + let snapshot_producer = SnapshotProducer::new( + table, + self.commit_uuid.unwrap_or_else(Uuid::now_v7), + self.snapshot_properties.clone(), + self.added_data_files.clone(), + self.added_delete_files.clone(), + ); + snapshot_producer.validate_added_files(&self.added_data_files)?; + snapshot_producer.validate_added_files(&self.added_delete_files)?; + snapshot_producer.stage(&FastAppendOperation).await + } +} + #[async_trait] impl TransactionAction for FastAppendAction { async fn commit(self: Arc, table: &Table) -> Result { @@ -92,11 +131,13 @@ impl TransactionAction for FastAppendAction { table, self.commit_uuid.unwrap_or_else(Uuid::now_v7), self.snapshot_properties.clone(), - self.dedupe_added_files(), + Self::dedupe_added_files(&self.added_data_files), + Self::dedupe_added_files(&self.added_delete_files), ); - // validate added files - snapshot_producer.validate_added_data_files()?; + // Both vecs follow the same partition-spec / partition-value rules. + snapshot_producer.validate_added_files(&self.added_data_files)?; + snapshot_producer.validate_added_files(&self.added_delete_files)?; // Checks duplicate files if self.check_duplicate { @@ -125,7 +166,7 @@ impl SnapshotProduceOperation for FastAppendOperation { async fn existing_manifest( &self, - snapshot_produce: &SnapshotProducer<'_>, + snapshot_produce: &mut SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { return Ok(vec![]); @@ -794,6 +835,298 @@ mod tests { ); } + /// FastAppend now accepts mixed data + equality-delete files and routes + /// them into separate manifests at commit time. The Iceberg manifest list + /// should end up with two entries — one with `ManifestContentType::Data`, + /// one with `Deletes`. + #[tokio::test] + async fn test_fast_append_routes_equality_deletes_into_separate_manifest() { + use crate::spec::{ManifestContentType, NestedField, PrimitiveType, Schema, Type}; + + // Build a minimal in-memory table with FileIO::new_with_memory so the + // manifest writer can actually persist files. + let schema = Schema::builder() + .with_schema_id(0) + .with_identifier_field_ids([1]) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "qty", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + use crate::CatalogBuilder; + let memory: crate::memory::MemoryCatalog = crate::memory::MemoryCatalogBuilder::default() + .load( + "test", + HashMap::from([( + crate::memory::MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )]), + ) + .await + .unwrap(); + use crate::Catalog; + let ns = crate::NamespaceIdent::from_strs(["public"]).unwrap(); + memory.create_namespace(&ns, HashMap::new()).await.unwrap(); + let creation = crate::TableCreation::builder() + .name("orders".to_string()) + .schema(schema) + .build(); + let table = memory.create_table(&ns, creation).await.unwrap(); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(format!("{}/data/0001.parquet", table.metadata().location())) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(2048) + .record_count(7) + .partition(Struct::empty()) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap(); + + let delete_file = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path(format!( + "{}/deletes/0001.parquet", + table.metadata().location() + )) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(64) + .record_count(1) + .equality_ids(Some(vec![1])) + .partition(Struct::empty()) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap(); + + use crate::transaction::ApplyTransactionAction; + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![data_file.clone(), delete_file.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&memory).await.unwrap(); + + let snap = table + .metadata() + .current_snapshot() + .expect("commit must produce a snapshot"); + let manifest_list = table.manifest_list_reader(snap).load().await.unwrap(); + + // Two manifests — one Data, one Deletes. + assert_eq!(manifest_list.entries().len(), 2); + let mut content_types: Vec = + manifest_list.entries().iter().map(|e| e.content).collect(); + content_types.sort_by_key(|c| match c { + ManifestContentType::Data => 0, + ManifestContentType::Deletes => 1, + }); + assert_eq!(content_types, vec![ + ManifestContentType::Data, + ManifestContentType::Deletes + ]); + + // Walk each manifest and confirm the right file landed in the right + // bucket. + for entry in manifest_list.entries() { + let manifest = table.manifest_reader().read(entry).await.unwrap(); + assert_eq!(manifest.entries().len(), 1); + let me = &manifest.entries()[0]; + match entry.content { + ManifestContentType::Data => { + assert_eq!(me.data_file().content_type(), DataContentType::Data); + assert_eq!(me.data_file().record_count(), 7); + } + ManifestContentType::Deletes => { + assert_eq!( + me.data_file().content_type(), + DataContentType::EqualityDeletes + ); + assert_eq!(me.data_file().equality_ids(), Some(vec![1])); + } + } + } + } + + /// A fresh in-memory table, and its catalog. + async fn memory_table() -> (crate::memory::MemoryCatalog, Table) { + use crate::spec::{NestedField, PrimitiveType, Schema, Type}; + use crate::{Catalog, CatalogBuilder}; + let schema = Schema::builder() + .with_schema_id(0) + .with_identifier_field_ids([1]) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let memory: crate::memory::MemoryCatalog = crate::memory::MemoryCatalogBuilder::default() + .load( + "test", + HashMap::from([( + crate::memory::MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )]), + ) + .await + .unwrap(); + let ns = crate::NamespaceIdent::from_strs(["public"]).unwrap(); + memory.create_namespace(&ns, HashMap::new()).await.unwrap(); + let creation = crate::TableCreation::builder() + .name("orders".to_string()) + .schema(schema) + .build(); + let table = memory.create_table(&ns, creation).await.unwrap(); + (memory, table) + } + + /// Step `n`'s files: a data file, and an equality-delete file. + fn step_files(table: &Table, n: usize) -> Vec { + let file = |content, kind: &str| { + let mut builder = DataFileBuilder::default(); + builder + .content(content) + .file_path(format!( + "{}/{kind}/{n}.parquet", + table.metadata().location() + )) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(n as u64 + 1) + .partition(Struct::empty()) + .partition_spec_id(table.metadata().default_partition_spec_id()); + if content == DataContentType::EqualityDeletes { + builder.equality_ids(Some(vec![1])); + } + builder.build().unwrap() + }; + vec![ + file(DataContentType::Data, "data"), + file(DataContentType::EqualityDeletes, "deletes"), + ] + } + + /// Each snapshot as its manifest list has it: per manifest, its content, + /// sequence numbers and the files it holds, with their sequence numbers. + async fn snapshots_as_listed(table: &Table) -> Vec<(i64, Vec)> { + let mut out = Vec::new(); + for snapshot in table.metadata().snapshots() { + let list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let mut manifests = Vec::new(); + for m in list.entries() { + let manifest = table.manifest_reader().read(m).await.unwrap(); + let files: Vec<_> = manifest + .entries() + .iter() + .map(|e| (e.file_path().to_string(), e.sequence_number())) + .collect(); + manifests.push(format!( + "{:?} seq {} min {} {:?}", + m.content, m.sequence_number, m.min_sequence_number, files + )); + } + manifests.sort(); + out.push((snapshot.sequence_number(), manifests)); + } + out.sort(); + out + } + + /// A chain of appends staged and written together — every snapshot's + /// manifests at once, then every manifest list, each naming its + /// parent's manifests and its own — lands as the same table as the + /// appends committed one after another. + #[tokio::test] + async fn a_staged_chain_of_appends_lands_as_appends_one_by_one() { + use crate::transaction::ApplyTransactionAction; + use crate::{Catalog, TableCommit}; + const STEPS: usize = 3; + + let (one_by_one_catalog, mut one_by_one) = memory_table().await; + for n in 0..STEPS { + let tx = Transaction::new(&one_by_one); + let files = step_files(&one_by_one, n); + let tx = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(files) + .apply(tx) + .unwrap(); + one_by_one = tx.commit(&one_by_one_catalog).await.unwrap(); + } + + let (staged_catalog, table) = memory_table().await; + let mut local = table.clone(); + let mut updates = Vec::new(); + let mut requirements = Vec::new(); + let mut staged = Vec::new(); + for n in 0..STEPS { + let files = step_files(&local, n); + let (commit, snapshot) = Transaction::new(&local) + .fast_append() + .with_check_duplicate(false) + .add_data_files(files) + .stage(&local) + .await + .unwrap(); + // The table as the commit finds it: the first step's. + let mut later = Vec::new(); + let step_requirements = if n == 0 { + &mut requirements + } else { + &mut later + }; + local = Transaction::apply(local, commit, &mut updates, step_requirements).unwrap(); + staged.push(snapshot); + } + let added = futures::future::try_join_all(staged.iter_mut().map(|s| s.write_manifests())) + .await + .unwrap(); + let mut parent = Vec::new(); + let mut lists = Vec::new(); + for (snapshot, added) in staged.into_iter().zip(added) { + let manifests: Vec = + parent.into_iter().chain(added).collect(); + parent = manifests.clone(); + lists.push(snapshot.write_manifest_list(manifests)); + } + futures::future::try_join_all(lists).await.unwrap(); + let staged_table = staged_catalog + .update_table( + TableCommit::builder() + .ident(table.identifier().clone()) + .updates(updates) + .requirements(requirements) + .build(), + ) + .await + .unwrap(); + + assert_eq!(staged_table.metadata().snapshots().count(), STEPS); + assert_eq!( + snapshots_as_listed(&staged_table).await, + snapshots_as_listed(&one_by_one).await + ); + let summaries = |t: &Table| { + let mut s: Vec<_> = t + .metadata() + .snapshots() + .map(|s| { + ( + s.sequence_number(), + s.summary().additional_properties.clone(), + ) + }) + .collect(); + s.sort_by_key(|(seq, _)| *seq); + s + }; + assert_eq!(summaries(&staged_table), summaries(&one_by_one)); + } + #[tokio::test] async fn test_fast_append_file_with_incompatible_partition_value() { let table = make_v2_minimal_table(); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 3622471277..3c006352cf 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -53,8 +53,11 @@ mod action; pub use action::*; +pub use snapshot::StagedSnapshot; mod append; mod expire_snapshots; +mod replace_schema; +mod rewrite; mod snapshot; mod sort_order; mod update_location; @@ -72,9 +75,10 @@ pub use update_schema::AddColumn; use crate::error::Result; use crate::spec::TableProperties; use crate::table::Table; -use crate::transaction::action::BoxedTransactionAction; pub use crate::transaction::append::FastAppendAction; pub use crate::transaction::expire_snapshots::ExpireSnapshotsAction; +pub use crate::transaction::replace_schema::ReplaceSchemaAction; +pub use crate::transaction::rewrite::RewriteFilesAction; pub use crate::transaction::sort_order::ReplaceSortOrderAction; pub use crate::transaction::update_location::UpdateLocationAction; pub use crate::transaction::update_properties::UpdatePropertiesAction; @@ -88,6 +92,9 @@ use crate::{Catalog, TableCommit, TableRequirement, TableUpdate}; pub struct Transaction { table: Table, actions: Vec, + /// Whether `table` was loaded just before: the first attempt commits + /// on it rather than reloading it. + fresh_base: bool, } impl Transaction { @@ -96,9 +103,21 @@ impl Transaction { Self { table: table.clone(), actions: vec![], + fresh_base: false, } } + /// Declares the table this transaction was created with as current — + /// loaded just before — so its first commit attempt doesn't load it + /// again: a round trip to the catalog saved. If the table changed + /// meanwhile, that attempt fails with a conflict, and the retry + /// reloads it and applies the actions again, as every attempt does + /// otherwise. + pub fn with_fresh_base(mut self) -> Self { + self.fresh_base = true; + self + } + fn update_table_metadata(table: Table, updates: &[TableUpdate]) -> Result
{ let mut metadata_builder = table.metadata().clone().into_builder(None); for update in updates { @@ -151,6 +170,13 @@ impl Transaction { FastAppendAction::new() } + /// Creates a `RewriteFilesAction` for compaction-style replace + /// operations: drops a set of existing files and adds new ones in one + /// `Operation::Replace` snapshot. + pub fn rewrite_files(&self) -> RewriteFilesAction { + RewriteFilesAction::new() + } + /// Creates replace sort order action. pub fn replace_sort_order(&self) -> ReplaceSortOrderAction { ReplaceSortOrderAction::new() @@ -171,6 +197,15 @@ impl Transaction { ExpireSnapshotsAction::new() } + /// Replace the table's current schema with a new target schema. + /// + /// The caller is responsible for constructing a schema that's a valid + /// evolution of the current one — this action does not enforce + /// schema-evolution compatibility rules. + pub fn replace_schema(&self) -> ReplaceSchemaAction { + ReplaceSchemaAction::new() + } + /// Commit transaction. pub async fn commit(self, catalog: &dyn Catalog) -> Result
{ if self.actions.is_empty() { @@ -208,13 +243,15 @@ impl Transaction { } async fn do_commit(&mut self, catalog: &dyn Catalog) -> Result
{ - let refreshed = catalog.load_table(self.table.identifier()).await?; - - if self.table.metadata() != refreshed.metadata() - || self.table.metadata_location() != refreshed.metadata_location() - { - // current base is stale, use refreshed as base and re-apply transaction actions - self.table = refreshed.clone(); + if !std::mem::take(&mut self.fresh_base) { + let refreshed = catalog.load_table(self.table.identifier()).await?; + + if self.table.metadata() != refreshed.metadata() + || self.table.metadata_location() != refreshed.metadata_location() + { + // current base is stale, use refreshed as base and re-apply transaction actions + self.table = refreshed.clone(); + } } let mut current_table = self.table.clone(); @@ -458,6 +495,63 @@ mod tests { mock_catalog } + /// A mock catalog whose first `conflicts` commits fail with a + /// retryable conflict, and that expects `loads` table loads. + fn mock_catalog_counting_loads(conflicts: u32, loads: usize) -> MockCatalog { + let mut mock_catalog = MockCatalog::new(); + mock_catalog + .expect_load_table() + .times(loads) + .returning_st(|_| Box::pin(async move { Ok(make_v2_table()) })); + let attempts = AtomicU32::new(0); + mock_catalog + .expect_update_table() + .times(conflicts as usize + 1) + .returning_st(move |_| { + if attempts.fetch_add(1, Ordering::SeqCst) < conflicts { + Box::pin(async move { + Err( + Error::new(ErrorKind::CatalogCommitConflicts, "Commit conflict") + .with_retryable(true), + ) + }) + } else { + Box::pin(async move { Ok(make_v2_table()) }) + } + }); + mock_catalog + } + + #[tokio::test] + async fn a_fresh_base_commits_without_reloading_the_table() { + let table = setup_test_table("3"); + let tx = create_test_transaction(&table).with_fresh_base(); + + let mock_catalog = mock_catalog_counting_loads(0, 0); + + assert!(tx.commit(&mock_catalog).await.is_ok()); + } + + #[tokio::test] + async fn a_fresh_base_reloads_the_table_to_retry_a_conflict() { + let table = setup_test_table("3"); + let tx = create_test_transaction(&table).with_fresh_base(); + + let mock_catalog = mock_catalog_counting_loads(1, 1); + + assert!(tx.commit(&mock_catalog).await.is_ok()); + } + + #[tokio::test] + async fn without_a_fresh_base_every_attempt_reloads_the_table() { + let table = setup_test_table("3"); + let tx = create_test_transaction(&table); + + let mock_catalog = mock_catalog_counting_loads(1, 2); + + assert!(tx.commit(&mock_catalog).await.is_ok()); + } + #[tokio::test] async fn test_commit_retryable_error() { // Create a test table with retry properties diff --git a/crates/iceberg/src/transaction/replace_schema.rs b/crates/iceberg/src/transaction/replace_schema.rs new file mode 100644 index 0000000000..10129ab7d4 --- /dev/null +++ b/crates/iceberg/src/transaction/replace_schema.rs @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::spec::Schema; +use crate::table::Table; +use crate::transaction::action::{ActionCommit, TransactionAction}; +use crate::{Error, ErrorKind, Result, TableRequirement, TableUpdate}; + +/// A transaction action that replaces the table's current schema with a new +/// target schema. (Upstream's `UpdateSchemaAction` evolves it column by +/// column, and can't rename, promote or make a column optional.) +/// +/// Callers compute the desired schema (e.g. by adding/dropping columns) and +/// hand it to [`Self::set_schema`]. On commit, this emits two table updates: +/// +/// - [`TableUpdate::AddSchema`] with the target schema. +/// - [`TableUpdate::SetCurrentSchema`] with `schema_id = -1` (the "last added +/// schema" sentinel), so we don't have to allocate a non-clashing schema id +/// ourselves — `TableMetadataBuilder` does that. +/// +/// Three requirements are attached so concurrent schema/column changes fail +/// loudly rather than silently overwrite each other: +/// +/// - `UuidMatch` — guards against the table being recreated under us. +/// - `CurrentSchemaIdMatch` — guards against a concurrent schema swap. +/// - `LastAssignedFieldIdMatch` — guards against a concurrent column add. +/// +/// Note: this action does not enforce schema-evolution compatibility rules +/// (e.g. "you may add columns but not remove them"). The metadata builder +/// also doesn't — it's the caller's responsibility to construct a schema +/// that's a valid evolution of the current one. +#[derive(Debug, Default)] +pub struct ReplaceSchemaAction { + target_schema: Option, +} + +impl ReplaceSchemaAction { + /// Creates a new [`ReplaceSchemaAction`] with no schema set. + pub fn new() -> Self { + ReplaceSchemaAction { + target_schema: None, + } + } + + /// Sets the target schema this action will commit. + pub fn set_schema(mut self, schema: Schema) -> Self { + self.target_schema = Some(schema); + self + } +} + +#[async_trait] +impl TransactionAction for ReplaceSchemaAction { + async fn commit(self: Arc, table: &Table) -> Result { + let target = self.target_schema.clone().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "Schema is not set for ReplaceSchemaAction!", + ) + })?; + + let metadata = table.metadata(); + + let updates = vec![ + TableUpdate::AddSchema { + schema: target.clone(), + }, + // `-1` is the LAST_ADDED sentinel honored by + // `TableMetadataBuilder::set_current_schema`. + TableUpdate::SetCurrentSchema { schema_id: -1 }, + ]; + + let requirements = vec![ + TableRequirement::UuidMatch { + uuid: metadata.uuid(), + }, + TableRequirement::CurrentSchemaIdMatch { + current_schema_id: metadata.current_schema_id(), + }, + TableRequirement::LastAssignedFieldIdMatch { + last_assigned_field_id: metadata.last_column_id(), + }, + ]; + + Ok(ActionCommit::new(updates, requirements)) + } +} + +#[cfg(test)] +mod tests { + use as_any::Downcast; + + use super::ReplaceSchemaAction; + use crate::spec::{NestedField, PrimitiveType, Schema, Type}; + use crate::transaction::Transaction; + use crate::transaction::action::{ApplyTransactionAction, TransactionAction}; + use crate::transaction::tests::make_v2_table; + + fn schema_with_one_int_column() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap() + } + + #[test] + fn applies_action_to_transaction() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + let tx = ReplaceSchemaAction::new() + .set_schema(schema_with_one_int_column()) + .apply(tx) + .unwrap(); + + assert_eq!(tx.actions.len(), 1); + + let action = (*tx.actions[0]) + .downcast_ref::() + .expect("ReplaceSchemaAction was not applied to Transaction!"); + + assert!(action.target_schema.is_some()); + } + + #[tokio::test] + async fn commit_without_schema_set_errors() { + let table = make_v2_table(); + let action = std::sync::Arc::new(ReplaceSchemaAction::new()); + let result = action.commit(&table).await; + assert!(result.is_err()); + let err = result.err().unwrap(); + assert!(err.message().contains("Schema is not set")); + } + + #[tokio::test] + async fn commit_emits_add_schema_and_set_current_schema_updates() { + let table = make_v2_table(); + let action = std::sync::Arc::new( + ReplaceSchemaAction::new().set_schema(schema_with_one_int_column()), + ); + let mut commit = action.commit(&table).await.unwrap(); + let updates = commit.take_updates(); + let requirements = commit.take_requirements(); + + assert_eq!(updates.len(), 2); + assert!(matches!(updates[0], crate::TableUpdate::AddSchema { .. })); + assert!(matches!(updates[1], crate::TableUpdate::SetCurrentSchema { + schema_id: -1 + })); + + // Three requirements, in a fixed order. + assert_eq!(requirements.len(), 3); + assert!(matches!( + requirements[0], + crate::TableRequirement::UuidMatch { .. } + )); + assert!(matches!( + requirements[1], + crate::TableRequirement::CurrentSchemaIdMatch { .. } + )); + assert!(matches!( + requirements[2], + crate::TableRequirement::LastAssignedFieldIdMatch { .. } + )); + } +} diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs new file mode 100644 index 0000000000..bfcde53983 --- /dev/null +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -0,0 +1,808 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use futures::{StreamExt, TryStreamExt}; +use uuid::Uuid; + +use crate::error::{Error, ErrorKind, Result}; +use crate::spec::{DataContentType, DataFile, Manifest, ManifestEntry, ManifestFile, Operation}; +use crate::table::Table; +use crate::transaction::snapshot::{ + DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, +}; +use crate::transaction::{ActionCommit, TransactionAction}; + +/// `RewriteFilesAction` produces a snapshot with `operation = Replace`: +/// drops a set of existing data/delete files and adds a set of new files +/// in one atomic commit. Used by compaction to swap N small files for M +/// larger files without exposing intermediate state to readers. +/// +/// Removed files are identified by file path (exact match). Manifests from +/// the prior snapshot are walked: +/// - Manifests with no removed files carry forward unchanged. +/// - Manifests where every entry is removed are dropped from the new +/// manifest list entirely. +/// - Manifests with partial removal are rewritten: surviving entries land +/// in a new manifest with `ManifestStatus::Existing` (preserving each +/// entry's original `snapshot_id` and `sequence_number`). +/// +/// Added files go into a new `ManifestStatus::Added` manifest, routed by +/// `content_type()` (Data vs. EqualityDeletes/PositionDeletes) into +/// separate manifests as required by the spec. +/// +/// The commit fails if a removed file is no longer in the table it lands +/// on — another rewrite got there first, and adding this one's files too +/// would duplicate their rows. A retried commit checks again. +pub struct RewriteFilesAction { + commit_uuid: Option, + snapshot_properties: HashMap, + added_data_files: Vec, + added_delete_files: Vec, + removed_paths: HashSet, + data_sequence_number: Option, +} + +impl RewriteFilesAction { + pub(crate) fn new() -> Self { + Self { + commit_uuid: None, + snapshot_properties: HashMap::default(), + added_data_files: vec![], + added_delete_files: vec![], + removed_paths: HashSet::default(), + data_sequence_number: None, + } + } + + /// Add files produced by the rewrite (e.g. compacted data files). + /// Routed by `content_type()` into separate manifests. + pub fn add_data_files(mut self, data_files: impl IntoIterator) -> Self { + for file in data_files { + match file.content_type() { + DataContentType::Data => self.added_data_files.push(file), + DataContentType::EqualityDeletes | DataContentType::PositionDeletes => { + self.added_delete_files.push(file) + } + } + } + self + } + + /// Mark the given files as removed in the new snapshot. Matching is by + /// `file_path()`. + pub fn remove_data_files(mut self, files: impl IntoIterator) -> Self { + for file in files { + self.removed_paths.insert(file.file_path().to_string()); + } + self + } + + /// Mark the given paths as removed. Convenience for callers that + /// already track removals as `String`. + pub fn remove_paths(mut self, paths: impl IntoIterator) -> Self { + self.removed_paths.extend(paths); + self + } + + /// Set commit UUID for the snapshot. + pub fn set_commit_uuid(mut self, commit_uuid: Uuid) -> Self { + self.commit_uuid = Some(commit_uuid); + self + } + + /// Set snapshot summary properties. + pub fn set_snapshot_properties(mut self, snapshot_properties: HashMap) -> Self { + self.snapshot_properties = snapshot_properties; + self + } + + /// Give the added data files this data sequence number — the one of + /// the snapshot the rewrite read its files at — instead of the new + /// snapshot's. Their rows are the removed files' rows as of then, so + /// equality deletes committed since still apply to them; with the new + /// snapshot's number they would rise above those deletes and bring the + /// deleted rows back. Java's `RewriteFiles.dataSequenceNumber`. + pub fn set_data_sequence_number(mut self, data_sequence_number: i64) -> Self { + self.data_sequence_number = Some(data_sequence_number); + self + } +} + +#[async_trait] +impl TransactionAction for RewriteFilesAction { + async fn commit(self: Arc, table: &Table) -> Result { + // One read of the table this attempt would land on serves both + // what the rewrite removes and which manifests it keeps. + let manifests = current_manifests(table).await?; + let removed_files = self.removed_files(&manifests)?; + let snapshot_producer = SnapshotProducer::new( + table, + self.commit_uuid.unwrap_or_else(Uuid::now_v7), + self.snapshot_properties.clone(), + self.added_data_files.clone(), + self.added_delete_files.clone(), + ) + .with_added_data_sequence_number(self.data_sequence_number) + .with_removed_files(removed_files); + + snapshot_producer.validate_added_files(&self.added_data_files)?; + snapshot_producer.validate_added_files(&self.added_delete_files)?; + + snapshot_producer + .commit( + RewriteFilesOperation { + removed_paths: self.removed_paths.clone(), + manifests, + }, + DefaultManifestProcess, + ) + .await + } +} + +/// Most manifests a rewrite reads at once. +const MANIFEST_READ_CONCURRENCY: usize = 16; + +/// `table`'s current manifests, each with its manifest-list entry, in the +/// manifest list's order: read [`MANIFEST_READ_CONCURRENCY`] at a time, as +/// each is a round trip to the object store. +async fn current_manifests(table: &Table) -> Result> { + let Some(snapshot) = table.metadata().current_snapshot() else { + return Ok(Vec::new()); + }; + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + let entries: Vec = manifest_list.consume_entries().into_iter().collect(); + let runtime = table.runtime(); + futures::stream::iter(entries) + .map(|entry| { + let reader = table.manifest_reader(); + runtime.io().spawn(async move { + let manifest = reader.read(&entry).await?; + Ok::<_, Error>((entry, manifest)) + }) + }) + .buffered(MANIFEST_READ_CONCURRENCY) + // The task's own result, inside its join's. + .and_then(futures::future::ready) + .try_collect() + .await +} + +impl RewriteFilesAction { + /// The live files in `manifests` that `removed_paths` names — the + /// snapshot summary subtracts them. Fails if one isn't live: another + /// rewrite replaced it first, and committing this one's files as well + /// would duplicate their rows. Runs on every commit attempt, against + /// the manifests of the table the commit would land on. + fn removed_files(&self, manifests: &[(ManifestFile, Manifest)]) -> Result> { + let mut removed = Vec::new(); + let mut found: HashSet<&str> = HashSet::new(); + for (_, manifest) in manifests { + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { + if let Some(path) = self.removed_paths.get(entry.data_file().file_path()) { + found.insert(path); + removed.push(entry.data_file().clone()); + } + } + } + if found.len() < self.removed_paths.len() { + let mut missing: Vec<&str> = self + .removed_paths + .iter() + .map(String::as_str) + .filter(|p| !found.contains(p)) + .collect(); + missing.sort(); + return Err(Error::new( + ErrorKind::CatalogCommitConflicts, + format!("Rewrite removes files no longer in the table: {missing:?}"), + )); + } + Ok(removed) + } +} + +struct RewriteFilesOperation { + removed_paths: HashSet, + /// The current snapshot's manifests, as the commit read them. + manifests: Vec<(ManifestFile, Manifest)>, +} + +impl SnapshotProduceOperation for RewriteFilesOperation { + fn operation(&self) -> Operation { + Operation::Replace + } + + async fn delete_entries( + &self, + _snapshot_produce: &SnapshotProducer<'_>, + ) -> Result> { + // We track removals at manifest-rewrite time inside `existing_manifest` + // rather than emitting separate `Deleted` entries — the new manifest + // list simply doesn't reference the dropped files. Java/PyIceberg do + // emit `Deleted` entries for audit-trail purposes; we may add that + // later if expiration tooling needs them. + Ok(vec![]) + } + + async fn existing_manifest( + &self, + snapshot_produce: &mut SnapshotProducer<'_>, + ) -> Result> { + // Walk each prior manifest, decide carry-forward vs. partial-rewrite + // vs. drop. + // + // Pass 1: classify each manifest based on its loaded entries. We + // can't write rewritten manifests yet because that needs `&mut + // snapshot_produce`, and we also need the producer to compute + // file-IO paths during this loop. + + enum Plan { + CarryForward(ManifestFile), + DropEntirely, + Rewrite { + survivors: Vec, + content: crate::spec::ManifestContentType, + }, + } + let mut plans: Vec = Vec::with_capacity(self.manifests.len()); + for (mfile, manifest) in &self.manifests { + let total = manifest.entries().len(); + let survivors: Vec = manifest + .entries() + .iter() + .filter(|me| !self.removed_paths.contains(me.data_file().file_path())) + .map(|me| ManifestEntry::clone(me)) + .collect(); + + if survivors.len() == total { + plans.push(Plan::CarryForward(mfile.clone())); + } else if survivors.is_empty() { + plans.push(Plan::DropEntirely); + } else { + plans.push(Plan::Rewrite { + survivors, + content: mfile.content, + }); + } + } + + // Pass 2: materialize. Carry-forwards and rewrites land in the + // output; drops produce nothing. + let mut out: Vec = Vec::with_capacity(plans.len()); + for plan in plans { + match plan { + Plan::CarryForward(mf) => out.push(mf), + Plan::DropEntirely => {} + Plan::Rewrite { survivors, content } => { + let rewritten = snapshot_produce + .write_existing_manifest_for(survivors, content) + .await?; + out.push(rewritten); + } + } + } + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, ManifestContentType, NestedField, + Operation, PrimitiveType, Schema, Struct, Type, + }; + use crate::transaction::{ApplyTransactionAction, Transaction}; + use crate::{Catalog, CatalogBuilder, ErrorKind}; + + async fn make_memory_table() -> (crate::memory::MemoryCatalog, crate::table::Table) { + let schema = Schema::builder() + .with_schema_id(0) + .with_identifier_field_ids([1]) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "qty", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let memory: crate::memory::MemoryCatalog = crate::memory::MemoryCatalogBuilder::default() + .load( + "test", + HashMap::from([( + crate::memory::MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )]), + ) + .await + .unwrap(); + use crate::Catalog; + let ns = crate::NamespaceIdent::from_strs(["public"]).unwrap(); + memory.create_namespace(&ns, HashMap::new()).await.unwrap(); + let creation = crate::TableCreation::builder() + .name("orders".to_string()) + .schema(schema) + .build(); + let table = memory.create_table(&ns, creation).await.unwrap(); + (memory, table) + } + + fn data_file(path: &str, record_count: u64, byte_size: u64) -> crate::spec::DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(byte_size) + .record_count(record_count) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap() + } + + /// Snapshot 1 appends three files; snapshot 2 (rewrite) drops them all + /// and adds one compacted file. The new manifest list must reference + /// only the compacted file's manifest — the original manifest is + /// dropped because every entry was removed. + #[tokio::test] + async fn rewrite_drops_all_files_in_manifest_and_swaps_in_new_one() { + let (catalog, table) = make_memory_table().await; + + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let f3 = data_file("memory:///warehouse/public/orders/data/3.parquet", 30, 3072); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone(), f2.clone(), f3.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 60, + 5120, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted.clone()]) + .remove_data_files(vec![f1.clone(), f2.clone(), f3.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.summary().operation, Operation::Replace); + + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let data_manifests: Vec<_> = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .collect(); + assert_eq!(data_manifests.len(), 1); + + let manifest = table + .manifest_reader() + .read(data_manifests[0]) + .await + .unwrap(); + assert_eq!(manifest.entries().len(), 1); + assert_eq!( + manifest.entries()[0].data_file().file_path(), + compacted.file_path() + ); + } + + /// Partial-manifest removal: snapshot 1 appends three files; snapshot 2 + /// removes one and adds one. The new manifest list should reference + /// (a) a rewritten manifest carrying the two survivors as `Existing`, + /// (b) a new manifest with the added file as `Added`. + #[tokio::test] + async fn rewrite_partial_manifest_keeps_survivors_as_existing() { + let (catalog, table) = make_memory_table().await; + + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let f3 = data_file("memory:///warehouse/public/orders/data/3.parquet", 30, 3072); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone(), f2.clone(), f3.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Drop only f1, replace with a "smaller" compacted file. f2 and f3 + // survive in the rewritten manifest. + let new_file = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![new_file.clone()]) + .remove_data_files(vec![f1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.summary().operation, Operation::Replace); + + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let data_manifests: Vec<_> = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .collect(); + // Two data manifests: rewritten survivor manifest + new added manifest. + assert_eq!(data_manifests.len(), 2); + + let mut survivor_paths: Vec = Vec::new(); + let mut added_paths: Vec = Vec::new(); + for mf in &data_manifests { + let m = table.manifest_reader().read(mf).await.unwrap(); + for entry in m.entries() { + match entry.status() { + crate::spec::ManifestStatus::Existing => { + survivor_paths.push(entry.data_file().file_path().to_string()); + } + crate::spec::ManifestStatus::Added => { + added_paths.push(entry.data_file().file_path().to_string()); + } + crate::spec::ManifestStatus::Deleted => { + panic!("unexpected Deleted status in new manifest list"); + } + } + } + } + survivor_paths.sort(); + let mut want_survivors = vec![f2.file_path().to_string(), f3.file_path().to_string()]; + want_survivors.sort(); + assert_eq!(survivor_paths, want_survivors); + assert_eq!(added_paths, vec![new_file.file_path().to_string()]); + } + + /// Mixed rewrite: prior snapshot has a data manifest + a delete manifest; + /// rewrite drops one data file and keeps the delete manifest untouched. + /// The new manifest list should still have a `Deletes` manifest carrying + /// forward, plus a rewritten data manifest with survivors. + #[tokio::test] + async fn rewrite_carries_forward_unaffected_delete_manifest() { + let (catalog, table) = make_memory_table().await; + + let d1 = data_file( + "memory:///warehouse/public/orders/data/d1.parquet", + 10, + 1024, + ); + let d2 = data_file( + "memory:///warehouse/public/orders/data/d2.parquet", + 20, + 2048, + ); + let eq_delete = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path("memory:///warehouse/public/orders/deletes/eq.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(64) + .record_count(1) + .equality_ids(Some(vec![1])) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap(); + + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![d1.clone(), d2.clone(), eq_delete.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Rewrite: drop d1, keep d2 + eq_delete. Add a compacted file. + let new_file = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![new_file.clone()]) + .remove_data_files(vec![d1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + + // Counts: 2 data manifests (rewritten survivors + new added) and + // 1 delete manifest (unchanged carry-forward). + let data_count = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .count(); + let delete_count = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Deletes) + .count(); + assert_eq!(data_count, 2); + assert_eq!(delete_count, 1); + + // Every survivor and added file is reachable. + let mut all_paths: Vec = Vec::new(); + for mf in mlist.entries() { + let m = table.manifest_reader().read(mf).await.unwrap(); + for entry in m.entries() { + all_paths.push(entry.data_file().file_path().to_string()); + } + } + all_paths.sort(); + let mut want = vec![ + d2.file_path().to_string(), + new_file.file_path().to_string(), + eq_delete.file_path().to_string(), + ]; + want.sort(); + assert_eq!(all_paths, want); + } + + /// Rewrite with no `removed_paths` is an append disguised as a replace — + /// the prior manifest is carried forward and the new file joins it. + /// The snapshot is still tagged `Replace`. + #[tokio::test] + async fn rewrite_with_no_removals_carries_prior_manifest() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().add_data_files(vec![f2.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.summary().operation, Operation::Replace); + + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let data_manifest_count = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .count(); + assert_eq!(data_manifest_count, 2); + } + + /// Rewrite with empty add + empty remove is a no-op-ish state — the + /// SnapshotProducer requires *something* added (upstream contract). + /// Documents the existing behavior so callers know to skip the action + /// when there's nothing to compact. + #[tokio::test] + async fn rewrite_with_empty_input_errors() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx.rewrite_files(); + let tx = action.apply(tx).unwrap(); + assert!(tx.commit(&catalog).await.is_err()); + } + + /// With a data sequence number, the added files carry it rather than + /// the new snapshot's, which still becomes their file sequence number. + #[tokio::test] + async fn rewrite_gives_added_files_its_data_sequence_number() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let mut table = table; + for f in [&f1, &f2] { + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f.clone()]); + table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + } + + // Rewrite f1 as planned at sequence number 1. + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted.clone()]) + .remove_data_files(vec![f1.clone()]) + .set_data_sequence_number(1); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.sequence_number(), 3); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let mut seqs = Vec::new(); + for mf in mlist.entries() { + let m = table.manifest_reader().read(mf).await.unwrap(); + for entry in m.entries() { + seqs.push(( + entry.data_file().file_path().to_string(), + entry.sequence_number(), + entry.file_sequence_number, + )); + } + } + seqs.sort(); + assert_eq!(seqs, vec![ + (f2.file_path().to_string(), Some(2), Some(2)), + (compacted.file_path().to_string(), Some(1), Some(3)), + ]); + } + + /// A rewrite of a file another rewrite already removed fails: adding + /// its files too would duplicate the rows. + #[tokio::test] + async fn rewrite_of_a_file_no_longer_in_the_table_fails() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + // Two rewrites of f1, planned against the same table. + let rewrite = |path: &str| { + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![data_file(path, 10, 1024)]) + .remove_data_files(vec![f1.clone()]); + action.apply(tx).unwrap() + }; + let first = rewrite("memory:///warehouse/public/orders/data/compact-a.parquet"); + let second = rewrite("memory:///warehouse/public/orders/data/compact-b.parquet"); + let table = first.commit(&catalog).await.unwrap(); + // The second lands on the first's table, without f1. + let err = second.commit(&catalog).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::CatalogCommitConflicts); + assert!(!err.retryable()); + + let table = catalog.load_table(table.identifier()).await.unwrap(); + let snap = table.metadata().current_snapshot().unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let mut paths = Vec::new(); + for mf in mlist.entries() { + let m = table.manifest_reader().read(mf).await.unwrap(); + for entry in m.entries() { + paths.push(entry.data_file().file_path().to_string()); + } + } + assert_eq!(paths, vec![ + "memory:///warehouse/public/orders/data/compact-a.parquet".to_string() + ]); + } + + /// So does one removing a file from a table with no snapshot at all. + #[tokio::test] + async fn rewrite_removing_from_an_empty_table_fails() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted]) + .remove_data_files(vec![f1]); + let err = action + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::CatalogCommitConflicts); + } + + /// The snapshot summary keeps running totals: a rewrite subtracts the + /// files it removes, data and deletes, and adds its outputs. + #[tokio::test] + async fn rewrite_summary_keeps_running_totals() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let eq_delete = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path("memory:///warehouse/public/orders/deletes/eq.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(64) + .record_count(1) + .equality_ids(Some(vec![1])) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap(); + let mut table = table; + for files in [vec![f1.clone()], vec![f2.clone(), eq_delete.clone()]] { + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(files); + table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + } + let summary = |table: &crate::table::Table, key: &str| -> String { + let snap = table.metadata().current_snapshot().unwrap(); + snap.summary().additional_properties[key].clone() + }; + assert_eq!(summary(&table, "total-records"), "30"); + assert_eq!(summary(&table, "total-delete-files"), "1"); + + // f1 rewritten as 9 rows, retiring the delete. + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 9, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted]) + .remove_data_files(vec![f1.clone(), eq_delete.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + assert_eq!(summary(&table, "deleted-data-files"), "1"); + assert_eq!(summary(&table, "deleted-records"), "10"); + assert_eq!(summary(&table, "removed-delete-files"), "1"); + assert_eq!(summary(&table, "total-records"), "29"); + assert_eq!(summary(&table, "total-data-files"), "2"); + assert_eq!(summary(&table, "total-delete-files"), "0"); + assert_eq!(summary(&table, "total-equality-deletes"), "0"); + } +} diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index e4b4f370fd..f71501741c 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -81,9 +81,14 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { /// - **Append operations**: Typically include all existing manifests /// - **Overwrite operations**: May exclude manifests for partitions being overwritten /// - **Delete operations**: May exclude manifests for partitions being deleted + /// + /// Takes `&mut SnapshotProducer` so implementations that rewrite + /// manifests in place (e.g. `Replace`) can call helpers like + /// `write_existing_manifest_for` here. Append-style implementations + /// that only need read access can ignore the mutability. fn existing_manifest( &self, - snapshot_produce: &SnapshotProducer<'_>, + snapshot_produce: &mut SnapshotProducer<'_>, ) -> impl Future>> + Send; } @@ -113,6 +118,16 @@ pub(crate) struct SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, + /// Equality- and position-delete files added in this snapshot. Written + /// to a separate manifest from `added_data_files` because the Iceberg + /// spec forbids mixing data and delete entries in one manifest. + added_delete_files: Vec, + /// Data sequence number for `added_data_files`; `None` to inherit the + /// snapshot's. + added_data_sequence_number: Option, + /// Files this snapshot removes from the table, for its summary's + /// `deleted-*` counts and running `total-*`. + removed_files: Vec, // A counter used to generate unique manifest file names. // It starts from 0 and increments for each new manifest file. // Note: This counter is limited to the range of (0..u64::MAX). @@ -125,6 +140,7 @@ impl<'a> SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, + added_delete_files: Vec, ) -> Self { Self { table, @@ -132,18 +148,33 @@ impl<'a> SnapshotProducer<'a> { commit_uuid, snapshot_properties, added_data_files, + added_delete_files, + added_data_sequence_number: None, + removed_files: Vec::new(), manifest_counter: (0..), } } - pub(crate) fn validate_added_data_files(&self) -> Result<()> { - for data_file in &self.added_data_files { - if data_file.content_type() != crate::spec::DataContentType::Data { - return Err(Error::new( - ErrorKind::DataInvalid, - "Only data content type is allowed for fast append", - )); - } + /// Write the added data files with this data sequence number instead of + /// letting them inherit the snapshot's: their rows logically belong to + /// an older sequence number (see the spec's "Sequence Number + /// Inheritance"). Ignored for v1 tables, which have none. + pub(crate) fn with_added_data_sequence_number(mut self, seq: Option) -> Self { + self.added_data_sequence_number = seq; + self + } + + /// Record files this snapshot removes, so its summary subtracts them. + pub(crate) fn with_removed_files(mut self, files: Vec) -> Self { + self.removed_files = files; + self + } + + /// Validate partition spec / partition value compatibility for a slice of + /// added files. Generic over data vs. delete files — the Iceberg spec + /// validation rules are the same for both content types. + pub(crate) fn validate_added_files(&self, files: &[DataFile]) -> Result<()> { + for data_file in files { // Check if the data file partition spec id matches the table default partition spec id. if self.table.metadata().default_partition_spec_id() != data_file.partition_spec_id { return Err(Error::new( @@ -168,6 +199,7 @@ impl<'a> SnapshotProducer<'a> { let new_files: HashSet<&str> = self .added_data_files .iter() + .chain(self.added_delete_files.iter()) .map(|df| df.file_path.as_str()) .collect(); @@ -306,19 +338,80 @@ impl<'a> SnapshotProducer<'a> { Ok(()) } - // Write manifest file for added data files and return the ManifestFile for ManifestList. - async fn write_added_manifest(&mut self) -> Result { - let added_data_files = std::mem::take(&mut self.added_data_files); - if added_data_files.is_empty() { + /// Write a fresh manifest containing `survivors` as `Existing` entries. + /// Used by replace/rewrite operations that drop some files from a prior + /// manifest while preserving the rest. Each survivor must carry its + /// original `snapshot_id` and `sequence_number` (the manifest reader's inherit + /// pass populates these from the manifest list entry, so feeding raw + /// `manifest.entries()` here is safe). + pub(crate) async fn write_existing_manifest_for( + &mut self, + survivors: Vec, + content: ManifestContentType, + ) -> Result { + if survivors.is_empty() { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "No survivor entries when rewriting an existing manifest", + )); + } + let mut writer = self.new_manifest_writer(content)?; + for entry in survivors { + let snapshot_id = entry.snapshot_id().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "manifest entry missing snapshot_id when rewriting as existing", + ) + })?; + let sequence_number = entry.sequence_number().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "manifest entry missing sequence_number when rewriting as existing", + ) + })?; + writer.add_existing_file( + entry.data_file().clone(), + snapshot_id, + sequence_number, + entry.file_sequence_number, + )?; + } + writer.write_manifest_file().await + } + + /// Write a manifest file for a batch of added files, returning the + /// `ManifestFile` to splice into the manifest list. The caller picks the + /// content type — data and delete files must live in separate manifests + /// per the Iceberg spec. + async fn write_added_manifest_for( + &mut self, + added_files: Vec, + content: ManifestContentType, + data_sequence_number: Option, + ) -> Result { + self.added_manifest_writer(added_files, content, data_sequence_number)? + .write_manifest_file() + .await + } + + /// A writer holding the manifest for a batch of added files, not yet + /// written. + fn added_manifest_writer( + &mut self, + added_files: Vec, + content: ManifestContentType, + data_sequence_number: Option, + ) -> Result { + if added_files.is_empty() { return Err(Error::new( ErrorKind::PreconditionFailed, - "No added data files found when write an added manifest file", + "No added files found when writing a manifest file", )); } let snapshot_id = self.snapshot_id; let format_version = self.table.metadata().format_version(); - let manifest_entries = added_data_files.into_iter().map(|data_file| { + let manifest_entries = added_files.into_iter().map(|data_file| { let builder = ManifestEntry::builder() .status(crate::spec::ManifestStatus::Added) .data_file(data_file); @@ -327,14 +420,14 @@ impl<'a> SnapshotProducer<'a> { } else { // For format version > 1, we set the snapshot id at the inherited time to avoid rewrite the manifest file when // commit failed. - builder.build() + builder.sequence_number_opt(data_sequence_number).build() } }); - let mut writer = self.new_manifest_writer(ManifestContentType::Data)?; + let mut writer = self.new_manifest_writer(content)?; for entry in manifest_entries { writer.add_entry(entry)?; } - writer.write_manifest_file().await + Ok(writer) } /// Creates new manifests for data files added or removed, @@ -349,24 +442,43 @@ impl<'a> SnapshotProducer<'a> { // TODO: Allowing snapshot property setup with no added data files is a workaround. // We should clean it up after all necessary actions are supported. // For details, please refer to https://github.com/apache/iceberg-rust/issues/1548 - if self.added_data_files.is_empty() && self.snapshot_properties.is_empty() { + if self.added_data_files.is_empty() + && self.added_delete_files.is_empty() + && self.snapshot_properties.is_empty() + { return Err(Error::new( ErrorKind::PreconditionFailed, - "No added data files or added snapshot properties found when write a manifest file", + "No added files or snapshot properties found when writing a manifest file", )); } + // Existing-manifest selection runs first because Replace operations + // may write rewritten manifests via `&mut self` here; the added-file + // manifest write below comes after to keep ordering deterministic. let existing_manifests = snapshot_produce_operation.existing_manifest(self).await?; let mut manifest_files = existing_manifests; - // Process added entries. + // Data and delete files must go into separate manifests — Iceberg + // doesn't allow mixing content types in a single manifest. if !self.added_data_files.is_empty() { - let added_manifest = self.write_added_manifest().await?; - manifest_files.push(added_manifest); + let added_data_files = std::mem::take(&mut self.added_data_files); + let seq = self.added_data_sequence_number; + manifest_files.push( + self.write_added_manifest_for(added_data_files, ManifestContentType::Data, seq) + .await?, + ); + } + if !self.added_delete_files.is_empty() { + let added_delete_files = std::mem::take(&mut self.added_delete_files); + manifest_files.push( + self.write_added_manifest_for( + added_delete_files, + ManifestContentType::Deletes, + None, + ) + .await?, + ); } - - // # TODO - // Support process delete entries. let manifest_files = manifest_process.process_manifests(self, manifest_files); Ok(manifest_files) @@ -395,13 +507,24 @@ impl<'a> SnapshotProducer<'a> { summary_collector.set_partition_summary_limit(partition_summary_limit); - for data_file in &self.added_data_files { + for data_file in self + .added_data_files + .iter() + .chain(self.added_delete_files.iter()) + { summary_collector.add_file( data_file, table_metadata.current_schema().clone(), table_metadata.default_partition_spec().clone(), ); } + for data_file in &self.removed_files { + let spec = table_metadata + .partition_spec_by_id(data_file.partition_spec_id) + .unwrap_or_else(|| table_metadata.default_partition_spec()) + .clone(); + summary_collector.remove_file(data_file, table_metadata.current_schema().clone(), spec); + } let previous_snapshot = table_metadata.current_snapshot(); @@ -443,45 +566,13 @@ impl<'a> SnapshotProducer<'a> { process: MP, ) -> Result { let manifest_list_path = self.generate_manifest_list_file_path(0)?; - let next_seq_num = self.table.metadata().next_sequence_number(); let first_row_id = self.table.metadata().next_row_id(); + let (mut manifest_list_writer, encrypted_output) = + self.manifest_list_writer(&manifest_list_path).await?; - let raw_output = self - .table - .file_io() - .new_output(manifest_list_path.clone())?; - - let (writer, encrypted_output) = match self.table.encryption_manager() { - Some(em) => { - let encrypted_output = em.encrypt(raw_output); - ( - encrypted_output.writer().await?, - Some((em.clone(), encrypted_output)), - ) - } - None => (raw_output.writer().await?, None), - }; - - let parent_snapshot_id = self.table.metadata().current_snapshot_id(); - let mut manifest_list_writer = match self.table.metadata().format_version() { - FormatVersion::V1 => { - ManifestListWriter::v1(writer, self.snapshot_id, parent_snapshot_id) - } - FormatVersion::V2 => { - ManifestListWriter::v2(writer, self.snapshot_id, parent_snapshot_id, next_seq_num) - } - FormatVersion::V3 => ManifestListWriter::v3( - writer, - self.snapshot_id, - parent_snapshot_id, - next_seq_num, - Some(first_row_id), - ), - }; - - // Calling self.summary() before self.produce_manifests() is important because self.added_data_files - // will be set to an empty vec after self.produce_manifests() returns, resulting in an empty summary - // being generated. + // Calling self.summary() before self.produce_manifests() is important because the + // added_*_files vecs are drained by self.produce_manifests(), which would result in + // an empty summary being generated otherwise. let summary = self.summary(&snapshot_produce_operation).map_err(|err| { Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) })?; @@ -503,24 +594,149 @@ impl<'a> SnapshotProducer<'a> { None => None, }; + let row_range = writer_next_row_id.map(|writer_next_row_id| { + let assigned_rows = writer_next_row_id - self.table.metadata().next_row_id(); + (first_row_id, assigned_rows) + }); + Ok(self.snapshot_commit(manifest_list_path, summary, row_range, encryption_key_id)) + } + + /// Plan this snapshot without writing it: the [`ActionCommit`] that adds + /// it, and its files to write ([`StagedSnapshot`]). Its manifest list + /// carries the manifests the caller names when it writes it, rather + /// than reading the parent's: snapshots one commit chains can be + /// written together. V1 and v2 tables only — v3's row lineage takes + /// its range from the written manifest list — and unencrypted tables + /// only: an encrypted list's key, which the snapshot names, is only + /// known once the list is written. + pub(crate) async fn stage( + mut self, + snapshot_produce_operation: &OP, + ) -> Result<(ActionCommit, StagedSnapshot)> { + if self.table.metadata().format_version() == FormatVersion::V3 { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "staged snapshots don't support format version 3", + )); + } + if self.table.encryption_manager().is_some() { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "staged snapshots don't support encrypted tables", + )); + } + if self.added_data_files.is_empty() + && self.added_delete_files.is_empty() + && self.snapshot_properties.is_empty() + { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "No added files or snapshot properties found when writing a manifest file", + )); + } + let manifest_list_path = self.generate_manifest_list_file_path(0)?; + let (list, _) = self.manifest_list_writer(&manifest_list_path).await?; + // Before the added files are taken below. + let summary = self.summary(snapshot_produce_operation).map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) + })?; + let mut manifests = Vec::new(); + if !self.added_data_files.is_empty() { + let added_data_files = std::mem::take(&mut self.added_data_files); + let seq = self.added_data_sequence_number; + manifests.push(self.added_manifest_writer( + added_data_files, + ManifestContentType::Data, + seq, + )?); + } + if !self.added_delete_files.is_empty() { + let added_delete_files = std::mem::take(&mut self.added_delete_files); + manifests.push(self.added_manifest_writer( + added_delete_files, + ManifestContentType::Deletes, + None, + )?); + } + let staged = StagedSnapshot { + manifests, + list, + sequence_number: self.table.metadata().next_sequence_number(), + format_version: self.table.metadata().format_version(), + }; + Ok(( + self.snapshot_commit(manifest_list_path, summary, None, None), + staged, + )) + } + + /// A writer for this snapshot's manifest list at `path` — encrypted, on + /// a table with an encryption manager, which then names the list's key + /// once it's written. + async fn manifest_list_writer( + &self, + path: &str, + ) -> Result<(ManifestListWriter, ManifestListEncryption)> { + let next_seq_num = self.table.metadata().next_sequence_number(); + let raw_output = self.table.file_io().new_output(path)?; + let (writer, encrypted_output) = match self.table.encryption_manager() { + Some(em) => { + let encrypted_output = em.encrypt(raw_output); + ( + encrypted_output.writer().await?, + Some((em.clone(), encrypted_output)), + ) + } + None => (raw_output.writer().await?, None), + }; + let list = match self.table.metadata().format_version() { + FormatVersion::V1 => ManifestListWriter::v1( + writer, + self.snapshot_id, + self.table.metadata().current_snapshot_id(), + ), + FormatVersion::V2 => ManifestListWriter::v2( + writer, + self.snapshot_id, + self.table.metadata().current_snapshot_id(), + next_seq_num, + ), + FormatVersion::V3 => ManifestListWriter::v3( + writer, + self.snapshot_id, + self.table.metadata().current_snapshot_id(), + next_seq_num, + Some(self.table.metadata().next_row_id()), + ), + }; + Ok((list, encrypted_output)) + } + + /// The commit adding this snapshot, its manifest list at + /// `manifest_list_path`, and making it the main branch's. + fn snapshot_commit( + &self, + manifest_list_path: String, + summary: Summary, + row_range: Option<(u64, u64)>, + encryption_key_id: Option, + ) -> ActionCommit { let commit_ts = chrono::Utc::now().timestamp_millis(); let new_snapshot = Snapshot::builder() .with_manifest_list(manifest_list_path) .with_snapshot_id(self.snapshot_id) .with_parent_snapshot_id(self.table.metadata().current_snapshot_id()) - .with_sequence_number(next_seq_num) + .with_sequence_number(self.table.metadata().next_sequence_number()) .with_summary(summary) .with_schema_id(self.table.metadata().current_schema_id()) .with_encryption_key_id(encryption_key_id) .with_timestamp_ms(commit_ts); - let new_snapshot = if let Some(writer_next_row_id) = writer_next_row_id { - let assigned_rows = writer_next_row_id - self.table.metadata().next_row_id(); - new_snapshot + let new_snapshot = match row_range { + Some((first_row_id, assigned_rows)) => new_snapshot .with_row_range(first_row_id, assigned_rows) - .build() - } else { - new_snapshot.build() + .build(), + None => new_snapshot.build(), }; let encryption_key_updates: Vec = self @@ -562,6 +778,60 @@ impl<'a> SnapshotProducer<'a> { }, ]; - Ok(ActionCommit::new(updates, requirements)) + ActionCommit::new(updates, requirements) + } +} + +/// The table's encryption manager and the encrypted file a manifest list is +/// written to, if the table is encrypted. +type ManifestListEncryption = Option<( + std::sync::Arc, + crate::encryption::EncryptedOutputFile, +)>; + +/// A snapshot planned but not written ([`crate::transaction::FastAppendAction::stage`]): +/// its own manifests, written by [`Self::write_manifests`], then its +/// manifest list, by [`Self::write_manifest_list`]. Each is a round trip to +/// the object store; snapshots one commit chains can be written together, +/// their manifests first, then their lists. +pub struct StagedSnapshot { + manifests: Vec, + list: ManifestListWriter, + sequence_number: i64, + format_version: FormatVersion, +} + +impl StagedSnapshot { + /// Write the snapshot's own manifests, together. Returns them as its + /// manifest list records them — with its sequence number — so a later + /// snapshot's list can carry them. + pub async fn write_manifests(&mut self) -> Result> { + let written = std::mem::take(&mut self.manifests) + .into_iter() + .map(|writer| writer.write_manifest_file()) + .collect::>() + .try_collect::>() + .await?; + Ok(written + .into_iter() + .map(|mut manifest| { + if self.format_version != FormatVersion::V1 { + if manifest.sequence_number == crate::spec::UNASSIGNED_SEQUENCE_NUMBER { + manifest.sequence_number = self.sequence_number; + } + if manifest.min_sequence_number == crate::spec::UNASSIGNED_SEQUENCE_NUMBER { + manifest.min_sequence_number = self.sequence_number; + } + } + manifest + }) + .collect()) + } + + /// Write the snapshot's manifest list, naming `manifests`: the ones it + /// carries from its parent, then its own. + pub async fn write_manifest_list(mut self, manifests: Vec) -> Result<()> { + self.list.add_manifests(manifests.into_iter())?; + self.list.close().await.map(drop) } }