diff --git a/Cargo.lock b/Cargo.lock index 96787eccc0..ef45b92ad5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3872,6 +3872,10 @@ name = "iceberg-catalog-rest" version = "0.10.1" dependencies = [ "async-trait", + "aws-config", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-runtime-api", "chrono", "http 1.4.0", "iceberg", diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index e043c195ef..f51b47c87c 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -31,6 +31,13 @@ repository = { workspace = true } [dependencies] async-trait = { workspace = true } +# AWS SigV4 request signing for AWS-native Iceberg REST endpoints +# (S3 Tables, Glue). Enabled per-catalog via the `rest.sigv4-enabled` +# property; otherwise these are inert. +aws-config = "1.8" +aws-credential-types = "1.2" +aws-sigv4 = { version = "1.2", features = ["http1"] } +aws-smithy-runtime-api = { version = "1", features = ["client"] } chrono = { workspace = true } http = { workspace = true } iceberg = { workspace = true } diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 0626ce5061..daa2103edb 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -220,6 +220,37 @@ impl RestCatalogConfig { self.client.clone() } + /// AWS SigV4 signing config, read from props. Returns + /// `Some((region, signing_name))` when `rest.sigv4-enabled` is + /// `"true"`, else `None` (no signing). + /// + /// Used for AWS-native Iceberg REST endpoints (S3 Tables signing + /// name `s3tables`, Glue `glue`) which require every request to be + /// SigV4-signed. Region falls back to `s3.region`; signing name + /// defaults to `execute-api`. + pub(crate) fn sigv4_config(&self) -> Option<(String, String)> { + let enabled = self + .props + .get("rest.sigv4-enabled") + .map(|v| v == "true") + .unwrap_or(false); + if !enabled { + return None; + } + let region = self + .props + .get("rest.signing-region") + .or_else(|| self.props.get("s3.region")) + .cloned() + .unwrap_or_else(|| "us-east-1".to_string()); + let name = self + .props + .get("rest.signing-name") + .cloned() + .unwrap_or_else(|| "execute-api".to_string()); + Some((region, name)) + } + /// Get the token from the config. /// /// The client can use this token to send requests. @@ -848,6 +879,12 @@ impl Catalog for RestCatalog { } }; + // Stash the response-side config map BEFORE merging it with + // the catalog-level user_config — `Table::properties()` + // surfaces *only* the per-table response props so downstream + // consumers can spot vended credentials without re-issuing + // the request. + let response_config = response.config.clone(); let config = response .config .into_iter() @@ -862,7 +899,8 @@ impl Catalog for RestCatalog { .identifier(table_ident.clone()) .file_io(file_io) .metadata(response.metadata) - .runtime(self.runtime.clone()); + .runtime(self.runtime.clone()) + .properties(response_config); if let Some(metadata_location) = response.metadata_location { table_builder.metadata_location(metadata_location).build() diff --git a/crates/catalog/rest/src/client.rs b/crates/catalog/rest/src/client.rs index 07dc0620da..3a1550cd28 100644 --- a/crates/catalog/rest/src/client.rs +++ b/crates/catalog/rest/src/client.rs @@ -17,17 +17,137 @@ use std::collections::HashMap; use std::fmt::{Debug, Formatter}; +use std::str::FromStr; use http::StatusCode; use iceberg::{Error, ErrorKind, Result}; -use reqwest::header::HeaderMap; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use reqwest::{Client, IntoUrl, Method, Request, RequestBuilder, Response}; use serde::de::DeserializeOwned; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, OnceCell}; use crate::RestCatalogConfig; use crate::types::{ErrorResponse, TokenResponse}; +/// AWS SigV4 request signer for AWS-native Iceberg REST endpoints +/// (S3 Tables, Glue). Credentials come from the standard AWS chain +/// (env, ECS/EKS task role, EC2 IMDS, shared config), loaded lazily on +/// first use and cached for the process lifetime. +struct SigV4Signer { + region: String, + name: String, + provider: OnceCell, +} + +impl SigV4Signer { + fn new(region: String, name: String) -> Self { + Self { + region, + name, + provider: OnceCell::new(), + } + } + + async fn provider(&self) -> Result<&aws_credential_types::provider::SharedCredentialsProvider> { + self.provider + .get_or_try_init(|| async { + let cfg = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + cfg.credentials_provider().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "no AWS credentials provider for SigV4 signing", + ) + }) + }) + .await + } + + /// Sign `request` in place: adds `authorization`, `x-amz-date`, and + /// `x-amz-content-sha256` headers computed over the request as it + /// currently stands. Must be called after all other headers/body + /// are set and before sending. + async fn sign(&self, request: &mut Request) -> Result<()> { + use aws_credential_types::provider::ProvideCredentials; + use aws_sigv4::http_request::{ + PayloadChecksumKind, SignableBody, SignableRequest, SigningSettings, sign, + }; + use aws_sigv4::sign::v4; + + let creds = self + .provider() + .await? + .provide_credentials() + .await + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "load AWS credentials").with_source(e) + })?; + let identity: aws_smithy_runtime_api::client::identity::Identity = creds.into(); + + let mut settings = SigningSettings::default(); + // AWS-native catalogs (S3 Tables/Glue) expect the payload hash. + settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256; + + let params = v4::SigningParams::builder() + .identity(&identity) + .region(&self.region) + .name(&self.name) + .time(std::time::SystemTime::now()) + .settings(settings) + .build() + .map_err(|e| Error::new(ErrorKind::Unexpected, "build SigV4 params").with_source(e))? + .into(); + + let body_bytes = request + .body() + .and_then(|b| b.as_bytes()) + .unwrap_or(&[]) + .to_vec(); + let uri = request.url().as_str().to_string(); + let method = request.method().as_str().to_string(); + let headers: Vec<(String, String)> = request + .headers() + .iter() + .map(|(k, v)| (k.as_str().to_string(), v.to_str().unwrap_or("").to_string())) + .collect(); + + let signable = SignableRequest::new( + &method, + &uri, + headers.iter().map(|(k, v)| (k.as_str(), v.as_str())), + SignableBody::Bytes(&body_bytes), + ) + .map_err(|e| Error::new(ErrorKind::Unexpected, "build signable request").with_source(e))?; + + let (instructions, _signature) = sign(signable, ¶ms) + .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 sign").with_source(e))? + .into_parts(); + + // Apply the signing headers onto a throwaway http::Request that + // mirrors the outgoing one, then copy the resulting headers back. + let mut http_req = http::Request::builder() + .method(request.method().clone()) + .uri(uri.as_str()); + for (k, v) in request.headers().iter() { + http_req = http_req.header(k, v); + } + let mut http_req = http_req.body(()).map_err(|e| { + Error::new(ErrorKind::Unexpected, "build http request for signing").with_source(e) + })?; + instructions.apply_to_request_http1x(&mut http_req); + + for (k, v) in http_req.headers().iter() { + let name = HeaderName::from_str(k.as_str()).map_err(|e| { + Error::new(ErrorKind::Unexpected, "signed header name").with_source(e) + })?; + let val = HeaderValue::from_bytes(v.as_bytes()).map_err(|e| { + Error::new(ErrorKind::Unexpected, "signed header value").with_source(e) + })?; + request.headers_mut().insert(name, val); + } + Ok(()) + } +} + pub(crate) struct HttpClient { client: Client, @@ -45,6 +165,8 @@ pub(crate) struct HttpClient { extra_oauth_params: HashMap, /// Whether to disable header redaction in error logs (defaults to false for security). disable_header_redaction: bool, + /// AWS SigV4 signer, set when `rest.sigv4-enabled` is configured. + sigv4: Option, } impl Debug for HttpClient { @@ -68,6 +190,9 @@ impl HttpClient { extra_headers, extra_oauth_params: cfg.extra_oauth_params(), disable_header_redaction: cfg.disable_header_redaction(), + sigv4: cfg + .sigv4_config() + .map(|(region, name)| SigV4Signer::new(region, name)), }) } @@ -96,6 +221,10 @@ impl HttpClient { self.extra_oauth_params }, disable_header_redaction: cfg.disable_header_redaction(), + sigv4: cfg + .sigv4_config() + .map(|(region, name)| SigV4Signer::new(region, name)) + .or(self.sigv4), }) } @@ -253,6 +382,10 @@ impl HttpClient { /// Executes the given `Request` and returns a `Response`. pub async fn execute(&self, mut request: Request) -> Result { request.headers_mut().extend(self.extra_headers.clone()); + // SigV4 must sign last, over the final headers + body. + if let Some(signer) = &self.sigv4 { + signer.sign(&mut request).await?; + } Ok(self.client.execute(request).await?) } diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index c9fd022724..4c9e8a3189 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -334,9 +334,14 @@ pub(crate) fn update_snapshot_summaries( previous_summary: Option<&Summary>, truncate_full_table: bool, ) -> Result { - // Validate that the operation is supported + // Validate that the operation is supported. `Replace` is treated like + // `Overwrite` for summary-update purposes — both swap files in/out + // without changing logical contents — but we keep them distinct in + // the snapshot summary so MoR tooling can distinguish compaction + // (`replace`) from logical overwrites (`overwrite`). if summary.operation != Operation::Append && summary.operation != Operation::Overwrite + && summary.operation != Operation::Replace && summary.operation != Operation::Delete { return Err(Error::new( diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 31feade038..41c8993d42 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -17,6 +17,7 @@ //! Table API for Apache Iceberg +use std::collections::HashMap; use std::sync::Arc; use crate::arrow::ArrowReaderBuilder; @@ -41,6 +42,7 @@ pub struct TableBuilder { disable_cache: bool, cache_size_bytes: Option, runtime: Option, + properties: HashMap, } impl TableBuilder { @@ -55,6 +57,7 @@ impl TableBuilder { disable_cache: false, cache_size_bytes: None, runtime: None, + properties: HashMap::new(), } } @@ -118,6 +121,15 @@ impl TableBuilder { self } + /// optional - per-table properties returned by the catalog (e.g. the + /// `config` map from a REST `loadTable` response, which carries + /// vended S3 credentials when the catalog is asked for delegated + /// access). + pub fn properties(mut self, properties: HashMap) -> Self { + self.properties = properties; + self + } + /// build the Table pub fn build(self) -> Result { let Self { @@ -130,6 +142,7 @@ impl TableBuilder { disable_cache, cache_size_bytes, runtime, + properties, } = self; let Some(file_io) = file_io else { @@ -190,6 +203,7 @@ impl TableBuilder { object_cache, runtime, encryption_manager, + properties, }) } } @@ -205,6 +219,13 @@ pub struct Table { object_cache: Arc, runtime: Runtime, encryption_manager: Option>, + /// Per-table properties returned by the catalog. For REST catalogs, + /// this is the `config` map in the `loadTable` response — empty for + /// catalogs that don't surface table-scoped config (file, glue, + /// memory). Useful for downstream consumers that need to extract + /// vended credentials (`s3.access-key-id`, `s3.session-token`, + /// etc.) without re-issuing the REST request. + properties: HashMap, } impl Table { @@ -260,6 +281,17 @@ impl Table { &self.file_io } + /// Returns per-table properties returned by the catalog. For REST + /// catalogs, this is the `config` map from `loadTable`. Empty for + /// catalogs that don't surface table-scoped config. + /// + /// Polynya patch: lets downstream consumers (e.g. pg2iceberg's + /// vended-S3 router) extract per-table S3 credentials without + /// duplicating the REST request. + pub fn properties(&self) -> &HashMap { + &self.properties + } + /// Returns this table's object cache pub(crate) fn object_cache(&self) -> Arc { self.object_cache.clone() diff --git a/crates/iceberg/src/transaction/action.rs b/crates/iceberg/src/transaction/action.rs index aa0a05d0d9..7b50f3cfa6 100644 --- a/crates/iceberg/src/transaction/action.rs +++ b/crates/iceberg/src/transaction/action.rs @@ -26,7 +26,7 @@ use crate::transaction::Transaction; use crate::{Result, TableRequirement, TableUpdate}; /// A boxed, thread-safe reference to a `TransactionAction`. -pub(crate) type BoxedTransactionAction = Arc; +pub type BoxedTransactionAction = Arc; /// A trait representing an atomic action that can be part of a transaction. /// @@ -34,7 +34,7 @@ pub(crate) type BoxedTransactionAction = Arc; /// Each action is responsible for generating the updates and requirements needed /// to modify the table metadata. #[async_trait] -pub(crate) trait TransactionAction: AsAny + Sync + Send { +pub trait TransactionAction: AsAny + Sync + Send { /// Commits this action against the provided table and returns the resulting updates. /// NOTE: This function is intended for internal use only and should not be called directly by users. /// diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 9d94e18ede..2a5f2bc6e8 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -29,13 +29,20 @@ use crate::transaction::snapshot::{ }; use crate::transaction::{ActionCommit, TransactionAction}; -/// FastAppendAction is a transaction action for fast append data files to the table. +/// FastAppendAction is a transaction action that appends data files and/or +/// equality-/position-delete files to the table in a single snapshot. +/// +/// Files passed via [`Self::add_data_files`] are routed by `content_type()` +/// into separate manifests at commit time — `Data` files into a data +/// manifest, `EqualityDeletes`/`PositionDeletes` into a delete manifest. The +/// Iceberg spec forbids mixing content types in one manifest. pub struct FastAppendAction { check_duplicate: bool, // below are properties used to create SnapshotProducer when commit commit_uuid: Option, snapshot_properties: HashMap, added_data_files: Vec, + added_delete_files: Vec, } impl FastAppendAction { @@ -45,6 +52,7 @@ impl FastAppendAction { commit_uuid: None, snapshot_properties: HashMap::default(), added_data_files: vec![], + added_delete_files: vec![], } } @@ -54,9 +62,19 @@ impl FastAppendAction { self } - /// Add data files to the snapshot. + /// Add files to the snapshot. Files are routed by `content_type()` + /// into the data manifest (for `Data`) or the delete manifest (for + /// `EqualityDeletes` / `PositionDeletes`). pub fn add_data_files(mut self, data_files: impl IntoIterator) -> Self { - self.added_data_files.extend(data_files); + for file in data_files { + match file.content_type() { + crate::spec::DataContentType::Data => self.added_data_files.push(file), + crate::spec::DataContentType::EqualityDeletes + | crate::spec::DataContentType::PositionDeletes => { + self.added_delete_files.push(file) + } + } + } self } @@ -81,10 +99,12 @@ impl TransactionAction for FastAppendAction { self.commit_uuid.unwrap_or_else(Uuid::now_v7), self.snapshot_properties.clone(), self.added_data_files.clone(), + self.added_delete_files.clone(), ); - // validate added files - snapshot_producer.validate_added_data_files()?; + // Both vecs follow the same partition-spec / partition-value rules. + snapshot_producer.validate_added_files(&self.added_data_files)?; + snapshot_producer.validate_added_files(&self.added_delete_files)?; // Checks duplicate files if self.check_duplicate { @@ -113,7 +133,7 @@ impl SnapshotProduceOperation for FastAppendOperation { async fn existing_manifest( &self, - snapshot_produce: &SnapshotProducer<'_>, + snapshot_produce: &mut SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { return Ok(vec![]); @@ -486,6 +506,122 @@ mod tests { ); } + /// FastAppend now accepts mixed data + equality-delete files and routes + /// them into separate manifests at commit time. The Iceberg manifest list + /// should end up with two entries — one with `ManifestContentType::Data`, + /// one with `Deletes`. + #[tokio::test] + async fn test_fast_append_routes_equality_deletes_into_separate_manifest() { + use crate::spec::{ManifestContentType, NestedField, PrimitiveType, Schema, Type}; + + // Build a minimal in-memory table with FileIO::new_with_memory so the + // manifest writer can actually persist files. + let schema = Schema::builder() + .with_schema_id(0) + .with_identifier_field_ids([1]) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "qty", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + use crate::CatalogBuilder; + let memory: crate::memory::MemoryCatalog = crate::memory::MemoryCatalogBuilder::default() + .load( + "test", + HashMap::from([( + crate::memory::MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )]), + ) + .await + .unwrap(); + use crate::Catalog; + let ns = crate::NamespaceIdent::from_strs(["public"]).unwrap(); + memory.create_namespace(&ns, HashMap::new()).await.unwrap(); + let creation = crate::TableCreation::builder() + .name("orders".to_string()) + .schema(schema) + .build(); + let table = memory.create_table(&ns, creation).await.unwrap(); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(format!("{}/data/0001.parquet", table.metadata().location())) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(2048) + .record_count(7) + .partition(Struct::empty()) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap(); + + let delete_file = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path(format!( + "{}/deletes/0001.parquet", + table.metadata().location() + )) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(64) + .record_count(1) + .equality_ids(Some(vec![1])) + .partition(Struct::empty()) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap(); + + use crate::transaction::ApplyTransactionAction; + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![data_file.clone(), delete_file.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&memory).await.unwrap(); + + let snap = table + .metadata() + .current_snapshot() + .expect("commit must produce a snapshot"); + let manifest_list = table.manifest_list_reader(snap).load().await.unwrap(); + + // Two manifests — one Data, one Deletes. + assert_eq!(manifest_list.entries().len(), 2); + let mut content_types: Vec = + manifest_list.entries().iter().map(|e| e.content).collect(); + content_types.sort_by_key(|c| match c { + ManifestContentType::Data => 0, + ManifestContentType::Deletes => 1, + }); + assert_eq!(content_types, vec![ + ManifestContentType::Data, + ManifestContentType::Deletes + ]); + + // Walk each manifest and confirm the right file landed in the right + // bucket. + for entry in manifest_list.entries() { + let manifest = entry.load_manifest(table.file_io()).await.unwrap(); + assert_eq!(manifest.entries().len(), 1); + let me = &manifest.entries()[0]; + match entry.content { + ManifestContentType::Data => { + assert_eq!(me.data_file().content_type(), DataContentType::Data); + assert_eq!(me.data_file().record_count(), 7); + } + ManifestContentType::Deletes => { + assert_eq!( + me.data_file().content_type(), + DataContentType::EqualityDeletes + ); + assert_eq!(me.data_file().equality_ids(), Some(vec![1])); + } + } + } + } + #[tokio::test] async fn test_fast_append_file_with_incompatible_partition_value() { let table = make_v2_minimal_table(); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index d78f41cd42..226be3501f 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -55,6 +55,8 @@ mod action; pub use action::*; mod append; mod expire_snapshots; +mod replace_schema; +mod rewrite; mod snapshot; mod sort_order; mod update_location; @@ -72,9 +74,10 @@ pub use update_schema::AddColumn; use crate::error::Result; use crate::spec::TableProperties; use crate::table::Table; -use crate::transaction::action::BoxedTransactionAction; use crate::transaction::append::FastAppendAction; use crate::transaction::expire_snapshots::ExpireSnapshotsAction; +use crate::transaction::replace_schema::ReplaceSchemaAction; +use crate::transaction::rewrite::RewriteFilesAction; use crate::transaction::sort_order::ReplaceSortOrderAction; use crate::transaction::update_location::UpdateLocationAction; use crate::transaction::update_properties::UpdatePropertiesAction; @@ -151,6 +154,13 @@ impl Transaction { FastAppendAction::new() } + /// Creates a `RewriteFilesAction` for compaction-style replace + /// operations: drops a set of existing files and adds new ones in one + /// `Operation::Replace` snapshot. + pub fn rewrite_files(&self) -> RewriteFilesAction { + RewriteFilesAction::new() + } + /// Creates replace sort order action. pub fn replace_sort_order(&self) -> ReplaceSortOrderAction { ReplaceSortOrderAction::new() @@ -171,6 +181,15 @@ impl Transaction { ExpireSnapshotsAction::new() } + /// Replace the table's current schema with a new target schema. + /// + /// The caller is responsible for constructing a schema that's a valid + /// evolution of the current one — this action does not enforce + /// schema-evolution compatibility rules. + pub fn replace_schema(&self) -> ReplaceSchemaAction { + ReplaceSchemaAction::new() + } + /// Commit transaction. pub async fn commit(self, catalog: &dyn Catalog) -> Result
{ if self.actions.is_empty() { diff --git a/crates/iceberg/src/transaction/replace_schema.rs b/crates/iceberg/src/transaction/replace_schema.rs new file mode 100644 index 0000000000..10129ab7d4 --- /dev/null +++ b/crates/iceberg/src/transaction/replace_schema.rs @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::spec::Schema; +use crate::table::Table; +use crate::transaction::action::{ActionCommit, TransactionAction}; +use crate::{Error, ErrorKind, Result, TableRequirement, TableUpdate}; + +/// A transaction action that replaces the table's current schema with a new +/// target schema. (Upstream's `UpdateSchemaAction` evolves it column by +/// column, and can't rename, promote or make a column optional.) +/// +/// Callers compute the desired schema (e.g. by adding/dropping columns) and +/// hand it to [`Self::set_schema`]. On commit, this emits two table updates: +/// +/// - [`TableUpdate::AddSchema`] with the target schema. +/// - [`TableUpdate::SetCurrentSchema`] with `schema_id = -1` (the "last added +/// schema" sentinel), so we don't have to allocate a non-clashing schema id +/// ourselves — `TableMetadataBuilder` does that. +/// +/// Three requirements are attached so concurrent schema/column changes fail +/// loudly rather than silently overwrite each other: +/// +/// - `UuidMatch` — guards against the table being recreated under us. +/// - `CurrentSchemaIdMatch` — guards against a concurrent schema swap. +/// - `LastAssignedFieldIdMatch` — guards against a concurrent column add. +/// +/// Note: this action does not enforce schema-evolution compatibility rules +/// (e.g. "you may add columns but not remove them"). The metadata builder +/// also doesn't — it's the caller's responsibility to construct a schema +/// that's a valid evolution of the current one. +#[derive(Debug, Default)] +pub struct ReplaceSchemaAction { + target_schema: Option, +} + +impl ReplaceSchemaAction { + /// Creates a new [`ReplaceSchemaAction`] with no schema set. + pub fn new() -> Self { + ReplaceSchemaAction { + target_schema: None, + } + } + + /// Sets the target schema this action will commit. + pub fn set_schema(mut self, schema: Schema) -> Self { + self.target_schema = Some(schema); + self + } +} + +#[async_trait] +impl TransactionAction for ReplaceSchemaAction { + async fn commit(self: Arc, table: &Table) -> Result { + let target = self.target_schema.clone().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "Schema is not set for ReplaceSchemaAction!", + ) + })?; + + let metadata = table.metadata(); + + let updates = vec![ + TableUpdate::AddSchema { + schema: target.clone(), + }, + // `-1` is the LAST_ADDED sentinel honored by + // `TableMetadataBuilder::set_current_schema`. + TableUpdate::SetCurrentSchema { schema_id: -1 }, + ]; + + let requirements = vec![ + TableRequirement::UuidMatch { + uuid: metadata.uuid(), + }, + TableRequirement::CurrentSchemaIdMatch { + current_schema_id: metadata.current_schema_id(), + }, + TableRequirement::LastAssignedFieldIdMatch { + last_assigned_field_id: metadata.last_column_id(), + }, + ]; + + Ok(ActionCommit::new(updates, requirements)) + } +} + +#[cfg(test)] +mod tests { + use as_any::Downcast; + + use super::ReplaceSchemaAction; + use crate::spec::{NestedField, PrimitiveType, Schema, Type}; + use crate::transaction::Transaction; + use crate::transaction::action::{ApplyTransactionAction, TransactionAction}; + use crate::transaction::tests::make_v2_table; + + fn schema_with_one_int_column() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap() + } + + #[test] + fn applies_action_to_transaction() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + let tx = ReplaceSchemaAction::new() + .set_schema(schema_with_one_int_column()) + .apply(tx) + .unwrap(); + + assert_eq!(tx.actions.len(), 1); + + let action = (*tx.actions[0]) + .downcast_ref::() + .expect("ReplaceSchemaAction was not applied to Transaction!"); + + assert!(action.target_schema.is_some()); + } + + #[tokio::test] + async fn commit_without_schema_set_errors() { + let table = make_v2_table(); + let action = std::sync::Arc::new(ReplaceSchemaAction::new()); + let result = action.commit(&table).await; + assert!(result.is_err()); + let err = result.err().unwrap(); + assert!(err.message().contains("Schema is not set")); + } + + #[tokio::test] + async fn commit_emits_add_schema_and_set_current_schema_updates() { + let table = make_v2_table(); + let action = std::sync::Arc::new( + ReplaceSchemaAction::new().set_schema(schema_with_one_int_column()), + ); + let mut commit = action.commit(&table).await.unwrap(); + let updates = commit.take_updates(); + let requirements = commit.take_requirements(); + + assert_eq!(updates.len(), 2); + assert!(matches!(updates[0], crate::TableUpdate::AddSchema { .. })); + assert!(matches!(updates[1], crate::TableUpdate::SetCurrentSchema { + schema_id: -1 + })); + + // Three requirements, in a fixed order. + assert_eq!(requirements.len(), 3); + assert!(matches!( + requirements[0], + crate::TableRequirement::UuidMatch { .. } + )); + assert!(matches!( + requirements[1], + crate::TableRequirement::CurrentSchemaIdMatch { .. } + )); + assert!(matches!( + requirements[2], + crate::TableRequirement::LastAssignedFieldIdMatch { .. } + )); + } +} diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs new file mode 100644 index 0000000000..c0fb6682ba --- /dev/null +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -0,0 +1,789 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use uuid::Uuid; + +use crate::error::{Error, ErrorKind, Result}; +use crate::spec::{DataContentType, DataFile, ManifestEntry, ManifestFile, Operation}; +use crate::table::Table; +use crate::transaction::snapshot::{ + DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, +}; +use crate::transaction::{ActionCommit, TransactionAction}; + +/// `RewriteFilesAction` produces a snapshot with `operation = Replace`: +/// drops a set of existing data/delete files and adds a set of new files +/// in one atomic commit. Used by compaction to swap N small files for M +/// larger files without exposing intermediate state to readers. +/// +/// Removed files are identified by file path (exact match). Manifests from +/// the prior snapshot are walked: +/// - Manifests with no removed files carry forward unchanged. +/// - Manifests where every entry is removed are dropped from the new +/// manifest list entirely. +/// - Manifests with partial removal are rewritten: surviving entries land +/// in a new manifest with `ManifestStatus::Existing` (preserving each +/// entry's original `snapshot_id` and `sequence_number`). +/// +/// Added files go into a new `ManifestStatus::Added` manifest, routed by +/// `content_type()` (Data vs. EqualityDeletes/PositionDeletes) into +/// separate manifests as required by the spec. +/// +/// The commit fails if a removed file is no longer in the table it lands +/// on — another rewrite got there first, and adding this one's files too +/// would duplicate their rows. A retried commit checks again. +pub struct RewriteFilesAction { + commit_uuid: Option, + snapshot_properties: HashMap, + added_data_files: Vec, + added_delete_files: Vec, + removed_paths: HashSet, + data_sequence_number: Option, +} + +impl RewriteFilesAction { + pub(crate) fn new() -> Self { + Self { + commit_uuid: None, + snapshot_properties: HashMap::default(), + added_data_files: vec![], + added_delete_files: vec![], + removed_paths: HashSet::default(), + data_sequence_number: None, + } + } + + /// Add files produced by the rewrite (e.g. compacted data files). + /// Routed by `content_type()` into separate manifests. + pub fn add_data_files(mut self, data_files: impl IntoIterator) -> Self { + for file in data_files { + match file.content_type() { + DataContentType::Data => self.added_data_files.push(file), + DataContentType::EqualityDeletes | DataContentType::PositionDeletes => { + self.added_delete_files.push(file) + } + } + } + self + } + + /// Mark the given files as removed in the new snapshot. Matching is by + /// `file_path()`. + pub fn remove_data_files(mut self, files: impl IntoIterator) -> Self { + for file in files { + self.removed_paths.insert(file.file_path().to_string()); + } + self + } + + /// Mark the given paths as removed. Convenience for callers that + /// already track removals as `String`. + pub fn remove_paths(mut self, paths: impl IntoIterator) -> Self { + self.removed_paths.extend(paths); + self + } + + /// Set commit UUID for the snapshot. + pub fn set_commit_uuid(mut self, commit_uuid: Uuid) -> Self { + self.commit_uuid = Some(commit_uuid); + self + } + + /// Set snapshot summary properties. + pub fn set_snapshot_properties(mut self, snapshot_properties: HashMap) -> Self { + self.snapshot_properties = snapshot_properties; + self + } + + /// Give the added data files this data sequence number — the one of + /// the snapshot the rewrite read its files at — instead of the new + /// snapshot's. Their rows are the removed files' rows as of then, so + /// equality deletes committed since still apply to them; with the new + /// snapshot's number they would rise above those deletes and bring the + /// deleted rows back. Java's `RewriteFiles.dataSequenceNumber`. + pub fn set_data_sequence_number(mut self, data_sequence_number: i64) -> Self { + self.data_sequence_number = Some(data_sequence_number); + self + } +} + +#[async_trait] +impl TransactionAction for RewriteFilesAction { + async fn commit(self: Arc, table: &Table) -> Result { + let removed_files = self.removed_files(table).await?; + let snapshot_producer = SnapshotProducer::new( + table, + self.commit_uuid.unwrap_or_else(Uuid::now_v7), + self.snapshot_properties.clone(), + self.added_data_files.clone(), + self.added_delete_files.clone(), + ) + .with_added_data_sequence_number(self.data_sequence_number) + .with_removed_files(removed_files); + + snapshot_producer.validate_added_files(&self.added_data_files)?; + snapshot_producer.validate_added_files(&self.added_delete_files)?; + + snapshot_producer + .commit( + RewriteFilesOperation { + removed_paths: self.removed_paths.clone(), + }, + DefaultManifestProcess, + ) + .await + } +} + +impl RewriteFilesAction { + /// The live files of `table` that `removed_paths` names — the snapshot + /// summary subtracts them. Fails if one isn't live: another rewrite + /// replaced it first, and committing this one's files as well would + /// duplicate their rows. Runs on every commit attempt, against the + /// table the commit would land on. + async fn removed_files(&self, table: &Table) -> Result> { + let mut removed = Vec::new(); + let mut found: HashSet<&str> = HashSet::new(); + if let Some(snapshot) = table.metadata().current_snapshot() { + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + for mfile in manifest_list.entries() { + let manifest = mfile.load_manifest(table.file_io()).await?; + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { + if let Some(path) = self.removed_paths.get(entry.data_file().file_path()) { + found.insert(path); + removed.push(entry.data_file().clone()); + } + } + } + } + if found.len() < self.removed_paths.len() { + let mut missing: Vec<&str> = self + .removed_paths + .iter() + .map(String::as_str) + .filter(|p| !found.contains(p)) + .collect(); + missing.sort(); + return Err(Error::new( + ErrorKind::CatalogCommitConflicts, + format!("Rewrite removes files no longer in the table: {missing:?}"), + )); + } + Ok(removed) + } +} + +struct RewriteFilesOperation { + removed_paths: HashSet, +} + +impl SnapshotProduceOperation for RewriteFilesOperation { + fn operation(&self) -> Operation { + Operation::Replace + } + + async fn delete_entries( + &self, + _snapshot_produce: &SnapshotProducer<'_>, + ) -> Result> { + // We track removals at manifest-rewrite time inside `existing_manifest` + // rather than emitting separate `Deleted` entries — the new manifest + // list simply doesn't reference the dropped files. Java/PyIceberg do + // emit `Deleted` entries for audit-trail purposes; we may add that + // later if expiration tooling needs them. + Ok(vec![]) + } + + async fn existing_manifest( + &self, + snapshot_produce: &mut SnapshotProducer<'_>, + ) -> Result> { + let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { + return Ok(vec![]); + }; + + let manifest_list = snapshot_produce + .table + .manifest_list_reader(snapshot) + .load() + .await?; + + // Walk each prior manifest, decide carry-forward vs. partial-rewrite + // vs. drop. + // + // Pass 1: classify each manifest based on its loaded entries. We + // can't write rewritten manifests yet because that needs `&mut + // snapshot_produce`, and we also need the producer to compute + // file-IO paths during this loop. + let manifest_list_entries = manifest_list.entries().to_vec(); + let file_io = snapshot_produce.table.file_io().clone(); + + enum Plan { + CarryForward(ManifestFile), + DropEntirely, + Rewrite { + survivors: Vec, + content: crate::spec::ManifestContentType, + }, + } + let mut plans: Vec = Vec::with_capacity(manifest_list_entries.len()); + for mfile in &manifest_list_entries { + let manifest = mfile.load_manifest(&file_io).await?; + let total = manifest.entries().len(); + let survivors: Vec = manifest + .entries() + .iter() + .filter(|me| !self.removed_paths.contains(me.data_file().file_path())) + .map(|me| ManifestEntry::clone(me)) + .collect(); + + if survivors.len() == total { + plans.push(Plan::CarryForward(mfile.clone())); + } else if survivors.is_empty() { + plans.push(Plan::DropEntirely); + } else { + plans.push(Plan::Rewrite { + survivors, + content: mfile.content, + }); + } + } + + // Pass 2: materialize. Carry-forwards and rewrites land in the + // output; drops produce nothing. + let mut out: Vec = Vec::with_capacity(plans.len()); + for plan in plans { + match plan { + Plan::CarryForward(mf) => out.push(mf), + Plan::DropEntirely => {} + Plan::Rewrite { survivors, content } => { + let rewritten = snapshot_produce + .write_existing_manifest_for(survivors, content) + .await?; + out.push(rewritten); + } + } + } + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, ManifestContentType, NestedField, + Operation, PrimitiveType, Schema, Struct, Type, + }; + use crate::transaction::{ApplyTransactionAction, Transaction}; + use crate::{Catalog, CatalogBuilder, ErrorKind}; + + async fn make_memory_table() -> (crate::memory::MemoryCatalog, crate::table::Table) { + let schema = Schema::builder() + .with_schema_id(0) + .with_identifier_field_ids([1]) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "qty", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let memory: crate::memory::MemoryCatalog = crate::memory::MemoryCatalogBuilder::default() + .load( + "test", + HashMap::from([( + crate::memory::MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )]), + ) + .await + .unwrap(); + use crate::Catalog; + let ns = crate::NamespaceIdent::from_strs(["public"]).unwrap(); + memory.create_namespace(&ns, HashMap::new()).await.unwrap(); + let creation = crate::TableCreation::builder() + .name("orders".to_string()) + .schema(schema) + .build(); + let table = memory.create_table(&ns, creation).await.unwrap(); + (memory, table) + } + + fn data_file(path: &str, record_count: u64, byte_size: u64) -> crate::spec::DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(byte_size) + .record_count(record_count) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap() + } + + /// Snapshot 1 appends three files; snapshot 2 (rewrite) drops them all + /// and adds one compacted file. The new manifest list must reference + /// only the compacted file's manifest — the original manifest is + /// dropped because every entry was removed. + #[tokio::test] + async fn rewrite_drops_all_files_in_manifest_and_swaps_in_new_one() { + let (catalog, table) = make_memory_table().await; + + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let f3 = data_file("memory:///warehouse/public/orders/data/3.parquet", 30, 3072); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone(), f2.clone(), f3.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 60, + 5120, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted.clone()]) + .remove_data_files(vec![f1.clone(), f2.clone(), f3.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.summary().operation, Operation::Replace); + + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let data_manifests: Vec<_> = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .collect(); + assert_eq!(data_manifests.len(), 1); + + let manifest = data_manifests[0] + .load_manifest(table.file_io()) + .await + .unwrap(); + assert_eq!(manifest.entries().len(), 1); + assert_eq!( + manifest.entries()[0].data_file().file_path(), + compacted.file_path() + ); + } + + /// Partial-manifest removal: snapshot 1 appends three files; snapshot 2 + /// removes one and adds one. The new manifest list should reference + /// (a) a rewritten manifest carrying the two survivors as `Existing`, + /// (b) a new manifest with the added file as `Added`. + #[tokio::test] + async fn rewrite_partial_manifest_keeps_survivors_as_existing() { + let (catalog, table) = make_memory_table().await; + + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let f3 = data_file("memory:///warehouse/public/orders/data/3.parquet", 30, 3072); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone(), f2.clone(), f3.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Drop only f1, replace with a "smaller" compacted file. f2 and f3 + // survive in the rewritten manifest. + let new_file = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![new_file.clone()]) + .remove_data_files(vec![f1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.summary().operation, Operation::Replace); + + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let data_manifests: Vec<_> = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .collect(); + // Two data manifests: rewritten survivor manifest + new added manifest. + assert_eq!(data_manifests.len(), 2); + + let mut survivor_paths: Vec = Vec::new(); + let mut added_paths: Vec = Vec::new(); + for mf in &data_manifests { + let m = mf.load_manifest(table.file_io()).await.unwrap(); + for entry in m.entries() { + match entry.status() { + crate::spec::ManifestStatus::Existing => { + survivor_paths.push(entry.data_file().file_path().to_string()); + } + crate::spec::ManifestStatus::Added => { + added_paths.push(entry.data_file().file_path().to_string()); + } + crate::spec::ManifestStatus::Deleted => { + panic!("unexpected Deleted status in new manifest list"); + } + } + } + } + survivor_paths.sort(); + let mut want_survivors = vec![f2.file_path().to_string(), f3.file_path().to_string()]; + want_survivors.sort(); + assert_eq!(survivor_paths, want_survivors); + assert_eq!(added_paths, vec![new_file.file_path().to_string()]); + } + + /// Mixed rewrite: prior snapshot has a data manifest + a delete manifest; + /// rewrite drops one data file and keeps the delete manifest untouched. + /// The new manifest list should still have a `Deletes` manifest carrying + /// forward, plus a rewritten data manifest with survivors. + #[tokio::test] + async fn rewrite_carries_forward_unaffected_delete_manifest() { + let (catalog, table) = make_memory_table().await; + + let d1 = data_file( + "memory:///warehouse/public/orders/data/d1.parquet", + 10, + 1024, + ); + let d2 = data_file( + "memory:///warehouse/public/orders/data/d2.parquet", + 20, + 2048, + ); + let eq_delete = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path("memory:///warehouse/public/orders/deletes/eq.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(64) + .record_count(1) + .equality_ids(Some(vec![1])) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap(); + + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![d1.clone(), d2.clone(), eq_delete.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Rewrite: drop d1, keep d2 + eq_delete. Add a compacted file. + let new_file = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![new_file.clone()]) + .remove_data_files(vec![d1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + + // Counts: 2 data manifests (rewritten survivors + new added) and + // 1 delete manifest (unchanged carry-forward). + let data_count = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .count(); + let delete_count = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Deletes) + .count(); + assert_eq!(data_count, 2); + assert_eq!(delete_count, 1); + + // Every survivor and added file is reachable. + let mut all_paths: Vec = Vec::new(); + for mf in mlist.entries() { + let m = mf.load_manifest(table.file_io()).await.unwrap(); + for entry in m.entries() { + all_paths.push(entry.data_file().file_path().to_string()); + } + } + all_paths.sort(); + let mut want = vec![ + d2.file_path().to_string(), + new_file.file_path().to_string(), + eq_delete.file_path().to_string(), + ]; + want.sort(); + assert_eq!(all_paths, want); + } + + /// Rewrite with no `removed_paths` is an append disguised as a replace — + /// the prior manifest is carried forward and the new file joins it. + /// The snapshot is still tagged `Replace`. + #[tokio::test] + async fn rewrite_with_no_removals_carries_prior_manifest() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().add_data_files(vec![f2.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.summary().operation, Operation::Replace); + + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let data_manifest_count = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .count(); + assert_eq!(data_manifest_count, 2); + } + + /// Rewrite with empty add + empty remove is a no-op-ish state — the + /// SnapshotProducer requires *something* added (upstream contract). + /// Documents the existing behavior so callers know to skip the action + /// when there's nothing to compact. + #[tokio::test] + async fn rewrite_with_empty_input_errors() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx.rewrite_files(); + let tx = action.apply(tx).unwrap(); + assert!(tx.commit(&catalog).await.is_err()); + } + + /// With a data sequence number, the added files carry it rather than + /// the new snapshot's, which still becomes their file sequence number. + #[tokio::test] + async fn rewrite_gives_added_files_its_data_sequence_number() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let mut table = table; + for f in [&f1, &f2] { + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f.clone()]); + table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + } + + // Rewrite f1 as planned at sequence number 1. + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted.clone()]) + .remove_data_files(vec![f1.clone()]) + .set_data_sequence_number(1); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.sequence_number(), 3); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let mut seqs = Vec::new(); + for mf in mlist.entries() { + let m = mf.load_manifest(table.file_io()).await.unwrap(); + for entry in m.entries() { + seqs.push(( + entry.data_file().file_path().to_string(), + entry.sequence_number(), + entry.file_sequence_number, + )); + } + } + seqs.sort(); + assert_eq!(seqs, vec![ + (f2.file_path().to_string(), Some(2), Some(2)), + (compacted.file_path().to_string(), Some(1), Some(3)), + ]); + } + + /// A rewrite of a file another rewrite already removed fails: adding + /// its files too would duplicate the rows. + #[tokio::test] + async fn rewrite_of_a_file_no_longer_in_the_table_fails() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + // Two rewrites of f1, planned against the same table. + let rewrite = |path: &str| { + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![data_file(path, 10, 1024)]) + .remove_data_files(vec![f1.clone()]); + action.apply(tx).unwrap() + }; + let first = rewrite("memory:///warehouse/public/orders/data/compact-a.parquet"); + let second = rewrite("memory:///warehouse/public/orders/data/compact-b.parquet"); + let table = first.commit(&catalog).await.unwrap(); + // The second lands on the first's table, without f1. + let err = second.commit(&catalog).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::CatalogCommitConflicts); + assert!(!err.retryable()); + + let table = catalog.load_table(table.identifier()).await.unwrap(); + let snap = table.metadata().current_snapshot().unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); + let mut paths = Vec::new(); + for mf in mlist.entries() { + let m = mf.load_manifest(table.file_io()).await.unwrap(); + for entry in m.entries() { + paths.push(entry.data_file().file_path().to_string()); + } + } + assert_eq!(paths, vec![ + "memory:///warehouse/public/orders/data/compact-a.parquet".to_string() + ]); + } + + /// So does one removing a file from a table with no snapshot at all. + #[tokio::test] + async fn rewrite_removing_from_an_empty_table_fails() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted]) + .remove_data_files(vec![f1]); + let err = action + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::CatalogCommitConflicts); + } + + /// The snapshot summary keeps running totals: a rewrite subtracts the + /// files it removes, data and deletes, and adds its outputs. + #[tokio::test] + async fn rewrite_summary_keeps_running_totals() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let eq_delete = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path("memory:///warehouse/public/orders/deletes/eq.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(64) + .record_count(1) + .equality_ids(Some(vec![1])) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap(); + let mut table = table; + for files in [vec![f1.clone()], vec![f2.clone(), eq_delete.clone()]] { + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(files); + table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + } + let summary = |table: &crate::table::Table, key: &str| -> String { + let snap = table.metadata().current_snapshot().unwrap(); + snap.summary().additional_properties[key].clone() + }; + assert_eq!(summary(&table, "total-records"), "30"); + assert_eq!(summary(&table, "total-delete-files"), "1"); + + // f1 rewritten as 9 rows, retiring the delete. + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 9, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted]) + .remove_data_files(vec![f1.clone(), eq_delete.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + assert_eq!(summary(&table, "deleted-data-files"), "1"); + assert_eq!(summary(&table, "deleted-records"), "10"); + assert_eq!(summary(&table, "removed-delete-files"), "1"); + assert_eq!(summary(&table, "total-records"), "29"); + assert_eq!(summary(&table, "total-data-files"), "2"); + assert_eq!(summary(&table, "total-delete-files"), "0"); + assert_eq!(summary(&table, "total-equality-deletes"), "0"); + } +} diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index f8235e67d4..d0811a5bcf 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -83,9 +83,14 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { /// - **Append operations**: Typically include all existing manifests /// - **Overwrite operations**: May exclude manifests for partitions being overwritten /// - **Delete operations**: May exclude manifests for partitions being deleted + /// + /// Takes `&mut SnapshotProducer` so implementations that rewrite + /// manifests in place (e.g. `Replace`) can call helpers like + /// `write_existing_manifest_for` here. Append-style implementations + /// that only need read access can ignore the mutability. fn existing_manifest( &self, - snapshot_produce: &SnapshotProducer<'_>, + snapshot_produce: &mut SnapshotProducer<'_>, ) -> impl Future>> + Send; } @@ -115,6 +120,16 @@ pub(crate) struct SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, + /// Equality- and position-delete files added in this snapshot. Written + /// to a separate manifest from `added_data_files` because the Iceberg + /// spec forbids mixing data and delete entries in one manifest. + added_delete_files: Vec, + /// Data sequence number for `added_data_files`; `None` to inherit the + /// snapshot's. + added_data_sequence_number: Option, + /// Files this snapshot removes from the table, for its summary's + /// `deleted-*` counts and running `total-*`. + removed_files: Vec, // A counter used to generate unique manifest file names. // It starts from 0 and increments for each new manifest file. // Note: This counter is limited to the range of (0..u64::MAX). @@ -127,6 +142,7 @@ impl<'a> SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, + added_delete_files: Vec, ) -> Self { Self { table, @@ -134,18 +150,33 @@ impl<'a> SnapshotProducer<'a> { commit_uuid, snapshot_properties, added_data_files, + added_delete_files, + added_data_sequence_number: None, + removed_files: Vec::new(), manifest_counter: (0..), } } - pub(crate) fn validate_added_data_files(&self) -> Result<()> { - for data_file in &self.added_data_files { - if data_file.content_type() != crate::spec::DataContentType::Data { - return Err(Error::new( - ErrorKind::DataInvalid, - "Only data content type is allowed for fast append", - )); - } + /// Write the added data files with this data sequence number instead of + /// letting them inherit the snapshot's: their rows logically belong to + /// an older sequence number (see the spec's "Sequence Number + /// Inheritance"). Ignored for v1 tables, which have none. + pub(crate) fn with_added_data_sequence_number(mut self, seq: Option) -> Self { + self.added_data_sequence_number = seq; + self + } + + /// Record files this snapshot removes, so its summary subtracts them. + pub(crate) fn with_removed_files(mut self, files: Vec) -> Self { + self.removed_files = files; + self + } + + /// Validate partition spec / partition value compatibility for a slice of + /// added files. Generic over data vs. delete files — the Iceberg spec + /// validation rules are the same for both content types. + pub(crate) fn validate_added_files(&self, files: &[DataFile]) -> Result<()> { + for data_file in files { // Check if the data file partition spec id matches the table default partition spec id. if self.table.metadata().default_partition_spec_id() != data_file.partition_spec_id { return Err(Error::new( @@ -170,6 +201,7 @@ impl<'a> SnapshotProducer<'a> { let new_files: HashSet<&str> = self .added_data_files .iter() + .chain(self.added_delete_files.iter()) .map(|df| df.file_path.as_str()) .collect(); @@ -303,19 +335,67 @@ impl<'a> SnapshotProducer<'a> { Ok(()) } - // Write manifest file for added data files and return the ManifestFile for ManifestList. - async fn write_added_manifest(&mut self) -> Result { - let added_data_files = std::mem::take(&mut self.added_data_files); - if added_data_files.is_empty() { + /// Write a fresh manifest containing `survivors` as `Existing` entries. + /// Used by replace/rewrite operations that drop some files from a prior + /// manifest while preserving the rest. Each survivor must carry its + /// original `snapshot_id` and `sequence_number` (load_manifest's inherit + /// pass populates these from the manifest list entry, so feeding raw + /// `manifest.entries()` here is safe). + pub(crate) async fn write_existing_manifest_for( + &mut self, + survivors: Vec, + content: ManifestContentType, + ) -> Result { + if survivors.is_empty() { return Err(Error::new( ErrorKind::PreconditionFailed, - "No added data files found when write an added manifest file", + "No survivor entries when rewriting an existing manifest", + )); + } + let mut writer = self.new_manifest_writer(content)?; + for entry in survivors { + let snapshot_id = entry.snapshot_id().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "manifest entry missing snapshot_id when rewriting as existing", + ) + })?; + let sequence_number = entry.sequence_number().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "manifest entry missing sequence_number when rewriting as existing", + ) + })?; + writer.add_existing_file( + entry.data_file().clone(), + snapshot_id, + sequence_number, + entry.file_sequence_number, + )?; + } + writer.write_manifest_file().await + } + + /// Write a manifest file for a batch of added files, returning the + /// `ManifestFile` to splice into the manifest list. The caller picks the + /// content type — data and delete files must live in separate manifests + /// per the Iceberg spec. + async fn write_added_manifest_for( + &mut self, + added_files: Vec, + content: ManifestContentType, + data_sequence_number: Option, + ) -> Result { + if added_files.is_empty() { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "No added files found when writing a manifest file", )); } let snapshot_id = self.snapshot_id; let format_version = self.table.metadata().format_version(); - let manifest_entries = added_data_files.into_iter().map(|data_file| { + let manifest_entries = added_files.into_iter().map(|data_file| { let builder = ManifestEntry::builder() .status(crate::spec::ManifestStatus::Added) .data_file(data_file); @@ -324,10 +404,10 @@ impl<'a> SnapshotProducer<'a> { } else { // For format version > 1, we set the snapshot id at the inherited time to avoid rewrite the manifest file when // commit failed. - builder.build() + builder.sequence_number_opt(data_sequence_number).build() } }); - let mut writer = self.new_manifest_writer(ManifestContentType::Data)?; + let mut writer = self.new_manifest_writer(content)?; for entry in manifest_entries { writer.add_entry(entry)?; } @@ -344,24 +424,43 @@ impl<'a> SnapshotProducer<'a> { // TODO: Allowing snapshot property setup with no added data files is a workaround. // We should clean it up after all necessary actions are supported. // For details, please refer to https://github.com/apache/iceberg-rust/issues/1548 - if self.added_data_files.is_empty() && self.snapshot_properties.is_empty() { + if self.added_data_files.is_empty() + && self.added_delete_files.is_empty() + && self.snapshot_properties.is_empty() + { return Err(Error::new( ErrorKind::PreconditionFailed, - "No added data files or added snapshot properties found when write a manifest file", + "No added files or snapshot properties found when writing a manifest file", )); } + // Existing-manifest selection runs first because Replace operations + // may write rewritten manifests via `&mut self` here; the added-file + // manifest write below comes after to keep ordering deterministic. let existing_manifests = snapshot_produce_operation.existing_manifest(self).await?; let mut manifest_files = existing_manifests; - // Process added entries. + // Data and delete files must go into separate manifests — Iceberg + // doesn't allow mixing content types in a single manifest. if !self.added_data_files.is_empty() { - let added_manifest = self.write_added_manifest().await?; - manifest_files.push(added_manifest); + let added_data_files = std::mem::take(&mut self.added_data_files); + let seq = self.added_data_sequence_number; + manifest_files.push( + self.write_added_manifest_for(added_data_files, ManifestContentType::Data, seq) + .await?, + ); + } + if !self.added_delete_files.is_empty() { + let added_delete_files = std::mem::take(&mut self.added_delete_files); + manifest_files.push( + self.write_added_manifest_for( + added_delete_files, + ManifestContentType::Deletes, + None, + ) + .await?, + ); } - - // # TODO - // Support process delete entries. let manifest_files = manifest_process.process_manifests(self, manifest_files); Ok(manifest_files) @@ -390,13 +489,24 @@ impl<'a> SnapshotProducer<'a> { summary_collector.set_partition_summary_limit(partition_summary_limit); - for data_file in &self.added_data_files { + for data_file in self + .added_data_files + .iter() + .chain(self.added_delete_files.iter()) + { summary_collector.add_file( data_file, table_metadata.current_schema().clone(), table_metadata.default_partition_spec().clone(), ); } + for data_file in &self.removed_files { + let spec = table_metadata + .partition_spec_by_id(data_file.partition_spec_id) + .unwrap_or_else(|| table_metadata.default_partition_spec()) + .clone(); + summary_collector.remove_file(data_file, table_metadata.current_schema().clone(), spec); + } let previous_snapshot = table_metadata.current_snapshot(); @@ -468,9 +578,9 @@ impl<'a> SnapshotProducer<'a> { ), }; - // Calling self.summary() before self.manifest_file() is important because self.added_data_files - // will be set to an empty vec after self.manifest_file() returns, resulting in an empty summary - // being generated. + // Calling self.summary() before self.manifest_file() is important because the + // added_*_files vecs are drained by self.manifest_file(), which would result in + // an empty summary being generated otherwise. let summary = self.summary(&snapshot_produce_operation).map_err(|err| { Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) })?;