From f349354f8490636f2b9d1f71257e88bb71fec185 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Sun, 26 Apr 2026 02:29:23 +0800 Subject: [PATCH 1/8] transaction: expose TransactionAction publicly + add ReplaceSchemaAction Two related changes that let downstream crates author their own TransactionActions and drive schema evolution without going through the private TableCommit::builder. 1. Flip TransactionAction (and BoxedTransactionAction) from pub(crate) to pub. The blanket ApplyTransactionAction impl already accepts any T: TransactionAction, so external implementors can add custom actions to a Transaction without further changes. 2. Add ReplaceSchemaAction. Callers compute the desired Schema and hand it to set_schema(); on commit the action emits AddSchema + SetCurrentSchema(-1) so TableMetadataBuilder allocates the new schema id. Three requirements (UuidMatch, CurrentSchemaIdMatch, LastAssignedFieldIdMatch) make concurrent schema/column changes fail loudly. Upstream's UpdateSchemaAction (0.10) evolves a schema column by column and can't rename, promote or make a column optional, so this stays alongside it. Also add a Transaction::replace_schema() convenience method matching the existing fast_append / update_location / update_statistics shape. --- crates/iceberg/src/transaction/action.rs | 4 +- crates/iceberg/src/transaction/mod.rs | 12 +- .../iceberg/src/transaction/replace_schema.rs | 190 ++++++++++++++++++ 3 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 crates/iceberg/src/transaction/replace_schema.rs diff --git a/crates/iceberg/src/transaction/action.rs b/crates/iceberg/src/transaction/action.rs index aa0a05d0d9..7b50f3cfa6 100644 --- a/crates/iceberg/src/transaction/action.rs +++ b/crates/iceberg/src/transaction/action.rs @@ -26,7 +26,7 @@ use crate::transaction::Transaction; use crate::{Result, TableRequirement, TableUpdate}; /// A boxed, thread-safe reference to a `TransactionAction`. -pub(crate) type BoxedTransactionAction = Arc; +pub type BoxedTransactionAction = Arc; /// A trait representing an atomic action that can be part of a transaction. /// @@ -34,7 +34,7 @@ pub(crate) type BoxedTransactionAction = Arc; /// Each action is responsible for generating the updates and requirements needed /// to modify the table metadata. #[async_trait] -pub(crate) trait TransactionAction: AsAny + Sync + Send { +pub trait TransactionAction: AsAny + Sync + Send { /// Commits this action against the provided table and returns the resulting updates. /// NOTE: This function is intended for internal use only and should not be called directly by users. /// diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index d78f41cd42..06758b9037 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -59,6 +59,7 @@ mod snapshot; mod sort_order; mod update_location; mod update_properties; +mod replace_schema; mod update_schema; mod update_statistics; mod upgrade_format_version; @@ -72,12 +73,12 @@ pub use update_schema::AddColumn; use crate::error::Result; use crate::spec::TableProperties; use crate::table::Table; -use crate::transaction::action::BoxedTransactionAction; use crate::transaction::append::FastAppendAction; use crate::transaction::expire_snapshots::ExpireSnapshotsAction; use crate::transaction::sort_order::ReplaceSortOrderAction; use crate::transaction::update_location::UpdateLocationAction; use crate::transaction::update_properties::UpdatePropertiesAction; +use crate::transaction::replace_schema::ReplaceSchemaAction; use crate::transaction::update_schema::UpdateSchemaAction; use crate::transaction::update_statistics::UpdateStatisticsAction; use crate::transaction::upgrade_format_version::UpgradeFormatVersionAction; @@ -171,6 +172,15 @@ impl Transaction { ExpireSnapshotsAction::new() } + /// Replace the table's current schema with a new target schema. + /// + /// The caller is responsible for constructing a schema that's a valid + /// evolution of the current one — this action does not enforce + /// schema-evolution compatibility rules. + pub fn replace_schema(&self) -> ReplaceSchemaAction { + ReplaceSchemaAction::new() + } + /// Commit transaction. pub async fn commit(self, catalog: &dyn Catalog) -> Result { if self.actions.is_empty() { diff --git a/crates/iceberg/src/transaction/replace_schema.rs b/crates/iceberg/src/transaction/replace_schema.rs new file mode 100644 index 0000000000..969398577f --- /dev/null +++ b/crates/iceberg/src/transaction/replace_schema.rs @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::spec::Schema; +use crate::table::Table; +use crate::transaction::action::{ActionCommit, TransactionAction}; +use crate::{Error, ErrorKind, Result, TableRequirement, TableUpdate}; + +/// A transaction action that replaces the table's current schema with a new +/// target schema. (Upstream's `UpdateSchemaAction` evolves it column by +/// column, and can't rename, promote or make a column optional.) +/// +/// Callers compute the desired schema (e.g. by adding/dropping columns) and +/// hand it to [`Self::set_schema`]. On commit, this emits two table updates: +/// +/// - [`TableUpdate::AddSchema`] with the target schema. +/// - [`TableUpdate::SetCurrentSchema`] with `schema_id = -1` (the "last added +/// schema" sentinel), so we don't have to allocate a non-clashing schema id +/// ourselves — `TableMetadataBuilder` does that. +/// +/// Three requirements are attached so concurrent schema/column changes fail +/// loudly rather than silently overwrite each other: +/// +/// - `UuidMatch` — guards against the table being recreated under us. +/// - `CurrentSchemaIdMatch` — guards against a concurrent schema swap. +/// - `LastAssignedFieldIdMatch` — guards against a concurrent column add. +/// +/// Note: this action does not enforce schema-evolution compatibility rules +/// (e.g. "you may add columns but not remove them"). The metadata builder +/// also doesn't — it's the caller's responsibility to construct a schema +/// that's a valid evolution of the current one. +#[derive(Debug, Default)] +pub struct ReplaceSchemaAction { + target_schema: Option, +} + +impl ReplaceSchemaAction { + /// Creates a new [`ReplaceSchemaAction`] with no schema set. + pub fn new() -> Self { + ReplaceSchemaAction { + target_schema: None, + } + } + + /// Sets the target schema this action will commit. + pub fn set_schema(mut self, schema: Schema) -> Self { + self.target_schema = Some(schema); + self + } +} + +#[async_trait] +impl TransactionAction for ReplaceSchemaAction { + async fn commit(self: Arc, table: &Table) -> Result { + let target = self.target_schema.clone().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "Schema is not set for ReplaceSchemaAction!", + ) + })?; + + let metadata = table.metadata(); + + let updates = vec![ + TableUpdate::AddSchema { + schema: target.clone(), + }, + // `-1` is the LAST_ADDED sentinel honored by + // `TableMetadataBuilder::set_current_schema`. + TableUpdate::SetCurrentSchema { schema_id: -1 }, + ]; + + let requirements = vec![ + TableRequirement::UuidMatch { + uuid: metadata.uuid(), + }, + TableRequirement::CurrentSchemaIdMatch { + current_schema_id: metadata.current_schema_id(), + }, + TableRequirement::LastAssignedFieldIdMatch { + last_assigned_field_id: metadata.last_column_id(), + }, + ]; + + Ok(ActionCommit::new(updates, requirements)) + } +} + +#[cfg(test)] +mod tests { + use as_any::Downcast; + + use super::ReplaceSchemaAction; + use crate::spec::{NestedField, PrimitiveType, Schema, Type}; + use crate::transaction::Transaction; + use crate::transaction::action::{ApplyTransactionAction, TransactionAction}; + use crate::transaction::tests::make_v2_table; + + fn schema_with_one_int_column() -> Schema { + Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap() + } + + #[test] + fn applies_action_to_transaction() { + let table = make_v2_table(); + let tx = Transaction::new(&table); + let tx = ReplaceSchemaAction::new() + .set_schema(schema_with_one_int_column()) + .apply(tx) + .unwrap(); + + assert_eq!(tx.actions.len(), 1); + + let action = (*tx.actions[0]) + .downcast_ref::() + .expect("ReplaceSchemaAction was not applied to Transaction!"); + + assert!(action.target_schema.is_some()); + } + + #[tokio::test] + async fn commit_without_schema_set_errors() { + let table = make_v2_table(); + let action = std::sync::Arc::new(ReplaceSchemaAction::new()); + let result = action.commit(&table).await; + assert!(result.is_err()); + let err = result.err().unwrap(); + assert!(err.message().contains("Schema is not set")); + } + + #[tokio::test] + async fn commit_emits_add_schema_and_set_current_schema_updates() { + let table = make_v2_table(); + let action = std::sync::Arc::new( + ReplaceSchemaAction::new().set_schema(schema_with_one_int_column()), + ); + let mut commit = action.commit(&table).await.unwrap(); + let updates = commit.take_updates(); + let requirements = commit.take_requirements(); + + assert_eq!(updates.len(), 2); + assert!(matches!( + updates[0], + crate::TableUpdate::AddSchema { .. } + )); + assert!(matches!( + updates[1], + crate::TableUpdate::SetCurrentSchema { schema_id: -1 } + )); + + // Three requirements, in a fixed order. + assert_eq!(requirements.len(), 3); + assert!(matches!( + requirements[0], + crate::TableRequirement::UuidMatch { .. } + )); + assert!(matches!( + requirements[1], + crate::TableRequirement::CurrentSchemaIdMatch { .. } + )); + assert!(matches!( + requirements[2], + crate::TableRequirement::LastAssignedFieldIdMatch { .. } + )); + } +} From a45ac76e0872040c6852d61bc5e7d22b029a1a2d Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Sun, 26 Apr 2026 03:13:58 +0800 Subject: [PATCH 2/8] transaction: route equality/position-delete files through FastAppendAction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FastAppend previously rejected any DataFile whose content_type was not Data, blocking commits that include equality- or position-delete files. This change accepts mixed input and routes by content_type into separate manifests at commit time, matching the Iceberg spec requirement that one manifest must hold a single content type. - FastAppendAction.add_data_files() now routes incoming files into added_data_files (Data) or added_delete_files (EqualityDeletes / PositionDeletes). The old "Only data content type is allowed" check is gone. - SnapshotProducer carries added_delete_files alongside added_data_files and writes them to a separate delete manifest via build_v{2,3}_deletes. - validate_added_data_files renamed to validate_added_files and is now generic over a slice — the partition-spec / partition-value rules apply uniformly to data and delete files. - Snapshot summary collector iterates both vecs so file/record counts reflect everything added in the snapshot. - New test test_fast_append_routes_equality_deletes_into_separate_manifest exercises a mixed Data + EqualityDeletes commit through MemoryCatalog and asserts both manifests appear in the manifest list with the right ManifestContentType. --- crates/iceberg/src/transaction/append.rs | 146 ++++++++++++++++++++- crates/iceberg/src/transaction/snapshot.rs | 75 +++++++---- 2 files changed, 189 insertions(+), 32 deletions(-) diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 9d94e18ede..b8b2d9954e 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -29,13 +29,20 @@ use crate::transaction::snapshot::{ }; use crate::transaction::{ActionCommit, TransactionAction}; -/// FastAppendAction is a transaction action for fast append data files to the table. +/// FastAppendAction is a transaction action that appends data files and/or +/// equality-/position-delete files to the table in a single snapshot. +/// +/// Files passed via [`Self::add_data_files`] are routed by `content_type()` +/// into separate manifests at commit time — `Data` files into a data +/// manifest, `EqualityDeletes`/`PositionDeletes` into a delete manifest. The +/// Iceberg spec forbids mixing content types in one manifest. pub struct FastAppendAction { check_duplicate: bool, // below are properties used to create SnapshotProducer when commit commit_uuid: Option, snapshot_properties: HashMap, added_data_files: Vec, + added_delete_files: Vec, } impl FastAppendAction { @@ -45,6 +52,7 @@ impl FastAppendAction { commit_uuid: None, snapshot_properties: HashMap::default(), added_data_files: vec![], + added_delete_files: vec![], } } @@ -54,9 +62,19 @@ impl FastAppendAction { self } - /// Add data files to the snapshot. + /// Add files to the snapshot. Files are routed by `content_type()` + /// into the data manifest (for `Data`) or the delete manifest (for + /// `EqualityDeletes` / `PositionDeletes`). pub fn add_data_files(mut self, data_files: impl IntoIterator) -> Self { - self.added_data_files.extend(data_files); + for file in data_files { + match file.content_type() { + crate::spec::DataContentType::Data => self.added_data_files.push(file), + crate::spec::DataContentType::EqualityDeletes + | crate::spec::DataContentType::PositionDeletes => { + self.added_delete_files.push(file) + } + } + } self } @@ -81,10 +99,12 @@ impl TransactionAction for FastAppendAction { self.commit_uuid.unwrap_or_else(Uuid::now_v7), self.snapshot_properties.clone(), self.added_data_files.clone(), + self.added_delete_files.clone(), ); - // validate added files - snapshot_producer.validate_added_data_files()?; + // Both vecs follow the same partition-spec / partition-value rules. + snapshot_producer.validate_added_files(&self.added_data_files)?; + snapshot_producer.validate_added_files(&self.added_delete_files)?; // Checks duplicate files if self.check_duplicate { @@ -486,6 +506,122 @@ mod tests { ); } + /// FastAppend now accepts mixed data + equality-delete files and routes + /// them into separate manifests at commit time. The Iceberg manifest list + /// should end up with two entries — one with `ManifestContentType::Data`, + /// one with `Deletes`. + #[tokio::test] + async fn test_fast_append_routes_equality_deletes_into_separate_manifest() { + use crate::spec::{ManifestContentType, NestedField, PrimitiveType, Schema, Type}; + + // Build a minimal in-memory table with FileIO::new_with_memory so the + // manifest writer can actually persist files. + let schema = Schema::builder() + .with_schema_id(0) + .with_identifier_field_ids([1]) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "qty", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + use crate::CatalogBuilder; + let memory: crate::memory::MemoryCatalog = crate::memory::MemoryCatalogBuilder::default() + .load( + "test", + HashMap::from([( + crate::memory::MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )]), + ) + .await + .unwrap(); + use crate::Catalog; + let ns = crate::NamespaceIdent::from_strs(["public"]).unwrap(); + memory + .create_namespace(&ns, HashMap::new()) + .await + .unwrap(); + let creation = crate::TableCreation::builder() + .name("orders".to_string()) + .schema(schema) + .build(); + let table = memory.create_table(&ns, creation).await.unwrap(); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(format!("{}/data/0001.parquet", table.metadata().location())) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(2048) + .record_count(7) + .partition(Struct::empty()) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap(); + + let delete_file = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path(format!("{}/deletes/0001.parquet", table.metadata().location())) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(64) + .record_count(1) + .equality_ids(Some(vec![1])) + .partition(Struct::empty()) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap(); + + use crate::transaction::ApplyTransactionAction; + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![data_file.clone(), delete_file.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&memory).await.unwrap(); + + let snap = table + .metadata() + .current_snapshot() + .expect("commit must produce a snapshot"); + let manifest_list = table.manifest_list_reader(snap) + .load() + .await + .unwrap(); + + // Two manifests — one Data, one Deletes. + assert_eq!(manifest_list.entries().len(), 2); + let mut content_types: Vec = + manifest_list.entries().iter().map(|e| e.content).collect(); + content_types.sort_by_key(|c| match c { + ManifestContentType::Data => 0, + ManifestContentType::Deletes => 1, + }); + assert_eq!(content_types, vec![ManifestContentType::Data, ManifestContentType::Deletes]); + + // Walk each manifest and confirm the right file landed in the right + // bucket. + for entry in manifest_list.entries() { + let manifest = entry.load_manifest(table.file_io()).await.unwrap(); + assert_eq!(manifest.entries().len(), 1); + let me = &manifest.entries()[0]; + match entry.content { + ManifestContentType::Data => { + assert_eq!(me.data_file().content_type(), DataContentType::Data); + assert_eq!(me.data_file().record_count(), 7); + } + ManifestContentType::Deletes => { + assert_eq!( + me.data_file().content_type(), + DataContentType::EqualityDeletes + ); + assert_eq!(me.data_file().equality_ids(), Some(vec![1])); + } + } + } + } + #[tokio::test] async fn test_fast_append_file_with_incompatible_partition_value() { let table = make_v2_minimal_table(); diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index f8235e67d4..849f2fbf61 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -115,6 +115,10 @@ pub(crate) struct SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, + /// Equality- and position-delete files added in this snapshot. Written + /// to a separate manifest from `added_data_files` because the Iceberg + /// spec forbids mixing data and delete entries in one manifest. + added_delete_files: Vec, // A counter used to generate unique manifest file names. // It starts from 0 and increments for each new manifest file. // Note: This counter is limited to the range of (0..u64::MAX). @@ -127,6 +131,7 @@ impl<'a> SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, + added_delete_files: Vec, ) -> Self { Self { table, @@ -134,18 +139,16 @@ impl<'a> SnapshotProducer<'a> { commit_uuid, snapshot_properties, added_data_files, + added_delete_files, manifest_counter: (0..), } } - pub(crate) fn validate_added_data_files(&self) -> Result<()> { - for data_file in &self.added_data_files { - if data_file.content_type() != crate::spec::DataContentType::Data { - return Err(Error::new( - ErrorKind::DataInvalid, - "Only data content type is allowed for fast append", - )); - } + /// Validate partition spec / partition value compatibility for a slice of + /// added files. Generic over data vs. delete files — the Iceberg spec + /// validation rules are the same for both content types. + pub(crate) fn validate_added_files(&self, files: &[DataFile]) -> Result<()> { + for data_file in files { // Check if the data file partition spec id matches the table default partition spec id. if self.table.metadata().default_partition_spec_id() != data_file.partition_spec_id { return Err(Error::new( @@ -170,6 +173,7 @@ impl<'a> SnapshotProducer<'a> { let new_files: HashSet<&str> = self .added_data_files .iter() + .chain(self.added_delete_files.iter()) .map(|df| df.file_path.as_str()) .collect(); @@ -303,19 +307,25 @@ impl<'a> SnapshotProducer<'a> { Ok(()) } - // Write manifest file for added data files and return the ManifestFile for ManifestList. - async fn write_added_manifest(&mut self) -> Result { - let added_data_files = std::mem::take(&mut self.added_data_files); - if added_data_files.is_empty() { + /// Write a manifest file for a batch of added files, returning the + /// `ManifestFile` to splice into the manifest list. The caller picks the + /// content type — data and delete files must live in separate manifests + /// per the Iceberg spec. + async fn write_added_manifest_for( + &mut self, + added_files: Vec, + content: ManifestContentType, + ) -> Result { + if added_files.is_empty() { return Err(Error::new( ErrorKind::PreconditionFailed, - "No added data files found when write an added manifest file", + "No added files found when writing a manifest file", )); } let snapshot_id = self.snapshot_id; let format_version = self.table.metadata().format_version(); - let manifest_entries = added_data_files.into_iter().map(|data_file| { + let manifest_entries = added_files.into_iter().map(|data_file| { let builder = ManifestEntry::builder() .status(crate::spec::ManifestStatus::Added) .data_file(data_file); @@ -327,7 +337,7 @@ impl<'a> SnapshotProducer<'a> { builder.build() } }); - let mut writer = self.new_manifest_writer(ManifestContentType::Data)?; + let mut writer = self.new_manifest_writer(content)?; for entry in manifest_entries { writer.add_entry(entry)?; } @@ -344,24 +354,35 @@ impl<'a> SnapshotProducer<'a> { // TODO: Allowing snapshot property setup with no added data files is a workaround. // We should clean it up after all necessary actions are supported. // For details, please refer to https://github.com/apache/iceberg-rust/issues/1548 - if self.added_data_files.is_empty() && self.snapshot_properties.is_empty() { + if self.added_data_files.is_empty() + && self.added_delete_files.is_empty() + && self.snapshot_properties.is_empty() + { return Err(Error::new( ErrorKind::PreconditionFailed, - "No added data files or added snapshot properties found when write a manifest file", + "No added files or snapshot properties found when writing a manifest file", )); } let existing_manifests = snapshot_produce_operation.existing_manifest(self).await?; let mut manifest_files = existing_manifests; - // Process added entries. + // Data and delete files must go into separate manifests — Iceberg + // doesn't allow mixing content types in a single manifest. if !self.added_data_files.is_empty() { - let added_manifest = self.write_added_manifest().await?; - manifest_files.push(added_manifest); + let added_data_files = std::mem::take(&mut self.added_data_files); + manifest_files.push( + self.write_added_manifest_for(added_data_files, ManifestContentType::Data) + .await?, + ); + } + if !self.added_delete_files.is_empty() { + let added_delete_files = std::mem::take(&mut self.added_delete_files); + manifest_files.push( + self.write_added_manifest_for(added_delete_files, ManifestContentType::Deletes) + .await?, + ); } - - // # TODO - // Support process delete entries. let manifest_files = manifest_process.process_manifests(self, manifest_files); Ok(manifest_files) @@ -390,7 +411,7 @@ impl<'a> SnapshotProducer<'a> { summary_collector.set_partition_summary_limit(partition_summary_limit); - for data_file in &self.added_data_files { + for data_file in self.added_data_files.iter().chain(self.added_delete_files.iter()) { summary_collector.add_file( data_file, table_metadata.current_schema().clone(), @@ -468,9 +489,9 @@ impl<'a> SnapshotProducer<'a> { ), }; - // Calling self.summary() before self.manifest_file() is important because self.added_data_files - // will be set to an empty vec after self.manifest_file() returns, resulting in an empty summary - // being generated. + // Calling self.summary() before self.manifest_file() is important because the + // added_*_files vecs are drained by self.manifest_file(), which would result in + // an empty summary being generated otherwise. let summary = self.summary(&snapshot_produce_operation).map_err(|err| { Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) })?; From a15fabf24faff1d232c342e81f2231c52654110b Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Sun, 26 Apr 2026 17:50:55 +0800 Subject: [PATCH 3/8] transaction: add RewriteFilesAction for compaction-style replace commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a new transaction action that produces snapshots with operation = Replace: drops a set of existing data/delete files and adds new ones in one atomic commit. Wired as `Transaction::rewrite_files()`. Manifests from the prior snapshot are walked and classified per-manifest: - No removals: carry forward unchanged. - All entries removed: drop manifest from new manifest list. - Partial removal: rewrite manifest with survivors marked Existing (preserving original snapshot_id + sequence_number). Added files go into a new ManifestStatus::Added manifest, routed by content_type() into separate Data / Deletes manifests as the spec requires. To support partial-manifest rewrite, SnapshotProduceOperation::existing_manifest now takes &mut SnapshotProducer so impls can call write_existing_manifest_for (new helper) inline. FastAppendOperation's signature was updated to match; its body is unchanged. snapshot_summary::update_snapshot_summaries now accepts Operation::Replace alongside Append/Overwrite/Delete. Tests: - full-manifest drop + swap - partial rewrite preserves survivors as Existing - mixed data + delete manifests carry forward correctly - no-removal path is just a Replace-tagged append - empty-input action errors (matches SnapshotProducer's contract) Mirrors Java/PyIceberg's RewriteFiles semantics but keeps the implementation narrow — we don't emit Deleted entries for audit-trail purposes, since the new manifest list simply doesn't reference the dropped files. Can revisit when expiration tooling needs them. --- crates/iceberg/src/spec/snapshot_summary.rs | 7 +- crates/iceberg/src/transaction/append.rs | 2 +- crates/iceberg/src/transaction/mod.rs | 9 + crates/iceberg/src/transaction/rewrite.rs | 558 ++++++++++++++++++++ crates/iceberg/src/transaction/snapshot.rs | 51 +- 5 files changed, 624 insertions(+), 3 deletions(-) create mode 100644 crates/iceberg/src/transaction/rewrite.rs diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index c9fd022724..4c9e8a3189 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -334,9 +334,14 @@ pub(crate) fn update_snapshot_summaries( previous_summary: Option<&Summary>, truncate_full_table: bool, ) -> Result { - // Validate that the operation is supported + // Validate that the operation is supported. `Replace` is treated like + // `Overwrite` for summary-update purposes — both swap files in/out + // without changing logical contents — but we keep them distinct in + // the snapshot summary so MoR tooling can distinguish compaction + // (`replace`) from logical overwrites (`overwrite`). if summary.operation != Operation::Append && summary.operation != Operation::Overwrite + && summary.operation != Operation::Replace && summary.operation != Operation::Delete { return Err(Error::new( diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index b8b2d9954e..a4fc9df030 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -133,7 +133,7 @@ impl SnapshotProduceOperation for FastAppendOperation { async fn existing_manifest( &self, - snapshot_produce: &SnapshotProducer<'_>, + snapshot_produce: &mut SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { return Ok(vec![]); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 06758b9037..84784c303e 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -55,6 +55,7 @@ mod action; pub use action::*; mod append; mod expire_snapshots; +mod rewrite; mod snapshot; mod sort_order; mod update_location; @@ -75,6 +76,7 @@ use crate::spec::TableProperties; use crate::table::Table; use crate::transaction::append::FastAppendAction; use crate::transaction::expire_snapshots::ExpireSnapshotsAction; +use crate::transaction::rewrite::RewriteFilesAction; use crate::transaction::sort_order::ReplaceSortOrderAction; use crate::transaction::update_location::UpdateLocationAction; use crate::transaction::update_properties::UpdatePropertiesAction; @@ -152,6 +154,13 @@ impl Transaction { FastAppendAction::new() } + /// Creates a `RewriteFilesAction` for compaction-style replace + /// operations: drops a set of existing files and adds new ones in one + /// `Operation::Replace` snapshot. + pub fn rewrite_files(&self) -> RewriteFilesAction { + RewriteFilesAction::new() + } + /// Creates replace sort order action. pub fn replace_sort_order(&self) -> ReplaceSortOrderAction { ReplaceSortOrderAction::new() diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs new file mode 100644 index 0000000000..996000580b --- /dev/null +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -0,0 +1,558 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use uuid::Uuid; + +use crate::error::Result; +use crate::spec::{DataContentType, DataFile, ManifestEntry, ManifestFile, Operation}; +use crate::table::Table; +use crate::transaction::snapshot::{ + DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, +}; +use crate::transaction::{ActionCommit, TransactionAction}; + +/// `RewriteFilesAction` produces a snapshot with `operation = Replace`: +/// drops a set of existing data/delete files and adds a set of new files +/// in one atomic commit. Used by compaction to swap N small files for M +/// larger files without exposing intermediate state to readers. +/// +/// Removed files are identified by file path (exact match). Manifests from +/// the prior snapshot are walked: +/// - Manifests with no removed files carry forward unchanged. +/// - Manifests where every entry is removed are dropped from the new +/// manifest list entirely. +/// - Manifests with partial removal are rewritten: surviving entries land +/// in a new manifest with `ManifestStatus::Existing` (preserving each +/// entry's original `snapshot_id` and `sequence_number`). +/// +/// Added files go into a new `ManifestStatus::Added` manifest, routed by +/// `content_type()` (Data vs. EqualityDeletes/PositionDeletes) into +/// separate manifests as required by the spec. +pub struct RewriteFilesAction { + commit_uuid: Option, + snapshot_properties: HashMap, + added_data_files: Vec, + added_delete_files: Vec, + removed_paths: HashSet, +} + +impl RewriteFilesAction { + pub(crate) fn new() -> Self { + Self { + commit_uuid: None, + snapshot_properties: HashMap::default(), + added_data_files: vec![], + added_delete_files: vec![], + removed_paths: HashSet::default(), + } + } + + /// Add files produced by the rewrite (e.g. compacted data files). + /// Routed by `content_type()` into separate manifests. + pub fn add_data_files(mut self, data_files: impl IntoIterator) -> Self { + for file in data_files { + match file.content_type() { + DataContentType::Data => self.added_data_files.push(file), + DataContentType::EqualityDeletes | DataContentType::PositionDeletes => { + self.added_delete_files.push(file) + } + } + } + self + } + + /// Mark the given files as removed in the new snapshot. Matching is by + /// `file_path()`. + pub fn remove_data_files(mut self, files: impl IntoIterator) -> Self { + for file in files { + self.removed_paths.insert(file.file_path().to_string()); + } + self + } + + /// Mark the given paths as removed. Convenience for callers that + /// already track removals as `String`. + pub fn remove_paths(mut self, paths: impl IntoIterator) -> Self { + self.removed_paths.extend(paths); + self + } + + /// Set commit UUID for the snapshot. + pub fn set_commit_uuid(mut self, commit_uuid: Uuid) -> Self { + self.commit_uuid = Some(commit_uuid); + self + } + + /// Set snapshot summary properties. + pub fn set_snapshot_properties(mut self, snapshot_properties: HashMap) -> Self { + self.snapshot_properties = snapshot_properties; + self + } +} + +#[async_trait] +impl TransactionAction for RewriteFilesAction { + async fn commit(self: Arc, table: &Table) -> Result { + let snapshot_producer = SnapshotProducer::new( + table, + self.commit_uuid.unwrap_or_else(Uuid::now_v7), + self.snapshot_properties.clone(), + self.added_data_files.clone(), + self.added_delete_files.clone(), + ); + + snapshot_producer.validate_added_files(&self.added_data_files)?; + snapshot_producer.validate_added_files(&self.added_delete_files)?; + + snapshot_producer + .commit( + RewriteFilesOperation { + removed_paths: self.removed_paths.clone(), + }, + DefaultManifestProcess, + ) + .await + } +} + +struct RewriteFilesOperation { + removed_paths: HashSet, +} + +impl SnapshotProduceOperation for RewriteFilesOperation { + fn operation(&self) -> Operation { + Operation::Replace + } + + async fn delete_entries( + &self, + _snapshot_produce: &SnapshotProducer<'_>, + ) -> Result> { + // We track removals at manifest-rewrite time inside `existing_manifest` + // rather than emitting separate `Deleted` entries — the new manifest + // list simply doesn't reference the dropped files. Java/PyIceberg do + // emit `Deleted` entries for audit-trail purposes; we may add that + // later if expiration tooling needs them. + Ok(vec![]) + } + + async fn existing_manifest( + &self, + snapshot_produce: &mut SnapshotProducer<'_>, + ) -> Result> { + let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { + return Ok(vec![]); + }; + + let manifest_list = snapshot_produce + .table + .manifest_list_reader(snapshot) + .load() + .await?; + + // Walk each prior manifest, decide carry-forward vs. partial-rewrite + // vs. drop. + // + // Pass 1: classify each manifest based on its loaded entries. We + // can't write rewritten manifests yet because that needs `&mut + // snapshot_produce`, and we also need the producer to compute + // file-IO paths during this loop. + let manifest_list_entries = manifest_list.entries().to_vec(); + let file_io = snapshot_produce.table.file_io().clone(); + + enum Plan { + CarryForward(ManifestFile), + DropEntirely, + Rewrite { + survivors: Vec, + content: crate::spec::ManifestContentType, + }, + } + let mut plans: Vec = Vec::with_capacity(manifest_list_entries.len()); + for mfile in &manifest_list_entries { + let manifest = mfile.load_manifest(&file_io).await?; + let total = manifest.entries().len(); + let survivors: Vec = manifest + .entries() + .iter() + .filter(|me| !self.removed_paths.contains(me.data_file().file_path())) + .map(|me| ManifestEntry::clone(me)) + .collect(); + + if survivors.len() == total { + plans.push(Plan::CarryForward(mfile.clone())); + } else if survivors.is_empty() { + plans.push(Plan::DropEntirely); + } else { + plans.push(Plan::Rewrite { + survivors, + content: mfile.content, + }); + } + } + + // Pass 2: materialize. Carry-forwards and rewrites land in the + // output; drops produce nothing. + let mut out: Vec = Vec::with_capacity(plans.len()); + for plan in plans { + match plan { + Plan::CarryForward(mf) => out.push(mf), + Plan::DropEntirely => {} + Plan::Rewrite { survivors, content } => { + let rewritten = snapshot_produce + .write_existing_manifest_for(survivors, content) + .await?; + out.push(rewritten); + } + } + } + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use crate::CatalogBuilder; + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, ManifestContentType, NestedField, + Operation, PrimitiveType, Schema, Struct, Type, + }; + use crate::transaction::{ApplyTransactionAction, Transaction}; + + async fn make_memory_table() -> (crate::memory::MemoryCatalog, crate::table::Table) { + let schema = Schema::builder() + .with_schema_id(0) + .with_identifier_field_ids([1]) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "qty", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + let memory: crate::memory::MemoryCatalog = crate::memory::MemoryCatalogBuilder::default() + .load( + "test", + HashMap::from([( + crate::memory::MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )]), + ) + .await + .unwrap(); + use crate::Catalog; + let ns = crate::NamespaceIdent::from_strs(["public"]).unwrap(); + memory.create_namespace(&ns, HashMap::new()).await.unwrap(); + let creation = crate::TableCreation::builder() + .name("orders".to_string()) + .schema(schema) + .build(); + let table = memory.create_table(&ns, creation).await.unwrap(); + (memory, table) + } + + fn data_file(path: &str, record_count: u64, byte_size: u64) -> crate::spec::DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(byte_size) + .record_count(record_count) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap() + } + + /// Snapshot 1 appends three files; snapshot 2 (rewrite) drops them all + /// and adds one compacted file. The new manifest list must reference + /// only the compacted file's manifest — the original manifest is + /// dropped because every entry was removed. + #[tokio::test] + async fn rewrite_drops_all_files_in_manifest_and_swaps_in_new_one() { + let (catalog, table) = make_memory_table().await; + + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let f3 = data_file("memory:///warehouse/public/orders/data/3.parquet", 30, 3072); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone(), f2.clone(), f3.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 60, + 5120, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted.clone()]) + .remove_data_files(vec![f1.clone(), f2.clone(), f3.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.summary().operation, Operation::Replace); + + let mlist = table.manifest_list_reader(snap) + .load() + .await + .unwrap(); + let data_manifests: Vec<_> = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .collect(); + assert_eq!(data_manifests.len(), 1); + + let manifest = data_manifests[0] + .load_manifest(table.file_io()) + .await + .unwrap(); + assert_eq!(manifest.entries().len(), 1); + assert_eq!( + manifest.entries()[0].data_file().file_path(), + compacted.file_path() + ); + } + + /// Partial-manifest removal: snapshot 1 appends three files; snapshot 2 + /// removes one and adds one. The new manifest list should reference + /// (a) a rewritten manifest carrying the two survivors as `Existing`, + /// (b) a new manifest with the added file as `Added`. + #[tokio::test] + async fn rewrite_partial_manifest_keeps_survivors_as_existing() { + let (catalog, table) = make_memory_table().await; + + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let f3 = data_file("memory:///warehouse/public/orders/data/3.parquet", 30, 3072); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone(), f2.clone(), f3.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Drop only f1, replace with a "smaller" compacted file. f2 and f3 + // survive in the rewritten manifest. + let new_file = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![new_file.clone()]) + .remove_data_files(vec![f1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.summary().operation, Operation::Replace); + + let mlist = table.manifest_list_reader(snap) + .load() + .await + .unwrap(); + let data_manifests: Vec<_> = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .collect(); + // Two data manifests: rewritten survivor manifest + new added manifest. + assert_eq!(data_manifests.len(), 2); + + let mut survivor_paths: Vec = Vec::new(); + let mut added_paths: Vec = Vec::new(); + for mf in &data_manifests { + let m = mf.load_manifest(table.file_io()).await.unwrap(); + for entry in m.entries() { + match entry.status() { + crate::spec::ManifestStatus::Existing => { + survivor_paths.push(entry.data_file().file_path().to_string()); + } + crate::spec::ManifestStatus::Added => { + added_paths.push(entry.data_file().file_path().to_string()); + } + crate::spec::ManifestStatus::Deleted => { + panic!("unexpected Deleted status in new manifest list"); + } + } + } + } + survivor_paths.sort(); + let mut want_survivors = vec![f2.file_path().to_string(), f3.file_path().to_string()]; + want_survivors.sort(); + assert_eq!(survivor_paths, want_survivors); + assert_eq!(added_paths, vec![new_file.file_path().to_string()]); + } + + /// Mixed rewrite: prior snapshot has a data manifest + a delete manifest; + /// rewrite drops one data file and keeps the delete manifest untouched. + /// The new manifest list should still have a `Deletes` manifest carrying + /// forward, plus a rewritten data manifest with survivors. + #[tokio::test] + async fn rewrite_carries_forward_unaffected_delete_manifest() { + let (catalog, table) = make_memory_table().await; + + let d1 = data_file("memory:///warehouse/public/orders/data/d1.parquet", 10, 1024); + let d2 = data_file("memory:///warehouse/public/orders/data/d2.parquet", 20, 2048); + let eq_delete = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path("memory:///warehouse/public/orders/deletes/eq.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(64) + .record_count(1) + .equality_ids(Some(vec![1])) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap(); + + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![d1.clone(), d2.clone(), eq_delete.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Rewrite: drop d1, keep d2 + eq_delete. Add a compacted file. + let new_file = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![new_file.clone()]) + .remove_data_files(vec![d1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + let mlist = table.manifest_list_reader(snap) + .load() + .await + .unwrap(); + + // Counts: 2 data manifests (rewritten survivors + new added) and + // 1 delete manifest (unchanged carry-forward). + let data_count = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .count(); + let delete_count = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Deletes) + .count(); + assert_eq!(data_count, 2); + assert_eq!(delete_count, 1); + + // Every survivor and added file is reachable. + let mut all_paths: Vec = Vec::new(); + for mf in mlist.entries() { + let m = mf.load_manifest(table.file_io()).await.unwrap(); + for entry in m.entries() { + all_paths.push(entry.data_file().file_path().to_string()); + } + } + all_paths.sort(); + let mut want = vec![ + d2.file_path().to_string(), + new_file.file_path().to_string(), + eq_delete.file_path().to_string(), + ]; + want.sort(); + assert_eq!(all_paths, want); + } + + /// Rewrite with no `removed_paths` is an append disguised as a replace — + /// the prior manifest is carried forward and the new file joins it. + /// The snapshot is still tagged `Replace`. + #[tokio::test] + async fn rewrite_with_no_removals_carries_prior_manifest() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().add_data_files(vec![f2.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.summary().operation, Operation::Replace); + + let mlist = table.manifest_list_reader(snap) + .load() + .await + .unwrap(); + let data_manifest_count = mlist + .entries() + .iter() + .filter(|m| m.content == ManifestContentType::Data) + .count(); + assert_eq!(data_manifest_count, 2); + } + + /// Rewrite with empty add + empty remove is a no-op-ish state — the + /// SnapshotProducer requires *something* added (upstream contract). + /// Documents the existing behavior so callers know to skip the action + /// when there's nothing to compact. + #[tokio::test] + async fn rewrite_with_empty_input_errors() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx.rewrite_files(); + let tx = action.apply(tx).unwrap(); + assert!(tx.commit(&catalog).await.is_err()); + } +} diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 849f2fbf61..af8ada7a23 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -83,9 +83,14 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { /// - **Append operations**: Typically include all existing manifests /// - **Overwrite operations**: May exclude manifests for partitions being overwritten /// - **Delete operations**: May exclude manifests for partitions being deleted + /// Returns existing manifest files that should be included in the new + /// snapshot. Takes `&mut SnapshotProducer` so implementations that + /// rewrite manifests in place (e.g. `Replace`) can call helpers like + /// `write_existing_manifest_for` here. Append-style implementations + /// that only need read access can ignore the mutability. fn existing_manifest( &self, - snapshot_produce: &SnapshotProducer<'_>, + snapshot_produce: &mut SnapshotProducer<'_>, ) -> impl Future>> + Send; } @@ -307,6 +312,47 @@ impl<'a> SnapshotProducer<'a> { Ok(()) } + /// Write a fresh manifest containing `survivors` as `Existing` entries. + /// Used by replace/rewrite operations that drop some files from a prior + /// manifest while preserving the rest. Each survivor must carry its + /// original `snapshot_id` and `sequence_number` (load_manifest's inherit + /// pass populates these from the manifest list entry, so feeding raw + /// `manifest.entries()` here is safe). + pub(crate) async fn write_existing_manifest_for( + &mut self, + survivors: Vec, + content: ManifestContentType, + ) -> Result { + if survivors.is_empty() { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "No survivor entries when rewriting an existing manifest", + )); + } + let mut writer = self.new_manifest_writer(content)?; + for entry in survivors { + let snapshot_id = entry.snapshot_id().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "manifest entry missing snapshot_id when rewriting as existing", + ) + })?; + let sequence_number = entry.sequence_number().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "manifest entry missing sequence_number when rewriting as existing", + ) + })?; + writer.add_existing_file( + entry.data_file().clone(), + snapshot_id, + sequence_number, + entry.file_sequence_number, + )?; + } + writer.write_manifest_file().await + } + /// Write a manifest file for a batch of added files, returning the /// `ManifestFile` to splice into the manifest list. The caller picks the /// content type — data and delete files must live in separate manifests @@ -364,6 +410,9 @@ impl<'a> SnapshotProducer<'a> { )); } + // Existing-manifest selection runs first because Replace operations + // may write rewritten manifests via `&mut self` here; the added-file + // manifest write below comes after to keep ordering deterministic. let existing_manifests = snapshot_produce_operation.existing_manifest(self).await?; let mut manifest_files = existing_manifests; From b17c4722e296b686da15e6350b68638f9b2d8fb0 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Mon, 27 Apr 2026 20:35:38 +0800 Subject: [PATCH 4/8] table: expose per-table config map on Table for vended credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plumbs the REST loadTable response config through Table::properties() so downstream consumers can extract vended S3 credentials without duplicating the catalog request. Previous behavior: response.config was merged into FileIO and discarded. The new field is empty for catalogs that don't surface table-scoped config (file, glue, memory) — it's an additive change with no behavioral impact on existing callers. Used by pg2iceberg-rust to build per-table object stores for the vended-credentials S3 router (Polaris, Snowflake, Tabular). --- crates/catalog/rest/src/catalog.rs | 9 ++++++++- crates/iceberg/src/table.rs | 32 ++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 0626ce5061..138032678f 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -848,6 +848,12 @@ impl Catalog for RestCatalog { } }; + // Stash the response-side config map BEFORE merging it with + // the catalog-level user_config — `Table::properties()` + // surfaces *only* the per-table response props so downstream + // consumers can spot vended credentials without re-issuing + // the request. + let response_config = response.config.clone(); let config = response .config .into_iter() @@ -862,7 +868,8 @@ impl Catalog for RestCatalog { .identifier(table_ident.clone()) .file_io(file_io) .metadata(response.metadata) - .runtime(self.runtime.clone()); + .runtime(self.runtime.clone()) + .properties(response_config); if let Some(metadata_location) = response.metadata_location { table_builder.metadata_location(metadata_location).build() diff --git a/crates/iceberg/src/table.rs b/crates/iceberg/src/table.rs index 31feade038..41c8993d42 100644 --- a/crates/iceberg/src/table.rs +++ b/crates/iceberg/src/table.rs @@ -17,6 +17,7 @@ //! Table API for Apache Iceberg +use std::collections::HashMap; use std::sync::Arc; use crate::arrow::ArrowReaderBuilder; @@ -41,6 +42,7 @@ pub struct TableBuilder { disable_cache: bool, cache_size_bytes: Option, runtime: Option, + properties: HashMap, } impl TableBuilder { @@ -55,6 +57,7 @@ impl TableBuilder { disable_cache: false, cache_size_bytes: None, runtime: None, + properties: HashMap::new(), } } @@ -118,6 +121,15 @@ impl TableBuilder { self } + /// optional - per-table properties returned by the catalog (e.g. the + /// `config` map from a REST `loadTable` response, which carries + /// vended S3 credentials when the catalog is asked for delegated + /// access). + pub fn properties(mut self, properties: HashMap) -> Self { + self.properties = properties; + self + } + /// build the Table pub fn build(self) -> Result
{ let Self { @@ -130,6 +142,7 @@ impl TableBuilder { disable_cache, cache_size_bytes, runtime, + properties, } = self; let Some(file_io) = file_io else { @@ -190,6 +203,7 @@ impl TableBuilder { object_cache, runtime, encryption_manager, + properties, }) } } @@ -205,6 +219,13 @@ pub struct Table { object_cache: Arc, runtime: Runtime, encryption_manager: Option>, + /// Per-table properties returned by the catalog. For REST catalogs, + /// this is the `config` map in the `loadTable` response — empty for + /// catalogs that don't surface table-scoped config (file, glue, + /// memory). Useful for downstream consumers that need to extract + /// vended credentials (`s3.access-key-id`, `s3.session-token`, + /// etc.) without re-issuing the REST request. + properties: HashMap, } impl Table { @@ -260,6 +281,17 @@ impl Table { &self.file_io } + /// Returns per-table properties returned by the catalog. For REST + /// catalogs, this is the `config` map from `loadTable`. Empty for + /// catalogs that don't surface table-scoped config. + /// + /// Polynya patch: lets downstream consumers (e.g. pg2iceberg's + /// vended-S3 router) extract per-table S3 credentials without + /// duplicating the REST request. + pub fn properties(&self) -> &HashMap { + &self.properties + } + /// Returns this table's object cache pub(crate) fn object_cache(&self) -> Arc { self.object_cache.clone() From 9e39ca1ef0b27cc785107254a8c241e469a8392a Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Wed, 3 Jun 2026 00:58:19 +0800 Subject: [PATCH 5/8] feat(catalog-rest): AWS SigV4 request signing AWS-native Iceberg REST endpoints (S3 Tables, Glue) require every request to be SigV4-signed; the REST client previously only supported bearer/OAuth, so requests returned 403 MissingAuthenticationToken. Add an opt-in SigV4Signer in the REST HttpClient, enabled per-catalog via the `rest.sigv4-enabled` property (with `rest.signing-region` / `rest.signing-name`, falling back to `s3.region`). Credentials load lazily from the standard AWS chain (env, ECS/EKS task role, EC2 IMDS, shared config) and are cached for the process. Signing runs last in execute(), over the final headers + body, so it is inert unless enabled. --- Cargo.lock | 4 + crates/catalog/rest/Cargo.toml | 7 ++ crates/catalog/rest/src/catalog.rs | 31 +++++++ crates/catalog/rest/src/client.rs | 144 ++++++++++++++++++++++++++++- 4 files changed, 184 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 96787eccc0..ef45b92ad5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3872,6 +3872,10 @@ name = "iceberg-catalog-rest" version = "0.10.1" dependencies = [ "async-trait", + "aws-config", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-runtime-api", "chrono", "http 1.4.0", "iceberg", diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index e043c195ef..f51b47c87c 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -31,6 +31,13 @@ repository = { workspace = true } [dependencies] async-trait = { workspace = true } +# AWS SigV4 request signing for AWS-native Iceberg REST endpoints +# (S3 Tables, Glue). Enabled per-catalog via the `rest.sigv4-enabled` +# property; otherwise these are inert. +aws-config = "1.8" +aws-credential-types = "1.2" +aws-sigv4 = { version = "1.2", features = ["http1"] } +aws-smithy-runtime-api = { version = "1", features = ["client"] } chrono = { workspace = true } http = { workspace = true } iceberg = { workspace = true } diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 138032678f..daa2103edb 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -220,6 +220,37 @@ impl RestCatalogConfig { self.client.clone() } + /// AWS SigV4 signing config, read from props. Returns + /// `Some((region, signing_name))` when `rest.sigv4-enabled` is + /// `"true"`, else `None` (no signing). + /// + /// Used for AWS-native Iceberg REST endpoints (S3 Tables signing + /// name `s3tables`, Glue `glue`) which require every request to be + /// SigV4-signed. Region falls back to `s3.region`; signing name + /// defaults to `execute-api`. + pub(crate) fn sigv4_config(&self) -> Option<(String, String)> { + let enabled = self + .props + .get("rest.sigv4-enabled") + .map(|v| v == "true") + .unwrap_or(false); + if !enabled { + return None; + } + let region = self + .props + .get("rest.signing-region") + .or_else(|| self.props.get("s3.region")) + .cloned() + .unwrap_or_else(|| "us-east-1".to_string()); + let name = self + .props + .get("rest.signing-name") + .cloned() + .unwrap_or_else(|| "execute-api".to_string()); + Some((region, name)) + } + /// Get the token from the config. /// /// The client can use this token to send requests. diff --git a/crates/catalog/rest/src/client.rs b/crates/catalog/rest/src/client.rs index 07dc0620da..da19aa6a99 100644 --- a/crates/catalog/rest/src/client.rs +++ b/crates/catalog/rest/src/client.rs @@ -17,17 +17,144 @@ use std::collections::HashMap; use std::fmt::{Debug, Formatter}; +use std::str::FromStr; use http::StatusCode; use iceberg::{Error, ErrorKind, Result}; -use reqwest::header::HeaderMap; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use reqwest::{Client, IntoUrl, Method, Request, RequestBuilder, Response}; use serde::de::DeserializeOwned; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, OnceCell}; use crate::RestCatalogConfig; use crate::types::{ErrorResponse, TokenResponse}; +/// AWS SigV4 request signer for AWS-native Iceberg REST endpoints +/// (S3 Tables, Glue). Credentials come from the standard AWS chain +/// (env, ECS/EKS task role, EC2 IMDS, shared config), loaded lazily on +/// first use and cached for the process lifetime. +struct SigV4Signer { + region: String, + name: String, + provider: OnceCell, +} + +impl SigV4Signer { + fn new(region: String, name: String) -> Self { + Self { + region, + name, + provider: OnceCell::new(), + } + } + + async fn provider( + &self, + ) -> Result<&aws_credential_types::provider::SharedCredentialsProvider> { + self.provider + .get_or_try_init(|| async { + let cfg = + aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + cfg.credentials_provider().ok_or_else(|| { + Error::new( + ErrorKind::Unexpected, + "no AWS credentials provider for SigV4 signing", + ) + }) + }) + .await + } + + /// Sign `request` in place: adds `authorization`, `x-amz-date`, and + /// `x-amz-content-sha256` headers computed over the request as it + /// currently stands. Must be called after all other headers/body + /// are set and before sending. + async fn sign(&self, request: &mut Request) -> Result<()> { + use aws_credential_types::provider::ProvideCredentials; + use aws_sigv4::http_request::{ + sign, PayloadChecksumKind, SignableBody, SignableRequest, SigningSettings, + }; + use aws_sigv4::sign::v4; + + let creds = self + .provider() + .await? + .provide_credentials() + .await + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "load AWS credentials").with_source(e) + })?; + let identity: aws_smithy_runtime_api::client::identity::Identity = creds.into(); + + let mut settings = SigningSettings::default(); + // AWS-native catalogs (S3 Tables/Glue) expect the payload hash. + settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256; + + let params = v4::SigningParams::builder() + .identity(&identity) + .region(&self.region) + .name(&self.name) + .time(std::time::SystemTime::now()) + .settings(settings) + .build() + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "build SigV4 params").with_source(e) + })? + .into(); + + let body_bytes = request + .body() + .and_then(|b| b.as_bytes()) + .unwrap_or(&[]) + .to_vec(); + let uri = request.url().as_str().to_string(); + let method = request.method().as_str().to_string(); + let headers: Vec<(String, String)> = request + .headers() + .iter() + .map(|(k, v)| (k.as_str().to_string(), v.to_str().unwrap_or("").to_string())) + .collect(); + + let signable = SignableRequest::new( + &method, + &uri, + headers.iter().map(|(k, v)| (k.as_str(), v.as_str())), + SignableBody::Bytes(&body_bytes), + ) + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "build signable request").with_source(e) + })?; + + let (instructions, _signature) = sign(signable, ¶ms) + .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 sign").with_source(e))? + .into_parts(); + + // Apply the signing headers onto a throwaway http::Request that + // mirrors the outgoing one, then copy the resulting headers back. + let mut http_req = http::Request::builder() + .method(request.method().clone()) + .uri(uri.as_str()); + for (k, v) in request.headers().iter() { + http_req = http_req.header(k, v); + } + let mut http_req = http_req.body(()).map_err(|e| { + Error::new(ErrorKind::Unexpected, "build http request for signing").with_source(e) + })?; + instructions.apply_to_request_http1x(&mut http_req); + + for (k, v) in http_req.headers().iter() { + let name = HeaderName::from_str(k.as_str()).map_err(|e| { + Error::new(ErrorKind::Unexpected, "signed header name").with_source(e) + })?; + let val = HeaderValue::from_bytes(v.as_bytes()).map_err(|e| { + Error::new(ErrorKind::Unexpected, "signed header value").with_source(e) + })?; + request.headers_mut().insert(name, val); + } + Ok(()) + } +} + pub(crate) struct HttpClient { client: Client, @@ -45,6 +172,8 @@ pub(crate) struct HttpClient { extra_oauth_params: HashMap, /// Whether to disable header redaction in error logs (defaults to false for security). disable_header_redaction: bool, + /// AWS SigV4 signer, set when `rest.sigv4-enabled` is configured. + sigv4: Option, } impl Debug for HttpClient { @@ -68,6 +197,9 @@ impl HttpClient { extra_headers, extra_oauth_params: cfg.extra_oauth_params(), disable_header_redaction: cfg.disable_header_redaction(), + sigv4: cfg + .sigv4_config() + .map(|(region, name)| SigV4Signer::new(region, name)), }) } @@ -96,6 +228,10 @@ impl HttpClient { self.extra_oauth_params }, disable_header_redaction: cfg.disable_header_redaction(), + sigv4: cfg + .sigv4_config() + .map(|(region, name)| SigV4Signer::new(region, name)) + .or(self.sigv4), }) } @@ -253,6 +389,10 @@ impl HttpClient { /// Executes the given `Request` and returns a `Response`. pub async fn execute(&self, mut request: Request) -> Result { request.headers_mut().extend(self.extra_headers.clone()); + // SigV4 must sign last, over the final headers + body. + if let Some(signer) = &self.sigv4 { + signer.sign(&mut request).await?; + } Ok(self.client.execute(request).await?) } From c7aa5c686c2e8941215168dd88768f159835f036 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 6 Oct 2026 12:59:03 +0800 Subject: [PATCH 6/8] transaction: RewriteFilesAction data sequence number and removed-file check (#2) --- crates/iceberg/src/transaction/rewrite.rs | 181 ++++++++++++++++++++- crates/iceberg/src/transaction/snapshot.rs | 27 ++- 2 files changed, 201 insertions(+), 7 deletions(-) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 996000580b..83c8b1b2f9 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use async_trait::async_trait; use uuid::Uuid; -use crate::error::Result; +use crate::error::{Error, ErrorKind, Result}; use crate::spec::{DataContentType, DataFile, ManifestEntry, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::snapshot::{ @@ -46,12 +46,17 @@ use crate::transaction::{ActionCommit, TransactionAction}; /// Added files go into a new `ManifestStatus::Added` manifest, routed by /// `content_type()` (Data vs. EqualityDeletes/PositionDeletes) into /// separate manifests as required by the spec. +/// +/// The commit fails if a removed file is no longer in the table it lands +/// on — another rewrite got there first, and adding this one's files too +/// would duplicate their rows. A retried commit checks again. pub struct RewriteFilesAction { commit_uuid: Option, snapshot_properties: HashMap, added_data_files: Vec, added_delete_files: Vec, removed_paths: HashSet, + data_sequence_number: Option, } impl RewriteFilesAction { @@ -62,6 +67,7 @@ impl RewriteFilesAction { added_data_files: vec![], added_delete_files: vec![], removed_paths: HashSet::default(), + data_sequence_number: None, } } @@ -106,6 +112,17 @@ impl RewriteFilesAction { self.snapshot_properties = snapshot_properties; self } + + /// Give the added data files this data sequence number — the one of + /// the snapshot the rewrite read its files at — instead of the new + /// snapshot's. Their rows are the removed files' rows as of then, so + /// equality deletes committed since still apply to them; with the new + /// snapshot's number they would rise above those deletes and bring the + /// deleted rows back. Java's `RewriteFiles.dataSequenceNumber`. + pub fn set_data_sequence_number(mut self, data_sequence_number: i64) -> Self { + self.data_sequence_number = Some(data_sequence_number); + self + } } #[async_trait] @@ -117,7 +134,8 @@ impl TransactionAction for RewriteFilesAction { self.snapshot_properties.clone(), self.added_data_files.clone(), self.added_delete_files.clone(), - ); + ) + .with_added_data_sequence_number(self.data_sequence_number); snapshot_producer.validate_added_files(&self.added_data_files)?; snapshot_producer.validate_added_files(&self.added_delete_files)?; @@ -137,6 +155,23 @@ struct RewriteFilesOperation { removed_paths: HashSet, } +impl RewriteFilesOperation { + /// The error for removed paths not all among `found`, the table's. + fn missing(&self, found: &HashSet<&str>) -> Error { + let mut missing: Vec<&str> = self + .removed_paths + .iter() + .map(String::as_str) + .filter(|p| !found.contains(p)) + .collect(); + missing.sort(); + Error::new( + ErrorKind::CatalogCommitConflicts, + format!("Rewrite removes files no longer in the table: {missing:?}"), + ) + } +} + impl SnapshotProduceOperation for RewriteFilesOperation { fn operation(&self) -> Operation { Operation::Replace @@ -159,6 +194,9 @@ impl SnapshotProduceOperation for RewriteFilesOperation { snapshot_produce: &mut SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { + if !self.removed_paths.is_empty() { + return Err(self.missing(&HashSet::new())); + } return Ok(vec![]); }; @@ -187,8 +225,14 @@ impl SnapshotProduceOperation for RewriteFilesOperation { }, } let mut plans: Vec = Vec::with_capacity(manifest_list_entries.len()); + let mut found: HashSet<&str> = HashSet::new(); for mfile in &manifest_list_entries { let manifest = mfile.load_manifest(&file_io).await?; + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { + if let Some(path) = self.removed_paths.get(entry.data_file().file_path()) { + found.insert(path); + } + } let total = manifest.entries().len(); let survivors: Vec = manifest .entries() @@ -209,6 +253,10 @@ impl SnapshotProduceOperation for RewriteFilesOperation { } } + if found.len() < self.removed_paths.len() { + return Err(self.missing(&found)); + } + // Pass 2: materialize. Carry-forwards and rewrites land in the // output; drops produce nothing. let mut out: Vec = Vec::with_capacity(plans.len()); @@ -232,7 +280,7 @@ impl SnapshotProduceOperation for RewriteFilesOperation { mod tests { use std::collections::HashMap; - use crate::CatalogBuilder; + use crate::{Catalog, CatalogBuilder, ErrorKind}; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, ManifestContentType, NestedField, Operation, PrimitiveType, Schema, Struct, Type, @@ -555,4 +603,131 @@ mod tests { let tx = action.apply(tx).unwrap(); assert!(tx.commit(&catalog).await.is_err()); } + + /// With a data sequence number, the added files carry it rather than + /// the new snapshot's, which still becomes their file sequence number. + #[tokio::test] + async fn rewrite_gives_added_files_its_data_sequence_number() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let mut table = table; + for f in [&f1, &f2] { + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f.clone()]); + table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + } + + // Rewrite f1 as planned at sequence number 1. + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted.clone()]) + .remove_data_files(vec![f1.clone()]) + .set_data_sequence_number(1); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let snap = table.metadata().current_snapshot().unwrap(); + assert_eq!(snap.sequence_number(), 3); + let mlist = table.manifest_list_reader(snap) + .load() + .await + .unwrap(); + let mut seqs = Vec::new(); + for mf in mlist.entries() { + let m = mf.load_manifest(table.file_io()).await.unwrap(); + for entry in m.entries() { + seqs.push(( + entry.data_file().file_path().to_string(), + entry.sequence_number(), + entry.file_sequence_number, + )); + } + } + seqs.sort(); + assert_eq!(seqs, vec![ + (f2.file_path().to_string(), Some(2), Some(2)), + (compacted.file_path().to_string(), Some(1), Some(3)), + ]); + } + + /// A rewrite of a file another rewrite already removed fails: adding + /// its files too would duplicate the rows. + #[tokio::test] + async fn rewrite_of_a_file_no_longer_in_the_table_fails() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![f1.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + // Two rewrites of f1, planned against the same table. + let rewrite = |path: &str| { + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![data_file(path, 10, 1024)]) + .remove_data_files(vec![f1.clone()]); + action.apply(tx).unwrap() + }; + let first = rewrite("memory:///warehouse/public/orders/data/compact-a.parquet"); + let second = rewrite("memory:///warehouse/public/orders/data/compact-b.parquet"); + let table = first.commit(&catalog).await.unwrap(); + // The second lands on the first's table, without f1. + let err = second.commit(&catalog).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::CatalogCommitConflicts); + assert!(!err.retryable()); + + let table = catalog.load_table(table.identifier()).await.unwrap(); + let snap = table.metadata().current_snapshot().unwrap(); + let mlist = table.manifest_list_reader(snap) + .load() + .await + .unwrap(); + let mut paths = Vec::new(); + for mf in mlist.entries() { + let m = mf.load_manifest(table.file_io()).await.unwrap(); + for entry in m.entries() { + paths.push(entry.data_file().file_path().to_string()); + } + } + assert_eq!(paths, vec![ + "memory:///warehouse/public/orders/data/compact-a.parquet".to_string() + ]); + } + + /// So does one removing a file from a table with no snapshot at all. + #[tokio::test] + async fn rewrite_removing_from_an_empty_table_fails() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 10, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted]) + .remove_data_files(vec![f1]); + let err = action + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::CatalogCommitConflicts); + } } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index af8ada7a23..e1c07a1ed6 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -124,6 +124,9 @@ pub(crate) struct SnapshotProducer<'a> { /// to a separate manifest from `added_data_files` because the Iceberg /// spec forbids mixing data and delete entries in one manifest. added_delete_files: Vec, + /// Data sequence number for `added_data_files`; `None` to inherit the + /// snapshot's. + added_data_sequence_number: Option, // A counter used to generate unique manifest file names. // It starts from 0 and increments for each new manifest file. // Note: This counter is limited to the range of (0..u64::MAX). @@ -145,10 +148,20 @@ impl<'a> SnapshotProducer<'a> { snapshot_properties, added_data_files, added_delete_files, + added_data_sequence_number: None, manifest_counter: (0..), } } + /// Write the added data files with this data sequence number instead of + /// letting them inherit the snapshot's: their rows logically belong to + /// an older sequence number (see the spec's "Sequence Number + /// Inheritance"). Ignored for v1 tables, which have none. + pub(crate) fn with_added_data_sequence_number(mut self, seq: Option) -> Self { + self.added_data_sequence_number = seq; + self + } + /// Validate partition spec / partition value compatibility for a slice of /// added files. Generic over data vs. delete files — the Iceberg spec /// validation rules are the same for both content types. @@ -361,6 +374,7 @@ impl<'a> SnapshotProducer<'a> { &mut self, added_files: Vec, content: ManifestContentType, + data_sequence_number: Option, ) -> Result { if added_files.is_empty() { return Err(Error::new( @@ -380,7 +394,7 @@ impl<'a> SnapshotProducer<'a> { } else { // For format version > 1, we set the snapshot id at the inherited time to avoid rewrite the manifest file when // commit failed. - builder.build() + builder.sequence_number_opt(data_sequence_number).build() } }); let mut writer = self.new_manifest_writer(content)?; @@ -420,16 +434,21 @@ impl<'a> SnapshotProducer<'a> { // doesn't allow mixing content types in a single manifest. if !self.added_data_files.is_empty() { let added_data_files = std::mem::take(&mut self.added_data_files); + let seq = self.added_data_sequence_number; manifest_files.push( - self.write_added_manifest_for(added_data_files, ManifestContentType::Data) + self.write_added_manifest_for(added_data_files, ManifestContentType::Data, seq) .await?, ); } if !self.added_delete_files.is_empty() { let added_delete_files = std::mem::take(&mut self.added_delete_files); manifest_files.push( - self.write_added_manifest_for(added_delete_files, ManifestContentType::Deletes) - .await?, + self.write_added_manifest_for( + added_delete_files, + ManifestContentType::Deletes, + None, + ) + .await?, ); } From b904ed2409ef7cc14759bbc0ca84f80613c53dd1 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 6 Oct 2026 15:33:57 +0800 Subject: [PATCH 7/8] transaction: RewriteFilesAction keeps the summary's running totals The snapshot summary's `total-*` counts are carried from the previous snapshot plus this one's `added-*` and minus its `deleted-*` / `removed-*`. A rewrite recorded no removed files, so after a compaction `total-records`, `total-data-files` and the delete totals still counted everything it replaced. Engines read these: ClickHouse 26.2 answered `count()` from `total-records`. The rewrite now looks up the live files it removes before the snapshot is produced (the summary is built before the manifests are written) and the producer subtracts them, under each file's own partition spec. The same pass carries the existing check that every removed file is still live, still on every commit attempt. Co-Authored-By: Claude Opus 5.5 --- crates/iceberg/src/transaction/rewrite.rs | 129 ++++++++++++++++----- crates/iceberg/src/transaction/snapshot.rs | 17 +++ 2 files changed, 115 insertions(+), 31 deletions(-) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 83c8b1b2f9..fd7ae01c9d 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -128,6 +128,7 @@ impl RewriteFilesAction { #[async_trait] impl TransactionAction for RewriteFilesAction { async fn commit(self: Arc, table: &Table) -> Result { + let removed_files = self.removed_files(table).await?; let snapshot_producer = SnapshotProducer::new( table, self.commit_uuid.unwrap_or_else(Uuid::now_v7), @@ -135,7 +136,8 @@ impl TransactionAction for RewriteFilesAction { self.added_data_files.clone(), self.added_delete_files.clone(), ) - .with_added_data_sequence_number(self.data_sequence_number); + .with_added_data_sequence_number(self.data_sequence_number) + .with_removed_files(removed_files); snapshot_producer.validate_added_files(&self.added_data_files)?; snapshot_producer.validate_added_files(&self.added_delete_files)?; @@ -151,25 +153,46 @@ impl TransactionAction for RewriteFilesAction { } } -struct RewriteFilesOperation { - removed_paths: HashSet, +impl RewriteFilesAction { + /// The live files of `table` that `removed_paths` names — the snapshot + /// summary subtracts them. Fails if one isn't live: another rewrite + /// replaced it first, and committing this one's files as well would + /// duplicate their rows. Runs on every commit attempt, against the + /// table the commit would land on. + async fn removed_files(&self, table: &Table) -> Result> { + let mut removed = Vec::new(); + let mut found: HashSet<&str> = HashSet::new(); + if let Some(snapshot) = table.metadata().current_snapshot() { + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + for mfile in manifest_list.entries() { + let manifest = mfile.load_manifest(table.file_io()).await?; + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { + if let Some(path) = self.removed_paths.get(entry.data_file().file_path()) { + found.insert(path); + removed.push(entry.data_file().clone()); + } + } + } + } + if found.len() < self.removed_paths.len() { + let mut missing: Vec<&str> = self + .removed_paths + .iter() + .map(String::as_str) + .filter(|p| !found.contains(p)) + .collect(); + missing.sort(); + return Err(Error::new( + ErrorKind::CatalogCommitConflicts, + format!("Rewrite removes files no longer in the table: {missing:?}"), + )); + } + Ok(removed) + } } -impl RewriteFilesOperation { - /// The error for removed paths not all among `found`, the table's. - fn missing(&self, found: &HashSet<&str>) -> Error { - let mut missing: Vec<&str> = self - .removed_paths - .iter() - .map(String::as_str) - .filter(|p| !found.contains(p)) - .collect(); - missing.sort(); - Error::new( - ErrorKind::CatalogCommitConflicts, - format!("Rewrite removes files no longer in the table: {missing:?}"), - ) - } +struct RewriteFilesOperation { + removed_paths: HashSet, } impl SnapshotProduceOperation for RewriteFilesOperation { @@ -194,9 +217,6 @@ impl SnapshotProduceOperation for RewriteFilesOperation { snapshot_produce: &mut SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { - if !self.removed_paths.is_empty() { - return Err(self.missing(&HashSet::new())); - } return Ok(vec![]); }; @@ -225,14 +245,8 @@ impl SnapshotProduceOperation for RewriteFilesOperation { }, } let mut plans: Vec = Vec::with_capacity(manifest_list_entries.len()); - let mut found: HashSet<&str> = HashSet::new(); for mfile in &manifest_list_entries { let manifest = mfile.load_manifest(&file_io).await?; - for entry in manifest.entries().iter().filter(|e| e.is_alive()) { - if let Some(path) = self.removed_paths.get(entry.data_file().file_path()) { - found.insert(path); - } - } let total = manifest.entries().len(); let survivors: Vec = manifest .entries() @@ -253,10 +267,6 @@ impl SnapshotProduceOperation for RewriteFilesOperation { } } - if found.len() < self.removed_paths.len() { - return Err(self.missing(&found)); - } - // Pass 2: materialize. Carry-forwards and rewrites land in the // output; drops produce nothing. let mut out: Vec = Vec::with_capacity(plans.len()); @@ -730,4 +740,61 @@ mod tests { .unwrap_err(); assert_eq!(err.kind(), ErrorKind::CatalogCommitConflicts); } + + /// The snapshot summary keeps running totals: a rewrite subtracts the + /// files it removes, data and deletes, and adds its outputs. + #[tokio::test] + async fn rewrite_summary_keeps_running_totals() { + let (catalog, table) = make_memory_table().await; + let f1 = data_file("memory:///warehouse/public/orders/data/1.parquet", 10, 1024); + let f2 = data_file("memory:///warehouse/public/orders/data/2.parquet", 20, 2048); + let eq_delete = DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path("memory:///warehouse/public/orders/deletes/eq.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(64) + .record_count(1) + .equality_ids(Some(vec![1])) + .partition(Struct::empty()) + .partition_spec_id(0) + .build() + .unwrap(); + let mut table = table; + for files in [vec![f1.clone()], vec![f2.clone(), eq_delete.clone()]] { + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(files); + table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + } + let summary = |table: &crate::table::Table, key: &str| -> String { + let snap = table.metadata().current_snapshot().unwrap(); + snap.summary().additional_properties[key].clone() + }; + assert_eq!(summary(&table, "total-records"), "30"); + assert_eq!(summary(&table, "total-delete-files"), "1"); + + // f1 rewritten as 9 rows, retiring the delete. + let compacted = data_file( + "memory:///warehouse/public/orders/data/compact-0.parquet", + 9, + 1024, + ); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .add_data_files(vec![compacted]) + .remove_data_files(vec![f1.clone(), eq_delete.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + assert_eq!(summary(&table, "deleted-data-files"), "1"); + assert_eq!(summary(&table, "deleted-records"), "10"); + assert_eq!(summary(&table, "removed-delete-files"), "1"); + assert_eq!(summary(&table, "total-records"), "29"); + assert_eq!(summary(&table, "total-data-files"), "2"); + assert_eq!(summary(&table, "total-delete-files"), "0"); + assert_eq!(summary(&table, "total-equality-deletes"), "0"); + } + } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index e1c07a1ed6..a9b92081fb 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -127,6 +127,9 @@ pub(crate) struct SnapshotProducer<'a> { /// Data sequence number for `added_data_files`; `None` to inherit the /// snapshot's. added_data_sequence_number: Option, + /// Files this snapshot removes from the table, for its summary's + /// `deleted-*` counts and running `total-*`. + removed_files: Vec, // A counter used to generate unique manifest file names. // It starts from 0 and increments for each new manifest file. // Note: This counter is limited to the range of (0..u64::MAX). @@ -149,6 +152,7 @@ impl<'a> SnapshotProducer<'a> { added_data_files, added_delete_files, added_data_sequence_number: None, + removed_files: Vec::new(), manifest_counter: (0..), } } @@ -162,6 +166,12 @@ impl<'a> SnapshotProducer<'a> { self } + /// Record files this snapshot removes, so its summary subtracts them. + pub(crate) fn with_removed_files(mut self, files: Vec) -> Self { + self.removed_files = files; + self + } + /// Validate partition spec / partition value compatibility for a slice of /// added files. Generic over data vs. delete files — the Iceberg spec /// validation rules are the same for both content types. @@ -486,6 +496,13 @@ impl<'a> SnapshotProducer<'a> { table_metadata.default_partition_spec().clone(), ); } + for data_file in &self.removed_files { + let spec = table_metadata + .partition_spec_by_id(data_file.partition_spec_id) + .unwrap_or_else(|| table_metadata.default_partition_spec()) + .clone(); + summary_collector.remove_file(data_file, table_metadata.current_schema().clone(), spec); + } let previous_snapshot = table_metadata.current_snapshot(); From 00a01b49f39401eb58d9b2f024061f5a49c6a836 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 6 Oct 2026 15:34:08 +0800 Subject: [PATCH 8/8] style: rustfmt the fork's patches --- crates/catalog/rest/src/client.rs | 17 +++---- crates/iceberg/src/transaction/append.rs | 20 ++++----- crates/iceberg/src/transaction/mod.rs | 4 +- .../iceberg/src/transaction/replace_schema.rs | 12 ++--- crates/iceberg/src/transaction/rewrite.rs | 45 +++++++------------ crates/iceberg/src/transaction/snapshot.rs | 12 +++-- 6 files changed, 46 insertions(+), 64 deletions(-) diff --git a/crates/catalog/rest/src/client.rs b/crates/catalog/rest/src/client.rs index da19aa6a99..3a1550cd28 100644 --- a/crates/catalog/rest/src/client.rs +++ b/crates/catalog/rest/src/client.rs @@ -48,13 +48,10 @@ impl SigV4Signer { } } - async fn provider( - &self, - ) -> Result<&aws_credential_types::provider::SharedCredentialsProvider> { + async fn provider(&self) -> Result<&aws_credential_types::provider::SharedCredentialsProvider> { self.provider .get_or_try_init(|| async { - let cfg = - aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + let cfg = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; cfg.credentials_provider().ok_or_else(|| { Error::new( ErrorKind::Unexpected, @@ -72,7 +69,7 @@ impl SigV4Signer { async fn sign(&self, request: &mut Request) -> Result<()> { use aws_credential_types::provider::ProvideCredentials; use aws_sigv4::http_request::{ - sign, PayloadChecksumKind, SignableBody, SignableRequest, SigningSettings, + PayloadChecksumKind, SignableBody, SignableRequest, SigningSettings, sign, }; use aws_sigv4::sign::v4; @@ -97,9 +94,7 @@ impl SigV4Signer { .time(std::time::SystemTime::now()) .settings(settings) .build() - .map_err(|e| { - Error::new(ErrorKind::Unexpected, "build SigV4 params").with_source(e) - })? + .map_err(|e| Error::new(ErrorKind::Unexpected, "build SigV4 params").with_source(e))? .into(); let body_bytes = request @@ -121,9 +116,7 @@ impl SigV4Signer { headers.iter().map(|(k, v)| (k.as_str(), v.as_str())), SignableBody::Bytes(&body_bytes), ) - .map_err(|e| { - Error::new(ErrorKind::Unexpected, "build signable request").with_source(e) - })?; + .map_err(|e| Error::new(ErrorKind::Unexpected, "build signable request").with_source(e))?; let (instructions, _signature) = sign(signable, ¶ms) .map_err(|e| Error::new(ErrorKind::Unexpected, "SigV4 sign").with_source(e))? diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index a4fc9df030..2a5f2bc6e8 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -539,10 +539,7 @@ mod tests { .unwrap(); use crate::Catalog; let ns = crate::NamespaceIdent::from_strs(["public"]).unwrap(); - memory - .create_namespace(&ns, HashMap::new()) - .await - .unwrap(); + memory.create_namespace(&ns, HashMap::new()).await.unwrap(); let creation = crate::TableCreation::builder() .name("orders".to_string()) .schema(schema) @@ -562,7 +559,10 @@ mod tests { let delete_file = DataFileBuilder::default() .content(DataContentType::EqualityDeletes) - .file_path(format!("{}/deletes/0001.parquet", table.metadata().location())) + .file_path(format!( + "{}/deletes/0001.parquet", + table.metadata().location() + )) .file_format(DataFileFormat::Parquet) .file_size_in_bytes(64) .record_count(1) @@ -585,10 +585,7 @@ mod tests { .metadata() .current_snapshot() .expect("commit must produce a snapshot"); - let manifest_list = table.manifest_list_reader(snap) - .load() - .await - .unwrap(); + let manifest_list = table.manifest_list_reader(snap).load().await.unwrap(); // Two manifests — one Data, one Deletes. assert_eq!(manifest_list.entries().len(), 2); @@ -598,7 +595,10 @@ mod tests { ManifestContentType::Data => 0, ManifestContentType::Deletes => 1, }); - assert_eq!(content_types, vec![ManifestContentType::Data, ManifestContentType::Deletes]); + assert_eq!(content_types, vec![ + ManifestContentType::Data, + ManifestContentType::Deletes + ]); // Walk each manifest and confirm the right file landed in the right // bucket. diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 84784c303e..226be3501f 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -55,12 +55,12 @@ mod action; pub use action::*; mod append; mod expire_snapshots; +mod replace_schema; mod rewrite; mod snapshot; mod sort_order; mod update_location; mod update_properties; -mod replace_schema; mod update_schema; mod update_statistics; mod upgrade_format_version; @@ -76,11 +76,11 @@ use crate::spec::TableProperties; use crate::table::Table; use crate::transaction::append::FastAppendAction; use crate::transaction::expire_snapshots::ExpireSnapshotsAction; +use crate::transaction::replace_schema::ReplaceSchemaAction; use crate::transaction::rewrite::RewriteFilesAction; use crate::transaction::sort_order::ReplaceSortOrderAction; use crate::transaction::update_location::UpdateLocationAction; use crate::transaction::update_properties::UpdatePropertiesAction; -use crate::transaction::replace_schema::ReplaceSchemaAction; use crate::transaction::update_schema::UpdateSchemaAction; use crate::transaction::update_statistics::UpdateStatisticsAction; use crate::transaction::upgrade_format_version::UpgradeFormatVersionAction; diff --git a/crates/iceberg/src/transaction/replace_schema.rs b/crates/iceberg/src/transaction/replace_schema.rs index 969398577f..10129ab7d4 100644 --- a/crates/iceberg/src/transaction/replace_schema.rs +++ b/crates/iceberg/src/transaction/replace_schema.rs @@ -163,14 +163,10 @@ mod tests { let requirements = commit.take_requirements(); assert_eq!(updates.len(), 2); - assert!(matches!( - updates[0], - crate::TableUpdate::AddSchema { .. } - )); - assert!(matches!( - updates[1], - crate::TableUpdate::SetCurrentSchema { schema_id: -1 } - )); + assert!(matches!(updates[0], crate::TableUpdate::AddSchema { .. })); + assert!(matches!(updates[1], crate::TableUpdate::SetCurrentSchema { + schema_id: -1 + })); // Three requirements, in a fixed order. assert_eq!(requirements.len(), 3); diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index fd7ae01c9d..c0fb6682ba 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -290,12 +290,12 @@ impl SnapshotProduceOperation for RewriteFilesOperation { mod tests { use std::collections::HashMap; - use crate::{Catalog, CatalogBuilder, ErrorKind}; use crate::spec::{ DataContentType, DataFileBuilder, DataFileFormat, ManifestContentType, NestedField, Operation, PrimitiveType, Schema, Struct, Type, }; use crate::transaction::{ApplyTransactionAction, Transaction}; + use crate::{Catalog, CatalogBuilder, ErrorKind}; async fn make_memory_table() -> (crate::memory::MemoryCatalog, crate::table::Table) { let schema = Schema::builder() @@ -377,10 +377,7 @@ mod tests { let snap = table.metadata().current_snapshot().unwrap(); assert_eq!(snap.summary().operation, Operation::Replace); - let mlist = table.manifest_list_reader(snap) - .load() - .await - .unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); let data_manifests: Vec<_> = mlist .entries() .iter() @@ -436,10 +433,7 @@ mod tests { let snap = table.metadata().current_snapshot().unwrap(); assert_eq!(snap.summary().operation, Operation::Replace); - let mlist = table.manifest_list_reader(snap) - .load() - .await - .unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); let data_manifests: Vec<_> = mlist .entries() .iter() @@ -481,8 +475,16 @@ mod tests { async fn rewrite_carries_forward_unaffected_delete_manifest() { let (catalog, table) = make_memory_table().await; - let d1 = data_file("memory:///warehouse/public/orders/data/d1.parquet", 10, 1024); - let d2 = data_file("memory:///warehouse/public/orders/data/d2.parquet", 20, 2048); + let d1 = data_file( + "memory:///warehouse/public/orders/data/d1.parquet", + 10, + 1024, + ); + let d2 = data_file( + "memory:///warehouse/public/orders/data/d2.parquet", + 20, + 2048, + ); let eq_delete = DataFileBuilder::default() .content(DataContentType::EqualityDeletes) .file_path("memory:///warehouse/public/orders/deletes/eq.parquet".to_string()) @@ -518,10 +520,7 @@ mod tests { let table = tx.commit(&catalog).await.unwrap(); let snap = table.metadata().current_snapshot().unwrap(); - let mlist = table.manifest_list_reader(snap) - .load() - .await - .unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); // Counts: 2 data manifests (rewritten survivors + new added) and // 1 delete manifest (unchanged carry-forward). @@ -580,10 +579,7 @@ mod tests { let snap = table.metadata().current_snapshot().unwrap(); assert_eq!(snap.summary().operation, Operation::Replace); - let mlist = table.manifest_list_reader(snap) - .load() - .await - .unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); let data_manifest_count = mlist .entries() .iter() @@ -647,10 +643,7 @@ mod tests { let snap = table.metadata().current_snapshot().unwrap(); assert_eq!(snap.sequence_number(), 3); - let mlist = table.manifest_list_reader(snap) - .load() - .await - .unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); let mut seqs = Vec::new(); for mf in mlist.entries() { let m = mf.load_manifest(table.file_io()).await.unwrap(); @@ -701,10 +694,7 @@ mod tests { let table = catalog.load_table(table.identifier()).await.unwrap(); let snap = table.metadata().current_snapshot().unwrap(); - let mlist = table.manifest_list_reader(snap) - .load() - .await - .unwrap(); + let mlist = table.manifest_list_reader(snap).load().await.unwrap(); let mut paths = Vec::new(); for mf in mlist.entries() { let m = mf.load_manifest(table.file_io()).await.unwrap(); @@ -796,5 +786,4 @@ mod tests { assert_eq!(summary(&table, "total-delete-files"), "0"); assert_eq!(summary(&table, "total-equality-deletes"), "0"); } - } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index a9b92081fb..d0811a5bcf 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -83,9 +83,9 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { /// - **Append operations**: Typically include all existing manifests /// - **Overwrite operations**: May exclude manifests for partitions being overwritten /// - **Delete operations**: May exclude manifests for partitions being deleted - /// Returns existing manifest files that should be included in the new - /// snapshot. Takes `&mut SnapshotProducer` so implementations that - /// rewrite manifests in place (e.g. `Replace`) can call helpers like + /// + /// Takes `&mut SnapshotProducer` so implementations that rewrite + /// manifests in place (e.g. `Replace`) can call helpers like /// `write_existing_manifest_for` here. Append-style implementations /// that only need read access can ignore the mutability. fn existing_manifest( @@ -489,7 +489,11 @@ impl<'a> SnapshotProducer<'a> { summary_collector.set_partition_summary_limit(partition_summary_limit); - for data_file in self.added_data_files.iter().chain(self.added_delete_files.iter()) { + for data_file in self + .added_data_files + .iter() + .chain(self.added_delete_files.iter()) + { summary_collector.add_file( data_file, table_metadata.current_schema().clone(),