From 6e4fc17a486abb5769efbcc77f0b1bc0da6ef7ba Mon Sep 17 00:00:00 2001 From: hotcache Date: Fri, 21 Aug 2026 13:49:30 +0900 Subject: [PATCH 01/28] feat(transaction): add MergingSnapshotProducer and RewriteFilesAction Add support for atomically replacing data files (compaction). - MergingSnapshotProducer: shared engine for operations that both add and delete files, with manifest filtering and summary computation - ManifestFilterManager: rewrites existing manifests to remove deleted file entries, with encryption support - RewriteFilesAction: thin action layer using Operation::Replace - Refactor SnapshotProducer to extract write_snapshot() shared method - Allow Operation::Replace in snapshot summary computation Closes #1607 Related: #2242, #624 --- crates/iceberg/src/spec/snapshot_summary.rs | 1 + crates/iceberg/src/transaction/merging.rs | 386 ++++++++++++++++++++ crates/iceberg/src/transaction/mod.rs | 12 + crates/iceberg/src/transaction/rewrite.rs | 323 ++++++++++++++++ crates/iceberg/src/transaction/snapshot.rs | 52 ++- 5 files changed, 761 insertions(+), 13 deletions(-) create mode 100644 crates/iceberg/src/transaction/merging.rs create mode 100644 crates/iceberg/src/transaction/rewrite.rs diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index c9fd022724..c7669eda21 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -338,6 +338,7 @@ pub(crate) fn update_snapshot_summaries( if summary.operation != Operation::Append && summary.operation != Operation::Overwrite && summary.operation != Operation::Delete + && summary.operation != Operation::Replace { return Err(Error::new( ErrorKind::DataInvalid, diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs new file mode 100644 index 0000000000..2c9ee86c03 --- /dev/null +++ b/crates/iceberg/src/transaction/merging.rs @@ -0,0 +1,386 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared engine for snapshot-producing operations that both add and delete files. +//! +//! [`MergingSnapshotProducer`] is the Rust equivalent of Java's +//! `MergingSnapshotProducer`. It handles manifest filtering, new manifest +//! creation, summary computation, and delegates the final snapshot commit to +//! [`SnapshotProducer`]. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use crate::error::Result; +use crate::io::OutputFile; +use crate::spec::{ + DataContentType, DataFile, DataFileFormat, FormatVersion, ManifestContentType, ManifestEntry, + ManifestFile, ManifestStatus, ManifestWriter, ManifestWriterBuilder, Operation, PartitionSpec, + SchemaRef, SnapshotSummaryCollector, Summary, update_snapshot_summaries, +}; +use crate::table::Table; +use crate::transaction::ActionCommit; +use crate::transaction::snapshot::SnapshotProducer; +use crate::{Error, ErrorKind}; + +/// Create a manifest writer that handles encryption when available. +fn new_manifest_writer( + table: &Table, + output_file: OutputFile, + snapshot_id: Option, + schema: SchemaRef, + partition_spec: PartitionSpec, +) -> Result { + let builder = if let Some(em) = table.encryption_manager() { + ManifestWriterBuilder::new_from_encrypted( + em.encrypt(output_file), + snapshot_id, + schema, + partition_spec, + )? + } else { + ManifestWriterBuilder::new(output_file, snapshot_id, schema, partition_spec) + }; + + match table.metadata().format_version() { + FormatVersion::V1 => Ok(builder.build_v1()), + FormatVersion::V2 => Ok(builder.build_v2_data()), + FormatVersion::V3 => Ok(builder.build_v3_data()), + } +} + +/// Filters existing manifests by removing entries for deleted data files. +/// +/// This is equivalent to Java's `ManifestFilterManager`. When a rewrite or +/// overwrite operation deletes files, the filter manager rewrites affected +/// manifests so that deleted entries are dropped and surviving entries are +/// re-emitted with [`ManifestStatus::Existing`]. +pub(crate) struct ManifestFilterManager { + deleted_file_paths: HashSet, + fail_missing_delete_paths: bool, +} + +impl ManifestFilterManager { + pub(crate) fn new(fail_missing_delete_paths: bool) -> Self { + Self { + deleted_file_paths: HashSet::new(), + fail_missing_delete_paths, + } + } + + pub(crate) fn add_delete(&mut self, path: String) { + self.deleted_file_paths.insert(path); + } + + /// Filter `manifests` by removing entries whose file path is in the delete + /// set. Returns the surviving manifests plus a [`SnapshotSummaryCollector`] + /// that recorded metrics for every removed file. + pub(crate) async fn filter_manifests( + &self, + table: &Table, + manifests: Vec, + snapshot_id: i64, + ) -> Result<(Vec, SnapshotSummaryCollector)> { + if self.deleted_file_paths.is_empty() { + return Ok((manifests, SnapshotSummaryCollector::default())); + } + + let mut result: Vec = Vec::with_capacity(manifests.len()); + let mut removed_collector = SnapshotSummaryCollector::default(); + let mut found_paths: HashSet = HashSet::new(); + + let schema = table.metadata().current_schema().clone(); + let partition_spec = table.metadata().default_partition_spec().clone(); + + for manifest_file in &manifests { + // Only filter data manifests; pass delete manifests through unchanged. + if manifest_file.content != ManifestContentType::Data { + result.push(manifest_file.clone()); + continue; + } + + let manifest = manifest_file.load_manifest(table.file_io()).await?; + + // Check whether this manifest contains any files we want to delete. + let has_deletes = manifest + .entries() + .iter() + .any(|e| e.is_alive() && self.deleted_file_paths.contains(e.file_path())); + + if !has_deletes { + // Manifest is unaffected — pass through verbatim. + result.push(manifest_file.clone()); + continue; + } + + // Manifest requires the current partition spec to match so we can + // rewrite it correctly. Non-default specs need a writer + // parameterised by the source spec, which is not yet implemented. + if manifest_file.partition_spec_id != table.metadata().default_partition_spec_id() { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "Cannot rewrite manifest with partition spec {} (table default is {}). \ + Rewriting manifests with non-default partition specs is not yet supported.", + manifest_file.partition_spec_id, + table.metadata().default_partition_spec_id(), + ), + )); + } + + // Rewrite: keep surviving entries as EXISTING, drop deleted ones. + let mut surviving_entries: Vec = Vec::new(); + for entry in manifest.entries() { + if entry.is_alive() && self.deleted_file_paths.contains(entry.file_path()) { + // Record removal metrics. + found_paths.insert(entry.file_path().to_string()); + removed_collector.remove_file( + entry.data_file(), + schema.clone(), + partition_spec.clone(), + ); + } else if entry.is_alive() { + // Surviving entry — re-emit as EXISTING with original ids preserved. + let existing = ManifestEntry::builder() + .status(ManifestStatus::Existing) + .snapshot_id( + entry + .snapshot_id() + .unwrap_or(manifest_file.added_snapshot_id), + ) + .sequence_number(entry.sequence_number().unwrap_or(0)) + .file_sequence_number(entry.file_sequence_number.unwrap_or(0)) + .data_file(entry.data_file().clone()) + .build(); + surviving_entries.push(existing); + } + // Already-deleted entries (status == Deleted) are dropped. + } + + if surviving_entries.is_empty() { + // Manifest is now empty — omit entirely. + continue; + } + + // Write the filtered manifest. + let new_manifest_path = format!( + "{}/{}-m-filter-{}.{}", + table.metadata().metadata_location()?, + Uuid::now_v7(), + manifest_file.partition_spec_id, + DataFileFormat::Avro, + ); + let output_file = table.file_io().new_output(new_manifest_path)?; + // Use the current snapshot_id so that the manifest list writer + // can assign sequence numbers to this rewritten manifest. + let mut writer = new_manifest_writer( + table, + output_file, + Some(snapshot_id), + schema.clone(), + partition_spec.as_ref().clone(), + )?; + for entry in surviving_entries { + writer.add_entry(entry)?; + } + let new_manifest = writer.write_manifest_file().await?; + result.push(new_manifest); + } + + // Validate that every delete target was found. + if self.fail_missing_delete_paths { + let missing: Vec<&str> = self + .deleted_file_paths + .iter() + .filter(|p| !found_paths.contains(p.as_str())) + .map(String::as_str) + .collect(); + if !missing.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Failed to find the following files to delete in the current snapshot: {}", + missing.join(", "), + ), + )); + } + } + + Ok((result, removed_collector)) + } +} + +/// Shared engine for operations that both add and remove data files. +/// +/// This struct is the Rust equivalent of Java's `MergingSnapshotProducer`. +/// It manages: +/// - Tracking files to add and delete +/// - Filtering existing manifests to remove deleted files +/// - Writing new manifests for added files +/// - Computing snapshot summaries that account for both additions and removals +/// - Delegating final snapshot creation to [`SnapshotProducer`] +/// +/// Concrete transaction actions like [`RewriteFilesAction`] own an instance +/// of this struct and configure it with the appropriate [`Operation`] type +/// and validation rules. +pub(crate) struct MergingSnapshotProducer { + operation: Operation, + pub(crate) added_data_files: Vec, + pub(crate) deleted_data_files: Vec, + filter_manager: ManifestFilterManager, + snapshot_properties: HashMap, + commit_uuid: Uuid, +} + +impl MergingSnapshotProducer { + pub(crate) fn new(operation: Operation) -> Self { + Self { + operation, + added_data_files: Vec::new(), + deleted_data_files: Vec::new(), + filter_manager: ManifestFilterManager::new(true), + snapshot_properties: HashMap::new(), + commit_uuid: Uuid::now_v7(), + } + } + + pub(crate) fn add_data_file(&mut self, file: DataFile) { + self.added_data_files.push(file); + } + + pub(crate) fn delete_data_file(&mut self, file: DataFile) { + self.filter_manager.add_delete(file.file_path.clone()); + self.deleted_data_files.push(file); + } + + /// Produce manifests, compute summary, and commit a new snapshot. + pub(crate) async fn commit_snapshot(&self, table: &Table) -> Result { + // Create the SnapshotProducer first so we can use its snapshot_id + // for new manifests. This ensures the manifest list writer can + // assign sequence numbers to manifests from this snapshot. + let snapshot_producer = + SnapshotProducer::new(table, self.commit_uuid, HashMap::new(), Vec::new()); + let snapshot_id = snapshot_producer.snapshot_id; + + // 1. Load existing manifests from the current snapshot. + let existing_manifests = match table.metadata().current_snapshot() { + Some(snapshot) => { + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + manifest_list + .entries() + .iter() + .filter(|e| { + e.has_added_files() || e.has_existing_files() || e.has_deleted_files() + }) + .cloned() + .collect() + } + None => Vec::new(), + }; + + // 2. Filter existing manifests — remove deleted file entries. + let (mut filtered_manifests, removed_collector) = self + .filter_manager + .filter_manifests(table, existing_manifests, snapshot_id) + .await?; + + // 3. Write a new manifest for added files. + if !self.added_data_files.is_empty() { + let added_manifest = self.write_added_manifest(table, snapshot_id).await?; + filtered_manifests.push(added_manifest); + } + + // 4. Compute summary (added + removed). + let summary = self.build_summary(table, removed_collector)?; + + // 5. Delegate to SnapshotProducer for manifest list + snapshot creation. + snapshot_producer + .commit_with_manifests(filtered_manifests, summary) + .await + } + + async fn write_added_manifest(&self, table: &Table, snapshot_id: i64) -> Result { + let new_manifest_path = format!( + "{}/{}-m-added.{}", + table.metadata().metadata_location()?, + self.commit_uuid, + DataFileFormat::Avro, + ); + let output_file = table.file_io().new_output(new_manifest_path)?; + let schema = table.metadata().current_schema().clone(); + let partition_spec = table.metadata().default_partition_spec().as_ref().clone(); + + let mut writer = new_manifest_writer( + table, + output_file, + Some(snapshot_id), + schema, + partition_spec, + )?; + + let format_version = table.metadata().format_version(); + for data_file in &self.added_data_files { + if data_file.content_type() != DataContentType::Data { + return Err(Error::new( + ErrorKind::DataInvalid, + "Only data content type is allowed for rewrite files", + )); + } + let entry_builder = ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(data_file.clone()); + let entry = if format_version == FormatVersion::V1 { + // V1 requires snapshot_id on each entry. + entry_builder.snapshot_id(snapshot_id).build() + } else { + entry_builder.build() + }; + writer.add_entry(entry)?; + } + + writer.write_manifest_file().await + } + + fn build_summary( + &self, + table: &Table, + removed_collector: SnapshotSummaryCollector, + ) -> Result { + let table_metadata = table.metadata_ref(); + let schema = table_metadata.current_schema().clone(); + let partition_spec = table_metadata.default_partition_spec().clone(); + + let mut collector = SnapshotSummaryCollector::default(); + for file in &self.added_data_files { + collector.add_file(file, schema.clone(), partition_spec.clone()); + } + // Merge removal metrics from the filter manager. + collector.merge(removed_collector); + + let mut additional_properties = self.snapshot_properties.clone(); + additional_properties.extend(collector.build()); + + let summary = Summary { + operation: self.operation.clone(), + additional_properties, + }; + + let previous_snapshot = table_metadata.current_snapshot(); + update_snapshot_summaries(summary, previous_snapshot.map(|s| s.summary()), false) + } +} diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 02acbc232c..b8f51a298a 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -55,6 +55,8 @@ mod action; pub use action::*; mod append; mod expire_snapshots; +mod merging; +mod rewrite; mod snapshot; mod sort_order; mod update_location; @@ -75,6 +77,7 @@ use crate::table::Table; use crate::transaction::action::BoxedTransactionAction; use crate::transaction::append::FastAppendAction; use crate::transaction::expire_snapshots::ExpireSnapshotsAction; +use crate::transaction::rewrite::RewriteFilesAction; use crate::transaction::sort_order::ReplaceSortOrderAction; use crate::transaction::update_location::UpdateLocationAction; use crate::transaction::update_properties::UpdatePropertiesAction; @@ -151,6 +154,15 @@ impl Transaction { FastAppendAction::new() } + /// Creates a rewrite files action for compaction. + /// + /// This action replaces a set of data files with a new set while keeping + /// the logical table contents unchanged. The resulting snapshot uses + /// [`Operation::Replace`](crate::spec::Operation::Replace). + pub fn rewrite_files(&self) -> RewriteFilesAction { + RewriteFilesAction::new() + } + /// Creates replace sort order action. pub fn replace_sort_order(&self) -> ReplaceSortOrderAction { ReplaceSortOrderAction::new() diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs new file mode 100644 index 0000000000..80f1d4ef8b --- /dev/null +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -0,0 +1,323 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Transaction action for rewriting data files (compaction). +//! +//! [`RewriteFilesAction`] replaces a set of data files with a new set while +//! keeping the logical table contents unchanged. This is used for compaction — +//! merging many small files into fewer large ones. +//! +//! The resulting snapshot uses [`Operation::Replace`] to indicate that files +//! were reorganised without changing the data. + +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::error::Result; +use crate::spec::{DataFile, Operation}; +use crate::table::Table; +use crate::transaction::merging::MergingSnapshotProducer; +use crate::transaction::{ActionCommit, TransactionAction}; +use crate::{Error, ErrorKind}; + +/// A transaction action that rewrites (replaces) data files. +/// +/// This is the Rust equivalent of Java's `BaseRewriteFiles`. It uses +/// [`MergingSnapshotProducer`] to handle manifest filtering and creation, +/// and commits a snapshot with [`Operation::Replace`]. +/// +/// # Example +/// +/// ```ignore +/// let tx = Transaction::new(&table); +/// let action = tx.rewrite_files() +/// .delete_file(old_file_1) +/// .delete_file(old_file_2) +/// .add_file(merged_file); +/// let tx = action.apply(tx)?; +/// let table = tx.commit(&catalog).await?; +/// ``` +pub struct RewriteFilesAction { + producer: MergingSnapshotProducer, +} + +impl RewriteFilesAction { + pub(crate) fn new() -> Self { + Self { + producer: MergingSnapshotProducer::new(Operation::Replace), + } + } + + /// Register a data file to be removed from the table. + /// + /// The file must exist in the current snapshot; otherwise the commit + /// will fail with a validation error. + pub fn delete_file(mut self, file: DataFile) -> Self { + self.producer.delete_data_file(file); + self + } + + /// Register a data file to be added to the table. + /// + /// Typically this is the merged output of the files being deleted. + pub fn add_file(mut self, file: DataFile) -> Self { + self.producer.add_data_file(file); + self + } + + fn validate(&self) -> Result<()> { + if self.producer.deleted_data_files.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + "Rewrite files requires at least one file to delete", + )); + } + if self.producer.added_data_files.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + "Rewrite files requires at least one file to add", + )); + } + Ok(()) + } +} + +#[async_trait] +impl TransactionAction for RewriteFilesAction { + async fn commit(self: Arc, table: &Table) -> Result { + self.validate()?; + self.producer.commit_snapshot(table).await + } +} + +#[cfg(test)] +mod tests { + use crate::Catalog; + use crate::memory::tests::new_memory_catalog; + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Literal, Operation, Struct, + }; + use crate::transaction::tests::make_v3_minimal_table_in_catalog; + use crate::transaction::{ApplyTransactionAction, Transaction}; + + fn make_data_file( + table: &crate::table::Table, + path: &str, + records: u64, + size: u64, + ) -> crate::spec::DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(size) + .record_count(records) + .partition(Struct::from_iter([Some(Literal::long(1))])) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap() + } + + /// Helper: append data files to a table and return the updated table. + async fn append_files( + catalog: &impl Catalog, + table: &crate::table::Table, + files: Vec, + ) -> crate::table::Table { + let tx = Transaction::new(table); + let tx = tx.fast_append().add_data_files(files).apply(tx).unwrap(); + tx.commit(catalog).await.unwrap() + } + + /// E2E: Compact 3 small files into 1 merged file. + /// Verify: operation=Replace, file counts, record counts. + #[tokio::test] + async fn test_rewrite_files_compaction() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + // Append 3 small files (10 records each). + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let f3 = make_data_file(&table, "test/3.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone(), f2.clone(), f3.clone()]).await; + + // Verify pre-compaction state. + let summary = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + assert_eq!(summary.get("total-data-files").unwrap(), "3"); + assert_eq!(summary.get("total-records").unwrap(), "30"); + + // Rewrite: delete 3 files, add 1 merged file (30 records). + let merged = make_data_file(&table, "test/merged.parquet", 30, 300); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(f1) + .delete_file(f2) + .delete_file(f3) + .add_file(merged); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Verify post-compaction snapshot. + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.summary().operation, Operation::Replace); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("total-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-records").unwrap(), "30"); + assert_eq!(summary.get("added-data-files").unwrap(), "1"); + assert_eq!(summary.get("deleted-data-files").unwrap(), "3"); + + // Verify manifest list: merged file is the only live file. + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let mut live_files: Vec = Vec::new(); + for manifest_entry in manifest_list.entries() { + let manifest = manifest_entry.load_manifest(table.file_io()).await.unwrap(); + for entry in manifest.entries() { + if entry.is_alive() { + live_files.push(entry.file_path().to_string()); + } + } + } + assert_eq!(live_files, vec!["test/merged.parquet"]); + } + + /// Rewrite with a non-existent delete target should fail. + #[tokio::test] + async fn test_rewrite_files_missing_delete_target() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + // Append 1 file. + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1]).await; + + // Try to delete a file that doesn't exist. + let ghost = make_data_file(&table, "test/ghost.parquet", 10, 100); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(ghost).add_file(merged); + let tx = action.apply(tx).unwrap(); + let result = tx.commit(&catalog).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.message() + .contains("Failed to find the following files to delete"), + "unexpected error: {}", + err.message() + ); + } + + /// Rewrite with no deletes should fail validation. + #[tokio::test] + async fn test_rewrite_files_no_deletes() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().add_file(merged); + let tx = action.apply(tx).unwrap(); + let result = tx.commit(&catalog).await; + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .message() + .contains("at least one file to delete") + ); + } + + /// Partial rewrite: delete 2 of 3 files, keeping one. + /// Verifies that the rewritten manifest for the surviving file + /// has the correct snapshot_id for sequence number assignment. + #[tokio::test] + async fn test_rewrite_files_partial() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let f3 = make_data_file(&table, "test/3.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone(), f2.clone(), f3.clone()]).await; + + // Rewrite only f1 and f2, keep f3. + let merged = make_data_file(&table, "test/merged.parquet", 20, 200); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(f1) + .delete_file(f2) + .add_file(merged); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.summary().operation, Operation::Replace); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("total-data-files").unwrap(), "2"); + assert_eq!(summary.get("total-records").unwrap(), "30"); + assert_eq!(summary.get("deleted-data-files").unwrap(), "2"); + assert_eq!(summary.get("added-data-files").unwrap(), "1"); + + // Verify live files: f3 (surviving) + merged (new). + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let mut live_files: Vec = Vec::new(); + for manifest_entry in manifest_list.entries() { + let manifest = manifest_entry.load_manifest(table.file_io()).await.unwrap(); + for entry in manifest.entries() { + if entry.is_alive() { + live_files.push(entry.file_path().to_string()); + } + } + } + live_files.sort(); + assert_eq!(live_files, vec!["test/3.parquet", "test/merged.parquet"]); + } + + /// Rewrite with no adds should fail validation. + #[tokio::test] + async fn test_rewrite_files_no_adds() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1); + let tx = action.apply(tx).unwrap(); + let result = tx.commit(&catalog).await; + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .message() + .contains("at least one file to add") + ); + } +} diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 1f09a79a08..532671a6d1 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -109,7 +109,7 @@ pub(crate) trait ManifestProcess: Send + Sync { pub(crate) struct SnapshotProducer<'a> { pub(crate) table: &'a Table, - snapshot_id: i64, + pub(crate) snapshot_id: i64, commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, @@ -441,6 +441,43 @@ impl<'a> SnapshotProducer<'a> { mut self, snapshot_produce_operation: OP, process: MP, + ) -> Result { + // Calling self.summary() before self.produce_manifests() is important because self.added_data_files + // will be set to an empty vec after self.produce_manifests() returns, resulting in an empty summary + // being generated. + let summary = self.summary(&snapshot_produce_operation).map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) + })?; + + let new_manifests = self + .produce_manifests(&snapshot_produce_operation, &process) + .await?; + + self.write_snapshot(new_manifests, summary).await + } + + /// Commit a snapshot from externally produced manifests and summary. + /// + /// This is used by [`MergingSnapshotProducer`] which manages manifest + /// creation (including filtering deleted files) on its own and only needs + /// `SnapshotProducer` to write the manifest list, build the snapshot object, + /// and return the [`ActionCommit`]. + pub(crate) async fn commit_with_manifests( + self, + manifests: Vec, + summary: Summary, + ) -> Result { + self.write_snapshot(manifests, summary).await + } + + /// Writes the manifest list, builds the snapshot, and returns the [`ActionCommit`]. + /// + /// This is the shared implementation used by both [`commit`] and + /// [`commit_with_manifests`]. + async fn write_snapshot( + self, + manifests: Vec, + summary: Summary, ) -> Result { let manifest_list_path = self.generate_manifest_list_file_path(0)?; let next_seq_num = self.table.metadata().next_sequence_number(); @@ -479,18 +516,7 @@ impl<'a> SnapshotProducer<'a> { ), }; - // Calling self.summary() before self.produce_manifests() is important because self.added_data_files - // will be set to an empty vec after self.produce_manifests() returns, resulting in an empty summary - // being generated. - let summary = self.summary(&snapshot_produce_operation).map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) - })?; - - let new_manifests = self - .produce_manifests(&snapshot_produce_operation, &process) - .await?; - - manifest_list_writer.add_manifests(new_manifests.into_iter())?; + manifest_list_writer.add_manifests(manifests.into_iter())?; let writer_next_row_id = manifest_list_writer.next_row_id(); manifest_list_writer.close().await?; From 64a5b1f0c83ecc1a1fb55057f89c54a5272b12a8 Mon Sep 17 00:00:00 2001 From: hotcache Date: Fri, 21 Aug 2026 22:56:38 +0900 Subject: [PATCH 02/28] fix(transaction): support non-default partition specs in manifest filtering Previously, ManifestFilterManager rejected manifests with non-default partition spec IDs with a FeatureUnsupported error. This broke tables that had undergone partition evolution, as old manifests carry their original spec ID. Now the filter manager resolves each manifest's own partition spec from table metadata and uses it for rewriting, allowing correct handling of partition-evolved tables. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/iceberg/src/transaction/merging.rs | 40 +++++++++++------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index 2c9ee86c03..b21df75c7c 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -104,9 +104,6 @@ impl ManifestFilterManager { let mut removed_collector = SnapshotSummaryCollector::default(); let mut found_paths: HashSet = HashSet::new(); - let schema = table.metadata().current_schema().clone(); - let partition_spec = table.metadata().default_partition_spec().clone(); - for manifest_file in &manifests { // Only filter data manifests; pass delete manifests through unchanged. if manifest_file.content != ManifestContentType::Data { @@ -128,20 +125,23 @@ impl ManifestFilterManager { continue; } - // Manifest requires the current partition spec to match so we can - // rewrite it correctly. Non-default specs need a writer - // parameterised by the source spec, which is not yet implemented. - if manifest_file.partition_spec_id != table.metadata().default_partition_spec_id() { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - format!( - "Cannot rewrite manifest with partition spec {} (table default is {}). \ - Rewriting manifests with non-default partition specs is not yet supported.", - manifest_file.partition_spec_id, - table.metadata().default_partition_spec_id(), - ), - )); - } + // Resolve the partition spec for this manifest. After partition + // evolution, old manifests may carry a non-default spec — we must + // use the manifest's own spec so the rewritten manifest stays valid. + let manifest_spec = table + .metadata() + .partition_spec_by_id(manifest_file.partition_spec_id) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Manifest references unknown partition spec {}", + manifest_file.partition_spec_id, + ), + ) + })? + .clone(); + let schema = table.metadata().current_schema().clone(); // Rewrite: keep surviving entries as EXISTING, drop deleted ones. let mut surviving_entries: Vec = Vec::new(); @@ -152,7 +152,7 @@ impl ManifestFilterManager { removed_collector.remove_file( entry.data_file(), schema.clone(), - partition_spec.clone(), + manifest_spec.clone(), ); } else if entry.is_alive() { // Surviving entry — re-emit as EXISTING with original ids preserved. @@ -177,7 +177,7 @@ impl ManifestFilterManager { continue; } - // Write the filtered manifest. + // Write the filtered manifest using the manifest's own partition spec. let new_manifest_path = format!( "{}/{}-m-filter-{}.{}", table.metadata().metadata_location()?, @@ -193,7 +193,7 @@ impl ManifestFilterManager { output_file, Some(snapshot_id), schema.clone(), - partition_spec.as_ref().clone(), + manifest_spec.as_ref().clone(), )?; for entry in surviving_entries { writer.add_entry(entry)?; From b02fb8eb24ad953644b41102e7fe44d617b83198 Mon Sep 17 00:00:00 2001 From: hotcache Date: Fri, 21 Aug 2026 23:02:04 +0900 Subject: [PATCH 03/28] refactor(transaction): encapsulate MergingSnapshotProducer fields Replace pub(crate) field access with accessor methods (has_added_data_files, has_deleted_data_files) to improve encapsulation. Downstream actions now use these methods instead of reaching into internal Vec fields directly. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/iceberg/src/transaction/merging.rs | 12 ++++++++++-- crates/iceberg/src/transaction/rewrite.rs | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index b21df75c7c..af4f844a38 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -240,8 +240,8 @@ impl ManifestFilterManager { /// and validation rules. pub(crate) struct MergingSnapshotProducer { operation: Operation, - pub(crate) added_data_files: Vec, - pub(crate) deleted_data_files: Vec, + added_data_files: Vec, + deleted_data_files: Vec, filter_manager: ManifestFilterManager, snapshot_properties: HashMap, commit_uuid: Uuid, @@ -268,6 +268,14 @@ impl MergingSnapshotProducer { self.deleted_data_files.push(file); } + pub(crate) fn has_added_data_files(&self) -> bool { + !self.added_data_files.is_empty() + } + + pub(crate) fn has_deleted_data_files(&self) -> bool { + !self.deleted_data_files.is_empty() + } + /// Produce manifests, compute summary, and commit a new snapshot. pub(crate) async fn commit_snapshot(&self, table: &Table) -> Result { // Create the SnapshotProducer first so we can use its snapshot_id diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 80f1d4ef8b..a184c8d093 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -81,13 +81,13 @@ impl RewriteFilesAction { } fn validate(&self) -> Result<()> { - if self.producer.deleted_data_files.is_empty() { + if !self.producer.has_deleted_data_files() { return Err(Error::new( ErrorKind::DataInvalid, "Rewrite files requires at least one file to delete", )); } - if self.producer.added_data_files.is_empty() { + if !self.producer.has_added_data_files() { return Err(Error::new( ErrorKind::DataInvalid, "Rewrite files requires at least one file to add", From 1f412f8cd3eea8e7072f7778204933659b0be4f2 Mon Sep 17 00:00:00 2001 From: hotcache Date: Fri, 21 Aug 2026 23:29:19 +0900 Subject: [PATCH 04/28] refactor(transaction): address review feedback for PR1 - Add ManifestContentType param to new_manifest_writer for future delete manifest support - Remove unused snapshot_properties field from MergingSnapshotProducer - Move test helpers (make_data_file, append_files) to mod.rs as pub(crate) for reuse across test modules - Add starting_snapshot_id to RewriteFilesAction for future conflict detection support - Fix action-specific error message in generic engine Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/iceberg/src/transaction/merging.rs | 24 ++++++----- crates/iceberg/src/transaction/mod.rs | 29 +++++++++++++- crates/iceberg/src/transaction/rewrite.rs | 49 ++++++----------------- 3 files changed, 54 insertions(+), 48 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index af4f844a38..d8fc7d9b1b 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -43,6 +43,7 @@ fn new_manifest_writer( table: &Table, output_file: OutputFile, snapshot_id: Option, + content: ManifestContentType, schema: SchemaRef, partition_spec: PartitionSpec, ) -> Result { @@ -59,8 +60,14 @@ fn new_manifest_writer( match table.metadata().format_version() { FormatVersion::V1 => Ok(builder.build_v1()), - FormatVersion::V2 => Ok(builder.build_v2_data()), - FormatVersion::V3 => Ok(builder.build_v3_data()), + FormatVersion::V2 => match content { + ManifestContentType::Data => Ok(builder.build_v2_data()), + ManifestContentType::Deletes => Ok(builder.build_v2_deletes()), + }, + FormatVersion::V3 => match content { + ManifestContentType::Data => Ok(builder.build_v3_data()), + ManifestContentType::Deletes => Ok(builder.build_v3_deletes()), + }, } } @@ -111,7 +118,7 @@ impl ManifestFilterManager { continue; } - let manifest = manifest_file.load_manifest(table.file_io()).await?; + let manifest = table.manifest_reader().read(manifest_file).await?; // Check whether this manifest contains any files we want to delete. let has_deletes = manifest @@ -192,6 +199,7 @@ impl ManifestFilterManager { table, output_file, Some(snapshot_id), + ManifestContentType::Data, schema.clone(), manifest_spec.as_ref().clone(), )?; @@ -243,7 +251,6 @@ pub(crate) struct MergingSnapshotProducer { added_data_files: Vec, deleted_data_files: Vec, filter_manager: ManifestFilterManager, - snapshot_properties: HashMap, commit_uuid: Uuid, } @@ -254,7 +261,6 @@ impl MergingSnapshotProducer { added_data_files: Vec::new(), deleted_data_files: Vec::new(), filter_manager: ManifestFilterManager::new(true), - snapshot_properties: HashMap::new(), commit_uuid: Uuid::now_v7(), } } @@ -337,6 +343,7 @@ impl MergingSnapshotProducer { table, output_file, Some(snapshot_id), + ManifestContentType::Data, schema, partition_spec, )?; @@ -346,7 +353,7 @@ impl MergingSnapshotProducer { if data_file.content_type() != DataContentType::Data { return Err(Error::new( ErrorKind::DataInvalid, - "Only data content type is allowed for rewrite files", + "Only data content type is allowed in added_data_files", )); } let entry_builder = ManifestEntry::builder() @@ -380,12 +387,9 @@ impl MergingSnapshotProducer { // Merge removal metrics from the filter manager. collector.merge(removed_collector); - let mut additional_properties = self.snapshot_properties.clone(); - additional_properties.extend(collector.build()); - let summary = Summary { operation: self.operation.clone(), - additional_properties, + additional_properties: collector.build(), }; let previous_snapshot = table_metadata.current_snapshot(); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index b8f51a298a..3fe1b161e7 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -160,7 +160,7 @@ impl Transaction { /// the logical table contents unchanged. The resulting snapshot uses /// [`Operation::Replace`](crate::spec::Operation::Replace). pub fn rewrite_files(&self) -> RewriteFilesAction { - RewriteFilesAction::new() + RewriteFilesAction::new(self.table.metadata().current_snapshot_id()) } /// Creates replace sort order action. @@ -266,7 +266,7 @@ mod tests { use crate::io::FileIO; use crate::memory::tests::new_memory_catalog; use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Literal, Struct, TableMetadata, + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, Struct, TableMetadata, TableProperties, }; use crate::table::Table; @@ -274,6 +274,31 @@ mod tests { use crate::transaction::{ApplyTransactionAction, Transaction}; use crate::{Catalog, Error, ErrorKind, TableCreation, TableIdent}; + /// Create a test data file with the given path, record count, and file size. + pub(crate) fn make_data_file(table: &Table, path: &str, records: u64, size: u64) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(size) + .record_count(records) + .partition(Struct::from_iter([Some(Literal::long(1))])) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap() + } + + /// Append data files to a table via fast_append and return the updated table. + pub(crate) async fn append_files( + catalog: &impl Catalog, + table: &Table, + files: Vec, + ) -> Table { + let tx = Transaction::new(table); + let tx = tx.fast_append().add_data_files(files).apply(tx).unwrap(); + tx.commit(catalog).await.unwrap() + } + pub fn make_v1_table() -> Table { let file = File::open(format!( "{}/testdata/table_metadata/{}", diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index a184c8d093..3bf7202ea9 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -54,12 +54,17 @@ use crate::{Error, ErrorKind}; /// ``` pub struct RewriteFilesAction { producer: MergingSnapshotProducer, + /// The snapshot ID at which this rewrite started reading. Used to + /// detect conflicting deletes added after this point. + #[allow(dead_code)] // Will be used for conflict detection in a follow-up PR. + starting_snapshot_id: Option, } impl RewriteFilesAction { - pub(crate) fn new() -> Self { + pub(crate) fn new(starting_snapshot_id: Option) -> Self { Self { producer: MergingSnapshotProducer::new(Operation::Replace), + starting_snapshot_id, } } @@ -107,43 +112,13 @@ impl TransactionAction for RewriteFilesAction { #[cfg(test)] mod tests { - use crate::Catalog; use crate::memory::tests::new_memory_catalog; - use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Literal, Operation, Struct, + use crate::spec::Operation; + use crate::transaction::tests::{ + append_files, make_data_file, make_v3_minimal_table_in_catalog, }; - use crate::transaction::tests::make_v3_minimal_table_in_catalog; use crate::transaction::{ApplyTransactionAction, Transaction}; - fn make_data_file( - table: &crate::table::Table, - path: &str, - records: u64, - size: u64, - ) -> crate::spec::DataFile { - DataFileBuilder::default() - .content(DataContentType::Data) - .file_path(path.to_string()) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(size) - .record_count(records) - .partition(Struct::from_iter([Some(Literal::long(1))])) - .partition_spec_id(table.metadata().default_partition_spec_id()) - .build() - .unwrap() - } - - /// Helper: append data files to a table and return the updated table. - async fn append_files( - catalog: &impl Catalog, - table: &crate::table::Table, - files: Vec, - ) -> crate::table::Table { - let tx = Transaction::new(table); - let tx = tx.fast_append().add_data_files(files).apply(tx).unwrap(); - tx.commit(catalog).await.unwrap() - } - /// E2E: Compact 3 small files into 1 merged file. /// Verify: operation=Replace, file counts, record counts. #[tokio::test] @@ -193,7 +168,8 @@ mod tests { let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); let mut live_files: Vec = Vec::new(); for manifest_entry in manifest_list.entries() { - let manifest = manifest_entry.load_manifest(table.file_io()).await.unwrap(); + let manifest = table.manifest_reader().read(manifest_entry).await.unwrap() +; for entry in manifest.entries() { if entry.is_alive() { live_files.push(entry.file_path().to_string()); @@ -289,7 +265,8 @@ mod tests { let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); let mut live_files: Vec = Vec::new(); for manifest_entry in manifest_list.entries() { - let manifest = manifest_entry.load_manifest(table.file_io()).await.unwrap(); + let manifest = table.manifest_reader().read(manifest_entry).await.unwrap() +; for entry in manifest.entries() { if entry.is_alive() { live_files.push(entry.file_path().to_string()); From 0fb348cda5be5e79b17942e500b937495a3dc406 Mon Sep 17 00:00:00 2001 From: hotcache Date: Sat, 22 Aug 2026 08:29:10 +0900 Subject: [PATCH 05/28] test(transaction): add coverage for empty table rewrite and manifest omission - test_rewrite_files_empty_table: rewrite on table with no snapshot - test_rewrite_files_removes_empty_manifest: verify empty manifests are omitted, not kept as empty files Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/iceberg/src/transaction/rewrite.rs | 51 +++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 3bf7202ea9..8b9e3f3136 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -277,6 +277,57 @@ mod tests { assert_eq!(live_files, vec!["test/3.parquet", "test/merged.parquet"]); } + /// Rewrite on an empty table (no snapshot) should fail because + /// the delete target doesn't exist. + #[tokio::test] + async fn test_rewrite_files_empty_table() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(merged); + let tx = action.apply(tx).unwrap(); + let result = tx.commit(&catalog).await; + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .message() + .contains("Failed to find the following files to delete"), + ); + } + + /// Rewrite where all files in a manifest are deleted should omit + /// the manifest entirely (not leave an empty one). + #[tokio::test] + async fn test_rewrite_files_removes_empty_manifest() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + // Append 1 file → 1 manifest. + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + + // Rewrite: delete f1, add merged → f1's manifest should be omitted. + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(merged); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + // Verify: only 1 manifest (the new one), not 2. + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + assert_eq!( + manifest_list.entries().len(), + 1, + "empty manifest should be omitted, not kept" + ); + } + /// Rewrite with no adds should fail validation. #[tokio::test] async fn test_rewrite_files_no_adds() { From 4208c44525330bfe8104d6bb78d89b473ac3679b Mon Sep 17 00:00:00 2001 From: hotcache Date: Tue, 25 Aug 2026 11:35:09 +0900 Subject: [PATCH 06/28] fix(transaction): address review feedback from #3046 - Return explicit errors instead of silently defaulting to 0 for sequence_number and file_sequence_number in manifest entries - Use standard manifest naming scheme (-m.avro) instead of custom names (-m-filter-, -m-added) Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/iceberg/src/transaction/merging.rs | 52 +++++++++++++++++++---- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index d8fc7d9b1b..70299a1001 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -23,6 +23,7 @@ //! [`SnapshotProducer`]. use std::collections::{HashMap, HashSet}; +use std::ops::RangeFrom; use uuid::Uuid; @@ -102,6 +103,8 @@ impl ManifestFilterManager { table: &Table, manifests: Vec, snapshot_id: i64, + commit_uuid: Uuid, + manifest_counter: &mut RangeFrom, ) -> Result<(Vec, SnapshotSummaryCollector)> { if self.deleted_file_paths.is_empty() { return Ok((manifests, SnapshotSummaryCollector::default())); @@ -163,6 +166,24 @@ impl ManifestFilterManager { ); } else if entry.is_alive() { // Surviving entry — re-emit as EXISTING with original ids preserved. + let seq = entry.sequence_number().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Manifest entry for {} is missing sequence_number", + entry.file_path(), + ), + ) + })?; + let file_seq = entry.file_sequence_number.ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Manifest entry for {} is missing file_sequence_number", + entry.file_path(), + ), + ) + })?; let existing = ManifestEntry::builder() .status(ManifestStatus::Existing) .snapshot_id( @@ -170,8 +191,8 @@ impl ManifestFilterManager { .snapshot_id() .unwrap_or(manifest_file.added_snapshot_id), ) - .sequence_number(entry.sequence_number().unwrap_or(0)) - .file_sequence_number(entry.file_sequence_number.unwrap_or(0)) + .sequence_number(seq) + .file_sequence_number(file_seq) .data_file(entry.data_file().clone()) .build(); surviving_entries.push(existing); @@ -186,10 +207,10 @@ impl ManifestFilterManager { // Write the filtered manifest using the manifest's own partition spec. let new_manifest_path = format!( - "{}/{}-m-filter-{}.{}", + "{}/{}-m{}.{}", table.metadata().metadata_location()?, - Uuid::now_v7(), - manifest_file.partition_spec_id, + commit_uuid, + manifest_counter.next().unwrap(), DataFileFormat::Avro, ); let output_file = table.file_io().new_output(new_manifest_path)?; @@ -290,6 +311,7 @@ impl MergingSnapshotProducer { let snapshot_producer = SnapshotProducer::new(table, self.commit_uuid, HashMap::new(), Vec::new()); let snapshot_id = snapshot_producer.snapshot_id; + let mut manifest_counter: RangeFrom = (0..); // 1. Load existing manifests from the current snapshot. let existing_manifests = match table.metadata().current_snapshot() { @@ -310,12 +332,18 @@ impl MergingSnapshotProducer { // 2. Filter existing manifests — remove deleted file entries. let (mut filtered_manifests, removed_collector) = self .filter_manager - .filter_manifests(table, existing_manifests, snapshot_id) + .filter_manifests( + table, + existing_manifests, + snapshot_id, + self.commit_uuid, + &mut manifest_counter, + ) .await?; // 3. Write a new manifest for added files. if !self.added_data_files.is_empty() { - let added_manifest = self.write_added_manifest(table, snapshot_id).await?; + let added_manifest = self.write_added_manifest(table, snapshot_id, &mut manifest_counter).await?; filtered_manifests.push(added_manifest); } @@ -328,11 +356,17 @@ impl MergingSnapshotProducer { .await } - async fn write_added_manifest(&self, table: &Table, snapshot_id: i64) -> Result { + async fn write_added_manifest( + &self, + table: &Table, + snapshot_id: i64, + manifest_counter: &mut RangeFrom, + ) -> Result { let new_manifest_path = format!( - "{}/{}-m-added.{}", + "{}/{}-m{}.{}", table.metadata().metadata_location()?, self.commit_uuid, + manifest_counter.next().unwrap(), DataFileFormat::Avro, ); let output_file = table.file_io().new_output(new_manifest_path)?; From 1dd97f04a9781f374df8dff85f85e7b1b07367ed Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:25:46 +0200 Subject: [PATCH 07/28] Preserve survivor entries as Existing when filtering manifests `ManifestFilterManager::filter_manifests` built each surviving entry with status Existing and the original snapshot id and sequence numbers, then handed it to `ManifestWriter::add_entry`, which rewrites the status to Added, replaces the snapshot id with the new snapshot's, and clears the file sequence number. Survivors of a rewrite were therefore recorded as newly added files of the rewrite snapshot. Use `add_existing_entry`, which only sets the status and leaves the ids the filter already resolved, and drop its `#[allow(dead_code)]`. --- crates/iceberg/src/spec/manifest/writer.rs | 4 ---- crates/iceberg/src/transaction/merging.rs | 2 +- crates/iceberg/src/transaction/rewrite.rs | 22 +++++++++++++++++++--- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index 7256692950..c58425ac31 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -371,10 +371,6 @@ impl ManifestWriter { /// Add an existing manifest entry. This method will update following status of the entry: /// - Update the entry status to `Existing` - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_existing_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Existing; diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index 70299a1001..2c594e60df 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -225,7 +225,7 @@ impl ManifestFilterManager { manifest_spec.as_ref().clone(), )?; for entry in surviving_entries { - writer.add_entry(entry)?; + writer.add_existing_entry(entry)?; } let new_manifest = writer.write_manifest_file().await?; result.push(new_manifest); diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 8b9e3f3136..c8b3524681 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -113,7 +113,7 @@ impl TransactionAction for RewriteFilesAction { #[cfg(test)] mod tests { use crate::memory::tests::new_memory_catalog; - use crate::spec::Operation; + use crate::spec::{ManifestStatus, Operation}; use crate::transaction::tests::{ append_files, make_data_file, make_v3_minimal_table_in_catalog, }; @@ -240,6 +240,7 @@ mod tests { let f2 = make_data_file(&table, "test/2.parquet", 10, 100); let f3 = make_data_file(&table, "test/3.parquet", 10, 100); let table = append_files(&catalog, &table, vec![f1.clone(), f2.clone(), f3.clone()]).await; + let append_snapshot = table.metadata().current_snapshot().unwrap().clone(); // Rewrite only f1 and f2, keep f3. let merged = make_data_file(&table, "test/merged.parquet", 20, 200); @@ -264,17 +265,32 @@ mod tests { // Verify live files: f3 (surviving) + merged (new). let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); let mut live_files: Vec = Vec::new(); + let mut survivor = None; for manifest_entry in manifest_list.entries() { - let manifest = table.manifest_reader().read(manifest_entry).await.unwrap() -; + let manifest = table.manifest_reader().read(manifest_entry).await.unwrap(); for entry in manifest.entries() { if entry.is_alive() { live_files.push(entry.file_path().to_string()); } + if entry.file_path() == "test/3.parquet" { + survivor = Some(entry.clone()); + } } } live_files.sort(); assert_eq!(live_files, vec!["test/3.parquet", "test/merged.parquet"]); + + let survivor = survivor.expect("surviving file should still be in a manifest"); + assert_eq!(survivor.status(), ManifestStatus::Existing); + assert_eq!(survivor.snapshot_id(), Some(append_snapshot.snapshot_id())); + assert_eq!( + survivor.sequence_number(), + Some(append_snapshot.sequence_number()) + ); + assert_eq!( + survivor.file_sequence_number, + Some(append_snapshot.sequence_number()) + ); } /// Rewrite on an empty table (no snapshot) should fail because From 243f8f32b676a16d5d2970b57b72911b53f2f0c2 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:28:56 +0200 Subject: [PATCH 08/28] Allow a rewrite to set the data sequence number of added files Java's `RewriteFiles.dataSequenceNumber` lets a compaction record the sequence number of the files it replaces, so the rewritten file does not shadow deletes written concurrently at a higher sequence number. Without it the added files inherit the rewrite snapshot's own sequence number. `RewriteFilesAction::data_sequence_number` sets it, and `write_added_manifest` then writes the added files through `ManifestWriter::add_file`, which records the given number on the entry. A number above the one the new snapshot will carry is rejected at commit; on a V1 table that is 0, matching the V1 entry layout, which records no sequence number at all. The parameter is `i64`, not `u64`, to match `ManifestWriter::add_file` and every other sequence number in the crate. --- crates/iceberg/src/transaction/merging.rs | 32 ++++++- crates/iceberg/src/transaction/rewrite.rs | 108 ++++++++++++++++++++-- 2 files changed, 131 insertions(+), 9 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index 2c594e60df..8759c7e0c7 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -273,6 +273,7 @@ pub(crate) struct MergingSnapshotProducer { deleted_data_files: Vec, filter_manager: ManifestFilterManager, commit_uuid: Uuid, + data_sequence_number: Option, } impl MergingSnapshotProducer { @@ -283,6 +284,7 @@ impl MergingSnapshotProducer { deleted_data_files: Vec::new(), filter_manager: ManifestFilterManager::new(true), commit_uuid: Uuid::now_v7(), + data_sequence_number: None, } } @@ -290,6 +292,10 @@ impl MergingSnapshotProducer { self.added_data_files.push(file); } + pub(crate) fn set_data_sequence_number(&mut self, sequence_number: i64) { + self.data_sequence_number = Some(sequence_number); + } + pub(crate) fn delete_data_file(&mut self, file: DataFile) { self.filter_manager.add_delete(file.file_path.clone()); self.deleted_data_files.push(file); @@ -303,8 +309,26 @@ impl MergingSnapshotProducer { !self.deleted_data_files.is_empty() } + /// Reject a data sequence number above the one the new snapshot will carry. + fn validate_data_sequence_number(&self, table: &Table) -> Result<()> { + let Some(sequence_number) = self.data_sequence_number else { + return Ok(()); + }; + let snapshot_sequence_number = table.metadata().next_sequence_number(); + if sequence_number > snapshot_sequence_number { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Data sequence number {sequence_number} is greater than the snapshot's {snapshot_sequence_number}." + ), + )); + } + Ok(()) + } + /// Produce manifests, compute summary, and commit a new snapshot. pub(crate) async fn commit_snapshot(&self, table: &Table) -> Result { + self.validate_data_sequence_number(table)?; // Create the SnapshotProducer first so we can use its snapshot_id // for new manifests. This ensures the manifest list writer can // assign sequence numbers to manifests from this snapshot. @@ -343,7 +367,9 @@ impl MergingSnapshotProducer { // 3. Write a new manifest for added files. if !self.added_data_files.is_empty() { - let added_manifest = self.write_added_manifest(table, snapshot_id, &mut manifest_counter).await?; + let added_manifest = self + .write_added_manifest(table, snapshot_id, &mut manifest_counter) + .await?; filtered_manifests.push(added_manifest); } @@ -390,6 +416,10 @@ impl MergingSnapshotProducer { "Only data content type is allowed in added_data_files", )); } + if let Some(sequence_number) = self.data_sequence_number { + writer.add_file(data_file.clone(), sequence_number)?; + continue; + } let entry_builder = ManifestEntry::builder() .status(ManifestStatus::Added) .data_file(data_file.clone()); diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index c8b3524681..9f4339c42f 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -85,6 +85,18 @@ impl RewriteFilesAction { self } + /// Set the data sequence number recorded for every added file. + /// + /// Without this the added files inherit the sequence number of the new + /// snapshot. A compaction that must not shadow concurrently written + /// deletes sets the sequence number of the files it replaces instead. + /// V1 manifest entries carry no sequence number, so this has no effect on + /// a V1 table. + pub fn data_sequence_number(mut self, sequence_number: i64) -> Self { + self.producer.set_data_sequence_number(sequence_number); + self + } + fn validate(&self) -> Result<()> { if !self.producer.has_deleted_data_files() { return Err(Error::new( @@ -112,13 +124,31 @@ impl TransactionAction for RewriteFilesAction { #[cfg(test)] mod tests { + use crate::ErrorKind; use crate::memory::tests::new_memory_catalog; - use crate::spec::{ManifestStatus, Operation}; + use crate::spec::{ManifestEntryRef, ManifestStatus, Operation, SnapshotRef}; + use crate::table::Table; use crate::transaction::tests::{ append_files, make_data_file, make_v3_minimal_table_in_catalog, }; use crate::transaction::{ApplyTransactionAction, Transaction}; + /// Read back the manifest entry for `path` from the manifests of `snapshot`. + async fn find_entry( + table: &Table, + snapshot: &SnapshotRef, + path: &str, + ) -> Option { + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + for manifest_file in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + if let Some(entry) = manifest.entries().iter().find(|e| e.file_path() == path) { + return Some(entry.clone()); + } + } + None + } + /// E2E: Compact 3 small files into 1 merged file. /// Verify: operation=Replace, file counts, record counts. #[tokio::test] @@ -168,8 +198,7 @@ mod tests { let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); let mut live_files: Vec = Vec::new(); for manifest_entry in manifest_list.entries() { - let manifest = table.manifest_reader().read(manifest_entry).await.unwrap() -; + let manifest = table.manifest_reader().read(manifest_entry).await.unwrap(); for entry in manifest.entries() { if entry.is_alive() { live_files.push(entry.file_path().to_string()); @@ -265,22 +294,20 @@ mod tests { // Verify live files: f3 (surviving) + merged (new). let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); let mut live_files: Vec = Vec::new(); - let mut survivor = None; for manifest_entry in manifest_list.entries() { let manifest = table.manifest_reader().read(manifest_entry).await.unwrap(); for entry in manifest.entries() { if entry.is_alive() { live_files.push(entry.file_path().to_string()); } - if entry.file_path() == "test/3.parquet" { - survivor = Some(entry.clone()); - } } } live_files.sort(); assert_eq!(live_files, vec!["test/3.parquet", "test/merged.parquet"]); - let survivor = survivor.expect("surviving file should still be in a manifest"); + let survivor = find_entry(&table, snapshot, "test/3.parquet") + .await + .expect("surviving file should still be in a manifest"); assert_eq!(survivor.status(), ManifestStatus::Existing); assert_eq!(survivor.snapshot_id(), Some(append_snapshot.snapshot_id())); assert_eq!( @@ -293,6 +320,71 @@ mod tests { ); } + /// An explicit data sequence number is recorded on the added file instead + /// of the one it would inherit from the rewrite snapshot. + #[tokio::test] + async fn test_rewrite_files_applies_data_sequence_number() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + let append_sequence_number = table + .metadata() + .current_snapshot() + .unwrap() + .sequence_number(); + + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(f1) + .add_file(merged) + .data_sequence_number(append_sequence_number); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_ne!(snapshot.sequence_number(), append_sequence_number); + + let merged_entry = find_entry(&table, snapshot, "test/merged.parquet") + .await + .expect("added file should be in a manifest"); + assert_eq!(merged_entry.sequence_number(), Some(append_sequence_number)); + } + + /// A data sequence number above the one the new snapshot will carry is + /// rejected. + #[tokio::test] + async fn test_rewrite_files_rejects_data_sequence_number_above_the_snapshot() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + let snapshot_sequence_number = table.metadata().next_sequence_number(); + + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(f1) + .add_file(merged) + .data_sequence_number(snapshot_sequence_number + 1); + let tx = action.apply(tx).unwrap(); + let err = tx.commit(&catalog).await.unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains(&format!( + "Data sequence number {} is greater than the snapshot's {snapshot_sequence_number}.", + snapshot_sequence_number + 1 + )), + "{err}" + ); + } + /// Rewrite on an empty table (no snapshot) should fail because /// the delete target doesn't exist. #[tokio::test] From 291b9b6813f79c96cac14ccec7715ea0a8c4fae6 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:33:23 +0200 Subject: [PATCH 09/28] Run the producer validations before committing a rewrite `commit_snapshot` handed `SnapshotProducer` an empty added-file list and went straight to `commit_with_manifests`, so neither `validate_added_data_files` nor `validate_duplicate_files` ever saw the files a rewrite adds. A file carrying a partition value the default spec cannot hold, or a path that is already live, was written into a manifest and committed. Pass the added files to the producer and run both validations first, and drop `write_added_manifest`'s own content-type check, which `validate_added_data_files` now makes unreachable. --- crates/iceberg/src/transaction/merging.rs | 22 +++++----- crates/iceberg/src/transaction/rewrite.rs | 51 ++++++++++++++++++++++- 2 files changed, 61 insertions(+), 12 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index 8759c7e0c7..f3f38a6f7b 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -30,9 +30,9 @@ use uuid::Uuid; use crate::error::Result; use crate::io::OutputFile; use crate::spec::{ - DataContentType, DataFile, DataFileFormat, FormatVersion, ManifestContentType, ManifestEntry, - ManifestFile, ManifestStatus, ManifestWriter, ManifestWriterBuilder, Operation, PartitionSpec, - SchemaRef, SnapshotSummaryCollector, Summary, update_snapshot_summaries, + DataFile, DataFileFormat, FormatVersion, ManifestContentType, ManifestEntry, ManifestFile, + ManifestStatus, ManifestWriter, ManifestWriterBuilder, Operation, PartitionSpec, SchemaRef, + SnapshotSummaryCollector, Summary, update_snapshot_summaries, }; use crate::table::Table; use crate::transaction::ActionCommit; @@ -332,8 +332,14 @@ impl MergingSnapshotProducer { // Create the SnapshotProducer first so we can use its snapshot_id // for new manifests. This ensures the manifest list writer can // assign sequence numbers to manifests from this snapshot. - let snapshot_producer = - SnapshotProducer::new(table, self.commit_uuid, HashMap::new(), Vec::new()); + let snapshot_producer = SnapshotProducer::new( + table, + self.commit_uuid, + HashMap::new(), + self.added_data_files.clone(), + ); + snapshot_producer.validate_added_data_files()?; + snapshot_producer.validate_duplicate_files().await?; let snapshot_id = snapshot_producer.snapshot_id; let mut manifest_counter: RangeFrom = (0..); @@ -410,12 +416,6 @@ impl MergingSnapshotProducer { let format_version = table.metadata().format_version(); for data_file in &self.added_data_files { - if data_file.content_type() != DataContentType::Data { - return Err(Error::new( - ErrorKind::DataInvalid, - "Only data content type is allowed in added_data_files", - )); - } if let Some(sequence_number) = self.data_sequence_number { writer.add_file(data_file.clone(), sequence_number)?; continue; diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 9f4339c42f..a0f89e6094 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -126,7 +126,7 @@ impl TransactionAction for RewriteFilesAction { mod tests { use crate::ErrorKind; use crate::memory::tests::new_memory_catalog; - use crate::spec::{ManifestEntryRef, ManifestStatus, Operation, SnapshotRef}; + use crate::spec::{Literal, ManifestEntryRef, ManifestStatus, Operation, SnapshotRef, Struct}; use crate::table::Table; use crate::transaction::tests::{ append_files, make_data_file, make_v3_minimal_table_in_catalog, @@ -436,6 +436,55 @@ mod tests { ); } + /// An added file whose partition value does not fit the default spec is + /// rejected before anything is written. + #[tokio::test] + async fn test_rewrite_files_rejects_incompatible_partition_value() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + + let mut merged = make_data_file(&table, "test/merged.parquet", 10, 100); + merged.partition = Struct::from_iter([Some(Literal::string("not-a-long"))]); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(merged); + let tx = action.apply(tx).unwrap(); + let err = tx.commit(&catalog).await.unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("Partition value is not compatible partition type"), + "{err}" + ); + } + + /// An added file that is already live in the current snapshot is rejected. + #[tokio::test] + async fn test_rewrite_files_rejects_already_referenced_file() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone(), f2.clone()]).await; + + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(f2); + let tx = action.apply(tx).unwrap(); + let err = tx.commit(&catalog).await.unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains( + "Cannot add files that are already referenced by table, files: test/2.parquet" + ), + "{err}" + ); + } + /// Rewrite with no adds should fail validation. #[tokio::test] async fn test_rewrite_files_no_adds() { From d969ab3746d7983ed4eceec51e1c357acc6c690e Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:42:32 +0200 Subject: [PATCH 10/28] Keep manifest names unique across commit attempts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `commit_snapshot` created a fresh `(0..)` counter on every call while the commit uuid stayed fixed for the life of the action, so a retried commit rewrote `{uuid}-m0.avro` over the manifest the previous attempt had written under the same name. Java keeps `manifestCount` on the snapshot producer and never resets it. The counter cannot live on Rust's `SnapshotProducer`: that is built fresh inside each `commit_snapshot` call and so resets with it. The object that survives a retry is the action, which the transaction re-invokes through an `Arc`, so the counter goes on `MergingSnapshotProducer` as an `AtomicU64` — `commit` only has `&self`. `fast_append` is unaffected: it draws a new commit uuid per attempt unless the caller pins one. --- crates/iceberg/src/transaction/merging.rs | 20 +++++---- crates/iceberg/src/transaction/rewrite.rs | 55 ++++++++++++++++++++++- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index f3f38a6f7b..5cd25e0b5d 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -23,7 +23,7 @@ //! [`SnapshotProducer`]. use std::collections::{HashMap, HashSet}; -use std::ops::RangeFrom; +use std::sync::atomic::{AtomicU64, Ordering}; use uuid::Uuid; @@ -104,7 +104,7 @@ impl ManifestFilterManager { manifests: Vec, snapshot_id: i64, commit_uuid: Uuid, - manifest_counter: &mut RangeFrom, + manifest_counter: &AtomicU64, ) -> Result<(Vec, SnapshotSummaryCollector)> { if self.deleted_file_paths.is_empty() { return Ok((manifests, SnapshotSummaryCollector::default())); @@ -210,7 +210,7 @@ impl ManifestFilterManager { "{}/{}-m{}.{}", table.metadata().metadata_location()?, commit_uuid, - manifest_counter.next().unwrap(), + manifest_counter.fetch_add(1, Ordering::Relaxed), DataFileFormat::Avro, ); let output_file = table.file_io().new_output(new_manifest_path)?; @@ -274,6 +274,10 @@ pub(crate) struct MergingSnapshotProducer { filter_manager: ManifestFilterManager, commit_uuid: Uuid, data_sequence_number: Option, + /// Names the manifests this producer writes. It is never reset, so a + /// commit retry cannot overwrite a manifest an earlier attempt wrote + /// under the same commit uuid. + manifest_counter: AtomicU64, } impl MergingSnapshotProducer { @@ -285,6 +289,7 @@ impl MergingSnapshotProducer { filter_manager: ManifestFilterManager::new(true), commit_uuid: Uuid::now_v7(), data_sequence_number: None, + manifest_counter: AtomicU64::new(0), } } @@ -341,7 +346,6 @@ impl MergingSnapshotProducer { snapshot_producer.validate_added_data_files()?; snapshot_producer.validate_duplicate_files().await?; let snapshot_id = snapshot_producer.snapshot_id; - let mut manifest_counter: RangeFrom = (0..); // 1. Load existing manifests from the current snapshot. let existing_manifests = match table.metadata().current_snapshot() { @@ -367,14 +371,14 @@ impl MergingSnapshotProducer { existing_manifests, snapshot_id, self.commit_uuid, - &mut manifest_counter, + &self.manifest_counter, ) .await?; // 3. Write a new manifest for added files. if !self.added_data_files.is_empty() { let added_manifest = self - .write_added_manifest(table, snapshot_id, &mut manifest_counter) + .write_added_manifest(table, snapshot_id, &self.manifest_counter) .await?; filtered_manifests.push(added_manifest); } @@ -392,13 +396,13 @@ impl MergingSnapshotProducer { &self, table: &Table, snapshot_id: i64, - manifest_counter: &mut RangeFrom, + manifest_counter: &AtomicU64, ) -> Result { let new_manifest_path = format!( "{}/{}-m{}.{}", table.metadata().metadata_location()?, self.commit_uuid, - manifest_counter.next().unwrap(), + manifest_counter.fetch_add(1, Ordering::Relaxed), DataFileFormat::Avro, ); let output_file = table.file_io().new_output(new_manifest_path)?; diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index a0f89e6094..86186a98fe 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -124,7 +124,9 @@ impl TransactionAction for RewriteFilesAction { #[cfg(test)] mod tests { - use crate::ErrorKind; + use std::sync::Arc; + + use super::*; use crate::memory::tests::new_memory_catalog; use crate::spec::{Literal, ManifestEntryRef, ManifestStatus, Operation, SnapshotRef, Struct}; use crate::table::Table; @@ -132,6 +134,7 @@ mod tests { append_files, make_data_file, make_v3_minimal_table_in_catalog, }; use crate::transaction::{ApplyTransactionAction, Transaction}; + use crate::{ErrorKind, TableUpdate}; /// Read back the manifest entry for `path` from the manifests of `snapshot`. async fn find_entry( @@ -436,6 +439,56 @@ mod tests { ); } + /// A retried commit must not reuse the manifest paths of the attempt + /// before it: the commit uuid is fixed, so only the counter separates them. + #[tokio::test] + async fn test_rewrite_files_writes_distinct_manifests_per_attempt() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone(), f2.clone()]).await; + + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = Arc::new(tx.rewrite_files().delete_file(f1).add_file(merged)); + + let first = manifest_paths(&table, Arc::clone(&action)).await; + let second = manifest_paths(&table, action).await; + + assert_eq!(first.len(), 2, "{first:?}"); + assert_eq!(first.len(), second.len()); + for path in &first { + assert!(!second.contains(path), "{path} was written twice"); + } + } + + /// Commit `action` against `table` and return the paths of the manifests + /// the resulting snapshot points at. + async fn manifest_paths(table: &Table, action: Arc) -> Vec { + let mut commit = action.commit(table).await.unwrap(); + let snapshot = commit + .take_updates() + .into_iter() + .find_map(|update| match update { + TableUpdate::AddSnapshot { snapshot } => Some(snapshot), + _ => None, + }) + .expect("commit should add a snapshot"); + + let manifest_list = table + .manifest_list_reader(&Arc::new(snapshot)) + .load() + .await + .unwrap(); + manifest_list + .entries() + .iter() + .map(|entry| entry.manifest_path.clone()) + .collect() + } + /// An added file whose partition value does not fit the default spec is /// rejected before anything is written. #[tokio::test] From 82ad3ae5ffcade761fab4b20e4f64c7ab39c04db Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:54:11 +0200 Subject: [PATCH 11/28] Drop manifests an earlier snapshot left all-deleted The filter carried a manifest forward on `has_deleted_files()`, so a manifest whose entries are all Deleted stayed in every following snapshot for the rest of the table's life. Java keeps an existing manifest on `hasAddedFiles() || hasExistingFiles()`; its third arm matches the snapshot being produced, which no manifest read from the current snapshot can carry. The manifests this commit rewrites are produced by the filter pass and kept on their own. --- crates/iceberg/src/transaction/merging.rs | 5 +- crates/iceberg/src/transaction/rewrite.rs | 123 +++++++++++++++++++++- 2 files changed, 124 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index 5cd25e0b5d..3b4363a409 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -354,9 +354,8 @@ impl MergingSnapshotProducer { manifest_list .entries() .iter() - .filter(|e| { - e.has_added_files() || e.has_existing_files() || e.has_deleted_files() - }) + // Java's third arm is the new snapshot's id, which no existing manifest has. + .filter(|e| e.has_added_files() || e.has_existing_files()) .cloned() .collect() } diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 86186a98fe..7348fdf4d6 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -126,9 +126,14 @@ impl TransactionAction for RewriteFilesAction { mod tests { use std::sync::Arc; + use uuid::Uuid; + use super::*; use crate::memory::tests::new_memory_catalog; - use crate::spec::{Literal, ManifestEntryRef, ManifestStatus, Operation, SnapshotRef, Struct}; + use crate::spec::{ + Literal, ManifestEntryRef, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, + Operation, SnapshotRef, Struct, + }; use crate::table::Table; use crate::transaction::tests::{ append_files, make_data_file, make_v3_minimal_table_in_catalog, @@ -439,6 +444,122 @@ mod tests { ); } + /// A manifest left with only deleted entries records nothing live, whether + /// the current snapshot or an older one left it that way. + #[tokio::test] + async fn test_rewrite_files_drops_all_deleted_manifests() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + let table = append_files(&catalog, &table, vec![f2]).await; + let current_snapshot_id = table.metadata().current_snapshot().unwrap().snapshot_id(); + let earlier_snapshot_id = table + .metadata() + .current_snapshot() + .unwrap() + .parent_snapshot_id() + .unwrap(); + let older = add_delete_only_manifest(&table, earlier_snapshot_id).await; + let current = add_delete_only_manifest(&table, current_snapshot_id).await; + + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(merged); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + for path in [older, current] { + assert!( + !manifest_list + .entries() + .iter() + .any(|m| m.manifest_path == path), + "{path} holds no live entry and should have been dropped" + ); + } + } + + /// Add a delete-only manifest, attributed to `added_snapshot_id`, to the + /// manifest list of the table's current snapshot, and return its path. + async fn add_delete_only_manifest(table: &Table, added_snapshot_id: i64) -> String { + let snapshot = table.metadata().current_snapshot().unwrap(); + let added_sequence_number = table + .metadata() + .snapshot_by_id(added_snapshot_id) + .unwrap() + .sequence_number(); + let output = table + .file_io() + .new_output(format!( + "{}/delete-only-{}.avro", + table.metadata().metadata_location().unwrap(), + Uuid::new_v4() + )) + .unwrap(); + let mut writer = ManifestWriterBuilder::new( + output, + Some(added_snapshot_id), + table.metadata().current_schema().clone(), + table.metadata().default_partition_spec().as_ref().clone(), + ) + .build_v3_data(); + writer + .add_delete_file( + make_data_file( + table, + &format!("test/removed-{added_snapshot_id}.parquet"), + 10, + 100, + ), + added_sequence_number, + Some(added_sequence_number), + ) + .unwrap(); + let mut delete_manifest = writer.write_manifest_file().await.unwrap(); + // The manifest list writer only assigns sequence numbers to manifests of + // the snapshot being written; this one belongs to an earlier snapshot. + delete_manifest.sequence_number = added_sequence_number; + delete_manifest.min_sequence_number = added_sequence_number; + let delete_manifest_path = delete_manifest.manifest_path.clone(); + assert!(!delete_manifest.has_added_files()); + assert!(!delete_manifest.has_existing_files()); + + let mut entries = table + .manifest_list_reader(snapshot) + .load() + .await + .unwrap() + .consume_entries() + .into_iter() + .collect::>(); + entries.push(delete_manifest); + + let mut manifest_list_writer = ManifestListWriter::v3( + table + .file_io() + .new_output(snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(), + snapshot.snapshot_id(), + snapshot.parent_snapshot_id(), + snapshot.sequence_number(), + Some(table.metadata().next_row_id()), + ); + manifest_list_writer + .add_manifests(entries.into_iter()) + .unwrap(); + manifest_list_writer.close().await.unwrap(); + + delete_manifest_path + } + /// A retried commit must not reuse the manifest paths of the attempt /// before it: the commit uuid is fixed, so only the counter separates them. #[tokio::test] From ddafede1a48c65b7cd80e56a1efb997232879d94 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:59:07 +0200 Subject: [PATCH 12/28] Refuse a rewrite when deletes landed after its starting snapshot `starting_snapshot_id` was recorded and never read. A rewrite planned against one snapshot and committed against a later one silently drops any delete written in between: the replacement file keeps the deleted rows. Java resolves this per file in `validateNoNewDeletesForDataFiles`, which needs a delete-file index this crate does not have yet. Until then the commit fails closed when the current snapshot carries a delete manifest with a sequence number above the starting snapshot's. --- crates/iceberg/src/transaction/rewrite.rs | 120 ++++++++++++++++++++-- 1 file changed, 111 insertions(+), 9 deletions(-) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 7348fdf4d6..9d264a4aa0 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -29,7 +29,7 @@ use std::sync::Arc; use async_trait::async_trait; use crate::error::Result; -use crate::spec::{DataFile, Operation}; +use crate::spec::{DataFile, ManifestContentType, Operation}; use crate::table::Table; use crate::transaction::merging::MergingSnapshotProducer; use crate::transaction::{ActionCommit, TransactionAction}; @@ -56,7 +56,6 @@ pub struct RewriteFilesAction { producer: MergingSnapshotProducer, /// The snapshot ID at which this rewrite started reading. Used to /// detect conflicting deletes added after this point. - #[allow(dead_code)] // Will be used for conflict detection in a follow-up PR. starting_snapshot_id: Option, } @@ -112,12 +111,49 @@ impl RewriteFilesAction { } Ok(()) } + + /// Reject the rewrite when a delete manifest was added after the snapshot + /// it was planned against. + /// + /// Java re-checks those deletes file by file in + /// `validateNoNewDeletesForDataFiles`. Until that exists here the commit + /// fails closed, because a rewritten file carries the deletes of the files + /// it replaces only if nothing was deleted from them in the meantime. + async fn validate_no_new_deletes(&self, table: &Table) -> Result<()> { + let (Some(starting_snapshot_id), Some(current_snapshot)) = + (self.starting_snapshot_id, table.metadata().current_snapshot()) + else { + return Ok(()); + }; + let starting_sequence_number = table + .metadata() + .snapshot_by_id(starting_snapshot_id) + .map_or(0, |snapshot| snapshot.sequence_number()); + + let manifest_list = table.manifest_list_reader(current_snapshot).load().await?; + let conflict = manifest_list.entries().iter().find(|entry| { + entry.content == ManifestContentType::Deletes + && entry.sequence_number > starting_sequence_number + }); + + match conflict { + Some(entry) => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Cannot rewrite files: delete manifest {} was added after the starting snapshot {starting_snapshot_id}.", + entry.manifest_path, + ), + )), + None => Ok(()), + } + } } #[async_trait] impl TransactionAction for RewriteFilesAction { async fn commit(self: Arc, table: &Table) -> Result { self.validate()?; + self.validate_no_new_deletes(table).await?; self.producer.commit_snapshot(table).await } } @@ -131,8 +167,8 @@ mod tests { use super::*; use crate::memory::tests::new_memory_catalog; use crate::spec::{ - Literal, ManifestEntryRef, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, - Operation, SnapshotRef, Struct, + DataContentType, Literal, ManifestEntryRef, ManifestFile, ManifestListWriter, + ManifestStatus, ManifestWriterBuilder, Operation, SnapshotRef, Struct, }; use crate::table::Table; use crate::transaction::tests::{ @@ -487,7 +523,6 @@ mod tests { /// Add a delete-only manifest, attributed to `added_snapshot_id`, to the /// manifest list of the table's current snapshot, and return its path. async fn add_delete_only_manifest(table: &Table, added_snapshot_id: i64) -> String { - let snapshot = table.metadata().current_snapshot().unwrap(); let added_sequence_number = table .metadata() .snapshot_by_id(added_snapshot_id) @@ -529,15 +564,54 @@ mod tests { assert!(!delete_manifest.has_added_files()); assert!(!delete_manifest.has_existing_files()); - let mut entries = table + append_to_manifest_list(table, delete_manifest).await; + delete_manifest_path + } + + /// Add a delete manifest, attributed to the table's current snapshot, to + /// that snapshot's manifest list. + async fn add_delete_manifest(table: &Table) -> String { + let snapshot = table.metadata().current_snapshot().unwrap(); + let output = table + .file_io() + .new_output(format!( + "{}/deletes-{}.avro", + table.metadata().metadata_location().unwrap(), + Uuid::new_v4() + )) + .unwrap(); + let mut writer = ManifestWriterBuilder::new( + output, + Some(snapshot.snapshot_id()), + table.metadata().current_schema().clone(), + table.metadata().default_partition_spec().as_ref().clone(), + ) + .build_v3_deletes(); + let mut delete_file = make_data_file(table, "test/1-deletes.parquet", 1, 100); + delete_file.content = DataContentType::PositionDeletes; + writer + .add_file(delete_file, snapshot.sequence_number()) + .unwrap(); + let delete_manifest = writer.write_manifest_file().await.unwrap(); + let delete_manifest_path = delete_manifest.manifest_path.clone(); + + append_to_manifest_list(table, delete_manifest).await; + delete_manifest_path + } + + /// Rewrite the manifest list of the table's current snapshot so it also + /// contains `manifest`. + async fn append_to_manifest_list(table: &Table, manifest: ManifestFile) { + let snapshot = table.metadata().current_snapshot().unwrap(); + let mut entries: Vec = table .manifest_list_reader(snapshot) .load() .await .unwrap() .consume_entries() .into_iter() - .collect::>(); - entries.push(delete_manifest); + .collect(); + entries.push(manifest); let mut manifest_list_writer = ManifestListWriter::v3( table @@ -556,8 +630,36 @@ mod tests { .add_manifests(entries.into_iter()) .unwrap(); manifest_list_writer.close().await.unwrap(); + } - delete_manifest_path + /// A delete manifest added after the snapshot the rewrite was planned + /// against must stop the commit. + #[tokio::test] + async fn test_rewrite_files_rejects_deletes_added_after_starting_snapshot() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + + // Plan the rewrite against this snapshot, then let another commit land. + let tx = Transaction::new(&table); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let action = Arc::new(tx.rewrite_files().delete_file(f1).add_file(merged)); + + let table = append_files(&catalog, &table, vec![f2]).await; + let delete_manifest_path = add_delete_manifest(&table).await; + + let Err(err) = action.commit(&table).await else { + panic!("a delete manifest newer than the starting snapshot must be rejected"); + }; + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains(&format!("delete manifest {delete_manifest_path} was added after")), + "{err}" + ); } /// A retried commit must not reuse the manifest paths of the attempt From 5f81639c91e06e48e79cc47666fe6ccc906f1000 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 18:01:53 +0200 Subject: [PATCH 13/28] Run the compaction and partial rewrites against a V2 table Every rewrite test used the V3 fixture, so nothing covered the V2 manifest and entry layout the action also writes. Both cases now run on V2 as well, through a V2 catalog fixture alongside the V3 one. --- crates/iceberg/src/transaction/mod.rs | 26 ++++++++++++++++-- crates/iceberg/src/transaction/rewrite.rs | 32 +++++++++++++++++++---- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 3fe1b161e7..12242cb364 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -359,7 +359,29 @@ mod tests { .unwrap() } + pub(crate) async fn make_v2_minimal_table_in_catalog(catalog: &impl Catalog) -> Table { + make_minimal_table_in_catalog( + catalog, + "TableMetadataV2ValidMinimal.json", + crate::spec::FormatVersion::V2, + ) + .await + } + pub(crate) async fn make_v3_minimal_table_in_catalog(catalog: &impl Catalog) -> Table { + make_minimal_table_in_catalog( + catalog, + "TableMetadataV3ValidMinimal.json", + crate::spec::FormatVersion::V3, + ) + .await + } + + async fn make_minimal_table_in_catalog( + catalog: &impl Catalog, + metadata_file: &str, + format_version: crate::spec::FormatVersion, + ) -> Table { let table_ident = TableIdent::from_strs([format!("ns1-{}", uuid::Uuid::new_v4()), "test1".to_string()]) .unwrap(); @@ -372,7 +394,7 @@ mod tests { let file = File::open(format!( "{}/testdata/table_metadata/{}", env!("CARGO_MANIFEST_DIR"), - "TableMetadataV3ValidMinimal.json" + metadata_file )) .unwrap(); let reader = BufReader::new(file); @@ -383,7 +405,7 @@ mod tests { .partition_spec((**base_metadata.default_partition_spec()).clone()) .sort_order((**base_metadata.default_sort_order()).clone()) .name(table_ident.name().to_string()) - .format_version(crate::spec::FormatVersion::V3) + .format_version(format_version) .build(); catalog diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 9d264a4aa0..6a509f075f 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -171,8 +171,10 @@ mod tests { ManifestStatus, ManifestWriterBuilder, Operation, SnapshotRef, Struct, }; use crate::table::Table; + use crate::Catalog; use crate::transaction::tests::{ - append_files, make_data_file, make_v3_minimal_table_in_catalog, + append_files, make_data_file, make_v2_minimal_table_in_catalog, + make_v3_minimal_table_in_catalog, }; use crate::transaction::{ApplyTransactionAction, Transaction}; use crate::{ErrorKind, TableUpdate}; @@ -199,12 +201,22 @@ mod tests { async fn test_rewrite_files_compaction() { let catalog = new_memory_catalog().await; let table = make_v3_minimal_table_in_catalog(&catalog).await; + rewrite_files_compaction(&catalog, table).await; + } + + #[tokio::test] + async fn test_rewrite_files_compaction_on_v2_table() { + let catalog = new_memory_catalog().await; + let table = make_v2_minimal_table_in_catalog(&catalog).await; + rewrite_files_compaction(&catalog, table).await; + } + async fn rewrite_files_compaction(catalog: &impl Catalog, table: Table) { // Append 3 small files (10 records each). let f1 = make_data_file(&table, "test/1.parquet", 10, 100); let f2 = make_data_file(&table, "test/2.parquet", 10, 100); let f3 = make_data_file(&table, "test/3.parquet", 10, 100); - let table = append_files(&catalog, &table, vec![f1.clone(), f2.clone(), f3.clone()]).await; + let table = append_files(catalog, &table, vec![f1.clone(), f2.clone(), f3.clone()]).await; // Verify pre-compaction state. let summary = &table @@ -226,7 +238,7 @@ mod tests { .delete_file(f3) .add_file(merged); let tx = action.apply(tx).unwrap(); - let table = tx.commit(&catalog).await.unwrap(); + let table = tx.commit(catalog).await.unwrap(); // Verify post-compaction snapshot. let snapshot = table.metadata().current_snapshot().unwrap(); @@ -308,11 +320,21 @@ mod tests { async fn test_rewrite_files_partial() { let catalog = new_memory_catalog().await; let table = make_v3_minimal_table_in_catalog(&catalog).await; + rewrite_files_partial(&catalog, table).await; + } + + #[tokio::test] + async fn test_rewrite_files_partial_on_v2_table() { + let catalog = new_memory_catalog().await; + let table = make_v2_minimal_table_in_catalog(&catalog).await; + rewrite_files_partial(&catalog, table).await; + } + async fn rewrite_files_partial(catalog: &impl Catalog, table: Table) { let f1 = make_data_file(&table, "test/1.parquet", 10, 100); let f2 = make_data_file(&table, "test/2.parquet", 10, 100); let f3 = make_data_file(&table, "test/3.parquet", 10, 100); - let table = append_files(&catalog, &table, vec![f1.clone(), f2.clone(), f3.clone()]).await; + let table = append_files(catalog, &table, vec![f1.clone(), f2.clone(), f3.clone()]).await; let append_snapshot = table.metadata().current_snapshot().unwrap().clone(); // Rewrite only f1 and f2, keep f3. @@ -324,7 +346,7 @@ mod tests { .delete_file(f2) .add_file(merged); let tx = action.apply(tx).unwrap(); - let table = tx.commit(&catalog).await.unwrap(); + let table = tx.commit(catalog).await.unwrap(); let snapshot = table.metadata().current_snapshot().unwrap(); assert_eq!(snapshot.summary().operation, Operation::Replace); From 021c3b3407ca5f18f82ce49ab75a57f1ed94fe22 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 18:11:13 +0200 Subject: [PATCH 14/28] Format the new code and record the rewrite action in the public API `cargo fmt` over the two new modules, and `public-api.txt` regenerated for `Transaction::rewrite_files`. The clippy warning on the unnecessary parentheses around the manifest counter went with the counter itself. --- crates/iceberg/public-api.txt | 1 + crates/iceberg/src/transaction/rewrite.rs | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 3d9abf2275..960672689e 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3235,6 +3235,7 @@ pub fn iceberg::transaction::Transaction::expire_snapshots(&self) -> iceberg::tr pub fn iceberg::transaction::Transaction::fast_append(&self) -> iceberg::transaction::append::FastAppendAction pub fn iceberg::transaction::Transaction::new(table: &iceberg::table::Table) -> Self pub fn iceberg::transaction::Transaction::replace_sort_order(&self) -> iceberg::transaction::sort_order::ReplaceSortOrderAction +pub fn iceberg::transaction::Transaction::rewrite_files(&self) -> iceberg::transaction::rewrite::RewriteFilesAction pub fn iceberg::transaction::Transaction::update_location(&self) -> iceberg::transaction::update_location::UpdateLocationAction pub fn iceberg::transaction::Transaction::update_schema(&self) -> iceberg::transaction::update_schema::UpdateSchemaAction pub fn iceberg::transaction::Transaction::update_statistics(&self) -> iceberg::transaction::update_statistics::UpdateStatisticsAction diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 6a509f075f..5c9c7104e6 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -120,9 +120,10 @@ impl RewriteFilesAction { /// fails closed, because a rewritten file carries the deletes of the files /// it replaces only if nothing was deleted from them in the meantime. async fn validate_no_new_deletes(&self, table: &Table) -> Result<()> { - let (Some(starting_snapshot_id), Some(current_snapshot)) = - (self.starting_snapshot_id, table.metadata().current_snapshot()) - else { + let (Some(starting_snapshot_id), Some(current_snapshot)) = ( + self.starting_snapshot_id, + table.metadata().current_snapshot(), + ) else { return Ok(()); }; let starting_sequence_number = table @@ -171,13 +172,12 @@ mod tests { ManifestStatus, ManifestWriterBuilder, Operation, SnapshotRef, Struct, }; use crate::table::Table; - use crate::Catalog; use crate::transaction::tests::{ append_files, make_data_file, make_v2_minimal_table_in_catalog, make_v3_minimal_table_in_catalog, }; use crate::transaction::{ApplyTransactionAction, Transaction}; - use crate::{ErrorKind, TableUpdate}; + use crate::{Catalog, ErrorKind, TableUpdate}; /// Read back the manifest entry for `path` from the manifests of `snapshot`. async fn find_entry( @@ -678,8 +678,9 @@ mod tests { }; assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!( - err.to_string() - .contains(&format!("delete manifest {delete_manifest_path} was added after")), + err.to_string().contains(&format!( + "delete manifest {delete_manifest_path} was added after" + )), "{err}" ); } From 7d630619eca4f9f47547cc3c01a49792f581ecbb Mon Sep 17 00:00:00 2001 From: hotcache Date: Mon, 28 Sep 2026 12:28:51 +0000 Subject: [PATCH 15/28] test(transaction): run the rewrite tests against a V1 table V1 manifest entries carry no sequence numbers, and `ManifestEntryV1` deserialization fills both in as 0. Nothing in the rewrite path treats that as missing, so a V1 table compacts like V2 and V3 and the surviving entries keep Existing status, their snapshot id and their zero sequence numbers. Cover it so it stays that way. --- crates/iceberg/src/transaction/mod.rs | 9 +++++++++ crates/iceberg/src/transaction/rewrite.rs | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index d13ccd080d..da80cf4167 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -359,6 +359,15 @@ mod tests { .unwrap() } + pub(crate) async fn make_v1_minimal_table_in_catalog(catalog: &impl Catalog) -> Table { + make_minimal_table_in_catalog( + catalog, + "TableMetadataV1Valid.json", + crate::spec::FormatVersion::V1, + ) + .await + } + pub(crate) async fn make_v2_minimal_table_in_catalog(catalog: &impl Catalog) -> Table { make_minimal_table_in_catalog( catalog, diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 5c9c7104e6..c76e0d0ceb 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -173,8 +173,8 @@ mod tests { }; use crate::table::Table; use crate::transaction::tests::{ - append_files, make_data_file, make_v2_minimal_table_in_catalog, - make_v3_minimal_table_in_catalog, + append_files, make_data_file, make_v1_minimal_table_in_catalog, + make_v2_minimal_table_in_catalog, make_v3_minimal_table_in_catalog, }; use crate::transaction::{ApplyTransactionAction, Transaction}; use crate::{Catalog, ErrorKind, TableUpdate}; @@ -204,6 +204,13 @@ mod tests { rewrite_files_compaction(&catalog, table).await; } + #[tokio::test] + async fn test_rewrite_files_compaction_on_v1_table() { + let catalog = new_memory_catalog().await; + let table = make_v1_minimal_table_in_catalog(&catalog).await; + rewrite_files_compaction(&catalog, table).await; + } + #[tokio::test] async fn test_rewrite_files_compaction_on_v2_table() { let catalog = new_memory_catalog().await; @@ -323,6 +330,13 @@ mod tests { rewrite_files_partial(&catalog, table).await; } + #[tokio::test] + async fn test_rewrite_files_partial_on_v1_table() { + let catalog = new_memory_catalog().await; + let table = make_v1_minimal_table_in_catalog(&catalog).await; + rewrite_files_partial(&catalog, table).await; + } + #[tokio::test] async fn test_rewrite_files_partial_on_v2_table() { let catalog = new_memory_catalog().await; From 318758c87ebd3eee5f218d39680c289b77f4d1fb Mon Sep 17 00:00:00 2001 From: hotcache Date: Mon, 28 Sep 2026 12:33:52 +0000 Subject: [PATCH 16/28] fix(transaction): name the file in the added-file content type error `validate_added_data_files` is shared by fast append and rewrite, but its content-type error said "for fast append", so a rewrite that was handed a delete file reported the wrong operation and not which file was wrong. Drop the operation from the message and name the file and its content type instead. --- crates/iceberg/src/transaction/rewrite.rs | 26 ++++++++++++++++++++++ crates/iceberg/src/transaction/snapshot.rs | 4 +++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index c76e0d0ceb..436be2fa2b 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -774,6 +774,32 @@ mod tests { ); } + /// A delete file handed to `add_file` is rejected: a rewrite replaces data + /// files, and the producer validation names the offending file. + #[tokio::test] + async fn test_rewrite_files_rejects_non_data_content_type() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + + let mut deletes = make_data_file(&table, "test/1-deletes.parquet", 1, 100); + deletes.content = DataContentType::PositionDeletes; + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(deletes); + let tx = action.apply(tx).unwrap(); + let err = tx.commit(&catalog).await.unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains( + "Only data content type is allowed in added data files, but test/1-deletes.parquet is PositionDeletes" + ), + "{err}" + ); + } + /// An added file that is already live in the current snapshot is rejected. #[tokio::test] async fn test_rewrite_files_rejects_already_referenced_file() { diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index aa5c779f43..2918e31b77 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -140,7 +140,9 @@ impl<'a> SnapshotProducer<'a> { for data_file in &self.added_data_files { if data_file.content_type() != crate::spec::DataContentType::Data { return Err(invalid_data!( - "Only data content type is allowed for fast append" + "Only data content type is allowed in added data files, but {} is {:?}", + data_file.file_path, + data_file.content_type() )); } // Check if the data file partition spec id matches the table default partition spec id. From 1af9f1d27fe1067cebf9ce4d606c6e9a7efa63c9 Mon Sep 17 00:00:00 2001 From: hotcache Date: Mon, 28 Sep 2026 12:39:59 +0000 Subject: [PATCH 17/28] fix(transaction): fail a rewrite whose starting snapshot is gone `validate_no_new_deletes` fell back to sequence number 0 when the starting snapshot was no longer in the table, which let a rewrite commit when the table held no delete manifest at all and otherwise blamed an unrelated delete manifest for the rejection. An expired starting snapshot means no delete can be ruled out, so say that instead. --- crates/iceberg/src/transaction/rewrite.rs | 60 +++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 436be2fa2b..c15ace74f3 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -126,10 +126,17 @@ impl RewriteFilesAction { ) else { return Ok(()); }; - let starting_sequence_number = table - .metadata() - .snapshot_by_id(starting_snapshot_id) - .map_or(0, |snapshot| snapshot.sequence_number()); + let Some(starting_snapshot) = table.metadata().snapshot_by_id(starting_snapshot_id) else { + // Without the starting snapshot there is no sequence number to + // compare against, so no delete can be ruled out. + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Cannot rewrite files: the starting snapshot {starting_snapshot_id} is no longer in the table, so deletes added since it cannot be ruled out.", + ), + )); + }; + let starting_sequence_number = starting_snapshot.sequence_number(); let manifest_list = table.manifest_list_reader(current_snapshot).load().await?; let conflict = manifest_list.entries().iter().find(|entry| { @@ -699,6 +706,51 @@ mod tests { ); } + /// A rewrite whose starting snapshot has been expired cannot rule out + /// deletes added since, and says so rather than comparing against zero. + #[tokio::test] + async fn test_rewrite_files_rejects_expired_starting_snapshot() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + let starting_snapshot_id = table.metadata().current_snapshot().unwrap().snapshot_id(); + + // Plan the rewrite against this snapshot, then expire it behind us. + let tx = Transaction::new(&table); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let action = Arc::new(tx.rewrite_files().delete_file(f1).add_file(merged)); + + let table = append_files(&catalog, &table, vec![f2]).await; + let tx = Transaction::new(&table); + let tx = tx + .expire_snapshots() + .expire_snapshot_ids([starting_snapshot_id]) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + assert!( + table + .metadata() + .snapshot_by_id(starting_snapshot_id) + .is_none(), + "the starting snapshot should have been expired" + ); + + let Err(err) = action.commit(&table).await else { + panic!("an expired starting snapshot must be rejected"); + }; + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains(&format!( + "the starting snapshot {starting_snapshot_id} is no longer in the table" + )), + "{err}" + ); + } + /// A retried commit must not reuse the manifest paths of the attempt /// before it: the commit uuid is fixed, so only the counter separates them. #[tokio::test] From 9c9f1edbb819c3f7e3a77350c46ccd7f39bcceae Mon Sep 17 00:00:00 2001 From: hotcache Date: Mon, 28 Sep 2026 12:42:59 +0000 Subject: [PATCH 18/28] fix(transaction): fail a rewrite planned before the table had a snapshot `validate_no_new_deletes` skipped the check whenever either the starting snapshot id or the current snapshot was absent. A missing starting snapshot id means the table held nothing when the rewrite was planned, so everything it holds at commit time, delete files included, arrived after that -- the case the check is least able to skip. Reject it, and keep returning early only when the table still has no snapshot, where the filter rejects the rewrite for its missing sources anyway. --- crates/iceberg/src/transaction/rewrite.rs | 42 ++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index c15ace74f3..6ad74e40df 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -120,12 +120,19 @@ impl RewriteFilesAction { /// fails closed, because a rewritten file carries the deletes of the files /// it replaces only if nothing was deleted from them in the meantime. async fn validate_no_new_deletes(&self, table: &Table) -> Result<()> { - let (Some(starting_snapshot_id), Some(current_snapshot)) = ( - self.starting_snapshot_id, - table.metadata().current_snapshot(), - ) else { + // Nothing has been committed, so there is nothing to conflict with. + // The filter rejects the rewrite for its missing sources instead. + let Some(current_snapshot) = table.metadata().current_snapshot() else { return Ok(()); }; + // The rewrite was planned against a table without a snapshot, so every + // snapshot it has now, deletes included, landed after that. + let Some(starting_snapshot_id) = self.starting_snapshot_id else { + return Err(Error::new( + ErrorKind::DataInvalid, + "Cannot rewrite files: the rewrite was planned against a table with no snapshot, and the table has been committed to since.", + )); + }; let Some(starting_snapshot) = table.metadata().snapshot_by_id(starting_snapshot_id) else { // Without the starting snapshot there is no sequence number to // compare against, so no delete can be ruled out. @@ -706,6 +713,33 @@ mod tests { ); } + /// A rewrite planned against a table with no snapshot knows nothing about + /// what the table holds once something is committed to it. + #[tokio::test] + async fn test_rewrite_files_rejects_commit_after_empty_starting_table() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + // Planned before the table had any snapshot. + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = Arc::new(tx.rewrite_files().delete_file(f1.clone()).add_file(merged)); + + // The file it wants to replace only exists because of a later commit. + let table = append_files(&catalog, &table, vec![f1]).await; + + let Err(err) = action.commit(&table).await else { + panic!("a rewrite planned against an empty table must be rejected"); + }; + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("planned against a table with no snapshot"), + "{err}" + ); + } + /// A rewrite whose starting snapshot has been expired cannot rule out /// deletes added since, and says so rather than comparing against zero. #[tokio::test] From 544a91f9e7242b4a2ca4d339a67403852ccdc2f3 Mon Sep 17 00:00:00 2001 From: hotcache Date: Mon, 28 Sep 2026 12:46:28 +0000 Subject: [PATCH 19/28] docs(transaction): describe when a rewrite refuses to commit The conflict check is table-wide and always on, so a compaction fails on a delete committed anywhere in the table while it ran, not only on the files it replaces. Say so on the action, along with the two other cases that cannot rule out a delete and what the check does not cover, so a caller can tell an expected rejection from a bug. Also stop the public docs linking to the private MergingSnapshotProducer, which rustdoc drops on the generated page. --- crates/iceberg/src/transaction/rewrite.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 6ad74e40df..1e9a5169d0 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -38,7 +38,7 @@ use crate::{Error, ErrorKind}; /// A transaction action that rewrites (replaces) data files. /// /// This is the Rust equivalent of Java's `BaseRewriteFiles`. It uses -/// [`MergingSnapshotProducer`] to handle manifest filtering and creation, +/// `MergingSnapshotProducer` to handle manifest filtering and creation, /// and commits a snapshot with [`Operation::Replace`]. /// /// # Example @@ -52,10 +52,27 @@ use crate::{Error, ErrorKind}; /// let tx = action.apply(tx)?; /// let table = tx.commit(&catalog).await?; /// ``` +/// +/// # Concurrent deletes +/// +/// The action records the table's current snapshot when it is created, and +/// refuses to commit if the table gained a delete manifest after it. Java +/// narrows that to the files being replaced in +/// `validateNoNewDeletesForDataFiles`; until that exists here the check is +/// table-wide, so a delete committed to an unrelated partition while the +/// rewrite ran also fails the commit with [`ErrorKind::DataInvalid`]. So do a +/// rewrite planned against a table that had no snapshot yet, and one whose +/// starting snapshot has since been expired: neither can rule out a delete. +/// Replan the rewrite against the current table and run it again. +/// +/// Deletes that were already committed when the rewrite was planned are not +/// covered by this check. Either apply them while rewriting, or keep them +/// applicable to the new file with +/// [`data_sequence_number`](RewriteFilesAction::data_sequence_number). pub struct RewriteFilesAction { producer: MergingSnapshotProducer, - /// The snapshot ID at which this rewrite started reading. Used to - /// detect conflicting deletes added after this point. + /// The snapshot the rewrite was planned against, if the table had one. + /// Deletes newer than it fail the commit. starting_snapshot_id: Option, } From ebf19464118702c9ddb71908ddce037afe0352d0 Mon Sep 17 00:00:00 2001 From: hotcache Date: Mon, 28 Sep 2026 13:01:45 +0000 Subject: [PATCH 20/28] refactor(transaction): build the rewrite errors with invalid_data! The macro from #2928 landed while this branch was open and snapshot.rs already uses it. Use it for the merging producer and the rewrite action too, so the whole transaction module reads the same way. --- crates/iceberg/src/transaction/merging.rs | 46 ++++++++--------------- crates/iceberg/src/transaction/rewrite.rs | 37 +++++++----------- 2 files changed, 29 insertions(+), 54 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index 3b4363a409..d3b3bf0edc 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -27,7 +27,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use uuid::Uuid; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::io::OutputFile; use crate::spec::{ DataFile, DataFileFormat, FormatVersion, ManifestContentType, ManifestEntry, ManifestFile, @@ -37,7 +37,6 @@ use crate::spec::{ use crate::table::Table; use crate::transaction::ActionCommit; use crate::transaction::snapshot::SnapshotProducer; -use crate::{Error, ErrorKind}; /// Create a manifest writer that handles encryption when available. fn new_manifest_writer( @@ -142,12 +141,9 @@ impl ManifestFilterManager { .metadata() .partition_spec_by_id(manifest_file.partition_spec_id) .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Manifest references unknown partition spec {}", - manifest_file.partition_spec_id, - ), + invalid_data!( + "Manifest references unknown partition spec {}", + manifest_file.partition_spec_id, ) })? .clone(); @@ -167,21 +163,15 @@ impl ManifestFilterManager { } else if entry.is_alive() { // Surviving entry — re-emit as EXISTING with original ids preserved. let seq = entry.sequence_number().ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Manifest entry for {} is missing sequence_number", - entry.file_path(), - ), + invalid_data!( + "Manifest entry for {} is missing sequence_number", + entry.file_path(), ) })?; let file_seq = entry.file_sequence_number.ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Manifest entry for {} is missing file_sequence_number", - entry.file_path(), - ), + invalid_data!( + "Manifest entry for {} is missing file_sequence_number", + entry.file_path(), ) })?; let existing = ManifestEntry::builder() @@ -240,12 +230,9 @@ impl ManifestFilterManager { .map(String::as_str) .collect(); if !missing.is_empty() { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Failed to find the following files to delete in the current snapshot: {}", - missing.join(", "), - ), + return Err(invalid_data!( + "Failed to find the following files to delete in the current snapshot: {}", + missing.join(", "), )); } } @@ -321,11 +308,8 @@ impl MergingSnapshotProducer { }; let snapshot_sequence_number = table.metadata().next_sequence_number(); if sequence_number > snapshot_sequence_number { - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Data sequence number {sequence_number} is greater than the snapshot's {snapshot_sequence_number}." - ), + return Err(invalid_data!( + "Data sequence number {sequence_number} is greater than the snapshot's {snapshot_sequence_number}." )); } Ok(()) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 1e9a5169d0..de31d723bc 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -28,12 +28,11 @@ use std::sync::Arc; use async_trait::async_trait; -use crate::error::Result; +use crate::error::{Result, invalid_data}; use crate::spec::{DataFile, ManifestContentType, Operation}; use crate::table::Table; use crate::transaction::merging::MergingSnapshotProducer; use crate::transaction::{ActionCommit, TransactionAction}; -use crate::{Error, ErrorKind}; /// A transaction action that rewrites (replaces) data files. /// @@ -60,7 +59,8 @@ use crate::{Error, ErrorKind}; /// narrows that to the files being replaced in /// `validateNoNewDeletesForDataFiles`; until that exists here the check is /// table-wide, so a delete committed to an unrelated partition while the -/// rewrite ran also fails the commit with [`ErrorKind::DataInvalid`]. So do a +/// rewrite ran also fails the commit with +/// [`ErrorKind::DataInvalid`](crate::ErrorKind::DataInvalid). So do a /// rewrite planned against a table that had no snapshot yet, and one whose /// starting snapshot has since been expired: neither can rule out a delete. /// Replan the rewrite against the current table and run it again. @@ -115,15 +115,13 @@ impl RewriteFilesAction { fn validate(&self) -> Result<()> { if !self.producer.has_deleted_data_files() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Rewrite files requires at least one file to delete", + return Err(invalid_data!( + "Rewrite files requires at least one file to delete" )); } if !self.producer.has_added_data_files() { - return Err(Error::new( - ErrorKind::DataInvalid, - "Rewrite files requires at least one file to add", + return Err(invalid_data!( + "Rewrite files requires at least one file to add" )); } Ok(()) @@ -145,19 +143,15 @@ impl RewriteFilesAction { // The rewrite was planned against a table without a snapshot, so every // snapshot it has now, deletes included, landed after that. let Some(starting_snapshot_id) = self.starting_snapshot_id else { - return Err(Error::new( - ErrorKind::DataInvalid, - "Cannot rewrite files: the rewrite was planned against a table with no snapshot, and the table has been committed to since.", + return Err(invalid_data!( + "Cannot rewrite files: the rewrite was planned against a table with no snapshot, and the table has been committed to since." )); }; let Some(starting_snapshot) = table.metadata().snapshot_by_id(starting_snapshot_id) else { // Without the starting snapshot there is no sequence number to // compare against, so no delete can be ruled out. - return Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot rewrite files: the starting snapshot {starting_snapshot_id} is no longer in the table, so deletes added since it cannot be ruled out.", - ), + return Err(invalid_data!( + "Cannot rewrite files: the starting snapshot {starting_snapshot_id} is no longer in the table, so deletes added since it cannot be ruled out." )); }; let starting_sequence_number = starting_snapshot.sequence_number(); @@ -169,12 +163,9 @@ impl RewriteFilesAction { }); match conflict { - Some(entry) => Err(Error::new( - ErrorKind::DataInvalid, - format!( - "Cannot rewrite files: delete manifest {} was added after the starting snapshot {starting_snapshot_id}.", - entry.manifest_path, - ), + Some(entry) => Err(invalid_data!( + "Cannot rewrite files: delete manifest {} was added after the starting snapshot {starting_snapshot_id}.", + entry.manifest_path, )), None => Ok(()), } From 43a09a5027c833cb3dbf9f9f993a8709d14d0e6c Mon Sep 17 00:00:00 2001 From: Rexwell Minnis Date: Mon, 28 Sep 2026 11:24:04 -0500 Subject: [PATCH 21/28] test(transaction): cover rewrites on a spec-evolved table The filter resolves each manifest's own partition spec, but every test ran on the default spec. Add an identity(ts) -> day(ts) fixture on V2 and V3: a survivor stays Existing in a spec-0 manifest with its timestamptz value and partition summary intact, one rewrite filters a manifest of each spec, and retiring the last old-spec file drops the spec-0 manifest with the chained totals still exact. --- crates/iceberg/src/transaction/rewrite.rs | 281 +++++++++++++++++++++- 1 file changed, 278 insertions(+), 3 deletions(-) diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index de31d723bc..508b432106 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -183,6 +183,7 @@ impl TransactionAction for RewriteFilesAction { #[cfg(test)] mod tests { + use std::collections::HashMap; use std::sync::Arc; use uuid::Uuid; @@ -190,8 +191,10 @@ mod tests { use super::*; use crate::memory::tests::new_memory_catalog; use crate::spec::{ - DataContentType, Literal, ManifestEntryRef, ManifestFile, ManifestListWriter, - ManifestStatus, ManifestWriterBuilder, Operation, SnapshotRef, Struct, + DataContentType, DataFileBuilder, DataFileFormat, Datum, FormatVersion, Literal, + ManifestEntryRef, ManifestFile, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, + NestedField, Operation, PrimitiveType, Schema, SnapshotRef, Struct, Transform, Type, + UnboundPartitionSpec, }; use crate::table::Table; use crate::transaction::tests::{ @@ -199,7 +202,7 @@ mod tests { make_v2_minimal_table_in_catalog, make_v3_minimal_table_in_catalog, }; use crate::transaction::{ApplyTransactionAction, Transaction}; - use crate::{Catalog, ErrorKind, TableUpdate}; + use crate::{Catalog, ErrorKind, NamespaceIdent, TableCommit, TableCreation, TableUpdate}; /// Read back the manifest entry for `path` from the manifests of `snapshot`. async fn find_entry( @@ -938,4 +941,276 @@ mod tests { .contains("at least one file to add") ); } + + // Partition-spec evolution: `ts` is identity-partitioned under spec 0, then + // `day(ts)` becomes the default spec. Old files keep spec 0 and a + // timestamptz partition value; new files carry spec 1 and a date. + + /// 2026-08-24T09:30:00Z and 2026-08-24T17:45:00Z, in microseconds. + const TS_MORNING: i64 = 1_787_563_800_000_000; + const TS_EVENING: i64 = 1_787_593_500_000_000; + /// 2026-08-24, in days since the epoch. + const DAY: i32 = 20_689; + + /// A table partitioned by `identity(ts)`, still on its first spec. + async fn make_identity_ts_table( + catalog: &impl Catalog, + format_version: FormatVersion, + ) -> Table { + let namespace = NamespaceIdent::new(format!("ns1-{}", Uuid::new_v4())); + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap(); + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "ts", Type::Primitive(PrimitiveType::Timestamptz)).into(), + ]) + .build() + .unwrap(); + let spec = UnboundPartitionSpec::builder() + .add_partition_field(2, "ts", Transform::Identity) + .unwrap() + .build(); + let creation = TableCreation::builder() + .name("events".to_string()) + .schema(schema) + .partition_spec(spec) + .format_version(format_version) + .build(); + catalog.create_table(&namespace, creation).await.unwrap() + } + + /// Replace `identity(ts)` with `day(ts)` as the default spec. + async fn evolve_to_day_ts(catalog: &impl Catalog, table: &Table) -> Table { + let spec = UnboundPartitionSpec::builder() + .add_partition_field(2, "ts_day", Transform::Day) + .unwrap() + .build(); + let commit = TableCommit::builder() + .ident(table.identifier().clone()) + .requirements(vec![]) + .updates(vec![ + TableUpdate::AddSpec { spec }, + TableUpdate::SetDefaultSpec { spec_id: -1 }, + ]) + .build(); + let table = catalog.update_table(commit).await.unwrap(); + assert_eq!(table.metadata().default_partition_spec_id(), 1); + table + } + + fn make_file_in_spec( + path: &str, + records: u64, + spec_id: i32, + partition_value: Literal, + ) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(records * 10) + .record_count(records) + .partition(Struct::from_iter([Some(partition_value)])) + .partition_spec_id(spec_id) + .build() + .unwrap() + } + + /// Live files of the current snapshot, keyed by path, with the spec id of + /// the manifest that lists them. + async fn live_files_by_manifest_spec(table: &Table) -> Vec<(String, i32)> { + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let mut live = Vec::new(); + for manifest_file in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { + assert_eq!( + entry.data_file().partition_spec_id, + manifest_file.partition_spec_id, + "{} is listed by a manifest of another spec", + entry.file_path() + ); + live.push(( + entry.file_path().to_string(), + manifest_file.partition_spec_id, + )); + } + } + live.sort(); + live + } + + /// Two old-spec files and one new-spec file; returns the table and the + /// snapshot that appended the old-spec files. + async fn spec_evolved_table_with_files( + catalog: &impl Catalog, + format_version: FormatVersion, + ) -> (Table, SnapshotRef, [DataFile; 3]) { + let table = make_identity_ts_table(catalog, format_version).await; + let old_1 = make_file_in_spec( + "test/old-1.parquet", + 10, + 0, + Literal::timestamptz(TS_MORNING), + ); + let old_2 = make_file_in_spec( + "test/old-2.parquet", + 10, + 0, + Literal::timestamptz(TS_EVENING), + ); + let table = append_files(catalog, &table, vec![old_1.clone(), old_2.clone()]).await; + let old_snapshot = table.metadata().current_snapshot().unwrap().clone(); + + let table = evolve_to_day_ts(catalog, &table).await; + let new_1 = make_file_in_spec("test/new-1.parquet", 10, 1, Literal::date(DAY)); + let table = append_files(catalog, &table, vec![new_1.clone()]).await; + (table, old_snapshot, [old_1, old_2, new_1]) + } + + /// Compacting one old-spec file keeps the other in a manifest written under + /// spec 0: its entry, partition value and the manifest's partition summary + /// are still those of `identity(ts)`, not re-encoded under `day(ts)`. + #[tokio::test] + async fn test_rewrite_files_keeps_survivors_on_their_partition_spec() { + for format_version in [FormatVersion::V2, FormatVersion::V3] { + let catalog = new_memory_catalog().await; + let (table, old_snapshot, [old_1, _old_2, _new_1]) = + spec_evolved_table_with_files(&catalog, format_version).await; + + let merged = make_file_in_spec("test/merged.parquet", 10, 1, Literal::date(DAY)); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(old_1).add_file(merged); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + assert_eq!( + live_files_by_manifest_spec(&table).await, + vec![ + ("test/merged.parquet".to_string(), 1), + ("test/new-1.parquet".to_string(), 1), + ("test/old-2.parquet".to_string(), 0), + ], + "{format_version}" + ); + + let snapshot = table.metadata().current_snapshot().unwrap(); + let survivor = find_entry(&table, snapshot, "test/old-2.parquet") + .await + .unwrap(); + assert_eq!(survivor.status(), ManifestStatus::Existing); + assert_eq!(survivor.snapshot_id(), Some(old_snapshot.snapshot_id())); + assert_eq!( + survivor.sequence_number(), + Some(old_snapshot.sequence_number()) + ); + assert_eq!( + survivor.data_file().partition, + Struct::from_iter([Some(Literal::timestamptz(TS_EVENING))]) + ); + + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let old_spec_manifest = manifest_list + .entries() + .iter() + .find(|m| m.partition_spec_id == 0) + .unwrap(); + let summary = &old_spec_manifest.partitions.as_ref().unwrap()[0]; + let evening = Datum::timestamptz_micros(TS_EVENING).to_bytes().unwrap(); + assert_eq!(summary.lower_bound.as_ref(), Some(&evening)); + assert_eq!(summary.upper_bound.as_ref(), Some(&evening)); + + let totals = &snapshot.summary().additional_properties; + assert_eq!(totals.get("total-data-files").unwrap(), "3"); + assert_eq!(totals.get("total-records").unwrap(), "30"); + } + } + + /// One rewrite removing a file from each spec: both manifests are filtered + /// under their own spec and the merged file lands under the default spec. + #[tokio::test] + async fn test_rewrite_files_across_partition_specs() { + for format_version in [FormatVersion::V2, FormatVersion::V3] { + let catalog = new_memory_catalog().await; + let (table, _, [old_1, _old_2, new_1]) = + spec_evolved_table_with_files(&catalog, format_version).await; + + let merged = make_file_in_spec("test/merged.parquet", 20, 1, Literal::date(DAY)); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(old_1) + .delete_file(new_1) + .add_file(merged); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + assert_eq!( + live_files_by_manifest_spec(&table).await, + vec![ + ("test/merged.parquet".to_string(), 1), + ("test/old-2.parquet".to_string(), 0), + ], + "{format_version}" + ); + let totals = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + assert_eq!(totals.get("deleted-data-files").unwrap(), "2"); + assert_eq!(totals.get("total-data-files").unwrap(), "2"); + assert_eq!(totals.get("total-records").unwrap(), "30"); + } + } + + /// A second compaction that removes the last old-spec file drops the spec-0 + /// manifest, and the totals chained through both rewrites stay exact. + #[tokio::test] + async fn test_rewrite_files_retires_the_old_spec() { + for format_version in [FormatVersion::V2, FormatVersion::V3] { + let catalog = new_memory_catalog().await; + let (table, _, [old_1, old_2, new_1]) = + spec_evolved_table_with_files(&catalog, format_version).await; + + let merged = make_file_in_spec("test/merged-1.parquet", 10, 1, Literal::date(DAY)); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(old_1).add_file(merged); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let merged = make_file_in_spec("test/merged-2.parquet", 20, 1, Literal::date(DAY)); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(old_2) + .delete_file(new_1) + .add_file(merged); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + assert_eq!( + live_files_by_manifest_spec(&table).await, + vec![ + ("test/merged-1.parquet".to_string(), 1), + ("test/merged-2.parquet".to_string(), 1), + ], + "{format_version}" + ); + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + assert!( + manifest_list + .entries() + .iter() + .all(|m| m.partition_spec_id == 1), + "{format_version}: a spec-0 manifest outlived its last file" + ); + let totals = &snapshot.summary().additional_properties; + assert_eq!(totals.get("total-data-files").unwrap(), "2"); + assert_eq!(totals.get("total-records").unwrap(), "30"); + } + } } From 89786a86e9a25ea60ba2c3b78acc247d55e6e440 Mon Sep 17 00:00:00 2001 From: Rexwell Minnis Date: Mon, 28 Sep 2026 11:24:34 -0500 Subject: [PATCH 22/28] fix(transaction): keep partition metrics in the rewrite summary build_summary merged the filter's removal collector into a fresh one; merge drops partition metrics unless both sides trust them and neither did, so a rewrite never reported changed-partition-count or the partitions.* summaries. Add the new files to the filter's collector instead, and honour write.summary.partition-limit as fast append does. --- crates/iceberg/src/transaction/merging.rs | 15 ++++-- crates/iceberg/src/transaction/rewrite.rs | 61 +++++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index d3b3bf0edc..3b619717d0 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -32,7 +32,7 @@ use crate::io::OutputFile; use crate::spec::{ DataFile, DataFileFormat, FormatVersion, ManifestContentType, ManifestEntry, ManifestFile, ManifestStatus, ManifestWriter, ManifestWriterBuilder, Operation, PartitionSpec, SchemaRef, - SnapshotSummaryCollector, Summary, update_snapshot_summaries, + SnapshotSummaryCollector, Summary, TableProperties, update_snapshot_summaries, }; use crate::table::Table; use crate::transaction::ActionCommit; @@ -431,12 +431,19 @@ impl MergingSnapshotProducer { let schema = table_metadata.current_schema().clone(); let partition_spec = table_metadata.default_partition_spec().clone(); - let mut collector = SnapshotSummaryCollector::default(); + // Add to the filter's collector rather than merging two: `merge` drops + // partition metrics unless both sides trust them, and neither does. + let mut collector = removed_collector; + collector.set_partition_summary_limit( + table_metadata + .properties() + .get(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT) + .and_then(|limit| limit.parse::().ok()) + .unwrap_or(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT), + ); for file in &self.added_data_files { collector.add_file(file, schema.clone(), partition_spec.clone()); } - // Merge removal metrics from the filter manager. - collector.merge(removed_collector); let summary = Summary { operation: self.operation.clone(), diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 508b432106..5f5f821d64 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -1213,4 +1213,65 @@ mod tests { assert_eq!(totals.get("total-records").unwrap(), "30"); } } + + /// The summary counts and names the partitions a rewrite changed, each + /// under its file's own spec: one `identity(ts)` partition lost a file, one + /// `day(ts)` partition gained one. + #[tokio::test] + async fn test_rewrite_files_summarizes_partitions_of_both_specs() { + let catalog = new_memory_catalog().await; + let (table, _, [old_1, _old_2, _new_1]) = + spec_evolved_table_with_files(&catalog, FormatVersion::V2).await; + // Fast append reports the partition it changed. + let appended = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + assert_eq!( + appended.get("changed-partition-count").map(String::as_str), + Some("1") + ); + + let tx = Transaction::new(&table); + let tx = tx + .update_table_properties() + .set( + "write.summary.partition-limit".to_string(), + "10".to_string(), + ) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let merged = make_file_in_spec("test/merged.parquet", 10, 1, Literal::date(DAY)); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(old_1).add_file(merged); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let summary = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + assert_eq!( + summary.get("changed-partition-count").map(String::as_str), + Some("2"), + "{summary:?}" + ); + // The removed file is named under `identity(ts)`, the added one under + // `day(ts)`. + let mut partitions: Vec<&str> = summary + .keys() + .filter_map(|k| k.strip_prefix("partitions.")) + .collect(); + partitions.sort(); + assert_eq!( + partitions, + vec!["ts=2026-08-24+09%3A30%3A00+UTC", "ts_day=2026-08-24"], + "{summary:?}" + ); + } } From 8a4f780596f296068a3bfe90afd564a4d3004df3 Mon Sep 17 00:00:00 2001 From: hotcache Date: Tue, 29 Sep 2026 14:29:43 +0000 Subject: [PATCH 23/28] fix(transaction): record removed files as DELETED manifest entries The filter dropped the entries of the files it removed, so nothing in the metadata said which snapshot removed them. Java's expiry walks exactly those entries -- IncrementalFileCleanup finds a data file to delete only through a DELETED entry whose snapshot has expired -- so a table compacted by this action kept every replaced file in storage for good, and a changelog reader saw no removals. Read back with Trino, the manifests reported deleted_data_files_count = 0 while the snapshot summary claimed deleted-data-files = 2. Write each removed entry with add_delete_entry, which stamps the removing snapshot and keeps the file's own sequence numbers. A manifest the commit empties is now written rather than omitted, holding only those entries; the next merging commit drops it, as it already did for manifests left all-deleted by an earlier snapshot. Reported by @rexminnis against Apache Polaris with Trino and PyIceberg. Two tests change with the contract. The emptied manifest is no longer absent, so they assert it has no live entry and that a later rewrite drops it -- an append carries it along, here as in Java, because only a merging commit rebuilds the manifest list. A filtered manifest's partition summary now spans its deleted entries too, matching Java, where every entry updates it. --- crates/iceberg/src/spec/manifest/writer.rs | 4 -- crates/iceberg/src/transaction/merging.rs | 16 ++++- crates/iceberg/src/transaction/rewrite.rs | 82 +++++++++++++++++++--- 3 files changed, 85 insertions(+), 17 deletions(-) diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index ba0d02b43f..106b462627 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -315,10 +315,6 @@ impl ManifestWriter { /// Add a delete manifest entry. This method will update following status of the entry: /// - Update the entry status to `Deleted` /// - Set the snapshot id to the current snapshot id - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_delete_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Deleted; diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index 3b619717d0..d66ae4df21 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -149,8 +149,11 @@ impl ManifestFilterManager { .clone(); let schema = table.metadata().current_schema().clone(); - // Rewrite: keep surviving entries as EXISTING, drop deleted ones. + // Rewrite: keep surviving entries as EXISTING, and record the + // removed ones as DELETED so that the file they name can be + // cleaned up when this snapshot expires. let mut surviving_entries: Vec = Vec::new(); + let mut removed_entries: Vec = Vec::new(); for entry in manifest.entries() { if entry.is_alive() && self.deleted_file_paths.contains(entry.file_path()) { // Record removal metrics. @@ -160,6 +163,7 @@ impl ManifestFilterManager { schema.clone(), manifest_spec.clone(), ); + removed_entries.push(entry.as_ref().clone()); } else if entry.is_alive() { // Surviving entry — re-emit as EXISTING with original ids preserved. let seq = entry.sequence_number().ok_or_else(|| { @@ -190,8 +194,8 @@ impl ManifestFilterManager { // Already-deleted entries (status == Deleted) are dropped. } - if surviving_entries.is_empty() { - // Manifest is now empty — omit entirely. + if surviving_entries.is_empty() && removed_entries.is_empty() { + // Nothing left to say about this manifest — omit entirely. continue; } @@ -217,6 +221,12 @@ impl ManifestFilterManager { for entry in surviving_entries { writer.add_existing_entry(entry)?; } + for entry in removed_entries { + // `add_delete_entry` stamps the removing snapshot on the entry + // and keeps the file's own sequence numbers, which is what an + // expiring reader needs to attribute the removal. + writer.add_delete_entry(entry)?; + } let new_manifest = writer.write_manifest_file().await?; result.push(new_manifest); } diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index 5f5f821d64..5ed8e36bc9 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -524,20 +524,52 @@ mod tests { let f1 = make_data_file(&table, "test/1.parquet", 10, 100); let table = append_files(&catalog, &table, vec![f1.clone()]).await; - // Rewrite: delete f1, add merged → f1's manifest should be omitted. + // Rewrite: delete f1, add merged. f1's manifest keeps no live entry, + // but it still has to record the removal for this snapshot. let merged = make_data_file(&table, "test/merged.parquet", 10, 100); let tx = Transaction::new(&table); - let action = tx.rewrite_files().delete_file(f1).add_file(merged); + let action = tx.rewrite_files().delete_file(f1.clone()).add_file(merged); let tx = action.apply(tx).unwrap(); let table = tx.commit(&catalog).await.unwrap(); - // Verify: only 1 manifest (the new one), not 2. let snapshot = table.metadata().current_snapshot().unwrap(); + let emptied = find_entry(&table, snapshot, "test/1.parquet") + .await + .expect("the removal has to be recorded somewhere"); + assert_eq!(emptied.status(), ManifestStatus::Deleted); + assert_eq!(emptied.snapshot_id(), Some(snapshot.snapshot_id())); let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); - assert_eq!( - manifest_list.entries().len(), - 1, - "empty manifest should be omitted, not kept" + assert_eq!(manifest_list.entries().len(), 2); + assert!( + manifest_list + .entries() + .iter() + .any(|m| !m.has_added_files() && !m.has_existing_files()), + "the emptied manifest holds only the deleted entry" + ); + + // A later rewrite drops it: it has nothing live, and no removal of its + // own left to record. An append would carry it along, as in Java, + // where only a merging commit rebuilds the manifest list. + let merged = find_entry(&table, snapshot, "test/merged.parquet") + .await + .unwrap() + .data_file() + .clone(); + let merged_again = make_data_file(&table, "test/merged-2.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(merged) + .add_file(merged_again); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + assert!( + find_entry(&table, snapshot, "test/1.parquet") + .await + .is_none(), + "an all-deleted manifest should not survive a later rewrite" ); } @@ -1119,9 +1151,12 @@ mod tests { .iter() .find(|m| m.partition_spec_id == 0) .unwrap(); + // The summary spans the deleted entry as well as the survivor, + // as it does in Java, where every entry updates it. let summary = &old_spec_manifest.partitions.as_ref().unwrap()[0]; + let morning = Datum::timestamptz_micros(TS_MORNING).to_bytes().unwrap(); let evening = Datum::timestamptz_micros(TS_EVENING).to_bytes().unwrap(); - assert_eq!(summary.lower_bound.as_ref(), Some(&evening)); + assert_eq!(summary.lower_bound.as_ref(), Some(&morning)); assert_eq!(summary.upper_bound.as_ref(), Some(&evening)); let totals = &snapshot.summary().additional_properties; @@ -1205,12 +1240,39 @@ mod tests { manifest_list .entries() .iter() - .all(|m| m.partition_spec_id == 1), - "{format_version}: a spec-0 manifest outlived its last file" + .filter(|m| m.partition_spec_id == 0) + .all(|m| !m.has_added_files() && !m.has_existing_files()), + "{format_version}: a spec-0 manifest kept a live file" ); + let totals = &snapshot.summary().additional_properties; assert_eq!(totals.get("total-data-files").unwrap(), "2"); assert_eq!(totals.get("total-records").unwrap(), "30"); + + // The spec-0 manifest only carries the removal made by the snapshot + // that emptied it. The next rewrite has no reason to keep it. + let merged = make_file_in_spec("test/merged-3.parquet", 10, 1, Literal::date(DAY)); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(make_file_in_spec( + "test/merged-1.parquet", + 10, + 1, + Literal::date(DAY), + )) + .add_file(merged); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + assert!( + manifest_list + .entries() + .iter() + .all(|m| m.partition_spec_id == 1), + "{format_version}: a spec-0 manifest outlived its last file" + ); } } From e864184896f21efec774eeb368760ea04a8d6505 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Tue, 13 Oct 2026 14:07:57 +0200 Subject: [PATCH 24/28] Allow rewrites without output files --- crates/iceberg/src/transaction/merging.rs | 4 -- crates/iceberg/src/transaction/rewrite.rs | 50 ++++++++++++++--------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index d66ae4df21..4af8c0ebfa 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -303,10 +303,6 @@ impl MergingSnapshotProducer { self.deleted_data_files.push(file); } - pub(crate) fn has_added_data_files(&self) -> bool { - !self.added_data_files.is_empty() - } - pub(crate) fn has_deleted_data_files(&self) -> bool { !self.deleted_data_files.is_empty() } diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index b8b297939c..a5b527bc7b 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -119,11 +119,6 @@ impl RewriteFilesAction { "Rewrite files requires at least one file to delete" )); } - if !self.producer.has_added_data_files() { - return Err(invalid_data!( - "Rewrite files requires at least one file to add" - )); - } Ok(()) } @@ -953,24 +948,41 @@ mod tests { ); } - /// Rewrite with no adds should fail validation. #[tokio::test] - async fn test_rewrite_files_no_adds() { + async fn test_rewrite_files_without_output_files() { let catalog = new_memory_catalog().await; let table = make_v3_minimal_table_in_catalog(&catalog).await; - - let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let removed = make_data_file(&table, "test/removed.parquet", 10, 100); + let kept = make_data_file(&table, "test/kept.parquet", 5, 50); + let table = append_files(&catalog, &table, vec![removed.clone(), kept]).await; let tx = Transaction::new(&table); - let action = tx.rewrite_files().delete_file(f1); - let tx = action.apply(tx).unwrap(); - let result = tx.commit(&catalog).await; - - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .message() - .contains("at least one file to add") + let table = tx + .rewrite_files() + .delete_file(removed) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let snapshot = table.metadata().current_snapshot().unwrap(); + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("deleted-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-records").unwrap(), "5"); + assert_eq!(summary.get("total-files-size").unwrap(), "50"); + assert_eq!( + find_entry(&table, snapshot, "test/removed.parquet") + .await + .unwrap() + .status(), + ManifestStatus::Deleted + ); + assert_eq!( + find_entry(&table, snapshot, "test/kept.parquet") + .await + .unwrap() + .status(), + ManifestStatus::Existing ); } From 6956f97d0ca032b92123fd83133309053671c983 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 27 Oct 2026 16:40:55 -0700 Subject: [PATCH 25/28] add msp --- crates/iceberg/src/spec/manifest/writer.rs | 8 - .../transaction/snapshot/manifest_filter.rs | 225 ++++++++ .../src/transaction/snapshot/merging.rs | 544 ++++++++++++++++++ .../iceberg/src/transaction/snapshot/mod.rs | 85 ++- .../src/transaction/snapshot/simple.rs | 48 +- 5 files changed, 847 insertions(+), 63 deletions(-) create mode 100644 crates/iceberg/src/transaction/snapshot/manifest_filter.rs create mode 100644 crates/iceberg/src/transaction/snapshot/merging.rs diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index a79312dfab..41a3d88e44 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -315,10 +315,6 @@ impl ManifestWriter { /// Add a delete manifest entry. This method will update following status of the entry: /// - Update the entry status to `Deleted` /// - Set the snapshot id to the current snapshot id - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_delete_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Deleted; @@ -350,10 +346,6 @@ impl ManifestWriter { /// Add an existing manifest entry. This method will update following status of the entry: /// - Update the entry status to `Existing` - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_existing_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Existing; diff --git a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs new file mode 100644 index 0000000000..1debd09553 --- /dev/null +++ b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Rewrites existing manifests to mark files as deleted. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use super::{manifest_path, new_manifest_writer}; +use crate::error::{Result, invalid_data}; +use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile}; +use crate::table::Table; + +/// Removes a fixed set of files from the manifests of one content type. +/// +/// One manager lives for one logical execution of an action and is driven +/// once per attempt via [`ManifestFilterManager::filter`]. A manifest that +/// references none of the requested files passes through untouched. One that +/// does is rewritten with those entries marked `Deleted` and the rest +/// `Existing`; the rewrite is keyed by the source manifest's path, which is +/// immutable, so a rebase only rewrites manifests not seen before. +pub(crate) struct ManifestFilterManager { + content: ManifestContentType, + /// Paths of the files to remove. Fixed for the execution. + requested_paths: HashSet, + /// Source manifest path → rewritten result. + filtered: HashMap, + /// Format version the rewrites were written under. Per spec, v1 entries + /// cannot inherit sequence numbers, so an upgrade invalidates them. + format_version: Option, + /// Metadata files written by this manager. + owned_artifacts: HashSet, +} + +/// One rewritten manifest and the files it marked as deleted. +#[derive(Clone)] +struct FilteredManifest { + manifest: ManifestFile, + removed_files: Vec, +} + +/// The result of one filter pass. +pub(crate) struct Filtered { + /// The resulting manifests, in input order. + pub manifests: Vec, + /// The files that were live and are now marked deleted, including their + /// metrics as read from the source manifests. + pub removed_files: Vec, +} + +impl ManifestFilterManager { + pub(crate) fn new(content: ManifestContentType, removed_files: &[DataFile]) -> Self { + Self { + content, + requested_paths: removed_files + .iter() + .map(|f| f.file_path().to_string()) + .collect(), + filtered: HashMap::new(), + format_version: None, + owned_artifacts: HashSet::new(), + } + } + + /// Metadata files written by this manager so far. + pub(crate) fn owned_artifacts(&self) -> &HashSet { + &self.owned_artifacts + } + + /// Applies the removals to `manifests`, rewriting those of this manager's + /// content type that reference a requested file. + /// + /// Fails if a requested file is not live in any manifest: it was never in + /// the table, or a concurrent commit already removed it. + /// + /// `next_manifest_number` allocates paths for rewritten manifests from the + /// producer's counter, so they never collide with other manifests it writes. + pub(crate) async fn filter( + &mut self, + table: &Table, + snapshot_id: i64, + commit_uuid: Uuid, + manifests: Vec, + next_manifest_number: &mut u64, + ) -> Result { + if self.requested_paths.is_empty() { + return Ok(Filtered { + manifests, + removed_files: Vec::new(), + }); + } + + let format_version = table.metadata().format_version(); + if self.format_version != Some(format_version) { + self.filtered.clear(); + self.format_version = Some(format_version); + } + + let mut out = Vec::with_capacity(manifests.len()); + let mut removed_files: Vec = Vec::new(); + + for source in manifests { + if source.content != self.content { + out.push(source); + continue; + } + + let result = match self.filtered.get(&source.manifest_path) { + Some(cached) => cached.clone(), + None => { + let result = self + .rewrite( + table, + snapshot_id, + commit_uuid, + &source, + next_manifest_number, + ) + .await?; + self.filtered + .insert(source.manifest_path.clone(), result.clone()); + result + } + }; + + removed_files.extend(result.removed_files); + out.push(result.manifest); + } + + let found: HashSet<&str> = removed_files.iter().map(|f| f.file_path()).collect(); + let mut missing: Vec<&str> = self + .requested_paths + .iter() + .map(String::as_str) + .filter(|p| !found.contains(p)) + .collect(); + if !missing.is_empty() { + missing.sort_unstable(); + return Err(invalid_data!( + "Cannot remove files that are not live in the table: {}", + missing.join(", ") + )); + } + + Ok(Filtered { + manifests: out, + removed_files, + }) + } + + /// Reads `source` and, if it references any requested file, writes a copy + /// with those entries marked `Deleted` and the rest `Existing`. + async fn rewrite( + &mut self, + table: &Table, + snapshot_id: i64, + commit_uuid: Uuid, + source: &ManifestFile, + next_manifest_number: &mut u64, + ) -> Result { + let manifest = table.object_cache().get_manifest(source).await?; + + let removed_files: Vec = manifest + .entries() + .iter() + .filter(|e| e.is_alive() && self.requested_paths.contains(e.file_path())) + .map(|e| e.data_file().clone()) + .collect(); + if removed_files.is_empty() { + return Ok(FilteredManifest { + manifest: source.clone(), + removed_files, + }); + } + + // Allocate the path before writing so a partial write is still owned. + let path = manifest_path(table, commit_uuid, *next_manifest_number)?; + *next_manifest_number += 1; + self.owned_artifacts.insert(path.clone()); + + // Keep the source manifest's spec: its entries' partition values are + // laid out for that spec, not the table's current default. + let spec = manifest.metadata().partition_spec().clone(); + let mut writer = new_manifest_writer( + table, + path, + snapshot_id, + self.content, + table.metadata().current_schema().clone(), + spec.into(), + )?; + for entry in manifest.entries() { + if !entry.is_alive() { + continue; + } + let entry = (**entry).clone(); + if self.requested_paths.contains(entry.file_path()) { + writer.add_delete_entry(entry)?; + } else { + writer.add_existing_entry(entry)?; + } + } + let manifest = writer.write_manifest_file().await?; + + Ok(FilteredManifest { + manifest, + removed_files, + }) + } +} diff --git a/crates/iceberg/src/transaction/snapshot/merging.rs b/crates/iceberg/src/transaction/snapshot/merging.rs new file mode 100644 index 0000000000..16e1ba85f4 --- /dev/null +++ b/crates/iceberg/src/transaction/snapshot/merging.rs @@ -0,0 +1,544 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Persistent producer for snapshot actions that add and remove data and +//! delete files. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use super::manifest_filter::ManifestFilterManager; +use super::{ + SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, + manifest_path, validate_partition_value, write_added_manifest, write_snapshot_commit, +}; +use crate::error::{Result, invalid_data}; +use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; +use crate::table::Table; +use crate::transaction::ActionCommit; + +/// What a merging action asks the producer to do. +/// +/// Fixed for one logical execution; the producer derives everything else from +/// it and the table of the current attempt. The action validates the +/// table-independent parts (content types, disjoint add/remove sets) before +/// each attempt. +pub(crate) struct MergingIntent { + /// Overrides the generated commit UUID. + pub commit_uuid: Option, + pub operation: Operation, + pub snapshot_properties: HashMap, + pub added_data_files: Vec, + pub added_delete_files: Vec, + pub removed_data_files: Vec, + pub removed_delete_files: Vec, + /// Whether the summary resets table totals (a full-table overwrite). + pub truncate_full_table: bool, +} + +/// Retry-persistent state for an action that adds and removes data and delete +/// files. +/// +/// One producer lives for one logical execution of an action and is driven once +/// per attempt via [`MergingSnapshotProducer::apply`]. Per attempt it takes the +/// current snapshot's manifests, rewrites those that reference a removed file, +/// appends manifests for the added files, and writes a manifest list. Identity +/// (commit UUID, snapshot id) is fixed on first use; rewritten and added +/// manifests are reused across attempts while their inputs are unchanged. +pub(crate) struct MergingSnapshotProducer { + intent: MergingIntent, + /// Fixed when the producer is created, so every attempt shares one UUID. + commit_uuid: Uuid, + /// Resolved against the table on the first attempt, then stable. + snapshot_id: Option, + /// Incremented per attempt; distinguishes manifest-list paths. + attempt: u64, + /// Numbers every manifest this producer writes, including rewrites. Never + /// reset, so paths never collide across attempts. + manifest_counter: u64, + data_filter: ManifestFilterManager, + delete_filter: ManifestFilterManager, + /// Manifests written for the added files, reused across attempts. + added_manifests: Option, + /// Metadata files written by this producer (manifests and every attempt's + /// manifest list). + // TODO(#3417): consumed by terminal cleanup once it is wired up. + owned_artifacts: HashSet, +} + +/// The manifests written for one execution's added data and delete files. +/// +/// Reusable while the table's format version is unchanged; see +/// `simple::AddedManifests` for why. +struct AddedManifests { + manifests: Vec, + format_version: FormatVersion, +} + +impl MergingSnapshotProducer { + pub(crate) fn new(intent: MergingIntent) -> Self { + Self { + commit_uuid: intent.commit_uuid.unwrap_or_else(Uuid::now_v7), + data_filter: ManifestFilterManager::new( + ManifestContentType::Data, + &intent.removed_data_files, + ), + delete_filter: ManifestFilterManager::new( + ManifestContentType::Deletes, + &intent.removed_delete_files, + ), + intent, + snapshot_id: None, + attempt: 0, + manifest_counter: 0, + added_manifests: None, + owned_artifacts: HashSet::new(), + } + } + + /// Produces the resulting manifest set for the current table and writes + /// this attempt's snapshot. + pub(crate) async fn apply(&mut self, table: &Table) -> Result { + let snapshot_id = self.snapshot_id(table); + + let manifests = current_manifests(table).await?; + + // Removals first: filtering only touches the current snapshot's + // manifests, never the ones this producer adds. + let data = self + .data_filter + .filter( + table, + snapshot_id, + self.commit_uuid, + manifests, + &mut self.manifest_counter, + ) + .await?; + let deletes = self + .delete_filter + .filter( + table, + snapshot_id, + self.commit_uuid, + data.manifests, + &mut self.manifest_counter, + ) + .await?; + let mut manifests = deletes.manifests; + self.owned_artifacts + .extend(self.data_filter.owned_artifacts().iter().cloned()); + self.owned_artifacts + .extend(self.delete_filter.owned_artifacts().iter().cloned()); + + manifests.extend(self.added_manifests(table, snapshot_id).await?); + + let manifest_list_path = self.next_manifest_list_path(table, snapshot_id)?; + + let added_files: Vec = self + .intent + .added_data_files + .iter() + .chain(&self.intent.added_delete_files) + .cloned() + .collect(); + + write_snapshot_commit(table, SnapshotCommitInput { + snapshot_id, + manifest_list_path, + operation: self.intent.operation.clone(), + properties: self.intent.snapshot_properties.clone(), + added_files: &added_files, + removed_files: &data.removed_files, + removed_delete_files: &deletes.removed_files, + manifests, + truncate_full_table: self.intent.truncate_full_table, + }) + .await + } + + /// Checks that every added file is written under the table's default + /// partition spec with a compatible partition value, and that delete files + /// are only added to tables whose format supports them. + /// + /// Runs against the table the manifests are written for; a later rebase + /// does not invalidate manifests already written under a spec the table + /// still has. + fn validate_added_files(&self, table: &Table) -> Result<()> { + let metadata = table.metadata(); + if !self.intent.added_delete_files.is_empty() + && metadata.format_version() == FormatVersion::V1 + { + return Err(invalid_data!( + "Cannot add delete files to a format version 1 table" + )); + } + for file in self + .intent + .added_data_files + .iter() + .chain(&self.intent.added_delete_files) + { + // TODO(#3500): accept any spec known to the table and group the + // files by spec when writing manifests. + if metadata.default_partition_spec_id() != file.partition_spec_id { + return Err(invalid_data!( + "File partition spec id does not match table default partition spec id" + )); + } + validate_partition_value(file.partition(), metadata.default_partition_type())?; + } + Ok(()) + } + + fn snapshot_id(&mut self, table: &Table) -> i64 { + *self + .snapshot_id + .get_or_insert_with(|| generate_snapshot_id(table)) + } + + fn next_manifest_list_path(&mut self, table: &Table, snapshot_id: i64) -> Result { + let path = manifest_list_path(table, snapshot_id, self.attempt, self.commit_uuid)?; + self.attempt += 1; + self.owned_artifacts.insert(path.clone()); + Ok(path) + } + + /// Returns the manifests for the added files, writing them only if no + /// reusable set exists for the current format version. Data and delete + /// files go into separate manifests. + async fn added_manifests( + &mut self, + table: &Table, + snapshot_id: i64, + ) -> Result> { + let format_version = table.metadata().format_version(); + + if let Some(cached) = &self.added_manifests + && cached.format_version == format_version + { + return Ok(cached.manifests.clone()); + } + + self.validate_added_files(table)?; + + let mut manifests = Vec::new(); + for (content, files) in [ + (ManifestContentType::Data, &self.intent.added_data_files), + ( + ManifestContentType::Deletes, + &self.intent.added_delete_files, + ), + ] { + if files.is_empty() { + continue; + } + // Allocate the path before writing so a partial write is still owned. + // + // TODO(#3500): group by partition spec. TODO(#3499): roll at the + // target size and write concurrently. + let path = manifest_path(table, self.commit_uuid, self.manifest_counter)?; + self.manifest_counter += 1; + self.owned_artifacts.insert(path.clone()); + manifests.push( + write_added_manifest( + table, + path, + snapshot_id, + content, + table.metadata().default_partition_spec_id(), + files, + ) + .await?, + ); + } + + self.added_manifests = Some(AddedManifests { + manifests: manifests.clone(), + format_version, + }); + Ok(manifests) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::super::tests::{append_to, data_file}; + use super::{MergingIntent, MergingSnapshotProducer}; + use crate::spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, ManifestContentType, + ManifestStatus, Operation, SnapshotRef, Struct, + }; + use crate::table::Table; + use crate::transaction::tests::{make_v1_table, make_v2_minimal_table}; + use crate::{ErrorKind, TableUpdate}; + + fn delete_file(table: &Table, path: &str) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::PositionDeletes) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(50) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap() + } + + fn intent(operation: Operation) -> MergingIntent { + MergingIntent { + commit_uuid: None, + operation, + snapshot_properties: HashMap::new(), + added_data_files: vec![], + added_delete_files: vec![], + removed_data_files: vec![], + removed_delete_files: vec![], + truncate_full_table: false, + } + } + + fn added_snapshot(commit: &mut crate::transaction::ActionCommit) -> SnapshotRef { + commit + .take_updates() + .into_iter() + .find_map(|u| match u { + TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot)), + _ => None, + }) + .expect("apply should add a snapshot") + } + + /// Reads every (status, path) pair in the manifest at `manifest_path`. + async fn entries_of( + table: &Table, + manifest: &crate::spec::ManifestFile, + ) -> Vec<(ManifestStatus, String)> { + let mut entries: Vec<_> = table + .manifest_reader() + .read(manifest) + .await + .unwrap() + .entries() + .iter() + .map(|e| (e.status(), e.file_path().to_string())) + .collect(); + entries.sort_by(|a, b| a.1.cmp(&b.1)); + entries + } + + /// Removing one of two files in a manifest rewrites it with one `Deleted` + /// and one `Existing` entry; a manifest without removed files is carried + /// forward by path. + #[tokio::test] + async fn test_remove_data_file_rewrites_only_its_manifest() { + let base = make_v2_minimal_table(); + // Two commits so the current snapshot has two data manifests. + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + let table = append_to(&table, vec![data_file(&base, "test/c.parquet", 1)]).await; + let current = table.metadata().current_snapshot().unwrap(); + let before = table.manifest_list_reader(current).load().await.unwrap(); + assert_eq!(before.entries().len(), 2); + let untouched = before + .entries() + .iter() + .find(|m| m.added_files_count == Some(1)) + .unwrap() + .manifest_path + .clone(); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let after = table.manifest_list_reader(&snapshot).load().await.unwrap(); + assert_eq!(after.entries().len(), 2); + assert!(after.entries().iter().any(|m| m.manifest_path == untouched)); + let rewritten = after + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .expect("a rewritten manifest belongs to the new snapshot"); + assert_eq!(rewritten.deleted_files_count, Some(1)); + assert_eq!(rewritten.existing_files_count, Some(1)); + assert_eq!(entries_of(&table, rewritten).await, vec![ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + ]); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("deleted-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-data-files").unwrap(), "2"); + } + + /// Added data and delete files go into separate manifests of the right + /// content type, and the summary counts both. + #[tokio::test] + async fn test_add_data_and_delete_files() { + let table = make_v2_minimal_table(); + let mut i = intent(Operation::Overwrite); + i.added_data_files = vec![data_file(&table, "test/d.parquet", 3)]; + i.added_delete_files = vec![delete_file(&table, "test/d-deletes.parquet")]; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let mut contents: Vec<_> = list.entries().iter().map(|m| m.content).collect(); + contents.sort_by_key(|c| *c as i32); + assert_eq!(contents, vec![ + ManifestContentType::Data, + ManifestContentType::Deletes + ]); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("added-data-files").unwrap(), "1"); + assert_eq!(summary.get("added-delete-files").unwrap(), "1"); + assert_eq!(summary.get("added-records").unwrap(), "3"); + } + + /// A retry reuses both the rewritten and the added manifests; only the + /// manifest list is written again. + #[tokio::test] + async fn test_retry_reuses_rewritten_and_added_manifests() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + let mut i = intent(Operation::Overwrite); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + i.added_data_files = vec![data_file(&base, "test/x.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + + let mut first = producer.apply(&table).await.unwrap(); + let mut second = producer.apply(&table).await.unwrap(); + let first = added_snapshot(&mut first); + let second = added_snapshot(&mut second); + + assert_eq!(first.snapshot_id(), second.snapshot_id()); + assert_ne!(first.manifest_list(), second.manifest_list()); + + let paths = |list: crate::spec::ManifestList| { + let mut p: Vec<_> = list + .entries() + .iter() + .map(|m| m.manifest_path.clone()) + .collect(); + p.sort(); + p + }; + let first_paths = paths(table.manifest_list_reader(&first).load().await.unwrap()); + let second_paths = paths(table.manifest_list_reader(&second).load().await.unwrap()); + assert_eq!(first_paths, second_paths); + // One rewrite plus one added manifest. + assert_eq!(producer.manifest_counter, 2); + assert_eq!(producer.attempt, 2); + } + + /// Removing a file that is not live anywhere fails and names the file. + #[tokio::test] + async fn test_remove_missing_file_fails() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/nope.parquet", 1)]; + let err = MergingSnapshotProducer::new(i) + .apply(&table) + .await + .err() + .expect("removing a file that is not live must fail"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("test/nope.parquet")); + } + + /// A file removed by a concurrent commit is no longer live on the rebased + /// table, so the retry fails rather than silently succeeding. + #[tokio::test] + async fn test_rebase_onto_table_without_removed_file_fails() { + let base = make_v2_minimal_table(); + let s1 = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + producer.apply(&s1).await.unwrap(); + + // Concurrent writer removed `a` first; our retry rebases onto that. + let mut other = intent(Operation::Delete); + other.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut other_commit = MergingSnapshotProducer::new(other) + .apply(&s1) + .await + .unwrap(); + let s2_snapshot = added_snapshot(&mut other_commit); + let s2 = { + use crate::spec::{MAIN_BRANCH, SnapshotReference, SnapshotRetention}; + let metadata = s1 + .metadata() + .clone() + .into_builder(None) + .add_snapshot(s2_snapshot.as_ref().clone()) + .unwrap() + .set_ref( + MAIN_BRANCH, + SnapshotReference::new( + s2_snapshot.snapshot_id(), + SnapshotRetention::branch(None, None, None), + ), + ) + .unwrap() + .build() + .unwrap() + .metadata; + s1.clone().with_metadata(std::sync::Arc::new(metadata)) + }; + + let err = producer + .apply(&s2) + .await + .err() + .expect("file already removed concurrently must not be removed twice"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_delete_files_rejected_on_v1_table() { + let table = make_v1_table(); + let mut i = intent(Operation::Overwrite); + i.added_delete_files = vec![delete_file(&table, "test/d-deletes.parquet")]; + let err = MergingSnapshotProducer::new(i) + .apply(&table) + .await + .err() + .expect("v1 tables cannot hold delete files"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } +} diff --git a/crates/iceberg/src/transaction/snapshot/mod.rs b/crates/iceberg/src/transaction/snapshot/mod.rs index 36616d0125..5d1215d65e 100644 --- a/crates/iceberg/src/transaction/snapshot/mod.rs +++ b/crates/iceberg/src/transaction/snapshot/mod.rs @@ -23,6 +23,12 @@ //! Iceberg metadata and files, never select operation policy, and never cache //! across calls. +// TODO(#1607): remove the allows once `RowDeltaAction` consumes the merging +// producer. +#[allow(dead_code)] +pub(crate) mod manifest_filter; +#[allow(dead_code)] +pub(crate) mod merging; pub(crate) mod simple; use std::collections::{HashMap, HashSet}; @@ -33,10 +39,11 @@ use uuid::Uuid; use crate::error::{Result, invalid_data}; use crate::spec::{ - DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestFile, - ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, PartitionSpecRef, - SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, - StructType, Summary, TableProperties, set_iceberg_version, update_snapshot_summaries, + DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, + ManifestFile, ManifestListWriter, ManifestStatus, ManifestWriter, ManifestWriterBuilder, + Operation, PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, + SnapshotSummaryCollector, Struct, StructType, Summary, TableProperties, set_iceberg_version, + update_snapshot_summaries, }; use crate::table::Table; use crate::transaction::ActionCommit; @@ -173,6 +180,46 @@ pub(crate) async fn current_manifests(table: &Table) -> Result .collect()) } +/// Writes one manifest at `path` holding `files` as `Added` entries, all of +/// which must belong to `partition_spec_id`, and returns its manifest-list entry. +pub(crate) async fn write_added_manifest( + table: &Table, + path: String, + snapshot_id: i64, + content: ManifestContentType, + partition_spec_id: i32, + files: &[DataFile], +) -> Result { + let metadata = table.metadata(); + let partition_spec = metadata + .partition_spec_by_id(partition_spec_id) + .ok_or_else(|| invalid_data!("Unknown partition spec id {partition_spec_id}"))?; + + let mut writer = new_manifest_writer( + table, + path, + snapshot_id, + content, + metadata.current_schema().clone(), + partition_spec.clone(), + )?; + let format_version = metadata.format_version(); + for file in files { + let builder = ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(file.clone()); + let entry = if format_version == FormatVersion::V1 { + builder.snapshot_id(snapshot_id).build() + } else { + // For format version > 1, the snapshot id is inherited from the + // manifest list, so a rebase does not require rewriting the manifest. + builder.build() + }; + writer.add_entry(entry)?; + } + writer.write_manifest_file().await +} + /// Fails if any of `data_files` is already live in the table's current snapshot. /// /// Must run against each attempt's base, since a concurrent commit may have @@ -238,6 +285,7 @@ fn build_summary( mut properties: HashMap, added_files: &[DataFile], removed_files: &[DataFile], + removed_delete_files: &[DataFile], truncate_full_table: bool, ) -> Result { let metadata = table.metadata(); @@ -259,7 +307,7 @@ fn build_summary( ); } - for data_file in removed_files { + for data_file in removed_files.iter().chain(removed_delete_files) { let partition_spec = metadata .partition_spec_by_id(data_file.partition_spec_id) .ok_or_else(|| { @@ -297,9 +345,12 @@ pub(crate) struct SnapshotCommitInput<'a> { pub manifest_list_path: String, pub operation: Operation, pub properties: HashMap, + /// Data and delete files added by this commit. pub added_files: &'a [DataFile], /// Must match the data files marked Deleted by this commit, including their metrics. pub removed_files: &'a [DataFile], + /// Must match the delete files marked Deleted by this commit, including their metrics. + pub removed_delete_files: &'a [DataFile], pub manifests: Vec, /// Whether the summary resets table totals (a full-table overwrite). pub truncate_full_table: bool, @@ -318,20 +369,29 @@ pub(crate) async fn write_snapshot_commit( properties, added_files, removed_files, + removed_delete_files, manifests, truncate_full_table, } = input; - debug_assert_eq!( + let deleted_count = |content: ManifestContentType| { manifests .iter() - .filter(|m| m.added_snapshot_id == snapshot_id && m.content == ManifestContentType::Data) + .filter(|m| m.added_snapshot_id == snapshot_id && m.content == content) .filter_map(|m| m.deleted_files_count) .map(u64::from) - .sum::(), + .sum::() + }; + debug_assert_eq!( + deleted_count(ManifestContentType::Data), removed_files.len() as u64, "deleted files must match this commit's manifest entries" ); + debug_assert_eq!( + deleted_count(ManifestContentType::Deletes), + removed_delete_files.len() as u64, + "deleted delete files must match this commit's manifest entries" + ); let metadata = table.metadata(); let parent_snapshot_id = metadata.current_snapshot_id(); @@ -344,6 +404,7 @@ pub(crate) async fn write_snapshot_commit( properties, added_files, removed_files, + removed_delete_files, truncate_full_table, ) .map_err(|err| { @@ -446,7 +507,7 @@ pub(crate) async fn write_snapshot_commit( } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::collections::HashMap; use std::sync::Arc; @@ -461,7 +522,7 @@ mod tests { use crate::transaction::tests::make_v2_minimal_table; use crate::{ErrorKind, TableUpdate}; - fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { + pub(super) fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { DataFileBuilder::default() .content(DataContentType::Data) .file_path(path.to_string()) @@ -480,7 +541,7 @@ mod tests { /// Commits `data_files` to `table` through a fresh producer, returning the /// table with that snapshot as current. - async fn append_to(table: &Table, data_files: Vec) -> Table { + pub(super) async fn append_to(table: &Table, data_files: Vec) -> Table { let mut producer = SimpleSnapshotProducer::new(AppendIntent { commit_uuid: None, snapshot_properties: HashMap::new(), @@ -567,6 +628,7 @@ mod tests { HashMap::new(), &[], &[deleted], + &[], false, ) .unwrap(); @@ -592,6 +654,7 @@ mod tests { properties: HashMap::new(), added_files: &[], removed_files: &[deleted], + removed_delete_files: &[], manifests: vec![], truncate_full_table: false, }) diff --git a/crates/iceberg/src/transaction/snapshot/simple.rs b/crates/iceberg/src/transaction/snapshot/simple.rs index 7aaa505399..bd810b846f 100644 --- a/crates/iceberg/src/transaction/snapshot/simple.rs +++ b/crates/iceberg/src/transaction/snapshot/simple.rs @@ -23,13 +23,10 @@ use uuid::Uuid; use super::{ SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, - manifest_path, new_manifest_writer, validate_partition_value, write_snapshot_commit, + manifest_path, validate_partition_value, write_added_manifest, write_snapshot_commit, }; use crate::error::{Result, invalid_data}; -use crate::spec::{ - DataFile, FormatVersion, ManifestContentType, ManifestEntry, ManifestFile, ManifestStatus, - Operation, -}; +use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::ActionCommit; @@ -122,6 +119,7 @@ impl SimpleSnapshotProducer { properties: self.intent.snapshot_properties.clone(), added_files: &self.intent.data_files, removed_files: &[], + removed_delete_files: &[], manifests, truncate_full_table: false, }) @@ -197,6 +195,7 @@ impl SimpleSnapshotProducer { table, path, snapshot_id, + ManifestContentType::Data, table.metadata().default_partition_spec_id(), &self.intent.data_files, ) @@ -211,45 +210,6 @@ impl SimpleSnapshotProducer { } } -/// Writes one data manifest at `path` for `data_files`, all of which must -/// belong to `partition_spec_id`, and returns its manifest-list entry. -async fn write_added_manifest( - table: &Table, - path: String, - snapshot_id: i64, - partition_spec_id: i32, - data_files: &[DataFile], -) -> Result { - let metadata = table.metadata(); - let partition_spec = metadata - .partition_spec_by_id(partition_spec_id) - .ok_or_else(|| invalid_data!("Unknown partition spec id {partition_spec_id}"))?; - - let mut writer = new_manifest_writer( - table, - path, - snapshot_id, - ManifestContentType::Data, - metadata.current_schema().clone(), - partition_spec.clone(), - )?; - let format_version = metadata.format_version(); - for data_file in data_files { - let builder = ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file(data_file.clone()); - let entry = if format_version == FormatVersion::V1 { - builder.snapshot_id(snapshot_id).build() - } else { - // For format version > 1, the snapshot id is inherited from the - // manifest list, so a rebase does not require rewriting the manifest. - builder.build() - }; - writer.add_entry(entry)?; - } - writer.write_manifest_file().await -} - #[cfg(test)] mod tests { use std::collections::HashMap; From ebb93575cffd25dceb4b100642ee2ff7a1aef81b Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 27 Oct 2026 17:49:51 -0700 Subject: [PATCH 26/28] robust caching --- crates/iceberg/src/io/object_cache.rs | 6 +- .../transaction/snapshot/manifest_filter.rs | 65 ++-- .../src/transaction/snapshot/merging.rs | 295 +++++++++++++++++- .../iceberg/src/transaction/snapshot/mod.rs | 33 +- .../src/transaction/snapshot/simple.rs | 21 +- 5 files changed, 350 insertions(+), 70 deletions(-) diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 874c9d3483..7ac2ecd3ca 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -52,6 +52,10 @@ pub(crate) enum CachedObjectKey { Manifest(ManifestKey), } +/// Identifies a parsed manifest: its path plus every manifest-list value the +/// reader inherits onto its entries. Two references to one physical manifest +/// with different inherited values parse to different entries, so anything +/// derived from parsed entries must be keyed by this, not by path alone. #[derive(Clone, Debug, Hash, Eq, PartialEq)] pub(crate) struct ManifestKey { path: String, @@ -61,7 +65,7 @@ pub(crate) struct ManifestKey { } impl ManifestKey { - fn new(manifest_file: &ManifestFile) -> Self { + pub(crate) fn new(manifest_file: &ManifestFile) -> Self { Self { path: manifest_file.manifest_path.clone(), added_snapshot_id: manifest_file.added_snapshot_id, diff --git a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs index 1debd09553..11eccfe393 100644 --- a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs +++ b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs @@ -23,6 +23,7 @@ use uuid::Uuid; use super::{manifest_path, new_manifest_writer}; use crate::error::{Result, invalid_data}; +use crate::io::object_cache::ManifestKey; use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile}; use crate::table::Table; @@ -30,21 +31,24 @@ use crate::table::Table; /// /// One manager lives for one logical execution of an action and is driven /// once per attempt via [`ManifestFilterManager::filter`]. A manifest that -/// references none of the requested files passes through untouched. One that -/// does is rewritten with those entries marked `Deleted` and the rest -/// `Existing`; the rewrite is keyed by the source manifest's path, which is -/// immutable, so a rebase only rewrites manifests not seen before. +/// references none of the files passes through untouched. One that does is +/// rewritten with those entries marked `Deleted` and the rest `Existing`. +/// +/// A rewrite is reused across attempts while its inputs are unchanged. Its +/// entries carry the sequence numbers and row ids the source's entries +/// inherited from the manifest list when they were read, so the reuse key is +/// the source's [`ManifestKey`], not its path alone: a transaction-local +/// source manifest keeps its path across attempts but inherits new values +/// after a rebase. pub(crate) struct ManifestFilterManager { content: ManifestContentType, /// Paths of the files to remove. Fixed for the execution. - requested_paths: HashSet, - /// Source manifest path → rewritten result. - filtered: HashMap, + files_to_remove: HashSet, + /// Source manifest → rewritten result. + filtered: HashMap, /// Format version the rewrites were written under. Per spec, v1 entries /// cannot inherit sequence numbers, so an upgrade invalidates them. format_version: Option, - /// Metadata files written by this manager. - owned_artifacts: HashSet, } /// One rewritten manifest and the files it marked as deleted. @@ -64,32 +68,27 @@ pub(crate) struct Filtered { } impl ManifestFilterManager { - pub(crate) fn new(content: ManifestContentType, removed_files: &[DataFile]) -> Self { + pub(crate) fn new(content: ManifestContentType, files_to_remove: &[DataFile]) -> Self { Self { content, - requested_paths: removed_files + files_to_remove: files_to_remove .iter() .map(|f| f.file_path().to_string()) .collect(), filtered: HashMap::new(), format_version: None, - owned_artifacts: HashSet::new(), } } - /// Metadata files written by this manager so far. - pub(crate) fn owned_artifacts(&self) -> &HashSet { - &self.owned_artifacts - } - /// Applies the removals to `manifests`, rewriting those of this manager's - /// content type that reference a requested file. + /// content type that reference a file to remove. /// - /// Fails if a requested file is not live in any manifest: it was never in + /// Fails if a file to remove is not live in any manifest: it was never in /// the table, or a concurrent commit already removed it. /// - /// `next_manifest_number` allocates paths for rewritten manifests from the - /// producer's counter, so they never collide with other manifests it writes. + /// Rewritten manifests take their paths from `next_manifest_number` and are + /// recorded in `owned_artifacts` before being written, so a failed write + /// still leaves its path tracked for cleanup. pub(crate) async fn filter( &mut self, table: &Table, @@ -97,8 +96,9 @@ impl ManifestFilterManager { commit_uuid: Uuid, manifests: Vec, next_manifest_number: &mut u64, + owned_artifacts: &mut HashSet, ) -> Result { - if self.requested_paths.is_empty() { + if self.files_to_remove.is_empty() { return Ok(Filtered { manifests, removed_files: Vec::new(), @@ -120,7 +120,8 @@ impl ManifestFilterManager { continue; } - let result = match self.filtered.get(&source.manifest_path) { + let key = ManifestKey::new(&source); + let result = match self.filtered.get(&key) { Some(cached) => cached.clone(), None => { let result = self @@ -130,10 +131,10 @@ impl ManifestFilterManager { commit_uuid, &source, next_manifest_number, + owned_artifacts, ) .await?; - self.filtered - .insert(source.manifest_path.clone(), result.clone()); + self.filtered.insert(key, result.clone()); result } }; @@ -144,7 +145,7 @@ impl ManifestFilterManager { let found: HashSet<&str> = removed_files.iter().map(|f| f.file_path()).collect(); let mut missing: Vec<&str> = self - .requested_paths + .files_to_remove .iter() .map(String::as_str) .filter(|p| !found.contains(p)) @@ -163,22 +164,23 @@ impl ManifestFilterManager { }) } - /// Reads `source` and, if it references any requested file, writes a copy + /// Reads `source` and, if it references any file to remove, writes a copy /// with those entries marked `Deleted` and the rest `Existing`. async fn rewrite( - &mut self, + &self, table: &Table, snapshot_id: i64, commit_uuid: Uuid, source: &ManifestFile, next_manifest_number: &mut u64, + owned_artifacts: &mut HashSet, ) -> Result { let manifest = table.object_cache().get_manifest(source).await?; let removed_files: Vec = manifest .entries() .iter() - .filter(|e| e.is_alive() && self.requested_paths.contains(e.file_path())) + .filter(|e| e.is_alive() && self.files_to_remove.contains(e.file_path())) .map(|e| e.data_file().clone()) .collect(); if removed_files.is_empty() { @@ -188,10 +190,9 @@ impl ManifestFilterManager { }); } - // Allocate the path before writing so a partial write is still owned. let path = manifest_path(table, commit_uuid, *next_manifest_number)?; *next_manifest_number += 1; - self.owned_artifacts.insert(path.clone()); + owned_artifacts.insert(path.clone()); // Keep the source manifest's spec: its entries' partition values are // laid out for that spec, not the table's current default. @@ -209,7 +210,7 @@ impl ManifestFilterManager { continue; } let entry = (**entry).clone(); - if self.requested_paths.contains(entry.file_path()) { + if self.files_to_remove.contains(entry.file_path()) { writer.add_delete_entry(entry)?; } else { writer.add_existing_entry(entry)?; diff --git a/crates/iceberg/src/transaction/snapshot/merging.rs b/crates/iceberg/src/transaction/snapshot/merging.rs index 16e1ba85f4..8d370befe3 100644 --- a/crates/iceberg/src/transaction/snapshot/merging.rs +++ b/crates/iceberg/src/transaction/snapshot/merging.rs @@ -24,8 +24,9 @@ use uuid::Uuid; use super::manifest_filter::ManifestFilterManager; use super::{ - SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, - manifest_path, validate_partition_value, write_added_manifest, write_snapshot_commit, + AddedManifests, SnapshotCommitInput, current_manifests, generate_snapshot_id, + manifest_list_path, manifest_path, validate_partition_value, write_added_manifest, + write_snapshot_commit, }; use crate::error::{Result, invalid_data}; use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; @@ -81,15 +82,6 @@ pub(crate) struct MergingSnapshotProducer { owned_artifacts: HashSet, } -/// The manifests written for one execution's added data and delete files. -/// -/// Reusable while the table's format version is unchanged; see -/// `simple::AddedManifests` for why. -struct AddedManifests { - manifests: Vec, - format_version: FormatVersion, -} - impl MergingSnapshotProducer { pub(crate) fn new(intent: MergingIntent) -> Self { Self { @@ -128,6 +120,7 @@ impl MergingSnapshotProducer { self.commit_uuid, manifests, &mut self.manifest_counter, + &mut self.owned_artifacts, ) .await?; let deletes = self @@ -138,13 +131,10 @@ impl MergingSnapshotProducer { self.commit_uuid, data.manifests, &mut self.manifest_counter, + &mut self.owned_artifacts, ) .await?; let mut manifests = deletes.manifests; - self.owned_artifacts - .extend(self.data_filter.owned_artifacts().iter().cloned()); - self.owned_artifacts - .extend(self.delete_filter.owned_artifacts().iter().cloned()); manifests.extend(self.added_manifests(table, snapshot_id).await?); @@ -529,6 +519,281 @@ mod tests { assert_eq!(err.kind(), ErrorKind::DataInvalid); } + /// Multi-action replay (RFC §5.2): action A's manifest keeps its path across + /// attempts, but after a rebase the manifest list assigns it a new sequence + /// number. Action B's rewrite of that manifest bakes the inherited sequence + /// numbers into its `Existing`/`Deleted` entries, so B must not reuse a + /// rewrite made under the previous sequence number. + #[tokio::test] + async fn test_rewrite_is_not_reused_when_source_manifest_sequence_number_changes() { + use super::super::simple::{AppendIntent, SimpleSnapshotProducer}; + use super::super::tests::with_snapshot; + + let s0 = make_v2_minimal_table(); + + // Action A appends x and y; action B removes x from A's manifest. + let mut a = SimpleSnapshotProducer::new(AppendIntent { + commit_uuid: None, + snapshot_properties: HashMap::new(), + data_files: vec![ + data_file(&s0, "test/x.parquet", 1), + data_file(&s0, "test/y.parquet", 1), + ], + }); + let mut b_intent = intent(Operation::Delete); + b_intent.removed_data_files = vec![data_file(&s0, "test/x.parquet", 1)]; + let mut b = MergingSnapshotProducer::new(b_intent); + + // Attempt 0 on S0. + let sa0 = added_snapshot(&mut a.apply(&s0).await.unwrap()); + let after_a0 = with_snapshot(&s0, sa0.clone()); + let sb0 = added_snapshot(&mut b.apply(&after_a0).await.unwrap()); + + // A concurrent commit lands on S0, taking the next sequence number. The + // retry rebases onto it: A reuses its manifest but it now inherits a new + // sequence number, and B sees the same manifest path under it. + let s1 = append_to(&s0, vec![data_file(&s0, "test/other.parquet", 1)]).await; + let sa1 = added_snapshot(&mut a.apply(&s1).await.unwrap()); + let after_a1 = with_snapshot(&s1, sa1.clone()); + let sb1 = added_snapshot(&mut b.apply(&after_a1).await.unwrap()); + assert_eq!(sa1.sequence_number(), sa0.sequence_number() + 1); + + // B's rewritten manifest in each attempt. + let rewritten_in = |table: Table, snapshot: SnapshotRef| async move { + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .cloned() + .unwrap(); + let manifest = table.manifest_reader().read(&rewritten).await.unwrap(); + let y = manifest + .entries() + .iter() + .find(|e| e.file_path() == "test/y.parquet") + .unwrap(); + (rewritten.manifest_path.clone(), y.sequence_number()) + }; + let (path0, y_seq0) = rewritten_in(after_a0, sb0).await; + let (path1, y_seq1) = rewritten_in(after_a1, sb1).await; + + // `y` stays live under A's snapshot, whose sequence number moved; the + // rewrite that keeps it as `Existing` must carry the new one. + assert_eq!(y_seq0, Some(sa0.sequence_number())); + assert_eq!(y_seq1, Some(sa1.sequence_number())); + assert_ne!( + path0, path1, + "a rewrite under a different sequence number is a new file" + ); + } + + /// Removing a data file from a v1 table: v1 entries read with sequence + /// number 0, which satisfies the writer, and v1 serialization drops the + /// field again. The rewritten manifest is readable and `b` stays live. + #[tokio::test] + async fn test_remove_data_file_from_v1_table() { + let base = make_v1_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut commit = MergingSnapshotProducer::new(i).apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + let manifest = table.manifest_reader().read(rewritten).await.unwrap(); + let mut live: Vec<_> = manifest + .entries() + .iter() + .filter(|e| e.is_alive()) + .map(|e| e.file_path().to_string()) + .collect(); + live.sort(); + assert_eq!(live, vec!["test/b.parquet".to_string()]); + // Both the rewritten entries and the snapshot stay at v1's sequence number 0. + assert!( + manifest + .entries() + .iter() + .all(|e| e.sequence_number() == Some(0)) + ); + assert_eq!(snapshot.sequence_number(), 0); + assert_eq!( + snapshot + .summary() + .additional_properties + .get("deleted-data-files") + .unwrap(), + "1" + ); + } + + /// A manifest written under an earlier schema is rewritten under the + /// current one; its partition spec still binds because partition source + /// columns cannot be dropped. + #[tokio::test] + async fn test_rewrite_manifest_written_under_earlier_schema() { + use crate::spec::{NestedField, PrimitiveType, Type}; + + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + // Evolve the schema after the manifest was written. + let new_schema = table + .metadata() + .current_schema() + .as_ref() + .clone() + .into_builder() + .with_fields([ + NestedField::optional(100, "added", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .add_current_schema(new_schema) + .unwrap() + .build() + .unwrap() + .metadata; + let evolved = table.clone().with_metadata(std::sync::Arc::new(metadata)); + assert_ne!( + evolved.metadata().current_schema_id(), + table.metadata().current_schema_id() + ); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut commit = MergingSnapshotProducer::new(i) + .apply(&evolved) + .await + .unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = evolved + .manifest_list_reader(&snapshot) + .load() + .await + .unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + assert_eq!( + rewritten.partition_spec_id, + table.metadata().default_partition_spec_id() + ); + assert_eq!(entries_of(&evolved, rewritten).await, vec![ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + ]); + // Partition values survive the rewrite under the new schema. + let manifest = evolved.manifest_reader().read(rewritten).await.unwrap(); + for entry in manifest.entries() { + assert_eq!( + entry.data_file().partition(), + &Struct::from_iter([Some(Literal::long(300))]) + ); + } + } + + /// On v3, existing entries keep the row ids they were assigned when first + /// committed, and the rewritten manifest gets its own range from the list + /// writer. + #[tokio::test] + async fn test_rewrite_preserves_first_row_ids_on_v3() { + use crate::transaction::tests::make_v3_minimal_table; + + let base = make_v3_minimal_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 5), + data_file(&base, "test/b.parquet", 7), + ]) + .await; + let before = table.metadata().current_snapshot().unwrap(); + let before_list = table.manifest_list_reader(before).load().await.unwrap(); + let before_manifest = table + .manifest_reader() + .read(&before_list.entries()[0]) + .await + .unwrap(); + let b_row_id = before_manifest + .entries() + .iter() + .find(|e| e.file_path() == "test/b.parquet") + .unwrap() + .data_file() + .first_row_id(); + assert_eq!(b_row_id, Some(5)); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 5)]; + let mut commit = MergingSnapshotProducer::new(i).apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + let manifest = table.manifest_reader().read(rewritten).await.unwrap(); + let b = manifest + .entries() + .iter() + .find(|e| e.file_path() == "test/b.parquet") + .unwrap(); + assert_eq!(b.data_file().first_row_id(), b_row_id); + // The new snapshot starts its range where the table's next_row_id was. + assert_eq!( + snapshot.first_row_id(), + Some(table.metadata().next_row_id()) + ); + } + + /// Every manifest the producer writes, including rewrites, is recorded as + /// owned before the write starts. + #[tokio::test] + async fn test_owned_artifacts_cover_rewrites_and_added_manifests() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Overwrite); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + i.added_data_files = vec![data_file(&base, "test/x.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + for m in list.entries() { + if m.added_snapshot_id == snapshot.snapshot_id() { + assert!(producer.owned_artifacts.contains(&m.manifest_path)); + } + } + assert!(producer.owned_artifacts.contains(snapshot.manifest_list())); + // One rewrite, one added manifest, one manifest list. + assert_eq!(producer.owned_artifacts.len(), 3); + } + #[tokio::test] async fn test_delete_files_rejected_on_v1_table() { let table = make_v1_table(); diff --git a/crates/iceberg/src/transaction/snapshot/mod.rs b/crates/iceberg/src/transaction/snapshot/mod.rs index 5d1215d65e..7ac1e4c7a8 100644 --- a/crates/iceberg/src/transaction/snapshot/mod.rs +++ b/crates/iceberg/src/transaction/snapshot/mod.rs @@ -180,6 +180,21 @@ pub(crate) async fn current_manifests(table: &Table) -> Result .collect()) } +/// The manifests a producer wrote for one execution's added files. +/// +/// Reusable while the table's format version is unchanged. Per spec, sequence +/// number inheritance does not apply to v1 manifests, so one written before a +/// concurrent upgrade cannot carry the v2 sequence number a retry assigns and +/// must be rewritten. The schema is not a guard: the manifest's embedded schema +/// describes the files it was written for, which do not change. Nor is the +/// default partition spec: the files name their own spec, which stays valid. +pub(crate) struct AddedManifests { + /// One per content type today. Several once added files are grouped by + /// partition spec (#3500) or rolled over at a target size (#3499). + pub manifests: Vec, + pub format_version: FormatVersion, +} + /// Writes one manifest at `path` holding `files` as `Added` entries, all of /// which must belong to `partition_spec_id`, and returns its manifest-list entry. pub(crate) async fn write_added_manifest( @@ -547,17 +562,25 @@ pub(crate) mod tests { snapshot_properties: HashMap::new(), data_files, }); - let snapshot = producer - .apply(table) - .await - .unwrap() + let commit = producer.apply(table).await.unwrap(); + with_snapshot(table, added_snapshot(commit)) + } + + /// Extracts the snapshot an `ActionCommit` adds. + pub(super) fn added_snapshot(mut commit: ActionCommit) -> SnapshotRef { + commit .take_updates() .into_iter() .find_map(|u| match u { TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot)), _ => None, }) - .expect("append should add a snapshot"); + .expect("commit should add a snapshot") + } + + /// Returns `table` with `snapshot` added and set as current, as a local + /// `TableCommit::apply` would. + pub(super) fn with_snapshot(table: &Table, snapshot: SnapshotRef) -> Table { let metadata = table .metadata() .clone() diff --git a/crates/iceberg/src/transaction/snapshot/simple.rs b/crates/iceberg/src/transaction/snapshot/simple.rs index bd810b846f..4da190e2f6 100644 --- a/crates/iceberg/src/transaction/snapshot/simple.rs +++ b/crates/iceberg/src/transaction/snapshot/simple.rs @@ -22,11 +22,12 @@ use std::collections::{HashMap, HashSet}; use uuid::Uuid; use super::{ - SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, - manifest_path, validate_partition_value, write_added_manifest, write_snapshot_commit, + AddedManifests, SnapshotCommitInput, current_manifests, generate_snapshot_id, + manifest_list_path, manifest_path, validate_partition_value, write_added_manifest, + write_snapshot_commit, }; use crate::error::{Result, invalid_data}; -use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; +use crate::spec::{DataFile, ManifestContentType, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::ActionCommit; @@ -73,20 +74,6 @@ pub(crate) struct SimpleSnapshotProducer { owned_artifacts: HashSet, } -/// The manifests written for one append's added files. -/// -/// Reusable while the table's format version is unchanged. Per spec, sequence -/// number inheritance does not apply to v1 manifests, so one written before a -/// concurrent upgrade cannot carry the v2 sequence number a retry assigns and -/// must be rewritten. The schema is not a guard: the manifest's embedded schema -/// describes the files it was written for, which do not change. -struct AddedManifests { - /// One manifest today. Several once added files are grouped by partition - /// spec (#3500) or rolled over at a target size (#3499). - manifests: Vec, - format_version: FormatVersion, -} - impl SimpleSnapshotProducer { pub(crate) fn new(intent: AppendIntent) -> Self { Self { From ff97a628d91f0b5656ccb65710bda34215cfcdc4 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Wed, 28 Oct 2026 13:40:31 -0700 Subject: [PATCH 27/28] use manifest's schema --- crates/iceberg/src/spec/transform.rs | 18 +++ .../transaction/snapshot/manifest_filter.rs | 11 +- .../src/transaction/snapshot/merging.rs | 114 ++++++++++++++++++ 3 files changed, 138 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index f52202b78b..576b6553a9 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -221,6 +221,9 @@ impl Transform { (Self::Hour, PrimitiveLiteral::Int(ordinal)) => Self::human_hour(ordinal), (_, value) => { let field_type = field_type.as_primitive_type().unwrap(); + // A value written before its source column was promoted (`int` + // to `long`, `float` to `double`) still carries the old type. + let value = value.promote_to(field_type).unwrap_or(value); Datum::new(field_type.clone(), value).to_human_string() } } @@ -1322,6 +1325,21 @@ mod tests { ); } + /// A partition value written before its source column was promoted keeps + /// the old literal type; rendering it under the promoted type must widen it + /// rather than panic. + #[test] + fn test_to_human_string_promotes_value_to_declared_type() { + assert_eq!(human(Transform::Identity, PrimitiveType::Long, 7), "7"); + assert_eq!( + Transform::Identity.to_human_string( + &Type::Primitive(PrimitiveType::Double), + Some(&Literal::float(1.5_f32)) + ), + "1.5" + ); + } + #[test] fn test_to_human_string_null_cases() { assert_eq!(human(Transform::Void, PrimitiveType::Int, 47), "null"); diff --git a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs index 11eccfe393..e977adb89b 100644 --- a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs +++ b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs @@ -194,16 +194,17 @@ impl ManifestFilterManager { *next_manifest_number += 1; owned_artifacts.insert(path.clone()); - // Keep the source manifest's spec: its entries' partition values are - // laid out for that spec, not the table's current default. - let spec = manifest.metadata().partition_spec().clone(); + // Write under the source manifest's own schema and spec: its entries' + // partition values and column bounds were encoded for that schema's + // types, which the table's current schema may since have promoted. + let source_metadata = manifest.metadata(); let mut writer = new_manifest_writer( table, path, snapshot_id, self.content, - table.metadata().current_schema().clone(), - spec.into(), + source_metadata.schema().clone(), + source_metadata.partition_spec().clone().into(), )?; for entry in manifest.entries() { if !entry.is_alive() { diff --git a/crates/iceberg/src/transaction/snapshot/merging.rs b/crates/iceberg/src/transaction/snapshot/merging.rs index 8d370befe3..3e93f8aba2 100644 --- a/crates/iceberg/src/transaction/snapshot/merging.rs +++ b/crates/iceberg/src/transaction/snapshot/merging.rs @@ -715,6 +715,120 @@ mod tests { } } + /// A manifest's partition values are encoded for the partition type its + /// spec had when it was written. Promoting the source column (`int` to + /// `long`) changes that type under the current schema; the rewrite must + /// keep the entries readable and their partition values intact. + #[tokio::test] + async fn test_rewrite_manifest_after_partition_source_type_promotion() { + use crate::TableIdent; + use crate::io::FileIO; + use crate::spec::{ + FormatVersion, NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, + TableMetadataBuilder, Transform, Type, + }; + use crate::test_utils::test_runtime; + + let int_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let spec = PartitionSpec::builder(int_schema.clone()) + .add_partition_field("x", "x", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let metadata = TableMetadataBuilder::new( + int_schema, + spec, + SortOrder::unsorted_order(), + "memory:///table".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + let base = Table::builder() + .metadata(metadata) + .metadata_location("memory:///table/metadata/v1.json") + .identifier(TableIdent::from_strs(["ns", "t"]).unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) + .build() + .unwrap(); + + let int_file = |path: &str| { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(base.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::int(7))])) + .build() + .unwrap() + }; + let table = append_to(&base, vec![ + int_file("test/a.parquet"), + int_file("test/b.parquet"), + ]) + .await; + + // Promote x: int -> long. + let long_schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .add_current_schema(long_schema) + .unwrap() + .build() + .unwrap() + .metadata; + let promoted = table.clone().with_metadata(std::sync::Arc::new(metadata)); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![int_file("test/a.parquet")]; + let mut commit = MergingSnapshotProducer::new(i) + .apply(&promoted) + .await + .unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = promoted + .manifest_list_reader(&snapshot) + .load() + .await + .unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + assert_eq!(entries_of(&promoted, rewritten).await, vec![ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + ]); + let manifest = promoted.manifest_reader().read(rewritten).await.unwrap(); + for entry in manifest.entries() { + assert_eq!( + entry.data_file().partition(), + &Struct::from_iter([Some(Literal::int(7))]) + ); + } + } + /// On v3, existing entries keep the row ids they were assigned when first /// committed, and the rewritten manifest gets its own range from the list /// writer. From 682a6150bd017e9a95c60d7e5dc44a902bc22d6e Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 30 Oct 2026 10:27:56 +0100 Subject: [PATCH 28/28] Keep the rewrite's snapshot id across commit attempts --- crates/iceberg/src/transaction/merging.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/iceberg/src/transaction/merging.rs b/crates/iceberg/src/transaction/merging.rs index 79b2e76c1d..b8be279a41 100644 --- a/crates/iceberg/src/transaction/merging.rs +++ b/crates/iceberg/src/transaction/merging.rs @@ -223,6 +223,7 @@ pub(crate) struct MergingSnapshotProducer { added_data_files: Vec, filter_manager: ManifestFilterManager, commit_uuid: Uuid, + snapshot_id: Option, data_sequence_number: Option, /// Names the manifests this producer writes. It is never reset, so a /// commit retry cannot overwrite a manifest an earlier attempt wrote @@ -247,6 +248,7 @@ impl MergingSnapshotProducer { added_data_files, filter_manager, commit_uuid: Uuid::now_v7(), + snapshot_id: None, data_sequence_number, manifest_counter: 0, attempt: 0, @@ -292,7 +294,9 @@ impl MergingSnapshotProducer { self.validate_data_sequence_number(table)?; self.validate_added_data_files(table)?; validate_files_not_live(table, &self.added_data_files).await?; - let snapshot_id = generate_snapshot_id(table); + let snapshot_id = *self + .snapshot_id + .get_or_insert_with(|| generate_snapshot_id(table)); // 1. Load existing manifests from the current snapshot. let existing_manifests = match table.metadata().current_snapshot() {