From 2641d626b213ef36e0d65403ddc405c92fecdc29 Mon Sep 17 00:00:00 2001 From: glitchy Date: Fri, 12 Jun 2026 22:09:59 -0400 Subject: [PATCH 01/50] feat(transaction): add OverwriteAction with CoW delete support Adds an OverwriteAction to Transaction for atomic replacement of data files via an Operation::Overwrite snapshot. add_data_files() stages new files; delete_data_files() rewrites affected manifests with matching entries marked ManifestStatus::Deleted (copy-on-write). Manifests are loaded through the Table::manifest_list_reader abstraction. --- crates/iceberg/src/spec/manifest/writer.rs | 8 + crates/iceberg/src/transaction/append.rs | 1 + crates/iceberg/src/transaction/mod.rs | 7 + crates/iceberg/src/transaction/overwrite.rs | 521 ++++++++++++++++++++ crates/iceberg/src/transaction/snapshot.rs | 15 + 5 files changed, 552 insertions(+) create mode 100644 crates/iceberg/src/transaction/overwrite.rs diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index 88da1a396c..3eae375022 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -382,6 +382,14 @@ impl ManifestWriter { Ok(()) } + /// Add a deleted manifest entry, preserving the original sequence numbers. + pub(crate) fn add_deleted_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { + self.check_data_file(&entry.data_file)?; + entry.status = ManifestStatus::Deleted; + self.add_entry_inner(entry)?; + Ok(()) + } + /// Add an file as existing manifest entry. The original data and file sequence numbers, snapshot ID, /// which were assigned at commit, must be preserved when adding an existing entry. pub fn add_existing_file( diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 546e1a2d64..ace32418d6 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -90,6 +90,7 @@ impl TransactionAction for FastAppendAction { self.key_metadata.clone(), self.snapshot_properties.clone(), self.added_data_files.clone(), + vec![], ); // validate added files diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index d78f41cd42..7493521e6f 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -55,6 +55,7 @@ mod action; pub use action::*; mod append; mod expire_snapshots; +mod overwrite; mod snapshot; mod sort_order; mod update_location; @@ -75,6 +76,7 @@ use crate::table::Table; use crate::transaction::action::BoxedTransactionAction; use crate::transaction::append::FastAppendAction; use crate::transaction::expire_snapshots::ExpireSnapshotsAction; +use crate::transaction::overwrite::OverwriteAction; use crate::transaction::sort_order::ReplaceSortOrderAction; use crate::transaction::update_location::UpdateLocationAction; use crate::transaction::update_properties::UpdatePropertiesAction; @@ -151,6 +153,11 @@ impl Transaction { FastAppendAction::new() } + /// Creates an overwrite action. + pub fn overwrite(&self) -> OverwriteAction { + OverwriteAction::new() + } + /// Creates replace sort order action. pub fn replace_sort_order(&self) -> ReplaceSortOrderAction { ReplaceSortOrderAction::new() diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs new file mode 100644 index 0000000000..013162c259 --- /dev/null +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -0,0 +1,521 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use uuid::Uuid; + +use crate::error::Result; +use crate::spec::{ + DataFile, FormatVersion, ManifestContentType, ManifestEntry, ManifestFile, + ManifestWriterBuilder, Operation, +}; +use crate::table::Table; +use crate::transaction::snapshot::{ + DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, +}; +use crate::transaction::{ActionCommit, TransactionAction}; + +/// OverwriteAction is a transaction action for overwriting data files in the table. +/// +/// Creates a snapshot with `Operation::Overwrite` semantics — adds new data files and +/// optionally removes existing data files by rewriting affected manifests with those +/// entries marked as `ManifestStatus::Deleted`. +pub struct OverwriteAction { + check_duplicate: bool, + commit_uuid: Option, + key_metadata: Option>, + snapshot_properties: HashMap, + added_data_files: Vec, + deleted_data_files: Vec, +} + +impl OverwriteAction { + pub(crate) fn new() -> Self { + Self { + check_duplicate: true, + commit_uuid: None, + key_metadata: None, + snapshot_properties: HashMap::default(), + added_data_files: vec![], + deleted_data_files: vec![], + } + } + + /// Set whether to check duplicate files. + pub fn with_check_duplicate(mut self, v: bool) -> Self { + self.check_duplicate = v; + self + } + + /// Add data files to the snapshot. + pub fn add_data_files(mut self, data_files: impl IntoIterator) -> Self { + self.added_data_files.extend(data_files); + self + } + + /// Specify data files to be removed from the table in this overwrite. + pub fn delete_data_files(mut self, data_files: impl IntoIterator) -> Self { + self.deleted_data_files.extend(data_files); + self + } + + /// Set commit UUID for the snapshot. + pub fn set_commit_uuid(mut self, commit_uuid: Uuid) -> Self { + self.commit_uuid = Some(commit_uuid); + self + } + + /// Set key metadata for manifest files. + pub fn set_key_metadata(mut self, key_metadata: Vec) -> Self { + self.key_metadata = Some(key_metadata); + self + } + + /// Set snapshot summary properties. + pub fn set_snapshot_properties(mut self, snapshot_properties: HashMap) -> Self { + self.snapshot_properties = snapshot_properties; + self + } +} + +#[async_trait] +impl TransactionAction for OverwriteAction { + async fn commit(self: Arc, table: &Table) -> Result { + let snapshot_producer = SnapshotProducer::new( + table, + self.commit_uuid.unwrap_or_else(Uuid::now_v7), + self.key_metadata.clone(), + self.snapshot_properties.clone(), + self.added_data_files.clone(), + self.deleted_data_files.clone(), + ); + + snapshot_producer.validate_added_data_files()?; + + if self.check_duplicate { + snapshot_producer.validate_duplicate_files().await?; + } + + let deleted_file_paths: HashSet = self + .deleted_data_files + .iter() + .map(|f| f.file_path.clone()) + .collect(); + + let snapshot_id = snapshot_producer.snapshot_id(); + snapshot_producer + .commit( + OverwriteOperation { + deleted_file_paths, + snapshot_id, + }, + DefaultManifestProcess, + ) + .await + } +} + +struct OverwriteOperation { + deleted_file_paths: HashSet, + snapshot_id: i64, +} + +impl SnapshotProduceOperation for OverwriteOperation { + fn operation(&self) -> Operation { + Operation::Overwrite + } + + async fn delete_entries( + &self, + _snapshot_produce: &SnapshotProducer<'_>, + ) -> Result> { + Ok(vec![]) + } + + async fn existing_manifest( + &self, + snapshot_produce: &SnapshotProducer<'_>, + ) -> Result> { + let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { + return Ok(vec![]); + }; + + let manifest_list = snapshot_produce + .table + .manifest_list_reader(snapshot) + .load() + .await?; + + if self.deleted_file_paths.is_empty() { + return Ok(manifest_list + .entries() + .iter() + .filter(|entry| entry.has_added_files() || entry.has_existing_files()) + .cloned() + .collect()); + } + + let mut result = Vec::new(); + + for manifest_file in manifest_list.entries() { + if !manifest_file.has_added_files() && !manifest_file.has_existing_files() { + continue; + } + + let manifest = manifest_file + .load_manifest(snapshot_produce.table.file_io()) + .await?; + + let has_deletes = manifest.entries().iter().any(|entry| { + entry.is_alive() && self.deleted_file_paths.contains(entry.file_path()) + }); + + if has_deletes { + let rewritten = self + .rewrite_manifest(snapshot_produce, manifest_file, &manifest) + .await?; + result.push(rewritten); + } else { + result.push(manifest_file.clone()); + } + } + + Ok(result) + } +} + +impl OverwriteOperation { + /// Rewrite a manifest, marking entries whose file paths are in `deleted_file_paths` + /// as `ManifestStatus::Deleted`. + async fn rewrite_manifest( + &self, + snapshot_produce: &SnapshotProducer<'_>, + manifest_file: &ManifestFile, + manifest: &crate::spec::Manifest, + ) -> Result { + let table = snapshot_produce.table; + + let new_manifest_path = format!( + "{}/metadata/{}-m-overwrite.avro", + table.metadata().location(), + Uuid::now_v7(), + ); + let output_file = table.file_io().new_output(&new_manifest_path)?; + let builder = ManifestWriterBuilder::new( + output_file, + Some(self.snapshot_id), + manifest_file.key_metadata.clone(), + table.metadata().current_schema().clone(), + table.metadata().default_partition_spec().as_ref().clone(), + ); + + let mut writer = match table.metadata().format_version() { + FormatVersion::V1 => builder.build_v1(), + FormatVersion::V2 => match manifest_file.content { + ManifestContentType::Data => builder.build_v2_data(), + ManifestContentType::Deletes => builder.build_v2_deletes(), + }, + FormatVersion::V3 => match manifest_file.content { + ManifestContentType::Data => builder.build_v3_data(), + ManifestContentType::Deletes => builder.build_v3_deletes(), + }, + }; + + for entry in manifest.entries() { + if entry.is_alive() && self.deleted_file_paths.contains(entry.file_path()) { + let mut deleted: ManifestEntry = (**entry).clone(); + deleted.snapshot_id = Some(self.snapshot_id); + writer.add_deleted_entry(deleted)?; + } else { + let cloned: ManifestEntry = (**entry).clone(); + writer.add_existing_entry(cloned)?; + } + } + + writer.write_manifest_file().await + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, ManifestStatus, + Operation, SnapshotRef, Struct, + }; + use crate::transaction::tests::make_v2_minimal_table; + use crate::transaction::{Transaction, TransactionAction}; + use crate::{TableRequirement, TableUpdate}; + + fn test_data_file(path: &str, partition_spec_id: i32) -> crate::spec::DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(partition_spec_id) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap() + } + + #[tokio::test] + async fn test_empty_data_overwrite_action() { + let table = make_v2_minimal_table(); + let tx = Transaction::new(&table); + let action = tx.overwrite().add_data_files(vec![]); + assert!(Arc::new(action).commit(&table).await.is_err()); + } + + #[tokio::test] + async fn test_overwrite_snapshot_properties() { + let table = make_v2_minimal_table(); + let tx = Transaction::new(&table); + + let mut snapshot_properties = HashMap::new(); + snapshot_properties.insert("key".to_string(), "val".to_string()); + + let data_file = test_data_file( + "test/1.parquet", + table.metadata().default_partition_spec_id(), + ); + + let action = tx + .overwrite() + .set_snapshot_properties(snapshot_properties) + .add_data_files(vec![data_file]); + let mut action_commit = Arc::new(action).commit(&table).await.unwrap(); + let updates = action_commit.take_updates(); + + let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { + snapshot + } else { + unreachable!() + }; + assert_eq!( + new_snapshot + .summary() + .additional_properties + .get("key") + .unwrap(), + "val" + ); + } + + #[tokio::test] + async fn test_overwrite_incompatible_partition_value() { + let table = make_v2_minimal_table(); + let tx = Transaction::new(&table); + + let data_file = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("test/3.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::string("test"))])) + .build() + .unwrap(); + + let action = tx.overwrite().add_data_files(vec![data_file]); + assert!(Arc::new(action).commit(&table).await.is_err()); + } + + #[tokio::test] + async fn test_overwrite_basic() { + let table = make_v2_minimal_table(); + let tx = Transaction::new(&table); + + let data_file = test_data_file( + "test/3.parquet", + table.metadata().default_partition_spec_id(), + ); + + let action = tx.overwrite().add_data_files(vec![data_file.clone()]); + let mut action_commit = Arc::new(action).commit(&table).await.unwrap(); + let updates = action_commit.take_updates(); + let requirements = action_commit.take_requirements(); + + assert!( + matches!((&updates[0],&updates[1]), (TableUpdate::AddSnapshot { snapshot },TableUpdate::SetSnapshotRef { reference,ref_name }) if snapshot.snapshot_id() == reference.snapshot_id && ref_name == MAIN_BRANCH) + ); + + assert_eq!( + vec![ + TableRequirement::UuidMatch { + uuid: table.metadata().uuid() + }, + TableRequirement::RefSnapshotIdMatch { + r#ref: MAIN_BRANCH.to_string(), + snapshot_id: table.metadata().current_snapshot_id + } + ], + requirements + ); + + let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { + SnapshotRef::new(snapshot.clone()) + } else { + unreachable!() + }; + assert_eq!(new_snapshot.summary().operation, Operation::Overwrite); + + let manifest_list = table + .manifest_list_reader(&new_snapshot) + .load() + .await + .unwrap(); + assert_eq!(1, manifest_list.entries().len()); + assert_eq!( + manifest_list.entries()[0].sequence_number, + new_snapshot.sequence_number() + ); + + let manifest = manifest_list.entries()[0] + .load_manifest(table.file_io()) + .await + .unwrap(); + assert_eq!(1, manifest.entries().len()); + assert_eq!( + new_snapshot.sequence_number(), + manifest.entries()[0] + .sequence_number() + .expect("Inherit sequence number by load manifest") + ); + assert_eq!( + new_snapshot.snapshot_id(), + manifest.entries()[0].snapshot_id().unwrap() + ); + assert_eq!(data_file, *manifest.entries()[0].data_file()); + } + + #[tokio::test] + async fn test_overwrite_with_deleted_files() { + use crate::memory::tests::new_memory_catalog; + use crate::transaction::ApplyTransactionAction; + use crate::transaction::tests::make_v3_minimal_table_in_catalog; + + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + + let original_file = test_data_file("test/original.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![original_file.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + assert_eq!(1, manifest_list.entries().len()); + + let replacement_file = test_data_file("test/replacement.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx + .overwrite() + .add_data_files(vec![replacement_file.clone()]) + .delete_data_files(vec![original_file.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.summary().operation, Operation::Overwrite); + + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + + assert_eq!(2, manifest_list.entries().len()); + + let mut all_entries = vec![]; + for manifest_file in manifest_list.entries() { + let manifest = manifest_file.load_manifest(table.file_io()).await.unwrap(); + for entry in manifest.entries() { + all_entries.push((entry.status(), entry.file_path().to_string())); + } + } + + assert!( + all_entries + .iter() + .any(|(status, path)| *status == ManifestStatus::Deleted + && path == "test/original.parquet"), + "Original file should be marked as Deleted, entries: {all_entries:?}", + ); + + assert!( + all_entries + .iter() + .any(|(status, path)| *status == ManifestStatus::Added + && path == "test/replacement.parquet"), + "Replacement file should be marked as Added, entries: {all_entries:?}", + ); + + // Verify snapshot summary reports the deleted file. + assert_eq!( + snapshot + .summary() + .additional_properties + .get("deleted-data-files") + .map(|s| s.as_str()), + Some("1") + ); + assert_eq!( + snapshot + .summary() + .additional_properties + .get("deleted-records") + .map(|s| s.as_str()), + Some("1") + ); + + // Step 3: Fast append after overwrite — delete-only manifest must survive. + let appended_file = test_data_file("test/appended.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![appended_file.clone()]); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + + // 3 manifests: rewritten (deleted entry), overwrite added, fast_append added. + assert_eq!(3, manifest_list.entries().len()); + + let mut all_entries = vec![]; + for manifest_file in manifest_list.entries() { + let manifest = manifest_file.load_manifest(table.file_io()).await.unwrap(); + for entry in manifest.entries() { + all_entries.push((entry.status(), entry.file_path().to_string())); + } + } + + // The deleted entry must still be present after fast_append. + assert!( + all_entries + .iter() + .any(|(status, path)| *status == ManifestStatus::Deleted + && path == "test/original.parquet"), + "Deleted entry should survive fast_append, entries: {all_entries:?}", + ); + } +} diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index d4a3a7fdee..72bb4673fe 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -116,6 +116,7 @@ pub(crate) struct SnapshotProducer<'a> { key_metadata: Option>, snapshot_properties: HashMap, added_data_files: Vec, + deleted_data_files: Vec, // A counter used to generate unique manifest file names. // It starts from 0 and increments for each new manifest file. // Note: This counter is limited to the range of (0..u64::MAX). @@ -123,12 +124,17 @@ pub(crate) struct SnapshotProducer<'a> { } impl<'a> SnapshotProducer<'a> { + pub(crate) fn snapshot_id(&self) -> i64 { + self.snapshot_id + } + pub(crate) fn new( table: &'a Table, commit_uuid: Uuid, key_metadata: Option>, snapshot_properties: HashMap, added_data_files: Vec, + deleted_data_files: Vec, ) -> Self { Self { table, @@ -137,6 +143,7 @@ impl<'a> SnapshotProducer<'a> { key_metadata, snapshot_properties, added_data_files, + deleted_data_files, manifest_counter: (0..), } } @@ -402,6 +409,14 @@ impl<'a> SnapshotProducer<'a> { ); } + for data_file in &self.deleted_data_files { + summary_collector.remove_file( + data_file, + table_metadata.current_schema().clone(), + table_metadata.default_partition_spec().clone(), + ); + } + let previous_snapshot = table_metadata.current_snapshot(); // User-supplied snapshot properties are applied first, then the computed From 4999d6935b646dc8da8113d71401501b8ded1574 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 11 Mar 2026 18:36:01 +0100 Subject: [PATCH 02/50] Get the previous snapshot directly from the table_metadata and add update unit test --- crates/iceberg/src/transaction/overwrite.rs | 43 ++++++++++++--------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 013162c259..b1d34b298b 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -420,9 +420,10 @@ mod tests { let table = make_v3_minimal_table_in_catalog(&catalog).await; let spec_id = table.metadata().default_partition_spec_id(); - let original_file = test_data_file("test/original.parquet", spec_id); + let original_file1 = test_data_file("test/original1.parquet", spec_id); + let original_file2 = test_data_file("test/original2.parquet", spec_id); let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(vec![original_file.clone()]); + let action = tx.fast_append().add_data_files(vec![original_file1.clone(), original_file2.clone()]); let tx = action.apply(tx).unwrap(); let table = tx.commit(&catalog).await.unwrap(); @@ -435,7 +436,7 @@ mod tests { let action = tx .overwrite() .add_data_files(vec![replacement_file.clone()]) - .delete_data_files(vec![original_file.clone()]); + .delete_data_files(vec![original_file1.clone(), original_file2.clone()]); let tx = action.apply(tx).unwrap(); let table = tx.commit(&catalog).await.unwrap(); @@ -454,13 +455,15 @@ mod tests { } } - assert!( - all_entries - .iter() - .any(|(status, path)| *status == ManifestStatus::Deleted - && path == "test/original.parquet"), - "Original file should be marked as Deleted, entries: {all_entries:?}", - ); + for original_file in ["test/original1.parquet", "test/original2.parquet"] { + assert!( + all_entries + .iter() + .any(|(status, path)| *status == ManifestStatus::Deleted + && path == original_file), + "Original file {original_file} should be marked as Deleted, entries: {all_entries:?}", + ); + } assert!( all_entries @@ -477,7 +480,7 @@ mod tests { .additional_properties .get("deleted-data-files") .map(|s| s.as_str()), - Some("1") + Some("2") ); assert_eq!( snapshot @@ -485,7 +488,7 @@ mod tests { .additional_properties .get("deleted-records") .map(|s| s.as_str()), - Some("1") + Some("2") ); // Step 3: Fast append after overwrite — delete-only manifest must survive. @@ -510,12 +513,14 @@ mod tests { } // The deleted entry must still be present after fast_append. - assert!( - all_entries - .iter() - .any(|(status, path)| *status == ManifestStatus::Deleted - && path == "test/original.parquet"), - "Deleted entry should survive fast_append, entries: {all_entries:?}", - ); + for original_file in ["test/original1.parquet", "test/original2.parquet"] { + assert!( + all_entries + .iter() + .any(|(status, path)| *status == ManifestStatus::Deleted + && path == original_file), + "Deleted entry should survive fast_append, entries: {all_entries:?}", + ); + } } } From a11e3b5a8b313f281405071d32934aebaea741a9 Mon Sep 17 00:00:00 2001 From: Gerald Berger Date: Thu, 28 May 2026 08:56:27 +0200 Subject: [PATCH 03/50] allow delete-only overwrite when no data files are added MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SnapshotProducer precondition check blocked snapshots with only deleted files and no added files or snapshot properties. A delete-only Overwrite snapshot is valid per the Iceberg spec — the existing manifests are rewritten with the target entries marked as ManifestStatus::Deleted. Relax the check to also allow the case when deleted_data_files is non-empty, so callers can clear a table without simultaneously adding new files. Co-Authored-By: Claude Sonnet 4.6 --- crates/iceberg/src/transaction/snapshot.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 72bb4673fe..61a90c8f6b 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -355,7 +355,13 @@ impl<'a> SnapshotProducer<'a> { // TODO: Allowing snapshot property setup with no added data files is a workaround. // We should clean it up after all necessary actions are supported. // For details, please refer to https://github.com/apache/iceberg-rust/issues/1548 - if self.added_data_files.is_empty() && self.snapshot_properties.is_empty() { + // + // A delete-only overwrite (no added files, no properties, but with deleted files) is + // valid per the Iceberg spec — the existing manifests are rewritten with deleted entries. + if self.added_data_files.is_empty() + && self.snapshot_properties.is_empty() + && self.deleted_data_files.is_empty() + { return Err(Error::new( ErrorKind::PreconditionFailed, "No added data files or added snapshot properties found when write a manifest file", From 5493b3a5ec61de6aa9bf4ac7f61cb514125be76a Mon Sep 17 00:00:00 2001 From: glitchy Date: Fri, 12 Jun 2026 22:19:24 -0400 Subject: [PATCH 04/50] chore(transaction): rustfmt and refresh public API snapshot for OverwriteAction --- crates/iceberg/public-api.txt | 1 + crates/iceberg/src/transaction/overwrite.rs | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 653649e6cf..6311d6b6a9 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3081,6 +3081,7 @@ pub async fn iceberg::transaction::Transaction::commit(self, catalog: &dyn icebe pub fn iceberg::transaction::Transaction::expire_snapshots(&self) -> iceberg::transaction::expire_snapshots::ExpireSnapshotsAction pub fn iceberg::transaction::Transaction::fast_append(&self) -> iceberg::transaction::append::FastAppendAction pub fn iceberg::transaction::Transaction::new(table: &iceberg::table::Table) -> Self +pub fn iceberg::transaction::Transaction::overwrite(&self) -> iceberg::transaction::overwrite::OverwriteAction pub fn iceberg::transaction::Transaction::replace_sort_order(&self) -> iceberg::transaction::sort_order::ReplaceSortOrderAction pub fn iceberg::transaction::Transaction::update_location(&self) -> iceberg::transaction::update_location::UpdateLocationAction pub fn iceberg::transaction::Transaction::update_schema(&self) -> iceberg::transaction::update_schema::UpdateSchemaAction diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index b1d34b298b..818637866a 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -423,7 +423,9 @@ mod tests { let original_file1 = test_data_file("test/original1.parquet", spec_id); let original_file2 = test_data_file("test/original2.parquet", spec_id); let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(vec![original_file1.clone(), original_file2.clone()]); + let action = tx + .fast_append() + .add_data_files(vec![original_file1.clone(), original_file2.clone()]); let tx = action.apply(tx).unwrap(); let table = tx.commit(&catalog).await.unwrap(); From 2b175f2c53dce4ed8f81848c9e612ca356f5bb4f Mon Sep 17 00:00:00 2001 From: glitchy Date: Mon, 29 Jun 2026 23:33:56 -0400 Subject: [PATCH 05/50] refactor(transaction): hoist inline spec imports in overwrite Import Manifest and DataFile at the top instead of using fully-qualified crate::spec:: paths inline in a signature and a test helper. --- crates/iceberg/src/transaction/overwrite.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 818637866a..961c3e43f8 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -23,7 +23,7 @@ use uuid::Uuid; use crate::error::Result; use crate::spec::{ - DataFile, FormatVersion, ManifestContentType, ManifestEntry, ManifestFile, + DataFile, FormatVersion, Manifest, ManifestContentType, ManifestEntry, ManifestFile, ManifestWriterBuilder, Operation, }; use crate::table::Table; @@ -208,7 +208,7 @@ impl OverwriteOperation { &self, snapshot_produce: &SnapshotProducer<'_>, manifest_file: &ManifestFile, - manifest: &crate::spec::Manifest, + manifest: &Manifest, ) -> Result { let table = snapshot_produce.table; @@ -259,14 +259,14 @@ mod tests { use std::sync::Arc; use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, ManifestStatus, - Operation, SnapshotRef, Struct, + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, + ManifestStatus, Operation, SnapshotRef, Struct, }; use crate::transaction::tests::make_v2_minimal_table; use crate::transaction::{Transaction, TransactionAction}; use crate::{TableRequirement, TableUpdate}; - fn test_data_file(path: &str, partition_spec_id: i32) -> crate::spec::DataFile { + fn test_data_file(path: &str, partition_spec_id: i32) -> DataFile { DataFileBuilder::default() .content(DataContentType::Data) .file_path(path.to_string()) From ede30bd7724844d12ba3406e66be7bd5deebaf2f Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:16:49 +0200 Subject: [PATCH 06/50] Adapt to current SnapshotProducer and manifest reader APIs --- crates/iceberg/src/transaction/overwrite.rs | 25 ++++++++------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 961c3e43f8..8c9fc4db5a 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -40,7 +40,6 @@ use crate::transaction::{ActionCommit, TransactionAction}; pub struct OverwriteAction { check_duplicate: bool, commit_uuid: Option, - key_metadata: Option>, snapshot_properties: HashMap, added_data_files: Vec, deleted_data_files: Vec, @@ -51,7 +50,6 @@ impl OverwriteAction { Self { check_duplicate: true, commit_uuid: None, - key_metadata: None, snapshot_properties: HashMap::default(), added_data_files: vec![], deleted_data_files: vec![], @@ -82,12 +80,6 @@ impl OverwriteAction { self } - /// Set key metadata for manifest files. - pub fn set_key_metadata(mut self, key_metadata: Vec) -> Self { - self.key_metadata = Some(key_metadata); - self - } - /// Set snapshot summary properties. pub fn set_snapshot_properties(mut self, snapshot_properties: HashMap) -> Self { self.snapshot_properties = snapshot_properties; @@ -101,7 +93,6 @@ impl TransactionAction for OverwriteAction { let snapshot_producer = SnapshotProducer::new( table, self.commit_uuid.unwrap_or_else(Uuid::now_v7), - self.key_metadata.clone(), self.snapshot_properties.clone(), self.added_data_files.clone(), self.deleted_data_files.clone(), @@ -179,8 +170,10 @@ impl SnapshotProduceOperation for OverwriteOperation { continue; } - let manifest = manifest_file - .load_manifest(snapshot_produce.table.file_io()) + let manifest = snapshot_produce + .table + .manifest_reader() + .read(manifest_file) .await?; let has_deletes = manifest.entries().iter().any(|entry| { @@ -221,7 +214,6 @@ impl OverwriteOperation { let builder = ManifestWriterBuilder::new( output_file, Some(self.snapshot_id), - manifest_file.key_metadata.clone(), table.metadata().current_schema().clone(), table.metadata().default_partition_spec().as_ref().clone(), ); @@ -392,8 +384,9 @@ mod tests { new_snapshot.sequence_number() ); - let manifest = manifest_list.entries()[0] - .load_manifest(table.file_io()) + let manifest = table + .manifest_reader() + .read(&manifest_list.entries()[0]) .await .unwrap(); assert_eq!(1, manifest.entries().len()); @@ -451,7 +444,7 @@ mod tests { let mut all_entries = vec![]; for manifest_file in manifest_list.entries() { - let manifest = manifest_file.load_manifest(table.file_io()).await.unwrap(); + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); for entry in manifest.entries() { all_entries.push((entry.status(), entry.file_path().to_string())); } @@ -508,7 +501,7 @@ mod tests { let mut all_entries = vec![]; for manifest_file in manifest_list.entries() { - let manifest = manifest_file.load_manifest(table.file_io()).await.unwrap(); + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); for entry in manifest.entries() { all_entries.push((entry.status(), entry.file_path().to_string())); } From 26645050628d2a205ddbafc1bff75b0d1a2b716b Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:25:49 +0200 Subject: [PATCH 07/50] Keep deleted entries deleted when rewriting a manifest A manifest rewritten by a second overwrite fell through to add_existing_entry for entries that were already Deleted, forcing their status back to Existing and resurrecting the files as live data. Guard on is_alive() first and re-add prior tombstones as Deleted with their original snapshot id, matching Java's filterManifestWithDeletedFiles, which only re-adds live entries. --- crates/iceberg/src/transaction/overwrite.rs | 104 +++++++++++++++++--- 1 file changed, 93 insertions(+), 11 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 8c9fc4db5a..093f26ca68 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -231,13 +231,17 @@ impl OverwriteOperation { }; for entry in manifest.entries() { - if entry.is_alive() && self.deleted_file_paths.contains(entry.file_path()) { - let mut deleted: ManifestEntry = (**entry).clone(); + let entry: ManifestEntry = (**entry).clone(); + if !entry.is_alive() { + // A tombstone keeps its original snapshot id so the delete stays attributed + // to the snapshot that made it. + writer.add_deleted_entry(entry)?; + } else if self.deleted_file_paths.contains(entry.file_path()) { + let mut deleted = entry; deleted.snapshot_id = Some(self.snapshot_id); writer.add_deleted_entry(deleted)?; } else { - let cloned: ManifestEntry = (**entry).clone(); - writer.add_existing_entry(cloned)?; + writer.add_existing_entry(entry)?; } } @@ -250,12 +254,14 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; + use crate::memory::tests::new_memory_catalog; use crate::spec::{ DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, - ManifestStatus, Operation, SnapshotRef, Struct, + ManifestEntryRef, ManifestStatus, Operation, SnapshotRef, Struct, }; - use crate::transaction::tests::make_v2_minimal_table; - use crate::transaction::{Transaction, TransactionAction}; + use crate::table::Table; + use crate::transaction::tests::{make_v2_minimal_table, make_v3_minimal_table_in_catalog}; + use crate::transaction::{ApplyTransactionAction, Transaction, TransactionAction}; use crate::{TableRequirement, TableUpdate}; fn test_data_file(path: &str, partition_spec_id: i32) -> DataFile { @@ -271,6 +277,33 @@ mod tests { .unwrap() } + async fn current_manifest_entries(table: &Table) -> Vec<(ManifestStatus, String)> { + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + + let mut entries = vec![]; + for manifest_file in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + for entry in manifest.entries() { + entries.push((entry.status(), entry.file_path().to_string())); + } + } + entries + } + + async fn current_manifest_entry(table: &Table, path: &str) -> ManifestEntryRef { + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + + for manifest_file in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + if let Some(entry) = manifest.entries().iter().find(|e| e.file_path() == path) { + return entry.clone(); + } + } + panic!("no manifest entry for {path}"); + } + #[tokio::test] async fn test_empty_data_overwrite_action() { let table = make_v2_minimal_table(); @@ -405,10 +438,6 @@ mod tests { #[tokio::test] async fn test_overwrite_with_deleted_files() { - use crate::memory::tests::new_memory_catalog; - use crate::transaction::ApplyTransactionAction; - use crate::transaction::tests::make_v3_minimal_table_in_catalog; - let catalog = new_memory_catalog().await; let table = make_v3_minimal_table_in_catalog(&catalog).await; let spec_id = table.metadata().default_partition_spec_id(); @@ -518,4 +547,57 @@ mod tests { ); } } + + #[tokio::test] + async fn test_second_overwrite_keeps_prior_deleted_entries_deleted() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + + let file_a = test_data_file("test/a.parquet", spec_id); + let file_b = test_data_file("test/b.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .add_data_files(vec![file_a.clone(), file_b.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx + .overwrite() + .add_data_files(vec![test_data_file("test/c.parquet", spec_id)]) + .delete_data_files(vec![file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + let tombstone = current_manifest_entry(&table, "test/a.parquet").await; + assert_eq!(ManifestStatus::Deleted, tombstone.status()); + + let tx = Transaction::new(&table); + let action = tx + .overwrite() + .add_data_files(vec![test_data_file("test/d.parquet", spec_id)]) + .delete_data_files(vec![file_b.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let carried = current_manifest_entry(&table, "test/a.parquet").await; + assert_eq!( + ManifestStatus::Deleted, + carried.status(), + "the first overwrite's tombstone must stay Deleted" + ); + assert_eq!( + tombstone.snapshot_id, carried.snapshot_id, + "the tombstone must keep the snapshot that deleted the file" + ); + assert_eq!( + (tombstone.sequence_number, tombstone.file_sequence_number), + (carried.sequence_number, carried.file_sequence_number), + "the tombstone must keep its original sequence numbers" + ); + + let entries = current_manifest_entries(&table).await; + assert!( + entries.contains(&(ManifestStatus::Deleted, "test/b.parquet".to_string())), + "the second overwrite's own delete must be a tombstone, entries: {entries:?}" + ); + } } From ca9bad14d2b17f33d1b76507b49b422a342356b8 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:28:32 +0200 Subject: [PATCH 08/50] Stamp a rewritten manifest with the schema and spec it was written under The rewrite used the table's current schema and default partition spec, so after a schema or partition evolution the Avro header carried the wrong schema-id and partition-spec-id and readers interpreted the partition values against the wrong mapping. Take both from the manifest being rewritten, as Java does via newManifestWriter(reader.spec()). --- crates/iceberg/src/transaction/overwrite.rs | 90 ++++++++++++++++++++- 1 file changed, 86 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 093f26ca68..9bd6bb526e 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -214,8 +214,8 @@ impl OverwriteOperation { let builder = ManifestWriterBuilder::new( output_file, Some(self.snapshot_id), - table.metadata().current_schema().clone(), - table.metadata().default_partition_spec().as_ref().clone(), + manifest.metadata().schema.clone(), + manifest.metadata().partition_spec.clone(), ); let mut writer = match table.metadata().format_version() { @@ -257,11 +257,12 @@ mod tests { use crate::memory::tests::new_memory_catalog; use crate::spec::{ DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, - ManifestEntryRef, ManifestStatus, Operation, SnapshotRef, Struct, + ManifestEntryRef, ManifestStatus, Operation, PrimitiveType, SnapshotRef, Struct, Transform, + Type, UnboundPartitionSpec, }; use crate::table::Table; use crate::transaction::tests::{make_v2_minimal_table, make_v3_minimal_table_in_catalog}; - use crate::transaction::{ApplyTransactionAction, Transaction, TransactionAction}; + use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction, TransactionAction}; use crate::{TableRequirement, TableUpdate}; fn test_data_file(path: &str, partition_spec_id: i32) -> DataFile { @@ -600,4 +601,85 @@ mod tests { "the second overwrite's own delete must be a tombstone, entries: {entries:?}" ); } + + #[tokio::test] + async fn test_rewritten_manifest_keeps_the_schema_and_spec_it_was_written_with() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let original_spec_id = table.metadata().default_partition_spec_id(); + let original_schema_id = table.metadata().current_schema_id(); + + let file_a = test_data_file("test/a.parquet", original_spec_id); + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx.update_schema().add_column(AddColumn::optional( + "extra", + Type::Primitive(PrimitiveType::Int), + )); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + // There is no transaction action for partition evolution yet, so the spec is replaced + // on the metadata directly. + let evolved = table + .metadata() + .clone() + .into_builder(None) + .add_default_partition_spec( + UnboundPartitionSpec::builder() + .add_partition_field(2, "y", Transform::Identity) + .unwrap() + .build(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + let table = table.with_metadata(Arc::new(evolved)); + assert_ne!(original_schema_id, table.metadata().current_schema_id()); + assert_ne!( + original_spec_id, + table.metadata().default_partition_spec_id() + ); + + let tx = Transaction::new(&table); + let action = tx + .overwrite() + .add_data_files(vec![test_data_file( + "test/b.parquet", + table.metadata().default_partition_spec_id(), + )]) + .delete_data_files(vec![file_a.clone()]); + let mut action_commit = Arc::new(action).commit(&table).await.unwrap(); + let updates = action_commit.take_updates(); + let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { + SnapshotRef::new(snapshot.clone()) + } else { + unreachable!() + }; + + let manifest_list = table + .manifest_list_reader(&new_snapshot) + .load() + .await + .unwrap(); + let mut rewritten = None; + for manifest_file in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + if manifest + .entries() + .iter() + .any(|entry| entry.status() == ManifestStatus::Deleted) + { + rewritten = Some((manifest_file.clone(), manifest)); + } + } + let (rewritten_file, rewritten_manifest) = + rewritten.expect("the manifest holding test/a.parquet must have been rewritten"); + + assert_eq!(original_schema_id, rewritten_manifest.metadata().schema_id); + assert_eq!(original_spec_id, rewritten_file.partition_spec_id); + } } From d782adc6755e058b3a7adc9125069b719be87969 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:30:45 +0200 Subject: [PATCH 09/50] Keep delete-only manifests when overwriting Both manifest filters dropped a manifest whose entries were all Deleted, which lets the removed files reappear as live data. This is the same fix FastAppend took in #2545 for #2148, applied to the no-deletes fast path and to the rewrite loop. --- crates/iceberg/src/transaction/overwrite.rs | 44 ++++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 9bd6bb526e..8a8f4766e4 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -158,7 +158,13 @@ impl SnapshotProduceOperation for OverwriteOperation { return Ok(manifest_list .entries() .iter() - .filter(|entry| entry.has_added_files() || entry.has_existing_files()) + .filter(|entry| { + // Delete-only manifests record which files were removed and must survive + // until `expire_snapshots` cleans them up (see #2148). + entry.has_added_files() + || entry.has_existing_files() + || entry.has_deleted_files() + }) .cloned() .collect()); } @@ -166,7 +172,10 @@ impl SnapshotProduceOperation for OverwriteOperation { let mut result = Vec::new(); for manifest_file in manifest_list.entries() { - if !manifest_file.has_added_files() && !manifest_file.has_existing_files() { + if !manifest_file.has_added_files() + && !manifest_file.has_existing_files() + && !manifest_file.has_deleted_files() + { continue; } @@ -682,4 +691,35 @@ mod tests { assert_eq!(original_schema_id, rewritten_manifest.metadata().schema_id); assert_eq!(original_spec_id, rewritten_file.partition_spec_id); } + + #[tokio::test] + async fn test_overwrite_preserves_delete_only_manifest() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + + let file_a = test_data_file("test/a.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx + .overwrite() + .add_data_files(vec![test_data_file("test/b.parquet", spec_id)]) + .delete_data_files(vec![file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx + .overwrite() + .add_data_files(vec![test_data_file("test/c.parquet", spec_id)]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let entries = current_manifest_entries(&table).await; + assert!( + entries.contains(&(ManifestStatus::Deleted, "test/a.parquet".to_string())), + "the delete-only manifest must survive a later overwrite, entries: {entries:?}" + ); + } } From cbae2456161d79f7d0b68e0de1e8301761d2cd61 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:36:29 +0200 Subject: [PATCH 10/50] Count the deletions the overwrite actually made The summary counted every file the caller passed to delete_data_files, so a path that was never committed or was passed twice inflated deleted-data-files and deleted-records. The rewrite pass now records the entries it marked Deleted, with the schema and partition spec of the manifest each was recorded in, and the summary counts those. Two changes follow from it. The summary is built after the manifests rather than before, since the counts are only known once the rewrite has run. And the full-table truncate path is now gated on an operation hook that a partial overwrite answers false, instead of on the operation being Overwrite; with truncate off, update_totals needs a saturating subtraction so an overwrite that deletes more than the previous total does not underflow. --- crates/iceberg/src/spec/snapshot_summary.rs | 2 +- crates/iceberg/src/transaction/append.rs | 2 +- crates/iceberg/src/transaction/overwrite.rs | 54 ++++++++++++++++++-- crates/iceberg/src/transaction/snapshot.rs | 55 +++++++++++++-------- 4 files changed, 87 insertions(+), 26 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index c9fd022724..568dc54dba 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -532,7 +532,7 @@ fn update_totals( return; }; - let new_total = previous_total + added - removed; + let new_total = (previous_total + added).saturating_sub(removed); summary .additional_properties .insert(total_property.to_string(), new_total.to_string()); diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index e510d4a4b8..a3dc6a31f1 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -125,7 +125,7 @@ impl SnapshotProduceOperation for FastAppendOperation { } async fn existing_manifest( - &self, + &mut self, snapshot_produce: &SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 8a8f4766e4..1d57844ebe 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -24,7 +24,7 @@ use uuid::Uuid; use crate::error::Result; use crate::spec::{ DataFile, FormatVersion, Manifest, ManifestContentType, ManifestEntry, ManifestFile, - ManifestWriterBuilder, Operation, + ManifestWriterBuilder, Operation, PartitionSpecRef, SchemaRef, }; use crate::table::Table; use crate::transaction::snapshot::{ @@ -116,6 +116,7 @@ impl TransactionAction for OverwriteAction { OverwriteOperation { deleted_file_paths, snapshot_id, + removed_data_files: vec![], }, DefaultManifestProcess, ) @@ -126,6 +127,7 @@ impl TransactionAction for OverwriteAction { struct OverwriteOperation { deleted_file_paths: HashSet, snapshot_id: i64, + removed_data_files: Vec<(DataFile, SchemaRef, PartitionSpecRef)>, } impl SnapshotProduceOperation for OverwriteOperation { @@ -140,8 +142,12 @@ impl SnapshotProduceOperation for OverwriteOperation { Ok(vec![]) } + fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { + &self.removed_data_files + } + async fn existing_manifest( - &self, + &mut self, snapshot_produce: &SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { @@ -207,7 +213,7 @@ impl OverwriteOperation { /// Rewrite a manifest, marking entries whose file paths are in `deleted_file_paths` /// as `ManifestStatus::Deleted`. async fn rewrite_manifest( - &self, + &mut self, snapshot_produce: &SnapshotProducer<'_>, manifest_file: &ManifestFile, manifest: &Manifest, @@ -220,6 +226,7 @@ impl OverwriteOperation { Uuid::now_v7(), ); let output_file = table.file_io().new_output(&new_manifest_path)?; + let partition_spec: PartitionSpecRef = Arc::new(manifest.metadata().partition_spec.clone()); let builder = ManifestWriterBuilder::new( output_file, Some(self.snapshot_id), @@ -248,6 +255,11 @@ impl OverwriteOperation { } else if self.deleted_file_paths.contains(entry.file_path()) { let mut deleted = entry; deleted.snapshot_id = Some(self.snapshot_id); + self.removed_data_files.push(( + deleted.data_file().clone(), + manifest.metadata().schema.clone(), + partition_spec.clone(), + )); writer.add_deleted_entry(deleted)?; } else { writer.add_existing_entry(entry)?; @@ -722,4 +734,40 @@ mod tests { "the delete-only manifest must survive a later overwrite, entries: {entries:?}" ); } + + #[tokio::test] + async fn test_overwrite_counts_a_duplicated_delete_path_once() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + + let file_a = test_data_file("test/a.parquet", spec_id); + let file_b = test_data_file("test/b.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .add_data_files(vec![file_a.clone(), file_b.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx + .overwrite() + .add_data_files(vec![test_data_file("test/c.parquet", spec_id)]) + .delete_data_files(vec![file_a.clone(), file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let summary = table.metadata().current_snapshot().unwrap().summary(); + assert_eq!( + summary.additional_properties.get("deleted-data-files"), + Some(&"1".to_string()) + ); + assert_eq!( + summary.additional_properties.get("deleted-records"), + Some(&"1".to_string()) + ); + assert_eq!( + summary.additional_properties.get("total-data-files"), + Some(&"2".to_string()) + ); + } } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index aa9cf3818a..1ea66bd202 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -26,9 +26,10 @@ use uuid::Uuid; use crate::error::Result; use crate::spec::{ DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, - ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, Snapshot, - SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, StructType, Summary, - TableProperties, update_snapshot_summaries, + ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, + PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, + SnapshotSummaryCollector, Struct, StructType, Summary, TableProperties, + update_snapshot_summaries, }; use crate::table::Table; use crate::transaction::ActionCommit; @@ -82,9 +83,24 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { /// - **Overwrite operations**: May exclude manifests for partitions being overwritten /// - **Delete operations**: May exclude manifests for partitions being deleted fn existing_manifest( - &self, + &mut self, snapshot_produce: &SnapshotProducer<'_>, ) -> impl Future>> + Send; + + /// Returns the data files this operation actually removed, each with the schema and + /// partition spec of the manifest that recorded it. + /// + /// Only populated once [`Self::existing_manifest`] has run, so the snapshot summary counts + /// what was really removed rather than what the caller asked to remove. + fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { + &[] + } + + /// Returns whether this operation replaces the whole table, dropping the previous totals + /// from the snapshot summary. A partial overwrite must return `false`. + fn truncate_full_table(&self) -> bool { + false + } } pub(crate) struct DefaultManifestProcess; @@ -315,7 +331,9 @@ impl<'a> SnapshotProducer<'a> { // Write manifest file for added data files and return the ManifestFile for ManifestList. async fn write_added_manifest(&mut self) -> Result { - let added_data_files = std::mem::take(&mut self.added_data_files); + // Cloned rather than taken: the summary is built after the manifests, so the added + // files must still be here. + let added_data_files = self.added_data_files.clone(); if added_data_files.is_empty() { return Err(Error::new( ErrorKind::PreconditionFailed, @@ -348,7 +366,7 @@ impl<'a> SnapshotProducer<'a> { /// and collects all of the manifests to be included in the new snapshot as [ManifestFile] entries. async fn produce_manifests( &mut self, - snapshot_produce_operation: &OP, + snapshot_produce_operation: &mut OP, manifest_process: &MP, ) -> Result> { // Assert current snapshot producer contains new content to add to new snapshot. @@ -416,12 +434,8 @@ impl<'a> SnapshotProducer<'a> { ); } - for data_file in &self.deleted_data_files { - summary_collector.remove_file( - data_file, - table_metadata.current_schema().clone(), - table_metadata.default_partition_spec().clone(), - ); + for (data_file, schema, partition_spec) in snapshot_produce_operation.removed_data_files() { + summary_collector.remove_file(data_file, schema.clone(), partition_spec.clone()); } let previous_snapshot = table_metadata.current_snapshot(); @@ -442,7 +456,7 @@ impl<'a> SnapshotProducer<'a> { update_snapshot_summaries( summary, previous_snapshot.map(|s| s.summary()), - snapshot_produce_operation.operation() == Operation::Overwrite, + snapshot_produce_operation.truncate_full_table(), ) } @@ -460,7 +474,7 @@ impl<'a> SnapshotProducer<'a> { /// Finished building the action and return the [`ActionCommit`] to the transaction. pub(crate) async fn commit( mut self, - snapshot_produce_operation: OP, + mut snapshot_produce_operation: OP, process: MP, ) -> Result { let manifest_list_path = self.generate_manifest_list_file_path(0)?; @@ -500,17 +514,16 @@ impl<'a> SnapshotProducer<'a> { ), }; - // Calling self.summary() before self.produce_manifests() is important because self.added_data_files - // will be set to an empty vec after self.produce_manifests() returns, resulting in an empty summary - // being generated. + // The manifests are produced first so the summary can count the entries the operation + // actually marked Deleted. + let new_manifests = self + .produce_manifests(&mut snapshot_produce_operation, &process) + .await?; + let summary = self.summary(&snapshot_produce_operation).map_err(|err| { Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) })?; - let new_manifests = self - .produce_manifests(&snapshot_produce_operation, &process) - .await?; - manifest_list_writer.add_manifests(new_manifests.into_iter())?; let writer_next_row_id = manifest_list_writer.next_row_id(); manifest_list_writer.close().await?; From 7f4f3ed56221ccdd9dc3c2371e1f73a3ed5bcd00 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:41:44 +0200 Subject: [PATCH 11/50] Name a rewritten manifest after the commit uuid The rewrite hardcoded {location}/metadata and a fresh Uuid::now_v7, so a table with a custom write.metadata.path got its rewritten manifests in the wrong prefix, and orphan cleanup could not find them after a failed commit because the name was not tied to the commit. Reuse the path SnapshotProducer already builds from metadata_location(), the commit uuid and the manifest counter; the counter becomes an AtomicU64 because the rewrite runs behind a shared reference. --- crates/iceberg/src/transaction/overwrite.rs | 51 ++++++++++++++++++--- crates/iceberg/src/transaction/snapshot.rs | 20 ++++---- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 1d57844ebe..49bb9c5bbf 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -220,12 +220,9 @@ impl OverwriteOperation { ) -> Result { let table = snapshot_produce.table; - let new_manifest_path = format!( - "{}/metadata/{}-m-overwrite.avro", - table.metadata().location(), - Uuid::now_v7(), - ); - let output_file = table.file_io().new_output(&new_manifest_path)?; + let output_file = table + .file_io() + .new_output(snapshot_produce.new_manifest_path()?)?; let partition_spec: PartitionSpecRef = Arc::new(manifest.metadata().partition_spec.clone()); let builder = ManifestWriterBuilder::new( output_file, @@ -275,6 +272,8 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; + use uuid::Uuid; + use crate::memory::tests::new_memory_catalog; use crate::spec::{ DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, @@ -770,4 +769,44 @@ mod tests { Some(&"2".to_string()) ); } + + #[tokio::test] + async fn test_rewritten_manifest_is_named_after_the_commit_uuid() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + + let file_a = test_data_file("test/a.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let commit_uuid = Uuid::now_v7(); + let tx = Transaction::new(&table); + let action = tx + .overwrite() + .set_commit_uuid(commit_uuid) + .add_data_files(vec![test_data_file("test/b.parquet", spec_id)]) + .delete_data_files(vec![file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let prefix = format!( + "{}/{}-m", + table.metadata().metadata_location().unwrap(), + commit_uuid + ); + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + assert_eq!(2, manifest_list.entries().len()); + for manifest_file in manifest_list.entries() { + let path = &manifest_file.manifest_path; + let counter = path + .strip_prefix(&prefix) + .and_then(|rest| rest.strip_suffix(".avro")); + assert!( + counter.is_some_and(|c| !c.is_empty() && c.chars().all(|c| c.is_ascii_digit())), + "manifest path must be {prefix}.avro, got {path}" + ); + } + } } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 1ea66bd202..2e4a68f135 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -17,7 +17,7 @@ use std::collections::{HashMap, HashSet}; use std::future::Future; -use std::ops::RangeFrom; +use std::sync::atomic::{AtomicU64, Ordering}; use futures::TryStreamExt; use futures::stream::FuturesUnordered; @@ -133,7 +133,7 @@ pub(crate) struct SnapshotProducer<'a> { // A counter used to generate unique manifest file names. // It starts from 0 and increments for each new manifest file. // Note: This counter is limited to the range of (0..u64::MAX). - manifest_counter: RangeFrom, + manifest_counter: AtomicU64, } impl<'a> SnapshotProducer<'a> { @@ -155,7 +155,7 @@ impl<'a> SnapshotProducer<'a> { snapshot_properties, added_data_files, deleted_data_files, - manifest_counter: (0..), + manifest_counter: AtomicU64::new(0), } } @@ -257,15 +257,19 @@ impl<'a> SnapshotProducer<'a> { snapshot_id } - fn new_manifest_writer(&mut self, content: ManifestContentType) -> Result { - let new_manifest_path = format!( + /// Returns the path for the next manifest file of this commit. + pub(crate) fn new_manifest_path(&self) -> Result { + Ok(format!( "{}/{}-m{}.{}", self.table.metadata().metadata_location()?, self.commit_uuid, - self.manifest_counter.next().unwrap(), + self.manifest_counter.fetch_add(1, Ordering::Relaxed), DataFileFormat::Avro - ); - let output_file = self.table.file_io().new_output(new_manifest_path)?; + )) + } + + fn new_manifest_writer(&mut self, content: ManifestContentType) -> Result { + let output_file = self.table.file_io().new_output(self.new_manifest_path()?)?; let partition_spec = self .table .metadata() From 774f62ab4609e4d5e17921c8fdceb94df94b30d2 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:44:13 +0200 Subject: [PATCH 12/50] Write rewritten manifests through the encryption manager rewrite_manifest built its own plain output file, so on an encrypted table the rewritten manifest went out in plaintext while its key metadata said otherwise. It now goes through SnapshotProducer::new_manifest_writer, which already branches on the table's encryption manager; the writer takes the schema and partition spec as arguments so the rewrite can keep the ones the manifest was written under. That leaves new_manifest_path with no caller outside this module, so it becomes private again. --- crates/iceberg/src/transaction/overwrite.rs | 29 ++++---------------- crates/iceberg/src/transaction/snapshot.rs | 30 +++++++++++++-------- 2 files changed, 24 insertions(+), 35 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 49bb9c5bbf..5eae6914b2 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -23,8 +23,7 @@ use uuid::Uuid; use crate::error::Result; use crate::spec::{ - DataFile, FormatVersion, Manifest, ManifestContentType, ManifestEntry, ManifestFile, - ManifestWriterBuilder, Operation, PartitionSpecRef, SchemaRef, + DataFile, Manifest, ManifestEntry, ManifestFile, Operation, PartitionSpecRef, SchemaRef, }; use crate::table::Table; use crate::transaction::snapshot::{ @@ -218,30 +217,12 @@ impl OverwriteOperation { manifest_file: &ManifestFile, manifest: &Manifest, ) -> Result { - let table = snapshot_produce.table; - - let output_file = table - .file_io() - .new_output(snapshot_produce.new_manifest_path()?)?; let partition_spec: PartitionSpecRef = Arc::new(manifest.metadata().partition_spec.clone()); - let builder = ManifestWriterBuilder::new( - output_file, - Some(self.snapshot_id), + let mut writer = snapshot_produce.new_manifest_writer( + manifest_file.content, manifest.metadata().schema.clone(), - manifest.metadata().partition_spec.clone(), - ); - - let mut writer = match table.metadata().format_version() { - FormatVersion::V1 => builder.build_v1(), - FormatVersion::V2 => match manifest_file.content { - ManifestContentType::Data => builder.build_v2_data(), - ManifestContentType::Deletes => builder.build_v2_deletes(), - }, - FormatVersion::V3 => match manifest_file.content { - ManifestContentType::Data => builder.build_v3_data(), - ManifestContentType::Deletes => builder.build_v3_deletes(), - }, - }; + partition_spec.as_ref().clone(), + )?; for entry in manifest.entries() { let entry: ManifestEntry = (**entry).clone(); diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 2e4a68f135..7f50cd9ac5 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -27,7 +27,7 @@ use crate::error::Result; use crate::spec::{ DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, - PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, + PartitionSpec, PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, StructType, Summary, TableProperties, update_snapshot_summaries, }; @@ -258,7 +258,7 @@ impl<'a> SnapshotProducer<'a> { } /// Returns the path for the next manifest file of this commit. - pub(crate) fn new_manifest_path(&self) -> Result { + fn new_manifest_path(&self) -> Result { Ok(format!( "{}/{}-m{}.{}", self.table.metadata().metadata_location()?, @@ -268,15 +268,15 @@ impl<'a> SnapshotProducer<'a> { )) } - fn new_manifest_writer(&mut self, content: ManifestContentType) -> Result { + /// Returns a writer for the next manifest file of this commit, routed through the table's + /// encryption manager when one is configured. + pub(crate) fn new_manifest_writer( + &self, + content: ManifestContentType, + schema: SchemaRef, + partition_spec: PartitionSpec, + ) -> Result { let output_file = self.table.file_io().new_output(self.new_manifest_path()?)?; - let partition_spec = self - .table - .metadata() - .default_partition_spec() - .as_ref() - .clone(); - let schema = self.table.metadata().current_schema().clone(); let builder = if let Some(em) = self.table.encryption_manager() { ManifestWriterBuilder::new_from_encrypted( @@ -359,7 +359,15 @@ impl<'a> SnapshotProducer<'a> { builder.build() } }); - let mut writer = self.new_manifest_writer(ManifestContentType::Data)?; + let mut writer = self.new_manifest_writer( + ManifestContentType::Data, + self.table.metadata().current_schema().clone(), + self.table + .metadata() + .default_partition_spec() + .as_ref() + .clone(), + )?; for entry in manifest_entries { writer.add_entry(entry)?; } From bd931fded5e5abf62ffe14d978df122534394060 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:46:16 +0200 Subject: [PATCH 13/50] Rename add_deleted_entry to add_tombstone_entry add_deleted_entry set the status but left snapshot_id to the caller, while add_delete_entry right above it stamps both; the two differed only by tense and a tombstone written with the wrong snapshot_id misattributes the delete, which expire_snapshots then acts on. Taking the reviewer's second option: rename rather than stamp, because carrying an earlier tombstone forward has to keep its original snapshot_id, and document that the caller owns the field. Named add_tombstone_entry rather than something closer to main's add_delete_entry (#2367) precisely so the two cannot be confused. --- crates/iceberg/src/spec/manifest/writer.rs | 9 +++++++-- crates/iceberg/src/transaction/overwrite.rs | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index e0044945a9..b17dee5617 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -382,8 +382,13 @@ impl ManifestWriter { Ok(()) } - /// Add a deleted manifest entry, preserving the original sequence numbers. - pub(crate) fn add_deleted_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { + /// Add a manifest entry as a tombstone, preserving its original sequence numbers and + /// snapshot id. + /// + /// The caller owns `snapshot_id`: set it to the deleting snapshot for a file this commit + /// removes, and leave it untouched when carrying an earlier tombstone forward. Use + /// `add_delete_entry` instead to have this manifest's snapshot id stamped on the entry. + pub(crate) fn add_tombstone_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Deleted; self.add_entry_inner(entry)?; diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 5eae6914b2..f3a813e65a 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -229,7 +229,7 @@ impl OverwriteOperation { if !entry.is_alive() { // A tombstone keeps its original snapshot id so the delete stays attributed // to the snapshot that made it. - writer.add_deleted_entry(entry)?; + writer.add_tombstone_entry(entry)?; } else if self.deleted_file_paths.contains(entry.file_path()) { let mut deleted = entry; deleted.snapshot_id = Some(self.snapshot_id); @@ -238,7 +238,7 @@ impl OverwriteOperation { manifest.metadata().schema.clone(), partition_spec.clone(), )); - writer.add_deleted_entry(deleted)?; + writer.add_tombstone_entry(deleted)?; } else { writer.add_existing_entry(entry)?; } From abad9a5c62db1d23182a9b8f5ff8bcb9ade02080 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:49:17 +0200 Subject: [PATCH 14/50] Report the operation an overwrite actually performed A delete-only overwrite reported Operation::Overwrite, so tools keying off the summary read it as a replacement. Match Java's BaseOverwriteFiles: DELETE when there are only deletes, APPEND when there are only adds, OVERWRITE when there are both. test_overwrite_basic adds without deleting, so its expected operation becomes APPEND. Also names deleted data files in the precondition error, which has allowed a delete-only commit since the guard was relaxed. --- crates/iceberg/src/transaction/overwrite.rs | 52 ++++++++++++++++++++- crates/iceberg/src/transaction/snapshot.rs | 2 +- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index f3a813e65a..84612142ad 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -114,6 +114,7 @@ impl TransactionAction for OverwriteAction { .commit( OverwriteOperation { deleted_file_paths, + has_added_data_files: !self.added_data_files.is_empty(), snapshot_id, removed_data_files: vec![], }, @@ -125,13 +126,21 @@ impl TransactionAction for OverwriteAction { struct OverwriteOperation { deleted_file_paths: HashSet, + has_added_data_files: bool, snapshot_id: i64, removed_data_files: Vec<(DataFile, SchemaRef, PartitionSpecRef)>, } impl SnapshotProduceOperation for OverwriteOperation { fn operation(&self) -> Operation { - Operation::Overwrite + match ( + self.has_added_data_files, + !self.deleted_file_paths.is_empty(), + ) { + (true, true) => Operation::Overwrite, + (false, true) => Operation::Delete, + _ => Operation::Append, + } } async fn delete_entries( @@ -406,7 +415,7 @@ mod tests { } else { unreachable!() }; - assert_eq!(new_snapshot.summary().operation, Operation::Overwrite); + assert_eq!(new_snapshot.summary().operation, Operation::Append); let manifest_list = table .manifest_list_reader(&new_snapshot) @@ -790,4 +799,43 @@ mod tests { ); } } + + #[tokio::test] + async fn test_delete_only_overwrite_reports_a_delete_operation() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + + let file_a = test_data_file("test/a.parquet", spec_id); + let file_b = test_data_file("test/b.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .add_data_files(vec![file_a.clone(), file_b.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx.overwrite().delete_data_files(vec![file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.summary().operation, Operation::Delete); + assert_eq!( + snapshot + .summary() + .additional_properties + .get("deleted-data-files"), + Some(&"1".to_string()) + ); + + let entries = current_manifest_entries(&table).await; + assert!( + entries.contains(&(ManifestStatus::Deleted, "test/a.parquet".to_string())), + "the deleted file must be a tombstone, entries: {entries:?}" + ); + assert!( + entries.contains(&(ManifestStatus::Existing, "test/b.parquet".to_string())), + "the untouched file must stay live, entries: {entries:?}" + ); + } } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 7f50cd9ac5..93dec9b44a 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -395,7 +395,7 @@ impl<'a> SnapshotProducer<'a> { { return Err(Error::new( ErrorKind::PreconditionFailed, - "No added data files or added snapshot properties found when write a manifest file", + "No added data files, deleted data files, or added snapshot properties found when write a manifest file", )); } From c20752e88cf6468999494485aa689b3834e6da40 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 17:51:46 +0200 Subject: [PATCH 15/50] Pin the precondition the empty overwrite test is about is_err() alone passes on any error, so the test would stay green if the commit started failing earlier for an unrelated reason. Assert the kind and the message instead. --- crates/iceberg/src/transaction/overwrite.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 84612142ad..43d6725726 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -273,7 +273,7 @@ mod tests { use crate::table::Table; use crate::transaction::tests::{make_v2_minimal_table, make_v3_minimal_table_in_catalog}; use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction, TransactionAction}; - use crate::{TableRequirement, TableUpdate}; + use crate::{ErrorKind, TableRequirement, TableUpdate}; fn test_data_file(path: &str, partition_spec_id: i32) -> DataFile { DataFileBuilder::default() @@ -320,7 +320,13 @@ mod tests { let table = make_v2_minimal_table(); let tx = Transaction::new(&table); let action = tx.overwrite().add_data_files(vec![]); - assert!(Arc::new(action).commit(&table).await.is_err()); + let err = Arc::new(action).commit(&table).await.err().unwrap(); + assert_eq!(err.kind(), ErrorKind::PreconditionFailed); + assert!( + err.to_string() + .contains("No added data files, deleted data files, or added snapshot properties"), + "{err}" + ); } #[tokio::test] From 0369a6208ea4e56fe3f42d7c7bc07f7e58c877db Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 18:25:06 +0200 Subject: [PATCH 16/50] Reject deletes of files the table does not have An overwrite silently ignored a delete path that was not live in any manifest, so a typo or an already-deleted file committed as a no-op delete. Java's BaseOverwriteFiles validates the same thing and fails with "Missing required files to delete"; the rewrite pass already records what it marked Deleted, so the check is a comparison against the requested set. --- crates/iceberg/src/transaction/overwrite.rs | 58 +++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 43d6725726..28ff313d84 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -30,6 +30,7 @@ use crate::transaction::snapshot::{ DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, }; use crate::transaction::{ActionCommit, TransactionAction}; +use crate::{Error, ErrorKind}; /// OverwriteAction is a transaction action for overwriting data files in the table. /// @@ -159,6 +160,7 @@ impl SnapshotProduceOperation for OverwriteOperation { snapshot_produce: &SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { + self.validate_deletes_matched()?; return Ok(vec![]); }; @@ -213,11 +215,39 @@ impl SnapshotProduceOperation for OverwriteOperation { } } + self.validate_deletes_matched()?; + Ok(result) } } impl OverwriteOperation { + /// Fails when a requested delete path was not live in any manifest, matching the existence + /// check Java's `BaseOverwriteFiles` performs before committing. + fn validate_deletes_matched(&self) -> Result<()> { + let removed: HashSet<&str> = self + .removed_data_files + .iter() + .map(|(data_file, _, _)| data_file.file_path.as_str()) + .collect(); + let mut missing: Vec<&str> = self + .deleted_file_paths + .iter() + .map(String::as_str) + .filter(|path| !removed.contains(path)) + .collect(); + + if missing.is_empty() { + return Ok(()); + } + + missing.sort_unstable(); + Err(Error::new( + ErrorKind::DataInvalid, + format!("Missing required files to delete: {}", missing.join(", ")), + )) + } + /// Rewrite a manifest, marking entries whose file paths are in `deleted_file_paths` /// as `ManifestStatus::Deleted`. async fn rewrite_manifest( @@ -844,4 +874,32 @@ mod tests { "the untouched file must stay live, entries: {entries:?}" ); } + + #[tokio::test] + async fn test_overwrite_rejects_deleting_a_file_the_table_does_not_have() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + + let file_a = test_data_file("test/a.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let tx = Transaction::new(&table); + let action = tx + .overwrite() + .add_data_files(vec![test_data_file("test/b.parquet", spec_id)]) + .delete_data_files(vec![ + file_a.clone(), + test_data_file("test/never-committed.parquet", spec_id), + ]); + let err = Arc::new(action).commit(&table).await.err().unwrap(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("Missing required files to delete: test/never-committed.parquet"), + "{err}" + ); + } } From 807a26897afbbbd5dd01558c74e1b7e786857fd3 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 18:09:24 +0200 Subject: [PATCH 17/50] Refresh the public API snapshot for OverwriteAction Regenerated with make generate-public-api (cargo-public-api 0.51.0); the snapshot was missing OverwriteAction and its builder methods. --- crates/iceberg/public-api.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 6786a80001..31cec20ec8 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3251,6 +3251,13 @@ pub fn iceberg::transaction::FastAppendAction::add_data_files(self, data_files: pub fn iceberg::transaction::FastAppendAction::set_commit_uuid(self, commit_uuid: uuid::Uuid) -> Self pub fn iceberg::transaction::FastAppendAction::set_snapshot_properties(self, snapshot_properties: std::collections::hash::map::HashMap) -> Self pub fn iceberg::transaction::FastAppendAction::with_check_duplicate(self, v: bool) -> Self +pub struct iceberg::transaction::OverwriteAction +impl iceberg::transaction::OverwriteAction +pub fn iceberg::transaction::OverwriteAction::add_data_files(self, data_files: impl core::iter::traits::collect::IntoIterator) -> Self +pub fn iceberg::transaction::OverwriteAction::delete_data_files(self, data_files: impl core::iter::traits::collect::IntoIterator) -> Self +pub fn iceberg::transaction::OverwriteAction::set_commit_uuid(self, commit_uuid: uuid::Uuid) -> Self +pub fn iceberg::transaction::OverwriteAction::set_snapshot_properties(self, snapshot_properties: std::collections::hash::map::HashMap) -> Self +pub fn iceberg::transaction::OverwriteAction::with_check_duplicate(self, v: bool) -> Self pub struct iceberg::transaction::ReplaceSortOrderAction impl iceberg::transaction::ReplaceSortOrderAction pub fn iceberg::transaction::ReplaceSortOrderAction::asc(self, name: &str, null_order: iceberg::spec::NullOrder) -> Self From fed895fde70456a11c4b17c1ce244ab79d215326 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 19:38:03 +0200 Subject: [PATCH 18/50] Let a snapshot operation report the files it removed The snapshot summary could only count what the caller asked to remove, so an operation that skips a path it never had, or is handed the same path twice, inflated deleted-data-files and deleted-records. SnapshotProduceOperation now reports the entries it actually marked Deleted, each with the schema and partition spec of the manifest that recorded it, and the summary counts those. Two changes follow from it. The manifests are produced before the summary, since the counts are only known once the operation has run, which means the added data files are cloned rather than taken. And the full-table truncate that drops the previous totals is gated on a new operation hook instead of on the operation being Overwrite: a partial overwrite keeps the totals, and with truncate off update_totals needs a saturating subtraction so removing more than the previous total does not underflow. --- crates/iceberg/src/spec/snapshot_summary.rs | 40 +++++- crates/iceberg/src/transaction/append.rs | 2 +- crates/iceberg/src/transaction/snapshot.rs | 135 +++++++++++++++++--- 3 files changed, 160 insertions(+), 17 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index c9fd022724..105952957f 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -532,7 +532,7 @@ fn update_totals( return; }; - let new_total = previous_total + added - removed; + let new_total = (previous_total + added).saturating_sub(removed); summary .additional_properties .insert(total_property.to_string(), new_total.to_string()); @@ -1274,4 +1274,42 @@ mod tests { assert_eq!(props.get(TOTAL_POSITION_DELETES).unwrap(), "2"); assert_eq!(props.get(TOTAL_EQUALITY_DELETES).unwrap(), "1"); } + + #[test] + fn test_update_totals_saturates_when_removed_exceeds_previous_total() { + let prev_props: HashMap = [ + (TOTAL_DATA_FILES, "2"), + (TOTAL_RECORDS, "10"), + (TOTAL_FILE_SIZE, "100"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let previous_summary = Summary { + operation: Operation::Append, + additional_properties: prev_props, + }; + + let new_props: HashMap = [ + (DELETED_DATA_FILES, "3"), + (DELETED_RECORDS, "25"), + (REMOVED_FILE_SIZE, "250"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let summary = Summary { + operation: Operation::Delete, + additional_properties: new_props, + }; + + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + let props = &updated.additional_properties; + + assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "0"); + assert_eq!(props.get(TOTAL_RECORDS).unwrap(), "0"); + assert_eq!(props.get(TOTAL_FILE_SIZE).unwrap(), "0"); + } } diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 872a1d89c1..3533c62d27 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -124,7 +124,7 @@ impl SnapshotProduceOperation for FastAppendOperation { } async fn existing_manifest( - &self, + &mut self, snapshot_produce: &SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 1f09a79a08..62ef9c6d60 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -26,9 +26,10 @@ use uuid::Uuid; use crate::error::Result; use crate::spec::{ DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, - ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, Snapshot, - SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, StructType, Summary, - TableProperties, update_snapshot_summaries, + ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, + PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, + SnapshotSummaryCollector, Struct, StructType, Summary, TableProperties, + update_snapshot_summaries, }; use crate::table::Table; use crate::transaction::ActionCommit; @@ -82,9 +83,24 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { /// - **Overwrite operations**: May exclude manifests for partitions being overwritten /// - **Delete operations**: May exclude manifests for partitions being deleted fn existing_manifest( - &self, + &mut self, snapshot_produce: &SnapshotProducer<'_>, ) -> impl Future>> + Send; + + /// Returns the data files this operation actually removed, each with the schema and + /// partition spec of the manifest that recorded it. + /// + /// Only populated once [`Self::existing_manifest`] has run, so the snapshot summary counts + /// what was really removed rather than what the caller asked to remove. + fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { + &[] + } + + /// Returns whether this operation replaces the whole table, dropping the previous totals + /// from the snapshot summary. A partial overwrite must return `false`. + fn truncate_full_table(&self) -> bool { + false + } } pub(crate) struct DefaultManifestProcess; @@ -308,7 +324,9 @@ impl<'a> SnapshotProducer<'a> { // Write manifest file for added data files and return the ManifestFile for ManifestList. async fn write_added_manifest(&mut self) -> Result { - let added_data_files = std::mem::take(&mut self.added_data_files); + // Cloned rather than taken: the summary is built after the manifests, so the added + // files must still be here. + let added_data_files = self.added_data_files.clone(); if added_data_files.is_empty() { return Err(Error::new( ErrorKind::PreconditionFailed, @@ -341,7 +359,7 @@ impl<'a> SnapshotProducer<'a> { /// and collects all of the manifests to be included in the new snapshot as [ManifestFile] entries. async fn produce_manifests( &mut self, - snapshot_produce_operation: &OP, + snapshot_produce_operation: &mut OP, manifest_process: &MP, ) -> Result> { // Assert current snapshot producer contains new content to add to new snapshot. @@ -403,6 +421,10 @@ impl<'a> SnapshotProducer<'a> { ); } + for (data_file, schema, partition_spec) in snapshot_produce_operation.removed_data_files() { + summary_collector.remove_file(data_file, schema.clone(), partition_spec.clone()); + } + let previous_snapshot = table_metadata.current_snapshot(); // User-supplied snapshot properties are applied first, then the computed @@ -421,7 +443,7 @@ impl<'a> SnapshotProducer<'a> { update_snapshot_summaries( summary, previous_snapshot.map(|s| s.summary()), - snapshot_produce_operation.operation() == Operation::Overwrite, + snapshot_produce_operation.truncate_full_table(), ) } @@ -439,7 +461,7 @@ impl<'a> SnapshotProducer<'a> { /// Finished building the action and return the [`ActionCommit`] to the transaction. pub(crate) async fn commit( mut self, - snapshot_produce_operation: OP, + mut snapshot_produce_operation: OP, process: MP, ) -> Result { let manifest_list_path = self.generate_manifest_list_file_path(0)?; @@ -479,17 +501,16 @@ impl<'a> SnapshotProducer<'a> { ), }; - // Calling self.summary() before self.produce_manifests() is important because self.added_data_files - // will be set to an empty vec after self.produce_manifests() returns, resulting in an empty summary - // being generated. + // The manifests are produced first so the summary can count the entries the operation + // actually marked Deleted. + let new_manifests = self + .produce_manifests(&mut snapshot_produce_operation, &process) + .await?; + let summary = self.summary(&snapshot_produce_operation).map_err(|err| { Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) })?; - let new_manifests = self - .produce_manifests(&snapshot_produce_operation, &process) - .await?; - manifest_list_writer.add_manifests(new_manifests.into_iter())?; let writer_next_row_id = manifest_list_writer.next_row_id(); manifest_list_writer.close().await?; @@ -556,3 +577,87 @@ impl<'a> SnapshotProducer<'a> { Ok(ActionCommit::new(updates, requirements)) } } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use uuid::Uuid; + + use super::*; + use crate::spec::{DataContentType, DataFileBuilder, Literal}; + use crate::transaction::tests::make_v2_minimal_table; + + /// An operation that adds no existing manifests and reports one removed data file. + struct RemoveOneFileOperation { + removed: Vec<(DataFile, SchemaRef, PartitionSpecRef)>, + } + + impl SnapshotProduceOperation for RemoveOneFileOperation { + fn operation(&self) -> Operation { + Operation::Delete + } + + async fn delete_entries(&self, _: &SnapshotProducer<'_>) -> Result> { + Ok(vec![]) + } + + async fn existing_manifest( + &mut self, + _: &SnapshotProducer<'_>, + ) -> Result> { + Ok(vec![]) + } + + fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { + &self.removed + } + } + + fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(record_count) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap() + } + + #[tokio::test] + async fn test_summary_counts_files_the_operation_reports_as_removed() { + let table = make_v2_minimal_table(); + let added = data_file(&table, "test/added.parquet", 3); + let removed = data_file(&table, "test/removed.parquet", 2); + + let operation = RemoveOneFileOperation { + removed: vec![( + removed, + table.metadata().current_schema().clone(), + table.metadata().default_partition_spec().clone(), + )], + }; + + let mut action_commit = + SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![added]) + .commit(operation, DefaultManifestProcess) + .await + .unwrap(); + + let updates = action_commit.take_updates(); + let TableUpdate::AddSnapshot { snapshot } = &updates[0] else { + unreachable!() + }; + let properties = &snapshot.summary().additional_properties; + + assert_eq!(properties.get("deleted-data-files").unwrap(), "1"); + assert_eq!(properties.get("deleted-records").unwrap(), "2"); + assert_eq!(properties.get("added-data-files").unwrap(), "1"); + assert_eq!(properties.get("added-records").unwrap(), "3"); + assert_eq!(properties.get("total-data-files").unwrap(), "0"); + assert_eq!(properties.get("total-records").unwrap(), "1"); + } +} From 3fc4e80621568c9f5601cb9e7b7003fe6d2eb053 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 19:38:47 +0200 Subject: [PATCH 19/50] Keep manifest names unique across a commit The manifest name is built inline in new_manifest_writer from the metadata location, the commit uuid and a counter, so anything else that needs to write a manifest for the same commit has to reinvent it, and a name invented elsewhere is neither unique within the commit nor findable by orphan cleanup after a failed one. Pull the name out into new_manifest_path, and make the counter an AtomicU64 so it can be bumped behind a shared reference. --- crates/iceberg/src/transaction/snapshot.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 62ef9c6d60..0347257290 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -17,7 +17,7 @@ use std::collections::{HashMap, HashSet}; use std::future::Future; -use std::ops::RangeFrom; +use std::sync::atomic::{AtomicU64, Ordering}; use futures::TryStreamExt; use futures::stream::FuturesUnordered; @@ -132,7 +132,7 @@ pub(crate) struct SnapshotProducer<'a> { // A counter used to generate unique manifest file names. // It starts from 0 and increments for each new manifest file. // Note: This counter is limited to the range of (0..u64::MAX). - manifest_counter: RangeFrom, + manifest_counter: AtomicU64, } impl<'a> SnapshotProducer<'a> { @@ -148,7 +148,7 @@ impl<'a> SnapshotProducer<'a> { commit_uuid, snapshot_properties, added_data_files, - manifest_counter: (0..), + manifest_counter: AtomicU64::new(0), } } @@ -250,15 +250,19 @@ impl<'a> SnapshotProducer<'a> { snapshot_id } - fn new_manifest_writer(&mut self, content: ManifestContentType) -> Result { - let new_manifest_path = format!( + /// Returns the path for the next manifest file of this commit. + fn new_manifest_path(&self) -> Result { + Ok(format!( "{}/{}-m{}.{}", self.table.metadata().metadata_location()?, self.commit_uuid, - self.manifest_counter.next().unwrap(), + self.manifest_counter.fetch_add(1, Ordering::Relaxed), DataFileFormat::Avro - ); - let output_file = self.table.file_io().new_output(new_manifest_path)?; + )) + } + + fn new_manifest_writer(&mut self, content: ManifestContentType) -> Result { + let output_file = self.table.file_io().new_output(self.new_manifest_path()?)?; let partition_spec = self .table .metadata() From 6325d2270424c92fb2f9734721de4ff91708c86e Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 18 Sep 2026 19:41:28 +0200 Subject: [PATCH 20/50] Write a manifest under the schema and spec it was written with new_manifest_writer hardcoded the table's current schema and default partition spec, so an operation that rewrites an existing manifest could not keep the schema and spec that manifest's entries were recorded under, and building a writer by hand instead means losing the encryption manager the method already branches on: on an encrypted table the manifest would go out in plaintext while its key metadata said otherwise. The schema and partition spec become arguments, and the added-files path passes the table's current ones. --- crates/iceberg/src/transaction/snapshot.rs | 89 +++++++++++++++++++--- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 0347257290..1770e12f6c 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -27,7 +27,7 @@ use crate::error::Result; use crate::spec::{ DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, - PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, + PartitionSpec, PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, StructType, Summary, TableProperties, update_snapshot_summaries, }; @@ -261,15 +261,15 @@ impl<'a> SnapshotProducer<'a> { )) } - fn new_manifest_writer(&mut self, content: ManifestContentType) -> Result { + /// Returns a writer for the next manifest file of this commit, routed through the table's + /// encryption manager when one is configured. + pub(crate) fn new_manifest_writer( + &self, + content: ManifestContentType, + schema: SchemaRef, + partition_spec: PartitionSpec, + ) -> Result { let output_file = self.table.file_io().new_output(self.new_manifest_path()?)?; - let partition_spec = self - .table - .metadata() - .default_partition_spec() - .as_ref() - .clone(); - let schema = self.table.metadata().current_schema().clone(); let builder = if let Some(em) = self.table.encryption_manager() { ManifestWriterBuilder::new_from_encrypted( @@ -352,7 +352,15 @@ impl<'a> SnapshotProducer<'a> { builder.build() } }); - let mut writer = self.new_manifest_writer(ManifestContentType::Data)?; + let mut writer = self.new_manifest_writer( + ManifestContentType::Data, + self.table.metadata().current_schema().clone(), + self.table + .metadata() + .default_partition_spec() + .as_ref() + .clone(), + )?; for entry in manifest_entries { writer.add_entry(entry)?; } @@ -585,11 +593,15 @@ impl<'a> SnapshotProducer<'a> { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; use uuid::Uuid; use super::*; - use crate::spec::{DataContentType, DataFileBuilder, Literal}; + use crate::spec::{ + DataContentType, DataFileBuilder, Literal, Manifest, NestedField, PartitionSpec, + PrimitiveType, Schema, Transform, Type, + }; use crate::transaction::tests::make_v2_minimal_table; /// An operation that adds no existing manifests and reports one removed data file. @@ -664,4 +676,59 @@ mod tests { assert_eq!(properties.get("total-data-files").unwrap(), "0"); assert_eq!(properties.get("total-records").unwrap(), "1"); } + + #[tokio::test] + async fn test_new_manifest_writer_uses_the_given_schema_and_spec() { + let table = make_v2_minimal_table(); + let schema = Arc::new( + Schema::builder() + .with_schema_id(7) + .with_fields(vec![ + NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(), + ); + let partition_spec = PartitionSpec::builder(schema.clone()) + .with_spec_id(3) + .add_partition_field("a", "a", Transform::Identity) + .unwrap() + .build() + .unwrap(); + + let producer = SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![]); + let first = producer + .new_manifest_writer( + ManifestContentType::Data, + schema.clone(), + partition_spec.clone(), + ) + .unwrap() + .write_manifest_file() + .await + .unwrap(); + let second = producer + .new_manifest_writer(ManifestContentType::Data, schema.clone(), partition_spec) + .unwrap() + .write_manifest_file() + .await + .unwrap(); + + assert_ne!(first.manifest_path, second.manifest_path); + + let bytes = table + .file_io() + .new_input(&first.manifest_path) + .unwrap() + .read() + .await + .unwrap(); + let metadata = Manifest::parse_avro(&bytes).unwrap().into_parts().1; + + // Not the table's defaults, which are schema 0 and spec 0. + assert_eq!(metadata.schema_id(), 7); + assert_eq!(metadata.schema().as_ref(), schema.as_ref()); + assert_eq!(metadata.partition_spec().spec_id(), 3); + assert_eq!(first.partition_spec_id, 3); + } } From 1cad7313308ad4bc7168d4111b12aa921a6379fc Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Tue, 29 Sep 2026 21:55:37 +0200 Subject: [PATCH 21/50] test(transaction): cover partial overwrite and per-file spec in snapshot summary Co-authored-by: Marco Kroll <216147982+mkroll-db@users.noreply.github.com> --- crates/iceberg/src/transaction/snapshot.rs | 200 ++++++++++++++++++++- 1 file changed, 198 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 1770e12f6c..c1b4feb40b 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -599,10 +599,11 @@ mod tests { use super::*; use crate::spec::{ - DataContentType, DataFileBuilder, Literal, Manifest, NestedField, PartitionSpec, - PrimitiveType, Schema, Transform, Type, + DataContentType, DataFileBuilder, Literal, Manifest, ManifestStatus, NestedField, + PartitionSpec, PrimitiveType, Schema, Transform, Type, }; use crate::transaction::tests::make_v2_minimal_table; + use crate::transaction::{Transaction, TransactionAction}; /// An operation that adds no existing manifests and reports one removed data file. struct RemoveOneFileOperation { @@ -630,6 +631,75 @@ mod tests { } } + struct RemoveExistingFileOperation { + file_path: String, + removed: Vec<(DataFile, SchemaRef, PartitionSpecRef)>, + } + + impl SnapshotProduceOperation for RemoveExistingFileOperation { + fn operation(&self) -> Operation { + Operation::Overwrite + } + + async fn delete_entries(&self, _: &SnapshotProducer<'_>) -> Result> { + Ok(vec![]) + } + + async fn existing_manifest( + &mut self, + producer: &SnapshotProducer<'_>, + ) -> Result> { + let snapshot = producer.table.metadata().current_snapshot().unwrap(); + let manifest_list = producer.table.manifest_list_reader(snapshot).load().await?; + // The initial append writes both files into one manifest. + assert_eq!(manifest_list.entries().len(), 1); + + let manifest = producer + .table + .manifest_reader() + .read(&manifest_list.entries()[0]) + .await?; + let schema = manifest.metadata().schema().clone(); + let partition_spec = manifest.metadata().partition_spec().clone(); + let mut writer = producer.new_manifest_writer( + ManifestContentType::Data, + schema.clone(), + partition_spec.clone(), + )?; + + let mut removed = Vec::new(); + for entry in manifest.entries() { + if entry.file_path() == self.file_path && entry.is_alive() { + writer.add_delete_entry(entry.as_ref().clone())?; + removed.push(( + entry.data_file().clone(), + schema.clone(), + Arc::new(partition_spec.clone()), + )); + } else if entry.is_alive() { + writer.add_existing_entry(entry.as_ref().clone())?; + } + } + assert_eq!(removed.len(), 1); + + let rewritten_manifest = writer.write_manifest_file().await?; + self.removed.extend(removed); + Ok(vec![rewritten_manifest]) + } + + fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { + &self.removed + } + } + + fn apply_commit(table: Table, mut commit: ActionCommit) -> Table { + let mut builder = table.metadata().clone().into_builder(None); + for update in commit.take_updates() { + builder = update.apply(builder).unwrap(); + } + table.with_metadata(Arc::new(builder.build().unwrap().metadata)) + } + fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { DataFileBuilder::default() .content(DataContentType::Data) @@ -677,6 +747,80 @@ mod tests { assert_eq!(properties.get("total-records").unwrap(), "1"); } + #[tokio::test] + async fn test_partial_overwrite_reports_actual_removed_file() { + let table = make_v2_minimal_table(); + let removed = data_file(&table, "test/removed.parquet", 2); + let kept = data_file(&table, "test/kept.parquet", 5); + let append = Transaction::new(&table) + .fast_append() + .add_data_files(vec![removed.clone(), kept]); + let initial_commit = Arc::new(append).commit(&table).await.unwrap(); + let table = apply_commit(table, initial_commit); + + let initial_summary = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + assert_eq!(initial_summary.get("total-data-files").unwrap(), "2"); + assert_eq!(initial_summary.get("total-records").unwrap(), "7"); + + let mut replacement = data_file(&table, "test/replacement.parquet", 3); + replacement.file_size_in_bytes = 300; + let operation = RemoveExistingFileOperation { + file_path: removed.file_path.clone(), + removed: vec![], + }; + let commit = + SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![replacement]) + .commit(operation, DefaultManifestProcess) + .await + .unwrap(); + let table = apply_commit(table, commit); + let snapshot = table.metadata().current_snapshot().unwrap(); + let properties = &snapshot.summary().additional_properties; + + assert_eq!(snapshot.summary().operation, Operation::Overwrite); + assert_eq!(properties.get("deleted-data-files").unwrap(), "1"); + assert_eq!(properties.get("deleted-records").unwrap(), "2"); + assert_eq!(properties.get("added-data-files").unwrap(), "1"); + assert_eq!(properties.get("added-records").unwrap(), "3"); + assert_eq!(properties.get("removed-files-size").unwrap(), "100"); + assert_eq!(properties.get("added-files-size").unwrap(), "300"); + assert_eq!(properties.get("total-data-files").unwrap(), "2"); + assert_eq!(properties.get("total-records").unwrap(), "8"); + assert_eq!(properties.get("total-files-size").unwrap(), "400"); + + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + assert_eq!(manifest_list.entries().len(), 2); + let mut statuses = HashMap::new(); + for manifest_file in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + for entry in manifest.entries() { + assert!( + statuses + .insert(entry.file_path().to_string(), entry.status()) + .is_none() + ); + } + } + assert_eq!(statuses.len(), 3); + assert_eq!( + statuses.get("test/removed.parquet"), + Some(&ManifestStatus::Deleted) + ); + assert_eq!( + statuses.get("test/kept.parquet"), + Some(&ManifestStatus::Existing) + ); + assert_eq!( + statuses.get("test/replacement.parquet"), + Some(&ManifestStatus::Added) + ); + } + #[tokio::test] async fn test_new_manifest_writer_uses_the_given_schema_and_spec() { let table = make_v2_minimal_table(); @@ -731,4 +875,56 @@ mod tests { assert_eq!(metadata.partition_spec().spec_id(), 3); assert_eq!(first.partition_spec_id, 3); } + + #[tokio::test] + async fn test_removed_file_summary_uses_the_reported_partition_spec() { + let table = make_v2_minimal_table(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .set_properties(HashMap::from([( + TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT.to_string(), + "10".to_string(), + )])) + .unwrap() + .build() + .unwrap() + .metadata; + let table = table.with_metadata(Arc::new(metadata)); + + let reported_schema = table.metadata().current_schema().clone(); + let reported_spec = Arc::new( + PartitionSpec::builder(reported_schema.clone()) + .with_spec_id(1) + .add_partition_field("y", "y", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); + + let added = data_file(&table, "test/added.parquet", 3); + let removed = data_file(&table, "test/removed.parquet", 2); + let operation = RemoveOneFileOperation { + removed: vec![(removed, reported_schema, reported_spec)], + }; + + let mut action_commit = + SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![added]) + .commit(operation, DefaultManifestProcess) + .await + .unwrap(); + + let updates = action_commit.take_updates(); + let TableUpdate::AddSnapshot { snapshot } = &updates[0] else { + unreachable!() + }; + let properties = &snapshot.summary().additional_properties; + + let removed_partition = properties + .get("partitions.y=300") + .expect("removal must be summarized under the reported spec's partition path"); + assert!(removed_partition.contains("deleted-data-files=1")); + assert!(properties.contains_key("partitions.x=300")); + } } From 7320fa4a03268b414ed8ab2fe3322afb4e8f9295 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Tue, 29 Sep 2026 21:57:29 +0200 Subject: [PATCH 22/50] Adapt the evolved-spec test to the UnboundPartitionField builder --- crates/iceberg/src/transaction/overwrite.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 28ff313d84..8597c65a7a 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -298,7 +298,7 @@ mod tests { use crate::spec::{ DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, ManifestEntryRef, ManifestStatus, Operation, PrimitiveType, SnapshotRef, Struct, Transform, - Type, UnboundPartitionSpec, + Type, UnboundPartitionField, UnboundPartitionSpec, }; use crate::table::Table; use crate::transaction::tests::{make_v2_minimal_table, make_v3_minimal_table_in_catalog}; @@ -675,7 +675,14 @@ mod tests { .into_builder(None) .add_default_partition_spec( UnboundPartitionSpec::builder() - .add_partition_field(2, "y", Transform::Identity) + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("y") + .transform(Transform::Identity) + .build() + .unwrap(), + ) .unwrap() .build(), ) From d5623e3c29b10db073fe60b48a39da32dd44423b Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:20:33 +0200 Subject: [PATCH 23/50] Drop the full-table truncation from the snapshot summary Java and PyIceberg derive totals from the files an operation marks deleted, with no truncation path. --- crates/iceberg/src/spec/snapshot_summary.rs | 283 +------------------- crates/iceberg/src/transaction/snapshot.rs | 12 +- 2 files changed, 8 insertions(+), 287 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index 105952957f..b13b35bf2a 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -330,9 +330,8 @@ where T: PartialOrd + Default + ToString { } pub(crate) fn update_snapshot_summaries( - summary: Summary, + mut summary: Summary, previous_summary: Option<&Summary>, - truncate_full_table: bool, ) -> Result { // Validate that the operation is supported if summary.operation != Operation::Append @@ -345,15 +344,6 @@ pub(crate) fn update_snapshot_summaries( )); } - let mut summary = match previous_summary { - Some(prev_summary) if truncate_full_table && summary.operation == Operation::Overwrite => { - truncate_table_summary(summary, prev_summary).map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to truncate table summary.") - .with_source(err) - })? - } - _ => summary, - }; update_totals( &mut summary, @@ -405,77 +395,6 @@ pub(crate) fn update_snapshot_summaries( Ok(summary) } -fn get_prop(previous_summary: &Summary, prop: &str) -> Result { - let value_str = previous_summary - .additional_properties - .get(prop) - .map(String::as_str) - .unwrap_or("0"); - value_str.parse::().map_err(|err| { - Error::new( - ErrorKind::Unexpected, - format!("Failed to parse summary property '{prop}' value '{value_str}' as u64."), - ) - .with_source(err) - }) -} - -fn truncate_table_summary(mut summary: Summary, previous_summary: &Summary) -> Result { - for prop in [ - TOTAL_DATA_FILES, - TOTAL_DELETE_FILES, - TOTAL_RECORDS, - TOTAL_FILE_SIZE, - TOTAL_POSITION_DELETES, - TOTAL_EQUALITY_DELETES, - ] { - summary - .additional_properties - .insert(prop.to_string(), "0".to_string()); - } - - let value = get_prop(previous_summary, TOTAL_DATA_FILES)?; - if value != 0 { - summary - .additional_properties - .insert(DELETED_DATA_FILES.to_string(), value.to_string()); - } - let value = get_prop(previous_summary, TOTAL_DELETE_FILES)?; - if value != 0 { - summary - .additional_properties - .insert(REMOVED_DELETE_FILES.to_string(), value.to_string()); - } - let value = get_prop(previous_summary, TOTAL_RECORDS)?; - if value != 0 { - summary - .additional_properties - .insert(DELETED_RECORDS.to_string(), value.to_string()); - } - let value = get_prop(previous_summary, TOTAL_FILE_SIZE)?; - if value != 0 { - summary - .additional_properties - .insert(REMOVED_FILE_SIZE.to_string(), value.to_string()); - } - - let value = get_prop(previous_summary, TOTAL_POSITION_DELETES)?; - if value != 0 { - summary - .additional_properties - .insert(REMOVED_POSITION_DELETES.to_string(), value.to_string()); - } - - let value = get_prop(previous_summary, TOTAL_EQUALITY_DELETES)?; - if value != 0 { - summary - .additional_properties - .insert(REMOVED_EQUALITY_DELETES.to_string(), value.to_string()); - } - - Ok(summary) -} - fn update_totals( summary: &mut Summary, previous_summary: Option<&Summary>, @@ -589,7 +508,7 @@ mod tests { additional_properties: new_props, }; - let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); assert_eq!( updated.additional_properties.get(TOTAL_DATA_FILES).unwrap(), @@ -626,194 +545,6 @@ mod tests { ); } - #[test] - fn test_truncate_table_summary() { - let prev_props: HashMap = [ - (TOTAL_DATA_FILES.to_string(), "10".to_string()), - (TOTAL_DELETE_FILES.to_string(), "5".to_string()), - (TOTAL_RECORDS.to_string(), "100".to_string()), - (TOTAL_FILE_SIZE.to_string(), "1000".to_string()), - (TOTAL_POSITION_DELETES.to_string(), "3".to_string()), - (TOTAL_EQUALITY_DELETES.to_string(), "2".to_string()), - ] - .into_iter() - .collect(); - - let previous_summary = Summary { - operation: Operation::Overwrite, - additional_properties: prev_props, - }; - - let mut new_props = HashMap::new(); - new_props.insert("dummy".to_string(), "value".to_string()); - let summary = Summary { - operation: Operation::Overwrite, - additional_properties: new_props, - }; - - let truncated = truncate_table_summary(summary, &previous_summary).unwrap(); - - assert_eq!( - truncated - .additional_properties - .get(TOTAL_DATA_FILES) - .unwrap(), - "0" - ); - assert_eq!( - truncated - .additional_properties - .get(TOTAL_DELETE_FILES) - .unwrap(), - "0" - ); - assert_eq!( - truncated.additional_properties.get(TOTAL_RECORDS).unwrap(), - "0" - ); - assert_eq!( - truncated - .additional_properties - .get(TOTAL_FILE_SIZE) - .unwrap(), - "0" - ); - assert_eq!( - truncated - .additional_properties - .get(TOTAL_POSITION_DELETES) - .unwrap(), - "0" - ); - assert_eq!( - truncated - .additional_properties - .get(TOTAL_EQUALITY_DELETES) - .unwrap(), - "0" - ); - - assert_eq!( - truncated - .additional_properties - .get(DELETED_DATA_FILES) - .unwrap(), - "10" - ); - assert_eq!( - truncated - .additional_properties - .get(REMOVED_DELETE_FILES) - .unwrap(), - "5" - ); - assert_eq!( - truncated - .additional_properties - .get(DELETED_RECORDS) - .unwrap(), - "100" - ); - assert_eq!( - truncated - .additional_properties - .get(REMOVED_FILE_SIZE) - .unwrap(), - "1000" - ); - assert_eq!( - truncated - .additional_properties - .get(REMOVED_POSITION_DELETES) - .unwrap(), - "3" - ); - assert_eq!( - truncated - .additional_properties - .get(REMOVED_EQUALITY_DELETES) - .unwrap(), - "2" - ); - } - - #[test] - fn test_update_snapshot_summaries_overwrite_truncate_handles_totals_above_i32_max() { - // A table can legitimately accumulate more than i32::MAX rows or files - // over its lifetime. Truncating such a table on overwrite must succeed - // and surface the previous totals into the deleted-* counters. - let big = (i32::MAX as u64 + 1).to_string(); // 2_147_483_648 - let prev_props: HashMap = [ - (TOTAL_DATA_FILES.to_string(), big.clone()), - (TOTAL_DELETE_FILES.to_string(), "0".to_string()), - (TOTAL_RECORDS.to_string(), big.clone()), - (TOTAL_FILE_SIZE.to_string(), "0".to_string()), - (TOTAL_POSITION_DELETES.to_string(), "0".to_string()), - (TOTAL_EQUALITY_DELETES.to_string(), "0".to_string()), - ] - .into_iter() - .collect(); - - let previous_summary = Summary { - operation: Operation::Overwrite, - additional_properties: prev_props, - }; - - let summary = Summary { - operation: Operation::Overwrite, - additional_properties: HashMap::new(), - }; - - let updated = update_snapshot_summaries(summary, Some(&previous_summary), true) - .expect("overwrite truncation should accept totals above i32::MAX"); - assert_eq!( - updated - .additional_properties - .get(DELETED_DATA_FILES) - .unwrap(), - &big - ); - assert_eq!( - updated.additional_properties.get(DELETED_RECORDS).unwrap(), - &big - ); - } - - #[test] - fn test_update_snapshot_summaries_overwrite_truncate_returns_err_on_malformed_total() { - // Non-numeric values in the previous summary (corruption, manual edits, - // a foreign implementation) must surface as a recoverable Err - not - // crash the process. - let prev_props: HashMap = [ - (TOTAL_DATA_FILES.to_string(), "not_a_number".to_string()), - (TOTAL_DELETE_FILES.to_string(), "0".to_string()), - (TOTAL_RECORDS.to_string(), "0".to_string()), - (TOTAL_FILE_SIZE.to_string(), "0".to_string()), - (TOTAL_POSITION_DELETES.to_string(), "0".to_string()), - (TOTAL_EQUALITY_DELETES.to_string(), "0".to_string()), - ] - .into_iter() - .collect(); - - let previous_summary = Summary { - operation: Operation::Overwrite, - additional_properties: prev_props, - }; - - let summary = Summary { - operation: Operation::Overwrite, - additional_properties: HashMap::new(), - }; - - let err = update_snapshot_summaries(summary, Some(&previous_summary), true) - .expect_err("malformed previous summary must produce an Err, not a panic"); - assert!( - err.message().contains("truncate table summary"), - "expected wrapped 'Failed to truncate table summary' context, got: {}", - err.message() - ); - } - #[test] fn test_snapshot_summary_collector_build() { let schema = Arc::new( @@ -1146,7 +877,7 @@ mod tests { additional_properties: new_props, }; - let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); let props = &updated.additional_properties; assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "12"); @@ -1192,7 +923,7 @@ mod tests { }; // Must not panic. - let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); let props = &updated.additional_properties; // The total whose added delta was unparsable is skipped... @@ -1220,7 +951,7 @@ mod tests { additional_properties: new_props, }; - let updated = update_snapshot_summaries(summary, None, false).unwrap(); + let updated = update_snapshot_summaries(summary, None).unwrap(); let props = &updated.additional_properties; assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "4"); @@ -1264,7 +995,7 @@ mod tests { additional_properties: new_props, }; - let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); let props = &updated.additional_properties; assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "8"); @@ -1305,7 +1036,7 @@ mod tests { additional_properties: new_props, }; - let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); + let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); let props = &updated.additional_properties; assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "0"); diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index c1b4feb40b..3aeb2af39a 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -95,12 +95,6 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { &[] } - - /// Returns whether this operation replaces the whole table, dropping the previous totals - /// from the snapshot summary. A partial overwrite must return `false`. - fn truncate_full_table(&self) -> bool { - false - } } pub(crate) struct DefaultManifestProcess; @@ -452,11 +446,7 @@ impl<'a> SnapshotProducer<'a> { additional_properties, }; - update_snapshot_summaries( - summary, - previous_snapshot.map(|s| s.summary()), - snapshot_produce_operation.truncate_full_table(), - ) + update_snapshot_summaries(summary, previous_snapshot.map(|s| s.summary())) } fn generate_manifest_list_file_path(&self, attempt: i64) -> Result { From 296532e6ba26caf89334ce1bcbacd0872674abc3 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:20:41 +0200 Subject: [PATCH 24/50] Omit a summary total that would go negative, as Java does --- crates/iceberg/src/spec/snapshot_summary.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index b13b35bf2a..89af2081ba 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -451,7 +451,10 @@ fn update_totals( return; }; - let new_total = (previous_total + added).saturating_sub(removed); + // Like Java, a total that would go negative is omitted rather than written as a false basis. + let Some(new_total) = previous_total.saturating_add(added).checked_sub(removed) else { + return; + }; summary .additional_properties .insert(total_property.to_string(), new_total.to_string()); @@ -1007,7 +1010,7 @@ mod tests { } #[test] - fn test_update_totals_saturates_when_removed_exceeds_previous_total() { + fn test_update_totals_omits_total_when_removed_exceeds_previous_total() { let prev_props: HashMap = [ (TOTAL_DATA_FILES, "2"), (TOTAL_RECORDS, "10"), @@ -1039,8 +1042,8 @@ mod tests { let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); let props = &updated.additional_properties; - assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "0"); - assert_eq!(props.get(TOTAL_RECORDS).unwrap(), "0"); - assert_eq!(props.get(TOTAL_FILE_SIZE).unwrap(), "0"); + assert!(!props.contains_key(TOTAL_DATA_FILES)); + assert!(!props.contains_key(TOTAL_RECORDS)); + assert!(!props.contains_key(TOTAL_FILE_SIZE)); } } From eeff802555161e76e8a8bfb7b2027307f3d005a1 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:21:11 +0200 Subject: [PATCH 25/50] Name the removed-file fields and take the manifest spec by ref --- crates/iceberg/src/transaction/snapshot.rs | 82 +++++++++++++--------- 1 file changed, 48 insertions(+), 34 deletions(-) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 3aeb2af39a..43345fdeb2 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -27,7 +27,7 @@ use crate::error::Result; use crate::spec::{ DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, - PartitionSpec, PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, + PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, StructType, Summary, TableProperties, update_snapshot_summaries, }; @@ -87,16 +87,23 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { snapshot_produce: &SnapshotProducer<'_>, ) -> impl Future>> + Send; - /// Returns the data files this operation actually removed, each with the schema and - /// partition spec of the manifest that recorded it. + /// Returns the data files this operation actually removed. /// /// Only populated once [`Self::existing_manifest`] has run, so the snapshot summary counts /// what was really removed rather than what the caller asked to remove. - fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { + fn removed_data_files(&self) -> &[RemovedDataFile] { &[] } } +/// A data file an operation removed, with the schema and partition spec of the manifest that +/// recorded it. +pub(crate) struct RemovedDataFile { + pub(crate) data_file: DataFile, + pub(crate) schema: SchemaRef, + pub(crate) partition_spec: PartitionSpecRef, +} + pub(crate) struct DefaultManifestProcess; impl ManifestProcess for DefaultManifestProcess { @@ -261,9 +268,10 @@ impl<'a> SnapshotProducer<'a> { &self, content: ManifestContentType, schema: SchemaRef, - partition_spec: PartitionSpec, + partition_spec: PartitionSpecRef, ) -> Result { let output_file = self.table.file_io().new_output(self.new_manifest_path()?)?; + let partition_spec = partition_spec.as_ref().clone(); let builder = if let Some(em) = self.table.encryption_manager() { ManifestWriterBuilder::new_from_encrypted( @@ -349,11 +357,7 @@ impl<'a> SnapshotProducer<'a> { let mut writer = self.new_manifest_writer( ManifestContentType::Data, self.table.metadata().current_schema().clone(), - self.table - .metadata() - .default_partition_spec() - .as_ref() - .clone(), + self.table.metadata().default_partition_spec().clone(), )?; for entry in manifest_entries { writer.add_entry(entry)?; @@ -427,8 +431,12 @@ impl<'a> SnapshotProducer<'a> { ); } - for (data_file, schema, partition_spec) in snapshot_produce_operation.removed_data_files() { - summary_collector.remove_file(data_file, schema.clone(), partition_spec.clone()); + for removed in snapshot_produce_operation.removed_data_files() { + summary_collector.remove_file( + &removed.data_file, + removed.schema.clone(), + removed.partition_spec.clone(), + ); } let previous_snapshot = table_metadata.current_snapshot(); @@ -597,7 +605,7 @@ mod tests { /// An operation that adds no existing manifests and reports one removed data file. struct RemoveOneFileOperation { - removed: Vec<(DataFile, SchemaRef, PartitionSpecRef)>, + removed: Vec, } impl SnapshotProduceOperation for RemoveOneFileOperation { @@ -616,14 +624,14 @@ mod tests { Ok(vec![]) } - fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { + fn removed_data_files(&self) -> &[RemovedDataFile] { &self.removed } } struct RemoveExistingFileOperation { file_path: String, - removed: Vec<(DataFile, SchemaRef, PartitionSpecRef)>, + removed: Vec, } impl SnapshotProduceOperation for RemoveExistingFileOperation { @@ -650,7 +658,7 @@ mod tests { .read(&manifest_list.entries()[0]) .await?; let schema = manifest.metadata().schema().clone(); - let partition_spec = manifest.metadata().partition_spec().clone(); + let partition_spec = Arc::new(manifest.metadata().partition_spec().clone()); let mut writer = producer.new_manifest_writer( ManifestContentType::Data, schema.clone(), @@ -661,11 +669,11 @@ mod tests { for entry in manifest.entries() { if entry.file_path() == self.file_path && entry.is_alive() { writer.add_delete_entry(entry.as_ref().clone())?; - removed.push(( - entry.data_file().clone(), - schema.clone(), - Arc::new(partition_spec.clone()), - )); + removed.push(RemovedDataFile { + data_file: entry.data_file().clone(), + schema: schema.clone(), + partition_spec: partition_spec.clone(), + }); } else if entry.is_alive() { writer.add_existing_entry(entry.as_ref().clone())?; } @@ -677,7 +685,7 @@ mod tests { Ok(vec![rewritten_manifest]) } - fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { + fn removed_data_files(&self) -> &[RemovedDataFile] { &self.removed } } @@ -710,11 +718,11 @@ mod tests { let removed = data_file(&table, "test/removed.parquet", 2); let operation = RemoveOneFileOperation { - removed: vec![( - removed, - table.metadata().current_schema().clone(), - table.metadata().default_partition_spec().clone(), - )], + removed: vec![RemovedDataFile { + data_file: removed, + schema: table.metadata().current_schema().clone(), + partition_spec: table.metadata().default_partition_spec().clone(), + }], }; let mut action_commit = @@ -823,12 +831,14 @@ mod tests { .build() .unwrap(), ); - let partition_spec = PartitionSpec::builder(schema.clone()) - .with_spec_id(3) - .add_partition_field("a", "a", Transform::Identity) - .unwrap() - .build() - .unwrap(); + let partition_spec = Arc::new( + PartitionSpec::builder(schema.clone()) + .with_spec_id(3) + .add_partition_field("a", "a", Transform::Identity) + .unwrap() + .build() + .unwrap(), + ); let producer = SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![]); let first = producer @@ -896,7 +906,11 @@ mod tests { let added = data_file(&table, "test/added.parquet", 3); let removed = data_file(&table, "test/removed.parquet", 2); let operation = RemoveOneFileOperation { - removed: vec![(removed, reported_schema, reported_spec)], + removed: vec![RemovedDataFile { + data_file: removed, + schema: reported_schema, + partition_spec: reported_spec, + }], }; let mut action_commit = From 495f3adc4e8bd2a349e59a6866a14b55b8ad8fdd Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:21:32 +0200 Subject: [PATCH 26/50] Take the added files again instead of cloning them for the summary --- crates/iceberg/src/transaction/snapshot.rs | 35 ++++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 43345fdeb2..5c335a5b9d 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -330,9 +330,7 @@ impl<'a> SnapshotProducer<'a> { // Write manifest file for added data files and return the ManifestFile for ManifestList. async fn write_added_manifest(&mut self) -> Result { - // Cloned rather than taken: the summary is built after the manifests, so the added - // files must still be here. - let added_data_files = self.added_data_files.clone(); + let added_data_files = std::mem::take(&mut self.added_data_files); if added_data_files.is_empty() { return Err(Error::new( ErrorKind::PreconditionFailed, @@ -400,11 +398,8 @@ impl<'a> SnapshotProducer<'a> { Ok(manifest_files) } - // Returns a `Summary` of the current snapshot - fn summary( - &self, - snapshot_produce_operation: &OP, - ) -> Result { + // Collects the added files, before `produce_manifests` takes them. + fn added_files_summary(&self) -> SnapshotSummaryCollector { let mut summary_collector = SnapshotSummaryCollector::default(); let table_metadata = self.table.metadata_ref(); @@ -430,6 +425,16 @@ impl<'a> SnapshotProducer<'a> { table_metadata.default_partition_spec().clone(), ); } + summary_collector + } + + // Returns a `Summary` of the current snapshot + fn summary( + &self, + mut summary_collector: SnapshotSummaryCollector, + snapshot_produce_operation: &OP, + ) -> Result { + let table_metadata = self.table.metadata_ref(); for removed in snapshot_produce_operation.removed_data_files() { summary_collector.remove_file( @@ -511,15 +516,19 @@ impl<'a> SnapshotProducer<'a> { ), }; - // The manifests are produced first so the summary can count the entries the operation - // actually marked Deleted. + // The manifests are produced before the summary is finished so it can count the entries + // the operation actually marked Deleted. + let summary_collector = self.added_files_summary(); let new_manifests = self .produce_manifests(&mut snapshot_produce_operation, &process) .await?; - let summary = self.summary(&snapshot_produce_operation).map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) - })?; + let summary = self + .summary(summary_collector, &snapshot_produce_operation) + .map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.") + .with_source(err) + })?; manifest_list_writer.add_manifests(new_manifests.into_iter())?; let writer_next_row_id = manifest_list_writer.next_row_id(); From 96da179920b3bf82c1fed8ff149b53b43c7ee3ed Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:23:03 +0200 Subject: [PATCH 27/50] Assert an operation reports exactly the files it marked deleted --- crates/iceberg/src/spec/snapshot_summary.rs | 1 - crates/iceberg/src/transaction/snapshot.rs | 31 +++++++++++++++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index 89af2081ba..ccdad4589a 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -344,7 +344,6 @@ pub(crate) fn update_snapshot_summaries( )); } - update_totals( &mut summary, previous_summary, diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 5c335a5b9d..5c6aca8c79 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -130,9 +130,7 @@ pub(crate) struct SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, - // A counter used to generate unique manifest file names. - // It starts from 0 and increments for each new manifest file. - // Note: This counter is limited to the range of (0..u64::MAX). + // Numbers this commit's manifest files. Atomic so `new_manifest_writer` can take `&self`. manifest_counter: AtomicU64, } @@ -394,6 +392,17 @@ impl<'a> SnapshotProducer<'a> { // # TODO // Support process delete entries. + debug_assert_eq!( + manifest_files + .iter() + .filter(|m| m.added_snapshot_id == self.snapshot_id + && m.content == ManifestContentType::Data) + .filter_map(|m| m.deleted_files_count) + .sum::() as usize, + snapshot_produce_operation.removed_data_files().len(), + "an operation must report exactly the data files it marked deleted" + ); + let manifest_files = manifest_process.process_manifests(self, manifest_files); Ok(manifest_files) } @@ -612,7 +621,7 @@ mod tests { use crate::transaction::tests::make_v2_minimal_table; use crate::transaction::{Transaction, TransactionAction}; - /// An operation that adds no existing manifests and reports one removed data file. + /// An operation that writes each of its removed files as a deleted entry in its own manifest. struct RemoveOneFileOperation { removed: Vec, } @@ -628,9 +637,19 @@ mod tests { async fn existing_manifest( &mut self, - _: &SnapshotProducer<'_>, + producer: &SnapshotProducer<'_>, ) -> Result> { - Ok(vec![]) + let mut manifests = vec![]; + for removed in &self.removed { + let mut writer = producer.new_manifest_writer( + ManifestContentType::Data, + removed.schema.clone(), + removed.partition_spec.clone(), + )?; + writer.add_delete_file(removed.data_file.clone(), 0, Some(0))?; + manifests.push(writer.write_manifest_file().await?); + } + Ok(manifests) } fn removed_data_files(&self) -> &[RemovedDataFile] { From a5653bceda2812696483842a1cc032a164ab8546 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:23:53 +0200 Subject: [PATCH 28/50] Say in the removed_data_files doc what it must match --- crates/iceberg/src/transaction/snapshot.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 5c6aca8c79..3c0c3c2f86 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -87,7 +87,7 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { snapshot_produce: &SnapshotProducer<'_>, ) -> impl Future>> + Send; - /// Returns the data files this operation actually removed. + /// Returns the data files this operation marked deleted in the manifests it wrote. /// /// Only populated once [`Self::existing_manifest`] has run, so the snapshot summary counts /// what was really removed rather than what the caller asked to remove. From 29772614f6e37c8338e46ed7c682297ad032da2d Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:29:21 +0200 Subject: [PATCH 29/50] Summarize deleted files from the producer's list, as pyiceberg does Drops the removed_data_files hook and the manifest-first reorder, and truncation defaults back to every overwrite. --- crates/iceberg/src/spec/snapshot_summary.rs | 315 +++++++++++++-- crates/iceberg/src/transaction/append.rs | 3 +- crates/iceberg/src/transaction/snapshot.rs | 403 ++++---------------- 3 files changed, 356 insertions(+), 365 deletions(-) diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index 7a639aa08e..d68a49d149 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -331,8 +331,9 @@ where T: PartialOrd + Default + ToString { } pub(crate) fn update_snapshot_summaries( - mut summary: Summary, + summary: Summary, previous_summary: Option<&Summary>, + truncate_full_table: bool, ) -> Result { // Validate that the operation is supported if summary.operation != Operation::Append @@ -342,6 +343,16 @@ pub(crate) fn update_snapshot_summaries( return Err(invalid_data!("Operation is not supported.")); } + let mut summary = match previous_summary { + Some(prev_summary) if truncate_full_table && summary.operation == Operation::Overwrite => { + truncate_table_summary(summary, prev_summary).map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to truncate table summary.") + .with_source(err) + })? + } + _ => summary, + }; + update_totals( &mut summary, previous_summary, @@ -392,6 +403,77 @@ pub(crate) fn update_snapshot_summaries( Ok(summary) } +fn get_prop(previous_summary: &Summary, prop: &str) -> Result { + let value_str = previous_summary + .additional_properties + .get(prop) + .map(String::as_str) + .unwrap_or("0"); + value_str.parse::().map_err(|err| { + Error::new( + ErrorKind::Unexpected, + format!("Failed to parse summary property '{prop}' value '{value_str}' as u64."), + ) + .with_source(err) + }) +} + +fn truncate_table_summary(mut summary: Summary, previous_summary: &Summary) -> Result { + for prop in [ + TOTAL_DATA_FILES, + TOTAL_DELETE_FILES, + TOTAL_RECORDS, + TOTAL_FILE_SIZE, + TOTAL_POSITION_DELETES, + TOTAL_EQUALITY_DELETES, + ] { + summary + .additional_properties + .insert(prop.to_string(), "0".to_string()); + } + + let value = get_prop(previous_summary, TOTAL_DATA_FILES)?; + if value != 0 { + summary + .additional_properties + .insert(DELETED_DATA_FILES.to_string(), value.to_string()); + } + let value = get_prop(previous_summary, TOTAL_DELETE_FILES)?; + if value != 0 { + summary + .additional_properties + .insert(REMOVED_DELETE_FILES.to_string(), value.to_string()); + } + let value = get_prop(previous_summary, TOTAL_RECORDS)?; + if value != 0 { + summary + .additional_properties + .insert(DELETED_RECORDS.to_string(), value.to_string()); + } + let value = get_prop(previous_summary, TOTAL_FILE_SIZE)?; + if value != 0 { + summary + .additional_properties + .insert(REMOVED_FILE_SIZE.to_string(), value.to_string()); + } + + let value = get_prop(previous_summary, TOTAL_POSITION_DELETES)?; + if value != 0 { + summary + .additional_properties + .insert(REMOVED_POSITION_DELETES.to_string(), value.to_string()); + } + + let value = get_prop(previous_summary, TOTAL_EQUALITY_DELETES)?; + if value != 0 { + summary + .additional_properties + .insert(REMOVED_EQUALITY_DELETES.to_string(), value.to_string()); + } + + Ok(summary) +} + fn update_totals( summary: &mut Summary, previous_summary: Option<&Summary>, @@ -508,7 +590,7 @@ mod tests { additional_properties: new_props, }; - let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); assert_eq!( updated.additional_properties.get(TOTAL_DATA_FILES).unwrap(), @@ -545,6 +627,194 @@ mod tests { ); } + #[test] + fn test_truncate_table_summary() { + let prev_props: HashMap = [ + (TOTAL_DATA_FILES.to_string(), "10".to_string()), + (TOTAL_DELETE_FILES.to_string(), "5".to_string()), + (TOTAL_RECORDS.to_string(), "100".to_string()), + (TOTAL_FILE_SIZE.to_string(), "1000".to_string()), + (TOTAL_POSITION_DELETES.to_string(), "3".to_string()), + (TOTAL_EQUALITY_DELETES.to_string(), "2".to_string()), + ] + .into_iter() + .collect(); + + let previous_summary = Summary { + operation: Operation::Overwrite, + additional_properties: prev_props, + }; + + let mut new_props = HashMap::new(); + new_props.insert("dummy".to_string(), "value".to_string()); + let summary = Summary { + operation: Operation::Overwrite, + additional_properties: new_props, + }; + + let truncated = truncate_table_summary(summary, &previous_summary).unwrap(); + + assert_eq!( + truncated + .additional_properties + .get(TOTAL_DATA_FILES) + .unwrap(), + "0" + ); + assert_eq!( + truncated + .additional_properties + .get(TOTAL_DELETE_FILES) + .unwrap(), + "0" + ); + assert_eq!( + truncated.additional_properties.get(TOTAL_RECORDS).unwrap(), + "0" + ); + assert_eq!( + truncated + .additional_properties + .get(TOTAL_FILE_SIZE) + .unwrap(), + "0" + ); + assert_eq!( + truncated + .additional_properties + .get(TOTAL_POSITION_DELETES) + .unwrap(), + "0" + ); + assert_eq!( + truncated + .additional_properties + .get(TOTAL_EQUALITY_DELETES) + .unwrap(), + "0" + ); + + assert_eq!( + truncated + .additional_properties + .get(DELETED_DATA_FILES) + .unwrap(), + "10" + ); + assert_eq!( + truncated + .additional_properties + .get(REMOVED_DELETE_FILES) + .unwrap(), + "5" + ); + assert_eq!( + truncated + .additional_properties + .get(DELETED_RECORDS) + .unwrap(), + "100" + ); + assert_eq!( + truncated + .additional_properties + .get(REMOVED_FILE_SIZE) + .unwrap(), + "1000" + ); + assert_eq!( + truncated + .additional_properties + .get(REMOVED_POSITION_DELETES) + .unwrap(), + "3" + ); + assert_eq!( + truncated + .additional_properties + .get(REMOVED_EQUALITY_DELETES) + .unwrap(), + "2" + ); + } + + #[test] + fn test_update_snapshot_summaries_overwrite_truncate_handles_totals_above_i32_max() { + // A table can legitimately accumulate more than i32::MAX rows or files + // over its lifetime. Truncating such a table on overwrite must succeed + // and surface the previous totals into the deleted-* counters. + let big = (i32::MAX as u64 + 1).to_string(); // 2_147_483_648 + let prev_props: HashMap = [ + (TOTAL_DATA_FILES.to_string(), big.clone()), + (TOTAL_DELETE_FILES.to_string(), "0".to_string()), + (TOTAL_RECORDS.to_string(), big.clone()), + (TOTAL_FILE_SIZE.to_string(), "0".to_string()), + (TOTAL_POSITION_DELETES.to_string(), "0".to_string()), + (TOTAL_EQUALITY_DELETES.to_string(), "0".to_string()), + ] + .into_iter() + .collect(); + + let previous_summary = Summary { + operation: Operation::Overwrite, + additional_properties: prev_props, + }; + + let summary = Summary { + operation: Operation::Overwrite, + additional_properties: HashMap::new(), + }; + + let updated = update_snapshot_summaries(summary, Some(&previous_summary), true) + .expect("overwrite truncation should accept totals above i32::MAX"); + assert_eq!( + updated + .additional_properties + .get(DELETED_DATA_FILES) + .unwrap(), + &big + ); + assert_eq!( + updated.additional_properties.get(DELETED_RECORDS).unwrap(), + &big + ); + } + + #[test] + fn test_update_snapshot_summaries_overwrite_truncate_returns_err_on_malformed_total() { + // Non-numeric values in the previous summary (corruption, manual edits, + // a foreign implementation) must surface as a recoverable Err - not + // crash the process. + let prev_props: HashMap = [ + (TOTAL_DATA_FILES.to_string(), "not_a_number".to_string()), + (TOTAL_DELETE_FILES.to_string(), "0".to_string()), + (TOTAL_RECORDS.to_string(), "0".to_string()), + (TOTAL_FILE_SIZE.to_string(), "0".to_string()), + (TOTAL_POSITION_DELETES.to_string(), "0".to_string()), + (TOTAL_EQUALITY_DELETES.to_string(), "0".to_string()), + ] + .into_iter() + .collect(); + + let previous_summary = Summary { + operation: Operation::Overwrite, + additional_properties: prev_props, + }; + + let summary = Summary { + operation: Operation::Overwrite, + additional_properties: HashMap::new(), + }; + + let err = update_snapshot_summaries(summary, Some(&previous_summary), true) + .expect_err("malformed previous summary must produce an Err, not a panic"); + assert!( + err.message().contains("truncate table summary"), + "expected wrapped 'Failed to truncate table summary' context, got: {}", + err.message() + ); + } + #[test] fn test_snapshot_summary_collector_build() { let schema = Arc::new( @@ -879,7 +1149,7 @@ mod tests { additional_properties: new_props, }; - let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); let props = &updated.additional_properties; assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "12"); @@ -925,7 +1195,7 @@ mod tests { }; // Must not panic. - let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); let props = &updated.additional_properties; // The total whose added delta was unparsable is skipped... @@ -953,7 +1223,7 @@ mod tests { additional_properties: new_props, }; - let updated = update_snapshot_summaries(summary, None).unwrap(); + let updated = update_snapshot_summaries(summary, None, false).unwrap(); let props = &updated.additional_properties; assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "4"); @@ -997,7 +1267,7 @@ mod tests { additional_properties: new_props, }; - let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); let props = &updated.additional_properties; assert_eq!(props.get(TOTAL_DATA_FILES).unwrap(), "8"); @@ -1010,39 +1280,20 @@ mod tests { #[test] fn test_update_totals_omits_total_when_removed_exceeds_previous_total() { - let prev_props: HashMap = [ - (TOTAL_DATA_FILES, "2"), - (TOTAL_RECORDS, "10"), - (TOTAL_FILE_SIZE, "100"), - ] - .into_iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - let previous_summary = Summary { operation: Operation::Append, - additional_properties: prev_props, + additional_properties: HashMap::from([(TOTAL_DATA_FILES.to_string(), "2".to_string())]), }; - - let new_props: HashMap = [ - (DELETED_DATA_FILES, "3"), - (DELETED_RECORDS, "25"), - (REMOVED_FILE_SIZE, "250"), - ] - .into_iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - let summary = Summary { operation: Operation::Delete, - additional_properties: new_props, + additional_properties: HashMap::from([( + DELETED_DATA_FILES.to_string(), + "3".to_string(), + )]), }; - let updated = update_snapshot_summaries(summary, Some(&previous_summary)).unwrap(); - let props = &updated.additional_properties; + let updated = update_snapshot_summaries(summary, Some(&previous_summary), false).unwrap(); - assert!(!props.contains_key(TOTAL_DATA_FILES)); - assert!(!props.contains_key(TOTAL_RECORDS)); - assert!(!props.contains_key(TOTAL_FILE_SIZE)); + assert!(!updated.additional_properties.contains_key(TOTAL_DATA_FILES)); } } diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 5701be3d13..881bd5d752 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -93,6 +93,7 @@ impl TransactionAction for FastAppendAction { self.commit_uuid.unwrap_or_else(Uuid::now_v7), self.snapshot_properties.clone(), self.dedupe_added_files(), + vec![], ); // validate added files @@ -124,7 +125,7 @@ impl SnapshotProduceOperation for FastAppendOperation { } async fn existing_manifest( - &mut self, + &self, snapshot_produce: &SnapshotProducer<'_>, ) -> Result> { let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index a3b6e70dbd..0a64594422 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -83,27 +83,17 @@ pub(crate) trait SnapshotProduceOperation: Send + Sync { /// - **Overwrite operations**: May exclude manifests for partitions being overwritten /// - **Delete operations**: May exclude manifests for partitions being deleted fn existing_manifest( - &mut self, + &self, snapshot_produce: &SnapshotProducer<'_>, ) -> impl Future>> + Send; - /// Returns the data files this operation marked deleted in the manifests it wrote. - /// - /// Only populated once [`Self::existing_manifest`] has run, so the snapshot summary counts - /// what was really removed rather than what the caller asked to remove. - fn removed_data_files(&self) -> &[RemovedDataFile] { - &[] + /// Returns whether the snapshot summary resets the table totals. Defaults to every + /// overwrite; an operation that overwrites only some files returns `false`. + fn truncate_full_table(&self) -> bool { + self.operation() == Operation::Overwrite } } -/// A data file an operation removed, with the schema and partition spec of the manifest that -/// recorded it. -pub(crate) struct RemovedDataFile { - pub(crate) data_file: DataFile, - pub(crate) schema: SchemaRef, - pub(crate) partition_spec: PartitionSpecRef, -} - pub(crate) struct DefaultManifestProcess; impl ManifestProcess for DefaultManifestProcess { @@ -130,6 +120,7 @@ pub(crate) struct SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, + deleted_data_files: Vec, // Numbers this commit's manifest files. Atomic so `new_manifest_writer` can take `&self`. manifest_counter: AtomicU64, } @@ -140,6 +131,7 @@ impl<'a> SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, + deleted_data_files: Vec, ) -> Self { Self { table, @@ -147,6 +139,7 @@ impl<'a> SnapshotProducer<'a> { commit_uuid, snapshot_properties, added_data_files, + deleted_data_files, manifest_counter: AtomicU64::new(0), } } @@ -244,26 +237,22 @@ impl<'a> SnapshotProducer<'a> { snapshot_id } - /// Returns the path for the next manifest file of this commit. - fn new_manifest_path(&self) -> Result { - Ok(format!( - "{}/{}-m{}.{}", - self.table.metadata().metadata_location()?, - self.commit_uuid, - self.manifest_counter.fetch_add(1, Ordering::Relaxed), - DataFileFormat::Avro - )) - } - - /// Returns a writer for the next manifest file of this commit, routed through the table's - /// encryption manager when one is configured. + /// Returns a writer for the next manifest file of this commit, written under the given + /// schema and partition spec. pub(crate) fn new_manifest_writer( &self, content: ManifestContentType, schema: SchemaRef, partition_spec: PartitionSpecRef, ) -> Result { - let output_file = self.table.file_io().new_output(self.new_manifest_path()?)?; + let new_manifest_path = format!( + "{}/{}-m{}.{}", + self.table.metadata().metadata_location()?, + self.commit_uuid, + self.manifest_counter.fetch_add(1, Ordering::Relaxed), + DataFileFormat::Avro + ); + let output_file = self.table.file_io().new_output(new_manifest_path)?; let partition_spec = partition_spec.as_ref().clone(); let builder = if let Some(em) = self.table.encryption_manager() { @@ -358,7 +347,7 @@ impl<'a> SnapshotProducer<'a> { /// and collects all of the manifests to be included in the new snapshot as [ManifestFile] entries. async fn produce_manifests( &mut self, - snapshot_produce_operation: &mut OP, + snapshot_produce_operation: &OP, manifest_process: &MP, ) -> Result> { // Assert current snapshot producer contains new content to add to new snapshot. @@ -366,10 +355,13 @@ impl<'a> SnapshotProducer<'a> { // TODO: Allowing snapshot property setup with no added data files is a workaround. // We should clean it up after all necessary actions are supported. // For details, please refer to https://github.com/apache/iceberg-rust/issues/1548 - if self.added_data_files.is_empty() && self.snapshot_properties.is_empty() { + if self.added_data_files.is_empty() + && self.deleted_data_files.is_empty() + && self.snapshot_properties.is_empty() + { return Err(Error::new( ErrorKind::PreconditionFailed, - "No added data files or added snapshot properties found when write a manifest file", + "No added or deleted data files or added snapshot properties found when write a manifest file", )); } @@ -385,23 +377,15 @@ impl<'a> SnapshotProducer<'a> { // # TODO // Support process delete entries. - debug_assert_eq!( - manifest_files - .iter() - .filter(|m| m.added_snapshot_id == self.snapshot_id - && m.content == ManifestContentType::Data) - .filter_map(|m| m.deleted_files_count) - .sum::() as usize, - snapshot_produce_operation.removed_data_files().len(), - "an operation must report exactly the data files it marked deleted" - ); - let manifest_files = manifest_process.process_manifests(self, manifest_files); Ok(manifest_files) } - // Collects the added files, before `produce_manifests` takes them. - fn added_files_summary(&self) -> SnapshotSummaryCollector { + // Returns a `Summary` of the current snapshot + fn summary( + &self, + snapshot_produce_operation: &OP, + ) -> Result { let mut summary_collector = SnapshotSummaryCollector::default(); let table_metadata = self.table.metadata_ref(); @@ -427,22 +411,17 @@ impl<'a> SnapshotProducer<'a> { table_metadata.default_partition_spec().clone(), ); } - summary_collector - } - - // Returns a `Summary` of the current snapshot - fn summary( - &self, - mut summary_collector: SnapshotSummaryCollector, - snapshot_produce_operation: &OP, - ) -> Result { - let table_metadata = self.table.metadata_ref(); - for removed in snapshot_produce_operation.removed_data_files() { + for data_file in &self.deleted_data_files { + let partition_spec = table_metadata + .partition_spec_by_id(data_file.partition_spec_id) + .ok_or_else(|| { + invalid_data!("Unknown partition spec id {}", data_file.partition_spec_id) + })?; summary_collector.remove_file( - &removed.data_file, - removed.schema.clone(), - removed.partition_spec.clone(), + data_file, + table_metadata.current_schema().clone(), + partition_spec.clone(), ); } @@ -461,7 +440,11 @@ impl<'a> SnapshotProducer<'a> { additional_properties, }; - update_snapshot_summaries(summary, previous_snapshot.map(|s| s.summary())) + update_snapshot_summaries( + summary, + previous_snapshot.map(|s| s.summary()), + snapshot_produce_operation.truncate_full_table(), + ) } fn generate_manifest_list_file_path(&self, attempt: i64) -> Result { @@ -478,7 +461,7 @@ impl<'a> SnapshotProducer<'a> { /// Finished building the action and return the [`ActionCommit`] to the transaction. pub(crate) async fn commit( mut self, - mut snapshot_produce_operation: OP, + snapshot_produce_operation: OP, process: MP, ) -> Result { let manifest_list_path = self.generate_manifest_list_file_path(0)?; @@ -518,20 +501,17 @@ impl<'a> SnapshotProducer<'a> { ), }; - // The manifests are produced before the summary is finished so it can count the entries - // the operation actually marked Deleted. - let summary_collector = self.added_files_summary(); + // Calling self.summary() before self.produce_manifests() is important because self.added_data_files + // will be set to an empty vec after self.produce_manifests() returns, resulting in an empty summary + // being generated. + let summary = self.summary(&snapshot_produce_operation).map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) + })?; + let new_manifests = self - .produce_manifests(&mut snapshot_produce_operation, &process) + .produce_manifests(&snapshot_produce_operation, &process) .await?; - let summary = self - .summary(summary_collector, &snapshot_produce_operation) - .map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.") - .with_source(err) - })?; - manifest_list_writer.add_manifests(new_manifests.into_iter())?; let writer_next_row_id = manifest_list_writer.next_row_id(); let file_metadata = manifest_list_writer.close().await?; @@ -617,18 +597,14 @@ mod tests { use super::*; use crate::spec::{ - DataContentType, DataFileBuilder, Literal, Manifest, ManifestStatus, NestedField, - PartitionSpec, PrimitiveType, Schema, Transform, Type, + DataContentType, DataFileBuilder, Literal, Manifest, NestedField, PartitionSpec, + PrimitiveType, Schema, Transform, Type, }; use crate::transaction::tests::make_v2_minimal_table; - use crate::transaction::{Transaction, TransactionAction}; - /// An operation that writes each of its removed files as a deleted entry in its own manifest. - struct RemoveOneFileOperation { - removed: Vec, - } + struct DeleteOperation; - impl SnapshotProduceOperation for RemoveOneFileOperation { + impl SnapshotProduceOperation for DeleteOperation { fn operation(&self) -> Operation { Operation::Delete } @@ -637,216 +613,34 @@ mod tests { Ok(vec![]) } - async fn existing_manifest( - &mut self, - producer: &SnapshotProducer<'_>, - ) -> Result> { - let mut manifests = vec![]; - for removed in &self.removed { - let mut writer = producer.new_manifest_writer( - ManifestContentType::Data, - removed.schema.clone(), - removed.partition_spec.clone(), - )?; - writer.add_delete_file(removed.data_file.clone(), 0, Some(0))?; - manifests.push(writer.write_manifest_file().await?); - } - Ok(manifests) - } - - fn removed_data_files(&self) -> &[RemovedDataFile] { - &self.removed - } - } - - struct RemoveExistingFileOperation { - file_path: String, - removed: Vec, - } - - impl SnapshotProduceOperation for RemoveExistingFileOperation { - fn operation(&self) -> Operation { - Operation::Overwrite - } - - async fn delete_entries(&self, _: &SnapshotProducer<'_>) -> Result> { + async fn existing_manifest(&self, _: &SnapshotProducer<'_>) -> Result> { Ok(vec![]) } - - async fn existing_manifest( - &mut self, - producer: &SnapshotProducer<'_>, - ) -> Result> { - let snapshot = producer.table.metadata().current_snapshot().unwrap(); - let manifest_list = producer.table.manifest_list_reader(snapshot).load().await?; - // The initial append writes both files into one manifest. - assert_eq!(manifest_list.entries().len(), 1); - - let manifest = producer - .table - .manifest_reader() - .read(&manifest_list.entries()[0]) - .await?; - let schema = manifest.metadata().schema().clone(); - let partition_spec = Arc::new(manifest.metadata().partition_spec().clone()); - let mut writer = producer.new_manifest_writer( - ManifestContentType::Data, - schema.clone(), - partition_spec.clone(), - )?; - - let mut removed = Vec::new(); - for entry in manifest.entries() { - if entry.file_path() == self.file_path && entry.is_alive() { - writer.add_delete_entry(entry.as_ref().clone())?; - removed.push(RemovedDataFile { - data_file: entry.data_file().clone(), - schema: schema.clone(), - partition_spec: partition_spec.clone(), - }); - } else if entry.is_alive() { - writer.add_existing_entry(entry.as_ref().clone())?; - } - } - assert_eq!(removed.len(), 1); - - let rewritten_manifest = writer.write_manifest_file().await?; - self.removed.extend(removed); - Ok(vec![rewritten_manifest]) - } - - fn removed_data_files(&self) -> &[RemovedDataFile] { - &self.removed - } } - fn apply_commit(table: Table, mut commit: ActionCommit) -> Table { - let mut builder = table.metadata().clone().into_builder(None); - for update in commit.take_updates() { - builder = update.apply(builder).unwrap(); - } - table.with_metadata(Arc::new(builder.build().unwrap().metadata)) - } - - fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { - DataFileBuilder::default() + #[test] + fn test_summary_counts_deleted_data_files() { + let table = make_v2_minimal_table(); + let deleted = DataFileBuilder::default() .content(DataContentType::Data) - .file_path(path.to_string()) + .file_path("test/deleted.parquet".to_string()) .file_format(DataFileFormat::Parquet) .file_size_in_bytes(100) - .record_count(record_count) + .record_count(2) .partition_spec_id(table.metadata().default_partition_spec_id()) .partition(Struct::from_iter([Some(Literal::long(300))])) .build() - .unwrap() - } - - #[tokio::test] - async fn test_summary_counts_files_the_operation_reports_as_removed() { - let table = make_v2_minimal_table(); - let added = data_file(&table, "test/added.parquet", 3); - let removed = data_file(&table, "test/removed.parquet", 2); - - let operation = RemoveOneFileOperation { - removed: vec![RemovedDataFile { - data_file: removed, - schema: table.metadata().current_schema().clone(), - partition_spec: table.metadata().default_partition_spec().clone(), - }], - }; - - let mut action_commit = - SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![added]) - .commit(operation, DefaultManifestProcess) - .await - .unwrap(); + .unwrap(); - let updates = action_commit.take_updates(); - let TableUpdate::AddSnapshot { snapshot } = &updates[0] else { - unreachable!() - }; - let properties = &snapshot.summary().additional_properties; + let producer = SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![], vec![ + deleted, + ]); + let summary = producer.summary(&DeleteOperation).unwrap(); + let properties = &summary.additional_properties; assert_eq!(properties.get("deleted-data-files").unwrap(), "1"); assert_eq!(properties.get("deleted-records").unwrap(), "2"); - assert_eq!(properties.get("added-data-files").unwrap(), "1"); - assert_eq!(properties.get("added-records").unwrap(), "3"); - assert_eq!(properties.get("total-data-files").unwrap(), "0"); - assert_eq!(properties.get("total-records").unwrap(), "1"); - } - - #[tokio::test] - async fn test_partial_overwrite_reports_actual_removed_file() { - let table = make_v2_minimal_table(); - let removed = data_file(&table, "test/removed.parquet", 2); - let kept = data_file(&table, "test/kept.parquet", 5); - let append = Transaction::new(&table) - .fast_append() - .add_data_files(vec![removed.clone(), kept]); - let initial_commit = Arc::new(append).commit(&table).await.unwrap(); - let table = apply_commit(table, initial_commit); - - let initial_summary = &table - .metadata() - .current_snapshot() - .unwrap() - .summary() - .additional_properties; - assert_eq!(initial_summary.get("total-data-files").unwrap(), "2"); - assert_eq!(initial_summary.get("total-records").unwrap(), "7"); - - let mut replacement = data_file(&table, "test/replacement.parquet", 3); - replacement.file_size_in_bytes = 300; - let operation = RemoveExistingFileOperation { - file_path: removed.file_path.clone(), - removed: vec![], - }; - let commit = - SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![replacement]) - .commit(operation, DefaultManifestProcess) - .await - .unwrap(); - let table = apply_commit(table, commit); - let snapshot = table.metadata().current_snapshot().unwrap(); - let properties = &snapshot.summary().additional_properties; - - assert_eq!(snapshot.summary().operation, Operation::Overwrite); - assert_eq!(properties.get("deleted-data-files").unwrap(), "1"); - assert_eq!(properties.get("deleted-records").unwrap(), "2"); - assert_eq!(properties.get("added-data-files").unwrap(), "1"); - assert_eq!(properties.get("added-records").unwrap(), "3"); assert_eq!(properties.get("removed-files-size").unwrap(), "100"); - assert_eq!(properties.get("added-files-size").unwrap(), "300"); - assert_eq!(properties.get("total-data-files").unwrap(), "2"); - assert_eq!(properties.get("total-records").unwrap(), "8"); - assert_eq!(properties.get("total-files-size").unwrap(), "400"); - - let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); - assert_eq!(manifest_list.entries().len(), 2); - let mut statuses = HashMap::new(); - for manifest_file in manifest_list.entries() { - let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); - for entry in manifest.entries() { - assert!( - statuses - .insert(entry.file_path().to_string(), entry.status()) - .is_none() - ); - } - } - assert_eq!(statuses.len(), 3); - assert_eq!( - statuses.get("test/removed.parquet"), - Some(&ManifestStatus::Deleted) - ); - assert_eq!( - statuses.get("test/kept.parquet"), - Some(&ManifestStatus::Existing) - ); - assert_eq!( - statuses.get("test/replacement.parquet"), - Some(&ManifestStatus::Added) - ); } #[tokio::test] @@ -870,7 +664,8 @@ mod tests { .unwrap(), ); - let producer = SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![]); + let producer = + SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![], vec![]); let first = producer .new_manifest_writer( ManifestContentType::Data, @@ -905,60 +700,4 @@ mod tests { assert_eq!(metadata.partition_spec().spec_id(), 3); assert_eq!(first.partition_spec_id, 3); } - - #[tokio::test] - async fn test_removed_file_summary_uses_the_reported_partition_spec() { - let table = make_v2_minimal_table(); - let metadata = table - .metadata() - .clone() - .into_builder(None) - .set_properties(HashMap::from([( - TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT.to_string(), - "10".to_string(), - )])) - .unwrap() - .build() - .unwrap() - .metadata; - let table = table.with_metadata(Arc::new(metadata)); - - let reported_schema = table.metadata().current_schema().clone(); - let reported_spec = Arc::new( - PartitionSpec::builder(reported_schema.clone()) - .with_spec_id(1) - .add_partition_field("y", "y", Transform::Identity) - .unwrap() - .build() - .unwrap(), - ); - - let added = data_file(&table, "test/added.parquet", 3); - let removed = data_file(&table, "test/removed.parquet", 2); - let operation = RemoveOneFileOperation { - removed: vec![RemovedDataFile { - data_file: removed, - schema: reported_schema, - partition_spec: reported_spec, - }], - }; - - let mut action_commit = - SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![added]) - .commit(operation, DefaultManifestProcess) - .await - .unwrap(); - - let updates = action_commit.take_updates(); - let TableUpdate::AddSnapshot { snapshot } = &updates[0] else { - unreachable!() - }; - let properties = &snapshot.summary().additional_properties; - - let removed_partition = properties - .get("partitions.y=300") - .expect("removal must be summarized under the reported spec's partition path"); - assert!(removed_partition.contains("deleted-data-files=1")); - assert!(properties.contains_key("partitions.x=300")); - } } From 1bb452b97ae4c491ae79ea9c5d8fdc97b74def1b Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:39:12 +0200 Subject: [PATCH 30/50] Collapse added and deleted files by path in the producer Overwrite gets the dedup fast append already had, and the shared error no longer says fast append. --- crates/iceberg/src/transaction/append.rs | 16 ++-------------- crates/iceberg/src/transaction/snapshot.rs | 14 +++++++++++--- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 881bd5d752..459f63811c 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; @@ -71,18 +71,6 @@ impl FastAppendAction { self.snapshot_properties = snapshot_properties; self } - - /// Collapse files sharing a path to their first occurrence, so a single - /// manifest never references the same file twice. Always runs (unlike the - /// `check_duplicate`-gated cross-snapshot check) since it is in-memory only. - fn dedupe_added_files(&self) -> Vec { - let mut seen = HashSet::with_capacity(self.added_data_files.len()); - self.added_data_files - .iter() - .filter(|data_file| seen.insert(data_file.file_path.as_str())) - .cloned() - .collect() - } } #[async_trait] @@ -92,7 +80,7 @@ impl TransactionAction for FastAppendAction { table, self.commit_uuid.unwrap_or_else(Uuid::now_v7), self.snapshot_properties.clone(), - self.dedupe_added_files(), + self.added_data_files.clone(), vec![], ); diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 0a64594422..d9aa05f90c 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -21,6 +21,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use futures::TryStreamExt; use futures::stream::FuturesUnordered; +use itertools::Itertools; use uuid::Uuid; use crate::error::{Result, invalid_data}; @@ -138,8 +139,15 @@ impl<'a> SnapshotProducer<'a> { snapshot_id: Self::generate_unique_snapshot_id(table), commit_uuid, snapshot_properties, - added_data_files, - deleted_data_files, + // Collapsed to the first file per path, so a manifest never references a file twice. + added_data_files: added_data_files + .into_iter() + .unique_by(|f| f.file_path.clone()) + .collect(), + deleted_data_files: deleted_data_files + .into_iter() + .unique_by(|f| f.file_path.clone()) + .collect(), manifest_counter: AtomicU64::new(0), } } @@ -148,7 +156,7 @@ impl<'a> SnapshotProducer<'a> { for data_file in &self.added_data_files { if data_file.content_type() != crate::spec::DataContentType::Data { return Err(invalid_data!( - "Only data content type is allowed for fast append" + "Only data content type is allowed" )); } // Check if the data file partition spec id matches the table default partition spec id. From 633142772c9697c80056cd14d865102c621ff45d Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:39:12 +0200 Subject: [PATCH 31/50] Rewrite only live entries and count deletes from the producer, as Java and pyiceberg do Drops add_tombstone_entry, the removed-files bookkeeping and the duplicate tests. --- crates/iceberg/src/spec/manifest/writer.rs | 21 - crates/iceberg/src/transaction/overwrite.rs | 568 ++++---------------- 2 files changed, 95 insertions(+), 494 deletions(-) diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index 874b2ee356..106b462627 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -315,10 +315,6 @@ impl ManifestWriter { /// Add a delete manifest entry. This method will update following status of the entry: /// - Update the entry status to `Deleted` /// - Set the snapshot id to the current snapshot id - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_delete_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Deleted; @@ -350,10 +346,6 @@ impl ManifestWriter { /// Add an existing manifest entry. This method will update following status of the entry: /// - Update the entry status to `Existing` - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_existing_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Existing; @@ -361,19 +353,6 @@ impl ManifestWriter { Ok(()) } - /// Add a manifest entry as a tombstone, preserving its original sequence numbers and - /// snapshot id. - /// - /// The caller owns `snapshot_id`: set it to the deleting snapshot for a file this commit - /// removes, and leave it untouched when carrying an earlier tombstone forward. Use - /// `add_delete_entry` instead to have this manifest's snapshot id stamped on the entry. - pub(crate) fn add_tombstone_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { - self.check_data_file(&entry.data_file)?; - entry.status = ManifestStatus::Deleted; - self.add_entry_inner(entry)?; - Ok(()) - } - /// Add an file as existing manifest entry. The original data and file sequence numbers, snapshot ID, /// which were assigned at commit, must be preserved when adding an existing entry. pub fn add_existing_file( diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 8597c65a7a..64e4d6f284 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -21,16 +21,13 @@ use std::sync::Arc; use async_trait::async_trait; use uuid::Uuid; -use crate::error::Result; -use crate::spec::{ - DataFile, Manifest, ManifestEntry, ManifestFile, Operation, PartitionSpecRef, SchemaRef, -}; +use crate::error::{Result, invalid_data}; +use crate::spec::{DataFile, ManifestEntry, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::snapshot::{ DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, }; use crate::transaction::{ActionCommit, TransactionAction}; -use crate::{Error, ErrorKind}; /// OverwriteAction is a transaction action for overwriting data files in the table. /// @@ -104,32 +101,23 @@ impl TransactionAction for OverwriteAction { snapshot_producer.validate_duplicate_files().await?; } - let deleted_file_paths: HashSet = self - .deleted_data_files - .iter() - .map(|f| f.file_path.clone()) - .collect(); - - let snapshot_id = snapshot_producer.snapshot_id(); + let operation = OverwriteOperation { + has_added_data_files: !self.added_data_files.is_empty(), + deleted_file_paths: self + .deleted_data_files + .iter() + .map(|f| f.file_path.clone()) + .collect(), + }; snapshot_producer - .commit( - OverwriteOperation { - deleted_file_paths, - has_added_data_files: !self.added_data_files.is_empty(), - snapshot_id, - removed_data_files: vec![], - }, - DefaultManifestProcess, - ) + .commit(operation, DefaultManifestProcess) .await } } struct OverwriteOperation { - deleted_file_paths: HashSet, has_added_data_files: bool, - snapshot_id: i64, - removed_data_files: Vec<(DataFile, SchemaRef, PartitionSpecRef)>, + deleted_file_paths: HashSet, } impl SnapshotProduceOperation for OverwriteOperation { @@ -140,10 +128,16 @@ impl SnapshotProduceOperation for OverwriteOperation { ) { (true, true) => Operation::Overwrite, (false, true) => Operation::Delete, + // Also a properties-only commit, the workaround from #1548. _ => Operation::Append, } } + // Only the listed files are replaced, so the table totals carry over. + fn truncate_full_table(&self) -> bool { + false + } + async fn delete_entries( &self, _snapshot_produce: &SnapshotProducer<'_>, @@ -151,159 +145,93 @@ impl SnapshotProduceOperation for OverwriteOperation { Ok(vec![]) } - fn removed_data_files(&self) -> &[(DataFile, SchemaRef, PartitionSpecRef)] { - &self.removed_data_files - } - async fn existing_manifest( - &mut self, + &self, snapshot_produce: &SnapshotProducer<'_>, ) -> Result> { - let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { - self.validate_deletes_matched()?; - return Ok(vec![]); - }; - - let manifest_list = snapshot_produce - .table - .manifest_list_reader(snapshot) - .load() - .await?; - - if self.deleted_file_paths.is_empty() { - return Ok(manifest_list - .entries() - .iter() - .filter(|entry| { - // Delete-only manifests record which files were removed and must survive - // until `expire_snapshots` cleans them up (see #2148). - entry.has_added_files() - || entry.has_existing_files() - || entry.has_deleted_files() - }) - .cloned() - .collect()); - } - - let mut result = Vec::new(); + let table = snapshot_produce.table; + let mut manifests = vec![]; + let mut matched: HashSet<&str> = HashSet::new(); + + if let Some(snapshot) = table.metadata().current_snapshot() { + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + for manifest_file in manifest_list.entries() { + // Delete-only manifests record which files were removed and must survive + // until `expire_snapshots` cleans them up (see #2148). + if !manifest_file.has_added_files() + && !manifest_file.has_existing_files() + && !manifest_file.has_deleted_files() + { + continue; + } + if self.deleted_file_paths.is_empty() { + manifests.push(manifest_file.clone()); + continue; + } - for manifest_file in manifest_list.entries() { - if !manifest_file.has_added_files() - && !manifest_file.has_existing_files() - && !manifest_file.has_deleted_files() - { - continue; - } + let manifest = table.manifest_reader().read(manifest_file).await?; + let deletes: Vec<&str> = manifest + .entries() + .iter() + .filter(|entry| entry.is_alive()) + .filter_map(|entry| self.deleted_file_paths.get(entry.file_path())) + .map(String::as_str) + .collect(); + if deletes.is_empty() { + manifests.push(manifest_file.clone()); + continue; + } + matched.extend(deletes); - let manifest = snapshot_produce - .table - .manifest_reader() - .read(manifest_file) - .await?; - - let has_deletes = manifest.entries().iter().any(|entry| { - entry.is_alive() && self.deleted_file_paths.contains(entry.file_path()) - }); - - if has_deletes { - let rewritten = self - .rewrite_manifest(snapshot_produce, manifest_file, &manifest) - .await?; - result.push(rewritten); - } else { - result.push(manifest_file.clone()); + let mut writer = snapshot_produce.new_manifest_writer( + manifest_file.content, + manifest.metadata().schema.clone(), + Arc::new(manifest.metadata().partition_spec.clone()), + )?; + // Like Java, only live entries are carried into the rewrite. + for entry in manifest.entries().iter().filter(|entry| entry.is_alive()) { + if self.deleted_file_paths.contains(entry.file_path()) { + writer.add_delete_entry(entry.as_ref().clone())?; + } else { + writer.add_existing_entry(entry.as_ref().clone())?; + } + } + manifests.push(writer.write_manifest_file().await?); } } - self.validate_deletes_matched()?; - - Ok(result) - } -} - -impl OverwriteOperation { - /// Fails when a requested delete path was not live in any manifest, matching the existence - /// check Java's `BaseOverwriteFiles` performs before committing. - fn validate_deletes_matched(&self) -> Result<()> { - let removed: HashSet<&str> = self - .removed_data_files - .iter() - .map(|(data_file, _, _)| data_file.file_path.as_str()) - .collect(); let mut missing: Vec<&str> = self .deleted_file_paths .iter() .map(String::as_str) - .filter(|path| !removed.contains(path)) + .filter(|path| !matched.contains(path)) .collect(); - - if missing.is_empty() { - return Ok(()); + if !missing.is_empty() { + missing.sort_unstable(); + return Err(invalid_data!( + "Missing required files to delete: {}", + missing.join(", ") + )); } - missing.sort_unstable(); - Err(Error::new( - ErrorKind::DataInvalid, - format!("Missing required files to delete: {}", missing.join(", ")), - )) - } - - /// Rewrite a manifest, marking entries whose file paths are in `deleted_file_paths` - /// as `ManifestStatus::Deleted`. - async fn rewrite_manifest( - &mut self, - snapshot_produce: &SnapshotProducer<'_>, - manifest_file: &ManifestFile, - manifest: &Manifest, - ) -> Result { - let partition_spec: PartitionSpecRef = Arc::new(manifest.metadata().partition_spec.clone()); - let mut writer = snapshot_produce.new_manifest_writer( - manifest_file.content, - manifest.metadata().schema.clone(), - partition_spec.as_ref().clone(), - )?; - - for entry in manifest.entries() { - let entry: ManifestEntry = (**entry).clone(); - if !entry.is_alive() { - // A tombstone keeps its original snapshot id so the delete stays attributed - // to the snapshot that made it. - writer.add_tombstone_entry(entry)?; - } else if self.deleted_file_paths.contains(entry.file_path()) { - let mut deleted = entry; - deleted.snapshot_id = Some(self.snapshot_id); - self.removed_data_files.push(( - deleted.data_file().clone(), - manifest.metadata().schema.clone(), - partition_spec.clone(), - )); - writer.add_tombstone_entry(deleted)?; - } else { - writer.add_existing_entry(entry)?; - } - } - - writer.write_manifest_file().await + Ok(manifests) } } #[cfg(test)] mod tests { - use std::collections::HashMap; use std::sync::Arc; - use uuid::Uuid; - use crate::memory::tests::new_memory_catalog; use crate::spec::{ - DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH, - ManifestEntryRef, ManifestStatus, Operation, PrimitiveType, SnapshotRef, Struct, Transform, - Type, UnboundPartitionField, UnboundPartitionSpec, + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, ManifestStatus, + Operation, PrimitiveType, SnapshotRef, Struct, Transform, Type, UnboundPartitionField, + UnboundPartitionSpec, }; use crate::table::Table; - use crate::transaction::tests::{make_v2_minimal_table, make_v3_minimal_table_in_catalog}; + use crate::transaction::tests::make_v3_minimal_table_in_catalog; use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction, TransactionAction}; - use crate::{ErrorKind, TableRequirement, TableUpdate}; + use crate::{ErrorKind, TableUpdate}; fn test_data_file(path: &str, partition_spec_id: i32) -> DataFile { DataFileBuilder::default() @@ -332,157 +260,6 @@ mod tests { entries } - async fn current_manifest_entry(table: &Table, path: &str) -> ManifestEntryRef { - let snapshot = table.metadata().current_snapshot().unwrap(); - let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); - - for manifest_file in manifest_list.entries() { - let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); - if let Some(entry) = manifest.entries().iter().find(|e| e.file_path() == path) { - return entry.clone(); - } - } - panic!("no manifest entry for {path}"); - } - - #[tokio::test] - async fn test_empty_data_overwrite_action() { - let table = make_v2_minimal_table(); - let tx = Transaction::new(&table); - let action = tx.overwrite().add_data_files(vec![]); - let err = Arc::new(action).commit(&table).await.err().unwrap(); - assert_eq!(err.kind(), ErrorKind::PreconditionFailed); - assert!( - err.to_string() - .contains("No added data files, deleted data files, or added snapshot properties"), - "{err}" - ); - } - - #[tokio::test] - async fn test_overwrite_snapshot_properties() { - let table = make_v2_minimal_table(); - let tx = Transaction::new(&table); - - let mut snapshot_properties = HashMap::new(); - snapshot_properties.insert("key".to_string(), "val".to_string()); - - let data_file = test_data_file( - "test/1.parquet", - table.metadata().default_partition_spec_id(), - ); - - let action = tx - .overwrite() - .set_snapshot_properties(snapshot_properties) - .add_data_files(vec![data_file]); - let mut action_commit = Arc::new(action).commit(&table).await.unwrap(); - let updates = action_commit.take_updates(); - - let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { - snapshot - } else { - unreachable!() - }; - assert_eq!( - new_snapshot - .summary() - .additional_properties - .get("key") - .unwrap(), - "val" - ); - } - - #[tokio::test] - async fn test_overwrite_incompatible_partition_value() { - let table = make_v2_minimal_table(); - let tx = Transaction::new(&table); - - let data_file = DataFileBuilder::default() - .content(DataContentType::Data) - .file_path("test/3.parquet".to_string()) - .file_format(DataFileFormat::Parquet) - .file_size_in_bytes(100) - .record_count(1) - .partition_spec_id(table.metadata().default_partition_spec_id()) - .partition(Struct::from_iter([Some(Literal::string("test"))])) - .build() - .unwrap(); - - let action = tx.overwrite().add_data_files(vec![data_file]); - assert!(Arc::new(action).commit(&table).await.is_err()); - } - - #[tokio::test] - async fn test_overwrite_basic() { - let table = make_v2_minimal_table(); - let tx = Transaction::new(&table); - - let data_file = test_data_file( - "test/3.parquet", - table.metadata().default_partition_spec_id(), - ); - - let action = tx.overwrite().add_data_files(vec![data_file.clone()]); - let mut action_commit = Arc::new(action).commit(&table).await.unwrap(); - let updates = action_commit.take_updates(); - let requirements = action_commit.take_requirements(); - - assert!( - matches!((&updates[0],&updates[1]), (TableUpdate::AddSnapshot { snapshot },TableUpdate::SetSnapshotRef { reference,ref_name }) if snapshot.snapshot_id() == reference.snapshot_id && ref_name == MAIN_BRANCH) - ); - - assert_eq!( - vec![ - TableRequirement::UuidMatch { - uuid: table.metadata().uuid() - }, - TableRequirement::RefSnapshotIdMatch { - r#ref: MAIN_BRANCH.to_string(), - snapshot_id: table.metadata().current_snapshot_id - } - ], - requirements - ); - - let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { - SnapshotRef::new(snapshot.clone()) - } else { - unreachable!() - }; - assert_eq!(new_snapshot.summary().operation, Operation::Append); - - let manifest_list = table - .manifest_list_reader(&new_snapshot) - .load() - .await - .unwrap(); - assert_eq!(1, manifest_list.entries().len()); - assert_eq!( - manifest_list.entries()[0].sequence_number, - new_snapshot.sequence_number() - ); - - let manifest = table - .manifest_reader() - .read(&manifest_list.entries()[0]) - .await - .unwrap(); - assert_eq!(1, manifest.entries().len()); - assert_eq!( - new_snapshot.sequence_number(), - manifest.entries()[0] - .sequence_number() - .expect("Inherit sequence number by load manifest") - ); - assert_eq!( - new_snapshot.snapshot_id(), - manifest.entries()[0].snapshot_id().unwrap() - ); - assert_eq!(data_file, *manifest.entries()[0].data_file()); - } - #[tokio::test] async fn test_overwrite_with_deleted_files() { let catalog = new_memory_catalog().await; @@ -498,10 +275,6 @@ mod tests { let tx = action.apply(tx).unwrap(); let table = tx.commit(&catalog).await.unwrap(); - let snapshot = table.metadata().current_snapshot().unwrap(); - let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); - assert_eq!(1, manifest_list.entries().len()); - let replacement_file = test_data_file("test/replacement.parquet", spec_id); let tx = Transaction::new(&table); let action = tx @@ -513,90 +286,26 @@ mod tests { let snapshot = table.metadata().current_snapshot().unwrap(); assert_eq!(snapshot.summary().operation, Operation::Overwrite); + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("deleted-data-files").unwrap(), "2"); + assert_eq!(summary.get("deleted-records").unwrap(), "2"); - let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); - - assert_eq!(2, manifest_list.entries().len()); - - let mut all_entries = vec![]; - for manifest_file in manifest_list.entries() { - let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); - for entry in manifest.entries() { - all_entries.push((entry.status(), entry.file_path().to_string())); - } - } - - for original_file in ["test/original1.parquet", "test/original2.parquet"] { - assert!( - all_entries - .iter() - .any(|(status, path)| *status == ManifestStatus::Deleted - && path == original_file), - "Original file {original_file} should be marked as Deleted, entries: {all_entries:?}", - ); - } - - assert!( - all_entries - .iter() - .any(|(status, path)| *status == ManifestStatus::Added - && path == "test/replacement.parquet"), - "Replacement file should be marked as Added, entries: {all_entries:?}", - ); - - // Verify snapshot summary reports the deleted file. - assert_eq!( - snapshot - .summary() - .additional_properties - .get("deleted-data-files") - .map(|s| s.as_str()), - Some("2") - ); - assert_eq!( - snapshot - .summary() - .additional_properties - .get("deleted-records") - .map(|s| s.as_str()), - Some("2") - ); - - // Step 3: Fast append after overwrite — delete-only manifest must survive. - let appended_file = test_data_file("test/appended.parquet", spec_id); - let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(vec![appended_file.clone()]); - let tx = action.apply(tx).unwrap(); - let table = tx.commit(&catalog).await.unwrap(); - - let snapshot = table.metadata().current_snapshot().unwrap(); - let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); - - // 3 manifests: rewritten (deleted entry), overwrite added, fast_append added. - assert_eq!(3, manifest_list.entries().len()); - - let mut all_entries = vec![]; - for manifest_file in manifest_list.entries() { - let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); - for entry in manifest.entries() { - all_entries.push((entry.status(), entry.file_path().to_string())); - } - } - - // The deleted entry must still be present after fast_append. - for original_file in ["test/original1.parquet", "test/original2.parquet"] { + let entries = current_manifest_entries(&table).await; + assert_eq!(3, entries.len(), "{entries:?}"); + for entry in [ + (ManifestStatus::Deleted, "test/original1.parquet"), + (ManifestStatus::Deleted, "test/original2.parquet"), + (ManifestStatus::Added, "test/replacement.parquet"), + ] { assert!( - all_entries - .iter() - .any(|(status, path)| *status == ManifestStatus::Deleted - && path == original_file), - "Deleted entry should survive fast_append, entries: {all_entries:?}", + entries.contains(&(entry.0, entry.1.to_string())), + "{entries:?}" ); } } #[tokio::test] - async fn test_second_overwrite_keeps_prior_deleted_entries_deleted() { + async fn test_second_overwrite_does_not_resurrect_a_deleted_file() { let catalog = new_memory_catalog().await; let table = make_v3_minimal_table_in_catalog(&catalog).await; let spec_id = table.metadata().default_partition_spec_id(); @@ -615,8 +324,6 @@ mod tests { .add_data_files(vec![test_data_file("test/c.parquet", spec_id)]) .delete_data_files(vec![file_a.clone()]); let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - let tombstone = current_manifest_entry(&table, "test/a.parquet").await; - assert_eq!(ManifestStatus::Deleted, tombstone.status()); let tx = Transaction::new(&table); let action = tx @@ -625,26 +332,12 @@ mod tests { .delete_data_files(vec![file_b.clone()]); let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - let carried = current_manifest_entry(&table, "test/a.parquet").await; - assert_eq!( - ManifestStatus::Deleted, - carried.status(), - "the first overwrite's tombstone must stay Deleted" - ); - assert_eq!( - tombstone.snapshot_id, carried.snapshot_id, - "the tombstone must keep the snapshot that deleted the file" - ); - assert_eq!( - (tombstone.sequence_number, tombstone.file_sequence_number), - (carried.sequence_number, carried.file_sequence_number), - "the tombstone must keep its original sequence numbers" - ); - let entries = current_manifest_entries(&table).await; assert!( - entries.contains(&(ManifestStatus::Deleted, "test/b.parquet".to_string())), - "the second overwrite's own delete must be a tombstone, entries: {entries:?}" + !entries.iter().any( + |(status, path)| path == "test/a.parquet" && *status != ManifestStatus::Deleted + ), + "the first overwrite's delete must not come back, entries: {entries:?}" ); } @@ -736,37 +429,6 @@ mod tests { assert_eq!(original_spec_id, rewritten_file.partition_spec_id); } - #[tokio::test] - async fn test_overwrite_preserves_delete_only_manifest() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let spec_id = table.metadata().default_partition_spec_id(); - - let file_a = test_data_file("test/a.parquet", spec_id); - let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(vec![file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let tx = Transaction::new(&table); - let action = tx - .overwrite() - .add_data_files(vec![test_data_file("test/b.parquet", spec_id)]) - .delete_data_files(vec![file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let tx = Transaction::new(&table); - let action = tx - .overwrite() - .add_data_files(vec![test_data_file("test/c.parquet", spec_id)]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let entries = current_manifest_entries(&table).await; - assert!( - entries.contains(&(ManifestStatus::Deleted, "test/a.parquet".to_string())), - "the delete-only manifest must survive a later overwrite, entries: {entries:?}" - ); - } - #[tokio::test] async fn test_overwrite_counts_a_duplicated_delete_path_once() { let catalog = new_memory_catalog().await; @@ -803,46 +465,6 @@ mod tests { ); } - #[tokio::test] - async fn test_rewritten_manifest_is_named_after_the_commit_uuid() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let spec_id = table.metadata().default_partition_spec_id(); - - let file_a = test_data_file("test/a.parquet", spec_id); - let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(vec![file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let commit_uuid = Uuid::now_v7(); - let tx = Transaction::new(&table); - let action = tx - .overwrite() - .set_commit_uuid(commit_uuid) - .add_data_files(vec![test_data_file("test/b.parquet", spec_id)]) - .delete_data_files(vec![file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let prefix = format!( - "{}/{}-m", - table.metadata().metadata_location().unwrap(), - commit_uuid - ); - let snapshot = table.metadata().current_snapshot().unwrap(); - let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); - assert_eq!(2, manifest_list.entries().len()); - for manifest_file in manifest_list.entries() { - let path = &manifest_file.manifest_path; - let counter = path - .strip_prefix(&prefix) - .and_then(|rest| rest.strip_suffix(".avro")); - assert!( - counter.is_some_and(|c| !c.is_empty() && c.chars().all(|c| c.is_ascii_digit())), - "manifest path must be {prefix}.avro, got {path}" - ); - } - } - #[tokio::test] async fn test_delete_only_overwrite_reports_a_delete_operation() { let catalog = new_memory_catalog().await; From 181f83417d5982ba87781b5e964f17889972eee2 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 2 Oct 2026 20:39:39 +0200 Subject: [PATCH 32/50] Fit the shared error on one line --- crates/iceberg/src/transaction/snapshot.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index d9aa05f90c..1c144fe936 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -155,9 +155,7 @@ impl<'a> SnapshotProducer<'a> { pub(crate) fn validate_added_data_files(&self) -> Result<()> { for data_file in &self.added_data_files { if data_file.content_type() != crate::spec::DataContentType::Data { - return Err(invalid_data!( - "Only data content type is allowed" - )); + return Err(invalid_data!("Only data content type is allowed")); } // Check if the data file partition spec id matches the table default partition spec id. if self.table.metadata().default_partition_spec_id() != data_file.partition_spec_id { From 683fae4fd1b4c43d44e699ca13cb872bf10a63d1 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Tue, 13 Oct 2026 14:07:32 +0200 Subject: [PATCH 33/50] Check that deletion summaries match manifest entries --- crates/iceberg/src/transaction/snapshot.rs | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 0a64594422..c301750012 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -120,6 +120,7 @@ pub(crate) struct SnapshotProducer<'a> { commit_uuid: Uuid, snapshot_properties: HashMap, added_data_files: Vec, + /// Must match the data files marked Deleted by this commit, including their metrics. deleted_data_files: Vec, // Numbers this commit's manifest files. Atomic so `new_manifest_writer` can take `&self`. manifest_counter: AtomicU64, @@ -377,6 +378,18 @@ impl<'a> SnapshotProducer<'a> { // # TODO // Support process delete entries. + debug_assert_eq!( + manifest_files + .iter() + .filter(|m| m.added_snapshot_id == self.snapshot_id + && m.content == ManifestContentType::Data) + .filter_map(|m| m.deleted_files_count) + .map(u64::from) + .sum::(), + self.deleted_data_files.len() as u64, + "deleted files must match this commit's manifest entries" + ); + let manifest_files = manifest_process.process_manifests(self, manifest_files); Ok(manifest_files) } @@ -643,6 +656,29 @@ mod tests { assert_eq!(properties.get("removed-files-size").unwrap(), "100"); } + #[cfg(debug_assertions)] + #[tokio::test] + #[should_panic(expected = "deleted files must match this commit's manifest entries")] + async fn test_commit_rejects_unrecorded_deletions() { + let table = make_v2_minimal_table(); + let deleted = DataFileBuilder::default() + .content(DataContentType::Data) + .file_path("test/deleted.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(2) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap(); + SnapshotProducer::new(&table, Uuid::now_v7(), HashMap::new(), vec![], vec![ + deleted, + ]) + .commit(DeleteOperation, DefaultManifestProcess) + .await + .unwrap(); + } + #[tokio::test] async fn test_new_manifest_writer_uses_the_given_schema_and_spec() { let table = make_v2_minimal_table(); From 5f3693808baeab1f04e4cdc295963a5326f76e28 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Tue, 13 Oct 2026 14:07:56 +0200 Subject: [PATCH 34/50] Format dropped-source integer partition values --- crates/iceberg/src/spec/partition.rs | 47 +++++++++++++++++++++++++ crates/iceberg/src/spec/values/datum.rs | 8 +++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/crates/iceberg/src/spec/partition.rs b/crates/iceberg/src/spec/partition.rs index 17aa3de968..fa8de99a59 100644 --- a/crates/iceberg/src/spec/partition.rs +++ b/crates/iceberg/src/spec/partition.rs @@ -1913,6 +1913,53 @@ mod tests { assert!(!spec.has_sequential_ids()); } + #[test] + fn test_partition_to_path_with_dropped_source_column() { + for (source_type, transform, value, expected) in [ + ( + PrimitiveType::Int, + Transform::Identity, + Literal::int(1), + "p=1", + ), + ( + PrimitiveType::Timestamp, + Transform::Identity, + Literal::long(1), + "p=1", + ), + ( + PrimitiveType::Int, + Transform::Truncate(10), + Literal::int(10), + "p=10", + ), + ] { + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "kept", PrimitiveType::Int.into()).into(), + NestedField::required(2, "dropped", source_type.into()).into(), + ]) + .build() + .unwrap(); + let spec = PartitionSpec::builder(schema) + .add_partition_field("dropped", "p", transform) + .unwrap() + .build() + .unwrap(); + let current_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "kept", PrimitiveType::Int.into()).into(), + ]) + .build() + .unwrap(); + assert_eq!( + spec.partition_to_path(&Struct::from_iter([Some(value)]), current_schema.into()), + expected + ); + } + } + #[test] fn test_partition_to_path() { let schema = Schema::builder() diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index 57d9675736..7fa54954c8 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -294,8 +294,12 @@ impl Display for Datum { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match (&self.r#type, &self.literal) { (_, PrimitiveLiteral::Boolean(val)) => write!(f, "{val}"), - (PrimitiveType::Int, PrimitiveLiteral::Int(val)) => write!(f, "{val}"), - (PrimitiveType::Long, PrimitiveLiteral::Long(val)) => write!(f, "{val}"), + (PrimitiveType::Int | PrimitiveType::Unknown, PrimitiveLiteral::Int(val)) => { + write!(f, "{val}") + } + (PrimitiveType::Long | PrimitiveType::Unknown, PrimitiveLiteral::Long(val)) => { + write!(f, "{val}") + } (_, PrimitiveLiteral::Float(val)) => write!(f, "{val}"), (_, PrimitiveLiteral::Double(val)) => write!(f, "{val}"), (PrimitiveType::Date, PrimitiveLiteral::Int(val)) => { From 543660f0d42c210ce7d44488cc2f146680139128 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Tue, 13 Oct 2026 14:09:32 +0200 Subject: [PATCH 35/50] Validate overwrite deletions before building their summary --- crates/iceberg/src/transaction/overwrite.rs | 278 ++++++++++++++++++-- crates/iceberg/src/transaction/snapshot.rs | 41 +-- 2 files changed, 272 insertions(+), 47 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 64e4d6f284..0577eb81ae 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -87,6 +87,13 @@ impl OverwriteAction { #[async_trait] impl TransactionAction for OverwriteAction { async fn commit(self: Arc, table: &Table) -> Result { + let mut deleted_files = HashMap::new(); + for file in &self.deleted_data_files { + if file.content_type() != crate::spec::DataContentType::Data { + return Err(invalid_data!("Only data content type is allowed")); + } + deleted_files.entry(file.file_path.as_str()).or_insert(file); + } let snapshot_producer = SnapshotProducer::new( table, self.commit_uuid.unwrap_or_else(Uuid::now_v7), @@ -103,11 +110,7 @@ impl TransactionAction for OverwriteAction { let operation = OverwriteOperation { has_added_data_files: !self.added_data_files.is_empty(), - deleted_file_paths: self - .deleted_data_files - .iter() - .map(|f| f.file_path.clone()) - .collect(), + deleted_files, }; snapshot_producer .commit(operation, DefaultManifestProcess) @@ -115,17 +118,14 @@ impl TransactionAction for OverwriteAction { } } -struct OverwriteOperation { +struct OverwriteOperation<'a> { has_added_data_files: bool, - deleted_file_paths: HashSet, + deleted_files: HashMap<&'a str, &'a DataFile>, } -impl SnapshotProduceOperation for OverwriteOperation { +impl SnapshotProduceOperation for OverwriteOperation<'_> { fn operation(&self) -> Operation { - match ( - self.has_added_data_files, - !self.deleted_file_paths.is_empty(), - ) { + match (self.has_added_data_files, !self.deleted_files.is_empty()) { (true, true) => Operation::Overwrite, (false, true) => Operation::Delete, // Also a properties-only commit, the workaround from #1548. @@ -164,19 +164,31 @@ impl SnapshotProduceOperation for OverwriteOperation { { continue; } - if self.deleted_file_paths.is_empty() { + if manifest_file.content == crate::spec::ManifestContentType::Deletes + || self.deleted_files.is_empty() + { manifests.push(manifest_file.clone()); continue; } let manifest = table.manifest_reader().read(manifest_file).await?; - let deletes: Vec<&str> = manifest - .entries() - .iter() - .filter(|entry| entry.is_alive()) - .filter_map(|entry| self.deleted_file_paths.get(entry.file_path())) - .map(String::as_str) - .collect(); + let mut deletes = vec![]; + for entry in manifest.entries().iter().filter(|entry| entry.is_alive()) { + if let Some(requested) = self.deleted_files.get(entry.file_path()) { + let actual = entry.data_file(); + if requested.record_count != actual.record_count + || requested.file_size_in_bytes != actual.file_size_in_bytes + || requested.partition_spec_id != actual.partition_spec_id + || requested.partition != actual.partition + { + return Err(invalid_data!( + "Delete file metadata does not match the table: {}", + entry.file_path() + )); + } + deletes.push(requested.file_path.as_str()); + } + } if deletes.is_empty() { manifests.push(manifest_file.clone()); continue; @@ -190,7 +202,7 @@ impl SnapshotProduceOperation for OverwriteOperation { )?; // Like Java, only live entries are carried into the rewrite. for entry in manifest.entries().iter().filter(|entry| entry.is_alive()) { - if self.deleted_file_paths.contains(entry.file_path()) { + if self.deleted_files.contains_key(entry.file_path()) { writer.add_delete_entry(entry.as_ref().clone())?; } else { writer.add_existing_entry(entry.as_ref().clone())?; @@ -201,9 +213,9 @@ impl SnapshotProduceOperation for OverwriteOperation { } let mut missing: Vec<&str> = self - .deleted_file_paths - .iter() - .map(String::as_str) + .deleted_files + .keys() + .copied() .filter(|path| !matched.contains(path)) .collect(); if !missing.is_empty() { @@ -224,9 +236,9 @@ mod tests { use crate::memory::tests::new_memory_catalog; use crate::spec::{ - DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, ManifestStatus, - Operation, PrimitiveType, SnapshotRef, Struct, Transform, Type, UnboundPartitionField, - UnboundPartitionSpec, + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, ManifestListWriter, + ManifestStatus, ManifestWriterBuilder, Operation, PrimitiveType, SnapshotRef, Struct, + Transform, Type, UnboundPartitionField, UnboundPartitionSpec, }; use crate::table::Table; use crate::transaction::tests::make_v3_minimal_table_in_catalog; @@ -531,4 +543,216 @@ mod tests { "{err}" ); } + + #[tokio::test] + async fn test_overwrite_rejects_incorrect_delete_metrics() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + let file = test_data_file("test/a.parquet", spec_id); + let tx = Transaction::new(&table); + let table = tx + .fast_append() + .add_data_files(vec![file.clone()]) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + for incorrect in [ + DataFile { + record_count: 99, + ..file.clone() + }, + DataFile { + file_size_in_bytes: 999, + ..file.clone() + }, + DataFile { + partition: Struct::from_iter([Some(Literal::long(999))]), + ..file.clone() + }, + DataFile { + partition_spec_id: 99, + ..file.clone() + }, + DataFile { + partition: Struct::from_iter([Some(Literal::string("invalid"))]), + ..file.clone() + }, + ] { + let action = Transaction::new(&table) + .overwrite() + .delete_data_files(vec![incorrect]); + let err = Arc::new(action).commit(&table).await.err().unwrap(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("Delete file metadata does not match") + ); + } + } + + #[tokio::test] + async fn test_overwrite_rejects_non_data_deletions() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + for content in [ + DataContentType::PositionDeletes, + DataContentType::EqualityDeletes, + ] { + let file = DataFile { + content, + ..test_data_file("test/deletes.parquet", spec_id) + }; + let action = Transaction::new(&table) + .overwrite() + .delete_data_files(vec![file]); + let err = Arc::new(action).commit(&table).await.err().unwrap(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("Only data content type is allowed")); + } + } + + #[tokio::test] + async fn test_overwrite_preserves_v3_row_lineage() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + let a = test_data_file("test/a.parquet", spec_id); + let b = test_data_file("test/b.parquet", spec_id); + let c = test_data_file("test/c.parquet", spec_id); + let tx = Transaction::new(&table); + let table = tx + .fast_append() + .add_data_files(vec![a, b.clone(), c]) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifests = table.manifest_list_reader(snapshot).load().await.unwrap(); + let manifest = table + .manifest_reader() + .read(&manifests.entries()[0]) + .await + .unwrap(); + let original_ids: Vec<_> = manifest + .entries() + .iter() + .map(|e| (e.file_path().to_string(), e.data_file().first_row_id)) + .collect(); + assert!(original_ids.iter().all(|(_, id)| id.is_some())); + let next_row_id = table.metadata().next_row_id(); + let replacement = test_data_file("test/d.parquet", spec_id); + let tx = Transaction::new(&table); + let table = tx + .overwrite() + .delete_data_files(vec![b]) + .add_data_files(vec![replacement]) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let manifests = table + .manifest_list_reader(table.metadata().current_snapshot().unwrap()) + .load() + .await + .unwrap(); + let mut actual_ids = vec![]; + for file in manifests.entries() { + let manifest = table.manifest_reader().read(file).await.unwrap(); + actual_ids.extend( + manifest + .entries() + .iter() + .filter(|e| e.is_alive()) + .map(|e| (e.file_path().to_string(), e.data_file().first_row_id)), + ); + } + for (path, id) in original_ids.iter().filter(|(p, _)| p != "test/b.parquet") { + assert!(actual_ids.contains(&(path.clone(), *id))); + } + // The rewritten manifest reserves IDs for its existing rows; their explicit IDs stay unchanged. + assert!(actual_ids.contains(&( + "test/d.parquet".to_string(), + Some((next_row_id + 2).try_into().unwrap()) + ))); + assert_eq!(table.metadata().next_row_id(), next_row_id + 3); + } + + #[tokio::test] + async fn test_overwrite_rejects_a_path_in_a_delete_manifest() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + let tx = Transaction::new(&table); + let table = tx + .fast_append() + .add_data_files(vec![test_data_file("test/a.parquet", spec_id)]) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let snapshot = table.metadata().current_snapshot().unwrap(); + let file = test_data_file("test/deletes.parquet", spec_id); + let output = table + .file_io() + .new_output("test/delete-manifest.avro") + .unwrap(); + let mut writer = ManifestWriterBuilder::new( + output, + Some(snapshot.snapshot_id()), + table.metadata().current_schema().clone(), + table.metadata().default_partition_spec().as_ref().clone(), + ) + .build_v3_deletes(); + writer + .add_file( + DataFile { + content: DataContentType::PositionDeletes, + ..file.clone() + }, + snapshot.sequence_number(), + ) + .unwrap(); + let delete_manifest = writer.write_manifest_file().await.unwrap(); + let mut manifests: Vec<_> = table + .manifest_list_reader(snapshot) + .load() + .await + .unwrap() + .consume_entries() + .into_iter() + .collect(); + manifests.push(delete_manifest); + let mut writer = ManifestListWriter::v3( + table + .file_io() + .new_output(snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(), + snapshot.snapshot_id(), + snapshot.parent_snapshot_id(), + snapshot.sequence_number(), + Some(table.metadata().next_row_id()), + ); + writer.add_manifests(manifests.into_iter()).unwrap(); + writer.close().await.unwrap(); + let action = Transaction::new(&table) + .overwrite() + .delete_data_files(vec![file]); + let err = Arc::new(action).commit(&table).await.err().unwrap(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("Missing required files to delete: test/deletes.parquet") + ); + } } diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 1c144fe936..1c69f07e34 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -21,7 +21,6 @@ use std::sync::atomic::{AtomicU64, Ordering}; use futures::TryStreamExt; use futures::stream::FuturesUnordered; -use itertools::Itertools; use uuid::Uuid; use crate::error::{Result, invalid_data}; @@ -139,19 +138,25 @@ impl<'a> SnapshotProducer<'a> { snapshot_id: Self::generate_unique_snapshot_id(table), commit_uuid, snapshot_properties, - // Collapsed to the first file per path, so a manifest never references a file twice. - added_data_files: added_data_files - .into_iter() - .unique_by(|f| f.file_path.clone()) - .collect(), - deleted_data_files: deleted_data_files - .into_iter() - .unique_by(|f| f.file_path.clone()) - .collect(), + added_data_files: Self::dedupe_data_files(added_data_files), + deleted_data_files: Self::dedupe_data_files(deleted_data_files), manifest_counter: AtomicU64::new(0), } } + fn dedupe_data_files(files: Vec) -> Vec { + let mut seen = HashSet::with_capacity(files.len()); + let keep: Vec = files + .iter() + .map(|f| seen.insert(f.file_path.as_str())) + .collect(); + files + .into_iter() + .zip(keep) + .filter_map(|(file, keep)| keep.then_some(file)) + .collect() + } + pub(crate) fn validate_added_data_files(&self) -> Result<()> { for data_file in &self.added_data_files { if data_file.content_type() != crate::spec::DataContentType::Data { @@ -355,7 +360,7 @@ impl<'a> SnapshotProducer<'a> { &mut self, snapshot_produce_operation: &OP, manifest_process: &MP, - ) -> Result> { + ) -> Result<(Vec, Summary)> { // Assert current snapshot producer contains new content to add to new snapshot. // // TODO: Allowing snapshot property setup with no added data files is a workaround. @@ -372,6 +377,9 @@ impl<'a> SnapshotProducer<'a> { } let existing_manifests = snapshot_produce_operation.existing_manifest(self).await?; + let summary = self.summary(snapshot_produce_operation).map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) + })?; let mut manifest_files = existing_manifests; // Process added entries. @@ -384,7 +392,7 @@ impl<'a> SnapshotProducer<'a> { // Support process delete entries. let manifest_files = manifest_process.process_manifests(self, manifest_files); - Ok(manifest_files) + Ok((manifest_files, summary)) } // Returns a `Summary` of the current snapshot @@ -507,14 +515,7 @@ impl<'a> SnapshotProducer<'a> { ), }; - // Calling self.summary() before self.produce_manifests() is important because self.added_data_files - // will be set to an empty vec after self.produce_manifests() returns, resulting in an empty summary - // being generated. - let summary = self.summary(&snapshot_produce_operation).map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) - })?; - - let new_manifests = self + let (new_manifests, summary) = self .produce_manifests(&snapshot_produce_operation, &process) .await?; From 3966f6bae3ea227d98c8e01123362c064e6ce90b Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Wed, 21 Oct 2026 22:20:24 +0200 Subject: [PATCH 36/50] Fix overwrite partition formatting after schema evolution --- crates/iceberg/src/spec/partition.rs | 15 ++ crates/iceberg/src/spec/values/datum.rs | 9 +- crates/iceberg/src/transaction/overwrite.rs | 268 +++++++++++++++++--- 3 files changed, 249 insertions(+), 43 deletions(-) diff --git a/crates/iceberg/src/spec/partition.rs b/crates/iceberg/src/spec/partition.rs index fdb5a4a7b8..2e440086dc 100644 --- a/crates/iceberg/src/spec/partition.rs +++ b/crates/iceberg/src/spec/partition.rs @@ -1934,6 +1934,21 @@ mod tests { Literal::int(10), "p=10", ), + ( + PrimitiveType::Decimal { + precision: 9, + scale: 2, + }, + Transform::Identity, + Literal::decimal(123), + "p=123", + ), + ( + PrimitiveType::Uuid, + Transform::Identity, + Literal::uuid_from_str("f79c3e09-677c-4bbd-a479-3f349cb785e7").unwrap(), + "p=f79c3e09-677c-4bbd-a479-3f349cb785e7", + ), ] { let schema = Schema::builder() .with_fields(vec![ diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index 2a7bd3f44a..a2128c0210 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -294,7 +294,10 @@ impl Display for Datum { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match (&self.r#type, &self.literal) { (_, PrimitiveLiteral::Boolean(val)) => write!(f, "{val}"), - (PrimitiveType::Int | PrimitiveType::Unknown, PrimitiveLiteral::Int(val)) => { + ( + PrimitiveType::Int | PrimitiveType::Long | PrimitiveType::Unknown, + PrimitiveLiteral::Int(val), + ) => { write!(f, "{val}") } (PrimitiveType::Long | PrimitiveType::Unknown, PrimitiveLiteral::Long(val)) => { @@ -321,7 +324,7 @@ impl Display for Datum { write!(f, "{}", timestamptz::nanoseconds_to_datetimetz(*val)) } (_, PrimitiveLiteral::String(val)) => write!(f, r#""{val}""#), - (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(val)) => { + (PrimitiveType::Uuid | PrimitiveType::Unknown, PrimitiveLiteral::UInt128(val)) => { write!(f, "{}", uuid::Uuid::from_u128(*val)) } (_, PrimitiveLiteral::Binary(val)) => display_bytes(val, f), @@ -334,6 +337,8 @@ impl Display for Datum { ) => { write!(f, "{}", decimal_from_i128_with_scale(*val, *scale)) } + // The scale lives on the dropped source type, so only the unscaled value is left. + (PrimitiveType::Unknown, PrimitiveLiteral::Int128(val)) => write!(f, "{val}"), (_, _) => { unreachable!() } diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 31d4f0ac33..750394c7bb 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -55,7 +55,8 @@ impl OverwriteAction { } } - /// Set whether to check duplicate files. + /// Set whether to reject added paths already live in the table (enabled by default). + /// This also rejects paths deleted and re-added in the same overwrite. pub fn with_check_duplicate(mut self, v: bool) -> Self { self.check_duplicate = v; self @@ -68,6 +69,15 @@ impl OverwriteAction { } /// Specify data files to be removed from the table in this overwrite. + /// + /// Files are matched by path against the current snapshot's live data files. + /// Their `record_count`, `file_size_in_bytes`, `partition_spec_id` and `partition` + /// must match the committed entry. Duplicate paths keep the first occurrence. + /// + /// # Errors + /// + /// The commit fails with [`crate::ErrorKind::DataInvalid`] for non-data files, + /// paths that are not live in the current snapshot, or mismatched metadata. pub fn delete_data_files(mut self, data_files: impl IntoIterator) -> Self { self.deleted_data_files.extend(data_files); self @@ -184,17 +194,34 @@ impl SnapshotProduceOperation for OverwriteOperation<'_> { for entry in manifest.entries().iter().filter(|entry| entry.is_alive()) { if let Some(requested) = self.deleted_files.get(entry.file_path()) { let actual = entry.data_file(); - if requested.record_count != actual.record_count - || requested.file_size_in_bytes != actual.file_size_in_bytes - || requested.partition_spec_id != actual.partition_spec_id - || requested.partition != actual.partition - { - return Err(invalid_data!( - "Delete file metadata does not match the table: {}", - entry.file_path() - )); - } - deletes.push(requested.file_path.as_str()); + let mismatch = if requested.record_count != actual.record_count { + format!( + "record_count: requested {}, table has {}", + requested.record_count, actual.record_count + ) + } else if requested.file_size_in_bytes != actual.file_size_in_bytes { + format!( + "file_size_in_bytes: requested {}, table has {}", + requested.file_size_in_bytes, actual.file_size_in_bytes + ) + } else if requested.partition_spec_id != actual.partition_spec_id { + format!( + "partition_spec_id: requested {}, table has {}", + requested.partition_spec_id, actual.partition_spec_id + ) + } else if requested.partition != actual.partition { + format!( + "partition: requested {:?}, table has {:?}", + requested.partition, actual.partition + ) + } else { + deletes.push(requested.file_path.as_str()); + continue; + }; + return Err(invalid_data!( + "Delete file metadata does not match the table: {} ({mismatch})", + entry.file_path() + )); } } if deletes.is_empty() { @@ -245,14 +272,14 @@ mod tests { use crate::memory::tests::new_memory_catalog; use crate::spec::{ DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, ManifestListWriter, - ManifestStatus, ManifestWriterBuilder, Operation, PrimitiveType, SnapshotRef, Struct, - Transform, Type, UnboundPartitionField, UnboundPartitionSpec, + ManifestStatus, ManifestWriterBuilder, NestedField, Operation, PrimitiveType, Schema, + SnapshotRef, Struct, Transform, Type, UnboundPartitionField, UnboundPartitionSpec, }; use crate::table::Table; use crate::test_utils::commit_with_fresh_state; use crate::transaction::tests::make_v3_minimal_table_in_catalog; use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction}; - use crate::{ErrorKind, TableUpdate}; + use crate::{Catalog, ErrorKind, TableCommit, TableCreation, TableUpdate}; fn test_data_file(path: &str, partition_spec_id: i32) -> DataFile { DataFileBuilder::default() @@ -553,6 +580,122 @@ mod tests { ); } + #[tokio::test] + async fn test_overwrite_deletes_files_after_partition_source_promotion() { + let catalog = new_memory_catalog().await; + let base = make_v3_minimal_table_in_catalog(&catalog).await; + for transform in [Transform::Identity, Transform::Truncate(10)] { + let schema = Schema::builder() + .with_fields([NestedField::required(1, "x", PrimitiveType::Int.into()).into()]) + .build() + .unwrap(); + let spec = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![1]) + .name("p") + .transform(transform) + .build() + .unwrap(), + ) + .unwrap() + .build(); + let table = catalog + .create_table( + base.identifier().namespace(), + TableCreation::builder() + .name(format!("promoted-{transform}")) + .schema(schema) + .partition_spec(spec) + .format_version(crate::spec::FormatVersion::V3) + .build(), + ) + .await + .unwrap(); + let file = DataFile { + partition: Struct::from_iter([Some(Literal::int(300))]), + ..test_data_file( + "test/a.parquet", + table.metadata().default_partition_spec_id(), + ) + }; + let tx = Transaction::new(&table); + let table = tx + .fast_append() + .add_data_files([file.clone()]) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let schema = Schema::builder() + .with_fields([NestedField::required(1, "x", PrimitiveType::Long.into()).into()]) + .build() + .unwrap(); + let table = catalog + .update_table( + TableCommit::builder() + .ident(table.identifier().clone()) + .requirements(vec![]) + .updates(vec![ + TableUpdate::AddSchema { schema }, + TableUpdate::SetCurrentSchema { schema_id: -1 }, + ]) + .build(), + ) + .await + .unwrap(); + let tx = Transaction::new(&table); + let table = tx + .overwrite() + .delete_data_files([file]) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.summary().operation, Operation::Delete); + assert_eq!( + snapshot.summary().additional_properties["deleted-records"], + "1" + ); + assert_eq!(current_manifest_entries(&table).await, [( + ManifestStatus::Deleted, + "test/a.parquet".to_string() + )]); + } + } + + #[tokio::test] + async fn test_overwrite_rejects_readding_a_deleted_path_by_default() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let file = test_data_file( + "test/a.parquet", + table.metadata().default_partition_spec_id(), + ); + let tx = Transaction::new(&table); + let table = tx + .fast_append() + .add_data_files([file.clone()]) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let action = Transaction::new(&table) + .overwrite() + .delete_data_files([file.clone()]) + .add_data_files([file]); + let err = commit_with_fresh_state(action, &table).await.err().unwrap(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.message() + .contains("Cannot add files that are already referenced by table") + ); + } + #[tokio::test] async fn test_overwrite_rejects_incorrect_delete_metrics() { let catalog = new_memory_catalog().await; @@ -568,37 +711,49 @@ mod tests { .commit(&catalog) .await .unwrap(); - for incorrect in [ - DataFile { - record_count: 99, - ..file.clone() - }, - DataFile { - file_size_in_bytes: 999, - ..file.clone() - }, - DataFile { - partition: Struct::from_iter([Some(Literal::long(999))]), - ..file.clone() - }, - DataFile { - partition_spec_id: 99, - ..file.clone() - }, - DataFile { - partition: Struct::from_iter([Some(Literal::string("invalid"))]), - ..file.clone() - }, + for (incorrect, mismatch) in [ + ( + DataFile { + record_count: 99, + ..file.clone() + }, + "record_count: requested 99, table has 1", + ), + ( + DataFile { + file_size_in_bytes: 999, + ..file.clone() + }, + "file_size_in_bytes: requested 999, table has 100", + ), + ( + DataFile { + partition: Struct::from_iter([Some(Literal::long(999))]), + ..file.clone() + }, + "partition: requested Struct { fields: [Some(Primitive(Long(999)))] }, table has Struct { fields: [Some(Primitive(Long(300)))] }", + ), + ( + DataFile { + partition_spec_id: 99, + ..file.clone() + }, + "partition_spec_id: requested 99, table has 0", + ), + ( + DataFile { + partition: Struct::from_iter([Some(Literal::string("invalid"))]), + ..file.clone() + }, + "partition: requested", + ), ] { let action = Transaction::new(&table) .overwrite() .delete_data_files(vec![incorrect]); let err = commit_with_fresh_state(action, &table).await.err().unwrap(); assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!( - err.message() - .contains("Delete file metadata does not match") - ); + assert!(err.message().contains(mismatch)); } } @@ -694,7 +849,7 @@ mod tests { } #[tokio::test] - async fn test_overwrite_rejects_a_path_in_a_delete_manifest() { + async fn test_overwrite_preserves_delete_manifests_and_rejects_their_paths() { let catalog = new_memory_catalog().await; let table = make_v3_minimal_table_in_catalog(&catalog).await; let spec_id = table.metadata().default_partition_spec_id(); @@ -763,5 +918,36 @@ mod tests { err.message() .contains("Missing required files to delete: test/deletes.parquet") ); + + let original = table + .manifest_list_reader(table.metadata().current_snapshot().unwrap()) + .load() + .await + .unwrap() + .entries() + .iter() + .find(|m| m.manifest_path == "test/delete-manifest.avro") + .unwrap() + .clone(); + let tx = Transaction::new(&table); + let table = tx + .overwrite() + .delete_data_files([test_data_file("test/a.parquet", spec_id)]) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let manifests = table + .manifest_list_reader(table.metadata().current_snapshot().unwrap()) + .load() + .await + .unwrap(); + let preserved = manifests + .entries() + .iter() + .find(|m| m.manifest_path == original.manifest_path) + .unwrap(); + assert_eq!(preserved, &original); } } From b106b0af66388e8aa296e6c7bfd8c190396e9e98 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Mon, 26 Oct 2026 13:47:43 +0100 Subject: [PATCH 37/50] Allow overwrite to re-add a path it deletes under check_duplicate The deleted entry is no longer live in the new snapshot, so reusing its path is not a duplicate. --- crates/iceberg/src/transaction/overwrite.rs | 24 ++++++++++++--------- crates/iceberg/src/transaction/snapshot.rs | 5 ++++- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 750394c7bb..e032249546 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -56,7 +56,7 @@ impl OverwriteAction { } /// Set whether to reject added paths already live in the table (enabled by default). - /// This also rejects paths deleted and re-added in the same overwrite. + /// Paths deleted in the same overwrite may be re-added. pub fn with_check_duplicate(mut self, v: bool) -> Self { self.check_duplicate = v; self @@ -668,7 +668,7 @@ mod tests { } #[tokio::test] - async fn test_overwrite_rejects_readding_a_deleted_path_by_default() { + async fn test_overwrite_readds_a_deleted_path_by_default() { let catalog = new_memory_catalog().await; let table = make_v3_minimal_table_in_catalog(&catalog).await; let file = test_data_file( @@ -684,16 +684,20 @@ mod tests { .commit(&catalog) .await .unwrap(); - let action = Transaction::new(&table) + let tx = Transaction::new(&table); + let table = tx .overwrite() .delete_data_files([file.clone()]) - .add_data_files([file]); - let err = commit_with_fresh_state(action, &table).await.err().unwrap(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!( - err.message() - .contains("Cannot add files that are already referenced by table") - ); + .add_data_files([file]) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + assert_eq!(current_manifest_entries(&table).await, [ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Added, "test/a.parquet".to_string()), + ]); } #[tokio::test] diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index 2bbb295464..73277bda66 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -183,11 +183,14 @@ impl<'a> SnapshotProducer<'a> { return Ok(()); }; - let new_files: HashSet<&str> = self + let mut new_files: HashSet<&str> = self .added_data_files .iter() .map(|df| df.file_path.as_str()) .collect(); + for df in &self.deleted_data_files { + new_files.remove(df.file_path.as_str()); + } let runtime = self.table.runtime(); let manifest_list = self From e5b660cf22cdda9781268dbcb5de75c937cf6b91 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 30 Oct 2026 10:17:24 +0100 Subject: [PATCH 38/50] Pin overwrite deleting a path live in two manifests --- crates/iceberg/src/transaction/overwrite.rs | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 2b04e45dc5..9f4a107319 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -544,6 +544,32 @@ mod tests { ); } + #[tokio::test] + async fn test_overwrite_deletes_every_live_copy_of_a_path() { + let catalog = new_memory_catalog().await; + let mut table = make_v3_minimal_table_in_catalog(&catalog).await; + let file_a = test_data_file( + "test/a.parquet", + table.metadata().default_partition_spec_id(), + ); + for _ in 0..2 { + let tx = Transaction::new(&table); + let action = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(vec![file_a.clone()]); + table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + } + + let tx = Transaction::new(&table); + let action = tx.overwrite().delete_data_files(vec![file_a]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let summary = table.metadata().current_snapshot().unwrap().summary(); + assert_eq!(summary.additional_properties["deleted-data-files"], "2"); + assert_eq!(summary.additional_properties["total-data-files"], "0"); + } + #[tokio::test] async fn test_delete_only_overwrite_reports_a_delete_operation() { let catalog = new_memory_catalog().await; From cf6cb1fd0ec13e5e207404e64cae867c65dec1dd Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 30 Oct 2026 10:20:24 +0100 Subject: [PATCH 39/50] Address overwrite doc and test nits --- crates/iceberg/src/transaction/overwrite.rs | 69 +++++++++++++-------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 9f4a107319..e4acc3efc1 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -34,11 +34,10 @@ use crate::transaction::snapshot::{ use crate::transaction::{ActionCommit, TransactionAction}; use crate::{Error, ErrorKind}; -/// OverwriteAction is a transaction action for overwriting data files in the table. +/// Adds data files and removes listed data files by path. /// -/// Creates a snapshot with `Operation::Overwrite` semantics — adds new data files and -/// optionally removes existing data files by rewriting affected manifests with those -/// entries marked as `ManifestStatus::Deleted`. +/// The operation is `Overwrite` when files are both added and removed, otherwise `Delete` or +/// `Append`. There is no row-filter mode, and files committed concurrently are not checked. #[derive(Clone)] pub struct OverwriteAction { check_duplicate: bool, @@ -302,13 +301,14 @@ mod tests { use crate::memory::tests::new_memory_catalog; use crate::spec::{ - DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, ManifestListWriter, - ManifestStatus, ManifestWriterBuilder, NestedField, Operation, PrimitiveType, Schema, - SnapshotRef, Struct, Transform, Type, UnboundPartitionField, UnboundPartitionSpec, + DataContentType, DataFile, DataFileBuilder, DataFileFormat, FormatVersion, Literal, + ManifestListWriter, ManifestStatus, ManifestWriterBuilder, NestedField, Operation, + PrimitiveType, Schema, SnapshotRef, Struct, Transform, Type, UnboundPartitionField, + UnboundPartitionSpec, }; use crate::table::Table; use crate::test_utils::commit_with_fresh_state; - use crate::transaction::tests::make_v3_minimal_table_in_catalog; + use crate::transaction::tests::{make_v2_minimal_table, make_v3_minimal_table_in_catalog}; use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction}; use crate::{Catalog, ErrorKind, TableCommit, TableCreation, TableUpdate}; @@ -386,7 +386,20 @@ mod tests { #[tokio::test] async fn test_second_overwrite_does_not_resurrect_a_deleted_file() { let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; + let base = make_v3_minimal_table_in_catalog(&catalog).await; + let v2 = make_v2_minimal_table(); + let table = catalog + .create_table( + base.identifier().namespace(), + TableCreation::builder() + .name("v2".to_string()) + .schema(v2.metadata().current_schema().as_ref().clone()) + .partition_spec(v2.metadata().default_partition_spec().as_ref().clone()) + .format_version(FormatVersion::V2) + .build(), + ) + .await + .unwrap(); let spec_id = table.metadata().default_partition_spec_id(); let file_a = test_data_file("test/a.parquet", spec_id); @@ -411,13 +424,11 @@ mod tests { .delete_data_files(vec![file_b.clone()]); let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - let entries = current_manifest_entries(&table).await; - assert!( - !entries.iter().any( - |(status, path)| path == "test/a.parquet" && *status != ManifestStatus::Deleted - ), - "the first overwrite's delete must not come back, entries: {entries:?}" - ); + assert_eq!(current_manifest_entries(&table).await, vec![ + (ManifestStatus::Deleted, "test/b.parquet".to_string()), + (ManifestStatus::Added, "test/c.parquet".to_string()), + (ManifestStatus::Added, "test/d.parquet".to_string()), + ]); } #[tokio::test] @@ -664,7 +675,7 @@ mod tests { .name(format!("promoted-{transform}")) .schema(schema) .partition_spec(spec) - .format_version(crate::spec::FormatVersion::V3) + .format_version(FormatVersion::V3) .build(), ) .await @@ -772,41 +783,49 @@ mod tests { .commit(&catalog) .await .unwrap(); + let partition_mismatch = |requested: &Struct| { + format!( + "partition: requested {requested:?}, table has {:?}", + file.partition + ) + }; + let long_partition = Struct::from_iter([Some(Literal::long(999))]); + let string_partition = Struct::from_iter([Some(Literal::string("invalid"))]); for (incorrect, mismatch) in [ ( DataFile { record_count: 99, ..file.clone() }, - "record_count: requested 99, table has 1", + "record_count: requested 99, table has 1".to_string(), ), ( DataFile { file_size_in_bytes: 999, ..file.clone() }, - "file_size_in_bytes: requested 999, table has 100", + "file_size_in_bytes: requested 999, table has 100".to_string(), ), ( DataFile { - partition: Struct::from_iter([Some(Literal::long(999))]), + partition: long_partition.clone(), ..file.clone() }, - "partition: requested Struct { fields: [Some(Primitive(Long(999)))] }, table has Struct { fields: [Some(Primitive(Long(300)))] }", + partition_mismatch(&long_partition), ), ( DataFile { partition_spec_id: 99, ..file.clone() }, - "partition_spec_id: requested 99, table has 0", + "partition_spec_id: requested 99, table has 0".to_string(), ), ( DataFile { - partition: Struct::from_iter([Some(Literal::string("invalid"))]), + partition: string_partition.clone(), ..file.clone() }, - "partition: requested", + partition_mismatch(&string_partition), ), ] { let action = Transaction::new(&table) @@ -814,7 +833,7 @@ mod tests { .delete_data_files(vec![incorrect]); let err = commit_with_fresh_state(action, &table).await.err().unwrap(); assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains(mismatch)); + assert!(err.message().contains(&mismatch), "{err}"); } } From 20693aa8460eddae9e38c5a2d31d75df2dd2b33a Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 30 Oct 2026 10:27:31 +0100 Subject: [PATCH 40/50] Keep overwrite identity across commit retries --- crates/iceberg/src/transaction/overwrite.rs | 92 ++++++++++++++++++--- 1 file changed, 79 insertions(+), 13 deletions(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index e4acc3efc1..e1de11cb98 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -143,11 +143,18 @@ impl OverwriteAction { #[async_trait] impl TransactionAction for OverwriteAction { - type State = (); - - fn new_state(&self) -> Self::State {} + type State = OverwriteState; + + fn new_state(&self) -> OverwriteState { + OverwriteState { + commit_uuid: self.commit_uuid.unwrap_or_else(Uuid::now_v7), + snapshot_id: None, + attempt: 0, + manifest_counter: 0, + } + } - async fn commit(&self, _state: &mut (), table: &Table) -> Result { + async fn commit(&self, state: &mut OverwriteState, table: &Table) -> Result { self.validate(table)?; let mut deleted_files = HashMap::new(); for file in &self.deleted_data_files { @@ -163,9 +170,9 @@ impl TransactionAction for OverwriteAction { validate_files_not_live(table, &added).await?; } - let snapshot_id = generate_snapshot_id(table); - let commit_uuid = self.commit_uuid.unwrap_or_else(Uuid::now_v7); - let mut manifest_counter = 0; + let snapshot_id = *state + .snapshot_id + .get_or_insert_with(|| generate_snapshot_id(table)); let mut manifests = vec![]; // One per tombstone written, so a path live in several manifests counts each copy. let mut removed_files = vec![]; @@ -199,13 +206,12 @@ impl TransactionAction for OverwriteAction { let mut writer = new_manifest_writer( table, - manifest_path(table, commit_uuid, manifest_counter)?, + state.next_manifest_path(table)?, snapshot_id, manifest_file.content, manifest.metadata().schema.clone(), Arc::new(manifest.metadata().partition_spec.clone()), )?; - manifest_counter += 1; // Like Java, only live entries are carried into the rewrite. for entry in live { if deleted_files.contains_key(entry.file_path()) { @@ -236,7 +242,7 @@ impl TransactionAction for OverwriteAction { manifests.push( write_added_manifest( table, - manifest_path(table, commit_uuid, manifest_counter)?, + state.next_manifest_path(table)?, snapshot_id, table.metadata().default_partition_spec_id(), &self.added_data_files, @@ -251,9 +257,12 @@ impl TransactionAction for OverwriteAction { // Also a properties-only commit, the workaround from #1548. _ => Operation::Append, }; + let manifest_list_path = + manifest_list_path(table, snapshot_id, state.attempt, state.commit_uuid)?; + state.attempt += 1; write_snapshot_commit(table, SnapshotCommitInput { snapshot_id, - manifest_list_path: manifest_list_path(table, snapshot_id, 0, commit_uuid)?, + manifest_list_path, operation, properties: self.snapshot_properties.clone(), added_files: &self.added_data_files, @@ -265,7 +274,23 @@ impl TransactionAction for OverwriteAction { .await } - async fn cleanup(self, _state: (), _table: &Table, _status: CommitStatus) {} + async fn cleanup(self, _state: OverwriteState, _table: &Table, _status: CommitStatus) {} +} + +/// Kept across retries, so no attempt overwrites a file an earlier attempt wrote. +pub(crate) struct OverwriteState { + commit_uuid: Uuid, + snapshot_id: Option, + attempt: u64, + manifest_counter: u64, +} + +impl OverwriteState { + fn next_manifest_path(&mut self, table: &Table) -> Result { + let path = manifest_path(table, self.commit_uuid, self.manifest_counter); + self.manifest_counter += 1; + path + } } /// Describes the first field where a requested delete differs from the table's entry. @@ -299,6 +324,8 @@ fn metadata_mismatch(requested: &DataFile, actual: &DataFile) -> Option mod tests { use std::sync::Arc; + use uuid::Uuid; + use crate::memory::tests::new_memory_catalog; use crate::spec::{ DataContentType, DataFile, DataFileBuilder, DataFileFormat, FormatVersion, Literal, @@ -309,7 +336,7 @@ mod tests { use crate::table::Table; use crate::test_utils::commit_with_fresh_state; use crate::transaction::tests::{make_v2_minimal_table, make_v3_minimal_table_in_catalog}; - use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction}; + use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction, TransactionAction}; use crate::{Catalog, ErrorKind, TableCommit, TableCreation, TableUpdate}; fn test_data_file(path: &str, partition_spec_id: i32) -> DataFile { @@ -431,6 +458,45 @@ mod tests { ]); } + #[tokio::test] + async fn test_overwrite_retry_keeps_its_snapshot_id_and_never_reuses_a_manifest_path() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + let file_a = test_data_file("test/a.parquet", spec_id); + let tx = Transaction::new(&table); + let action = tx.fast_append().add_data_files(vec![file_a.clone()]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let action = Transaction::new(&table) + .overwrite() + .set_commit_uuid(Uuid::nil()) + .add_data_files(vec![test_data_file("test/b.parquet", spec_id)]) + .delete_data_files(vec![file_a]); + let mut state = action.new_state(); + let mut attempts = vec![]; + for _ in 0..2 { + let mut action_commit = action.commit(&mut state, &table).await.unwrap(); + let Some(TableUpdate::AddSnapshot { snapshot }) = + action_commit.take_updates().into_iter().next() + else { + unreachable!() + }; + let snapshot = SnapshotRef::new(snapshot); + let manifest_list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let mut paths: Vec<_> = manifest_list + .entries() + .iter() + .map(|m| m.manifest_path.clone()) + .collect(); + paths.push(snapshot.manifest_list().to_string()); + attempts.push((snapshot.snapshot_id(), paths)); + } + + assert_eq!(attempts[0].0, attempts[1].0); + assert!(attempts[1].1.iter().all(|p| !attempts[0].1.contains(p))); + } + #[tokio::test] async fn test_rewritten_manifest_keeps_the_schema_and_spec_it_was_written_with() { let catalog = new_memory_catalog().await; From cc0c9e491fc70814ea5cae441e5e5b5f8bdfb8c1 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 30 Oct 2026 10:27:31 +0100 Subject: [PATCH 41/50] Drop overwrite manifest read TODO --- crates/iceberg/src/transaction/overwrite.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index e1de11cb98..58a72e396c 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -177,7 +177,6 @@ impl TransactionAction for OverwriteAction { // One per tombstone written, so a path live in several manifests counts each copy. let mut removed_files = vec![]; - // TODO: read these manifests concurrently, as `validate_files_not_live` does. for manifest_file in current_manifests(table).await? { if manifest_file.content == ManifestContentType::Deletes || deleted_files.is_empty() { manifests.push(manifest_file); From 36fd78505453a2148c5992512ea743831a71007a Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 30 Oct 2026 10:30:28 +0100 Subject: [PATCH 42/50] Assert both overwrite tombstones are written --- crates/iceberg/src/transaction/overwrite.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 58a72e396c..88cdd3e6c5 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -644,6 +644,10 @@ mod tests { let summary = table.metadata().current_snapshot().unwrap().summary(); assert_eq!(summary.additional_properties["deleted-data-files"], "2"); assert_eq!(summary.additional_properties["total-data-files"], "0"); + assert_eq!(current_manifest_entries(&table).await, [ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + ]); } #[tokio::test] From 604c6d149e27d42feab37a8301d8103c0d38bf69 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Tue, 3 Nov 2026 15:55:26 +0100 Subject: [PATCH 43/50] Match every requested delete before writing rewritten manifests Reads go through the object cache so a retry does not refetch manifests, and validate_files_not_live takes paths. --- crates/iceberg/src/transaction/append.rs | 3 +- crates/iceberg/src/transaction/overwrite.rs | 84 ++++++++++--------- .../iceberg/src/transaction/snapshot/mod.rs | 19 +++-- 3 files changed, 60 insertions(+), 46 deletions(-) diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 08ac900adf..a01a229bdb 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -130,7 +130,8 @@ impl TransactionAction for FastAppendAction { ) -> Result { self.validate()?; if self.check_duplicate { - validate_files_not_live(table, &self.added_data_files).await?; + validate_files_not_live(table, self.added_data_files.iter().map(DataFile::file_path)) + .await?; } state.apply(table).await } diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 88cdd3e6c5..0e26d543e0 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -161,33 +161,25 @@ impl TransactionAction for OverwriteAction { deleted_files.entry(file.file_path()).or_insert(file); } if self.check_duplicate { - let added: Vec = self - .added_data_files - .iter() - .filter(|f| !deleted_files.contains_key(f.file_path())) - .cloned() - .collect(); - validate_files_not_live(table, &added).await?; + let added = self.added_data_files.iter().map(DataFile::file_path); + validate_files_not_live(table, added.filter(|p| !deleted_files.contains_key(p))) + .await?; } - let snapshot_id = *state - .snapshot_id - .get_or_insert_with(|| generate_snapshot_id(table)); - let mut manifests = vec![]; - // One per tombstone written, so a path live in several manifests counts each copy. - let mut removed_files = vec![]; - + // Every requested delete is matched before any manifest is written, so a + // missing path or mismatched metadata fails without leaving files behind. + let mut matched = HashSet::new(); + let mut planned = vec![]; for manifest_file in current_manifests(table).await? { if manifest_file.content == ManifestContentType::Deletes || deleted_files.is_empty() { - manifests.push(manifest_file); + planned.push((manifest_file, None)); continue; } - let manifest = table.manifest_reader().read(&manifest_file).await?; - let live: Vec<_> = manifest.entries().iter().filter(|e| e.is_alive()).collect(); + let manifest = table.object_cache().get_manifest(&manifest_file).await?; let mut has_deletes = false; - for entry in &live { - let Some(requested) = deleted_files.get(entry.file_path()) else { + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { + let Some((path, requested)) = deleted_files.get_key_value(entry.file_path()) else { continue; }; if let Some(mismatch) = metadata_mismatch(requested, entry.data_file()) { @@ -196,13 +188,36 @@ impl TransactionAction for OverwriteAction { entry.file_path() )); } + matched.insert(*path); has_deletes = true; } - if !has_deletes { + planned.push((manifest_file, has_deletes.then_some(manifest))); + } + + let mut missing: Vec<&str> = deleted_files + .keys() + .copied() + .filter(|path| !matched.contains(path)) + .collect(); + if !missing.is_empty() { + missing.sort_unstable(); + return Err(invalid_data!( + "Missing required files to delete: {}", + missing.join(", ") + )); + } + + let snapshot_id = *state + .snapshot_id + .get_or_insert_with(|| generate_snapshot_id(table)); + let mut manifests = vec![]; + // One per tombstone written, so a path live in several manifests counts each copy. + let mut removed_files = vec![]; + for (manifest_file, rewrite) in planned { + let Some(manifest) = rewrite else { manifests.push(manifest_file); continue; - } - + }; let mut writer = new_manifest_writer( table, state.next_manifest_path(table)?, @@ -212,7 +227,7 @@ impl TransactionAction for OverwriteAction { Arc::new(manifest.metadata().partition_spec.clone()), )?; // Like Java, only live entries are carried into the rewrite. - for entry in live { + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { if deleted_files.contains_key(entry.file_path()) { removed_files.push(entry.data_file().clone()); writer.add_delete_entry(entry.as_ref().clone())?; @@ -223,20 +238,6 @@ impl TransactionAction for OverwriteAction { manifests.push(writer.write_manifest_file().await?); } - let removed_paths: HashSet<&str> = removed_files.iter().map(|f| f.file_path()).collect(); - let mut missing: Vec<&str> = deleted_files - .keys() - .copied() - .filter(|path| !removed_paths.contains(path)) - .collect(); - if !missing.is_empty() { - missing.sort_unstable(); - return Err(invalid_data!( - "Missing required files to delete: {}", - missing.join(", ") - )); - } - if !self.added_data_files.is_empty() { manifests.push( write_added_manifest( @@ -273,7 +274,10 @@ impl TransactionAction for OverwriteAction { .await } - async fn cleanup(self, _state: OverwriteState, _table: &Table, _status: CommitStatus) {} + async fn cleanup(self, _state: OverwriteState, _table: &Table, _status: CommitStatus) { + // TODO(#3417): remove this attempt's manifests once terminal cleanup is + // wired up. + } } /// Kept across retries, so no attempt overwrites a file an earlier attempt wrote. @@ -334,6 +338,7 @@ mod tests { }; use crate::table::Table; use crate::test_utils::commit_with_fresh_state; + use crate::transaction::snapshot::manifest_path; use crate::transaction::tests::{make_v2_minimal_table, make_v3_minimal_table_in_catalog}; use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction, TransactionAction}; use crate::{Catalog, ErrorKind, TableCommit, TableCreation, TableUpdate}; @@ -703,6 +708,7 @@ mod tests { let tx = Transaction::new(&table); let action = tx .overwrite() + .set_commit_uuid(Uuid::nil()) .add_data_files(vec![test_data_file("test/b.parquet", spec_id)]) .delete_data_files(vec![ file_a.clone(), @@ -715,6 +721,8 @@ mod tests { .contains("Missing required files to delete: test/never-committed.parquet"), "{err}" ); + let first_manifest = manifest_path(&table, Uuid::nil(), 0).unwrap(); + assert!(!table.file_io().exists(&first_manifest).await.unwrap()); } #[tokio::test] diff --git a/crates/iceberg/src/transaction/snapshot/mod.rs b/crates/iceberg/src/transaction/snapshot/mod.rs index 36616d0125..bdfcd2fc61 100644 --- a/crates/iceberg/src/transaction/snapshot/mod.rs +++ b/crates/iceberg/src/transaction/snapshot/mod.rs @@ -173,17 +173,20 @@ pub(crate) async fn current_manifests(table: &Table) -> Result .collect()) } -/// Fails if any of `data_files` is already live in the table's current snapshot. +/// Fails if any of the `file_paths` is already live in the table's current snapshot. /// /// Must run against each attempt's base, since a concurrent commit may have /// added one of these files. Manifest reads go through the table's object /// cache, so manifests unchanged by a rebase are not re-read. -pub(crate) async fn validate_files_not_live(table: &Table, data_files: &[DataFile]) -> Result<()> { +pub(crate) async fn validate_files_not_live( + table: &Table, + file_paths: impl IntoIterator, +) -> Result<()> { let Some(current_snapshot) = table.metadata().current_snapshot() else { return Ok(()); }; - let new_files: HashSet<&str> = data_files.iter().map(|df| df.file_path()).collect(); + let new_files: HashSet<&str> = file_paths.into_iter().collect(); if new_files.is_empty() { return Ok(()); } @@ -522,19 +525,21 @@ mod tests { // No current snapshot: nothing is live, so nothing is a duplicate. let table = make_v2_minimal_table(); let files = vec![data_file(&table, "test/dup.parquet", 1)]; - validate_files_not_live(&table, &files).await.unwrap(); + validate_files_not_live(&table, ["test/dup.parquet"]) + .await + .unwrap(); // A concurrent writer commits the same file; validating against that // base must detect it. - let rebased = append_to(&table, files.clone()).await; - let err = validate_files_not_live(&rebased, &files) + let rebased = append_to(&table, files).await; + let err = validate_files_not_live(&rebased, ["test/dup.parquet"]) .await .expect_err("file committed concurrently must be detected"); assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!(err.message().contains("test/dup.parquet")); // A different file is not flagged. - validate_files_not_live(&rebased, &[data_file(&table, "test/other.parquet", 1)]) + validate_files_not_live(&rebased, ["test/other.parquet"]) .await .unwrap(); } From a37f3df88aa4a6b01b982706f78379e7bbe21447 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 27 Oct 2026 16:40:55 -0700 Subject: [PATCH 44/50] add msp --- crates/iceberg/src/spec/manifest/writer.rs | 8 - .../transaction/snapshot/manifest_filter.rs | 225 ++++++++ .../src/transaction/snapshot/merging.rs | 544 ++++++++++++++++++ .../iceberg/src/transaction/snapshot/mod.rs | 85 ++- .../src/transaction/snapshot/simple.rs | 48 +- 5 files changed, 847 insertions(+), 63 deletions(-) create mode 100644 crates/iceberg/src/transaction/snapshot/manifest_filter.rs create mode 100644 crates/iceberg/src/transaction/snapshot/merging.rs diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index a79312dfab..41a3d88e44 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -315,10 +315,6 @@ impl ManifestWriter { /// Add a delete manifest entry. This method will update following status of the entry: /// - Update the entry status to `Deleted` /// - Set the snapshot id to the current snapshot id - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_delete_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Deleted; @@ -350,10 +346,6 @@ impl ManifestWriter { /// Add an existing manifest entry. This method will update following status of the entry: /// - Update the entry status to `Existing` - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_existing_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Existing; diff --git a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs new file mode 100644 index 0000000000..1debd09553 --- /dev/null +++ b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Rewrites existing manifests to mark files as deleted. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use super::{manifest_path, new_manifest_writer}; +use crate::error::{Result, invalid_data}; +use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile}; +use crate::table::Table; + +/// Removes a fixed set of files from the manifests of one content type. +/// +/// One manager lives for one logical execution of an action and is driven +/// once per attempt via [`ManifestFilterManager::filter`]. A manifest that +/// references none of the requested files passes through untouched. One that +/// does is rewritten with those entries marked `Deleted` and the rest +/// `Existing`; the rewrite is keyed by the source manifest's path, which is +/// immutable, so a rebase only rewrites manifests not seen before. +pub(crate) struct ManifestFilterManager { + content: ManifestContentType, + /// Paths of the files to remove. Fixed for the execution. + requested_paths: HashSet, + /// Source manifest path → rewritten result. + filtered: HashMap, + /// Format version the rewrites were written under. Per spec, v1 entries + /// cannot inherit sequence numbers, so an upgrade invalidates them. + format_version: Option, + /// Metadata files written by this manager. + owned_artifacts: HashSet, +} + +/// One rewritten manifest and the files it marked as deleted. +#[derive(Clone)] +struct FilteredManifest { + manifest: ManifestFile, + removed_files: Vec, +} + +/// The result of one filter pass. +pub(crate) struct Filtered { + /// The resulting manifests, in input order. + pub manifests: Vec, + /// The files that were live and are now marked deleted, including their + /// metrics as read from the source manifests. + pub removed_files: Vec, +} + +impl ManifestFilterManager { + pub(crate) fn new(content: ManifestContentType, removed_files: &[DataFile]) -> Self { + Self { + content, + requested_paths: removed_files + .iter() + .map(|f| f.file_path().to_string()) + .collect(), + filtered: HashMap::new(), + format_version: None, + owned_artifacts: HashSet::new(), + } + } + + /// Metadata files written by this manager so far. + pub(crate) fn owned_artifacts(&self) -> &HashSet { + &self.owned_artifacts + } + + /// Applies the removals to `manifests`, rewriting those of this manager's + /// content type that reference a requested file. + /// + /// Fails if a requested file is not live in any manifest: it was never in + /// the table, or a concurrent commit already removed it. + /// + /// `next_manifest_number` allocates paths for rewritten manifests from the + /// producer's counter, so they never collide with other manifests it writes. + pub(crate) async fn filter( + &mut self, + table: &Table, + snapshot_id: i64, + commit_uuid: Uuid, + manifests: Vec, + next_manifest_number: &mut u64, + ) -> Result { + if self.requested_paths.is_empty() { + return Ok(Filtered { + manifests, + removed_files: Vec::new(), + }); + } + + let format_version = table.metadata().format_version(); + if self.format_version != Some(format_version) { + self.filtered.clear(); + self.format_version = Some(format_version); + } + + let mut out = Vec::with_capacity(manifests.len()); + let mut removed_files: Vec = Vec::new(); + + for source in manifests { + if source.content != self.content { + out.push(source); + continue; + } + + let result = match self.filtered.get(&source.manifest_path) { + Some(cached) => cached.clone(), + None => { + let result = self + .rewrite( + table, + snapshot_id, + commit_uuid, + &source, + next_manifest_number, + ) + .await?; + self.filtered + .insert(source.manifest_path.clone(), result.clone()); + result + } + }; + + removed_files.extend(result.removed_files); + out.push(result.manifest); + } + + let found: HashSet<&str> = removed_files.iter().map(|f| f.file_path()).collect(); + let mut missing: Vec<&str> = self + .requested_paths + .iter() + .map(String::as_str) + .filter(|p| !found.contains(p)) + .collect(); + if !missing.is_empty() { + missing.sort_unstable(); + return Err(invalid_data!( + "Cannot remove files that are not live in the table: {}", + missing.join(", ") + )); + } + + Ok(Filtered { + manifests: out, + removed_files, + }) + } + + /// Reads `source` and, if it references any requested file, writes a copy + /// with those entries marked `Deleted` and the rest `Existing`. + async fn rewrite( + &mut self, + table: &Table, + snapshot_id: i64, + commit_uuid: Uuid, + source: &ManifestFile, + next_manifest_number: &mut u64, + ) -> Result { + let manifest = table.object_cache().get_manifest(source).await?; + + let removed_files: Vec = manifest + .entries() + .iter() + .filter(|e| e.is_alive() && self.requested_paths.contains(e.file_path())) + .map(|e| e.data_file().clone()) + .collect(); + if removed_files.is_empty() { + return Ok(FilteredManifest { + manifest: source.clone(), + removed_files, + }); + } + + // Allocate the path before writing so a partial write is still owned. + let path = manifest_path(table, commit_uuid, *next_manifest_number)?; + *next_manifest_number += 1; + self.owned_artifacts.insert(path.clone()); + + // Keep the source manifest's spec: its entries' partition values are + // laid out for that spec, not the table's current default. + let spec = manifest.metadata().partition_spec().clone(); + let mut writer = new_manifest_writer( + table, + path, + snapshot_id, + self.content, + table.metadata().current_schema().clone(), + spec.into(), + )?; + for entry in manifest.entries() { + if !entry.is_alive() { + continue; + } + let entry = (**entry).clone(); + if self.requested_paths.contains(entry.file_path()) { + writer.add_delete_entry(entry)?; + } else { + writer.add_existing_entry(entry)?; + } + } + let manifest = writer.write_manifest_file().await?; + + Ok(FilteredManifest { + manifest, + removed_files, + }) + } +} diff --git a/crates/iceberg/src/transaction/snapshot/merging.rs b/crates/iceberg/src/transaction/snapshot/merging.rs new file mode 100644 index 0000000000..16e1ba85f4 --- /dev/null +++ b/crates/iceberg/src/transaction/snapshot/merging.rs @@ -0,0 +1,544 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Persistent producer for snapshot actions that add and remove data and +//! delete files. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use super::manifest_filter::ManifestFilterManager; +use super::{ + SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, + manifest_path, validate_partition_value, write_added_manifest, write_snapshot_commit, +}; +use crate::error::{Result, invalid_data}; +use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; +use crate::table::Table; +use crate::transaction::ActionCommit; + +/// What a merging action asks the producer to do. +/// +/// Fixed for one logical execution; the producer derives everything else from +/// it and the table of the current attempt. The action validates the +/// table-independent parts (content types, disjoint add/remove sets) before +/// each attempt. +pub(crate) struct MergingIntent { + /// Overrides the generated commit UUID. + pub commit_uuid: Option, + pub operation: Operation, + pub snapshot_properties: HashMap, + pub added_data_files: Vec, + pub added_delete_files: Vec, + pub removed_data_files: Vec, + pub removed_delete_files: Vec, + /// Whether the summary resets table totals (a full-table overwrite). + pub truncate_full_table: bool, +} + +/// Retry-persistent state for an action that adds and removes data and delete +/// files. +/// +/// One producer lives for one logical execution of an action and is driven once +/// per attempt via [`MergingSnapshotProducer::apply`]. Per attempt it takes the +/// current snapshot's manifests, rewrites those that reference a removed file, +/// appends manifests for the added files, and writes a manifest list. Identity +/// (commit UUID, snapshot id) is fixed on first use; rewritten and added +/// manifests are reused across attempts while their inputs are unchanged. +pub(crate) struct MergingSnapshotProducer { + intent: MergingIntent, + /// Fixed when the producer is created, so every attempt shares one UUID. + commit_uuid: Uuid, + /// Resolved against the table on the first attempt, then stable. + snapshot_id: Option, + /// Incremented per attempt; distinguishes manifest-list paths. + attempt: u64, + /// Numbers every manifest this producer writes, including rewrites. Never + /// reset, so paths never collide across attempts. + manifest_counter: u64, + data_filter: ManifestFilterManager, + delete_filter: ManifestFilterManager, + /// Manifests written for the added files, reused across attempts. + added_manifests: Option, + /// Metadata files written by this producer (manifests and every attempt's + /// manifest list). + // TODO(#3417): consumed by terminal cleanup once it is wired up. + owned_artifacts: HashSet, +} + +/// The manifests written for one execution's added data and delete files. +/// +/// Reusable while the table's format version is unchanged; see +/// `simple::AddedManifests` for why. +struct AddedManifests { + manifests: Vec, + format_version: FormatVersion, +} + +impl MergingSnapshotProducer { + pub(crate) fn new(intent: MergingIntent) -> Self { + Self { + commit_uuid: intent.commit_uuid.unwrap_or_else(Uuid::now_v7), + data_filter: ManifestFilterManager::new( + ManifestContentType::Data, + &intent.removed_data_files, + ), + delete_filter: ManifestFilterManager::new( + ManifestContentType::Deletes, + &intent.removed_delete_files, + ), + intent, + snapshot_id: None, + attempt: 0, + manifest_counter: 0, + added_manifests: None, + owned_artifacts: HashSet::new(), + } + } + + /// Produces the resulting manifest set for the current table and writes + /// this attempt's snapshot. + pub(crate) async fn apply(&mut self, table: &Table) -> Result { + let snapshot_id = self.snapshot_id(table); + + let manifests = current_manifests(table).await?; + + // Removals first: filtering only touches the current snapshot's + // manifests, never the ones this producer adds. + let data = self + .data_filter + .filter( + table, + snapshot_id, + self.commit_uuid, + manifests, + &mut self.manifest_counter, + ) + .await?; + let deletes = self + .delete_filter + .filter( + table, + snapshot_id, + self.commit_uuid, + data.manifests, + &mut self.manifest_counter, + ) + .await?; + let mut manifests = deletes.manifests; + self.owned_artifacts + .extend(self.data_filter.owned_artifacts().iter().cloned()); + self.owned_artifacts + .extend(self.delete_filter.owned_artifacts().iter().cloned()); + + manifests.extend(self.added_manifests(table, snapshot_id).await?); + + let manifest_list_path = self.next_manifest_list_path(table, snapshot_id)?; + + let added_files: Vec = self + .intent + .added_data_files + .iter() + .chain(&self.intent.added_delete_files) + .cloned() + .collect(); + + write_snapshot_commit(table, SnapshotCommitInput { + snapshot_id, + manifest_list_path, + operation: self.intent.operation.clone(), + properties: self.intent.snapshot_properties.clone(), + added_files: &added_files, + removed_files: &data.removed_files, + removed_delete_files: &deletes.removed_files, + manifests, + truncate_full_table: self.intent.truncate_full_table, + }) + .await + } + + /// Checks that every added file is written under the table's default + /// partition spec with a compatible partition value, and that delete files + /// are only added to tables whose format supports them. + /// + /// Runs against the table the manifests are written for; a later rebase + /// does not invalidate manifests already written under a spec the table + /// still has. + fn validate_added_files(&self, table: &Table) -> Result<()> { + let metadata = table.metadata(); + if !self.intent.added_delete_files.is_empty() + && metadata.format_version() == FormatVersion::V1 + { + return Err(invalid_data!( + "Cannot add delete files to a format version 1 table" + )); + } + for file in self + .intent + .added_data_files + .iter() + .chain(&self.intent.added_delete_files) + { + // TODO(#3500): accept any spec known to the table and group the + // files by spec when writing manifests. + if metadata.default_partition_spec_id() != file.partition_spec_id { + return Err(invalid_data!( + "File partition spec id does not match table default partition spec id" + )); + } + validate_partition_value(file.partition(), metadata.default_partition_type())?; + } + Ok(()) + } + + fn snapshot_id(&mut self, table: &Table) -> i64 { + *self + .snapshot_id + .get_or_insert_with(|| generate_snapshot_id(table)) + } + + fn next_manifest_list_path(&mut self, table: &Table, snapshot_id: i64) -> Result { + let path = manifest_list_path(table, snapshot_id, self.attempt, self.commit_uuid)?; + self.attempt += 1; + self.owned_artifacts.insert(path.clone()); + Ok(path) + } + + /// Returns the manifests for the added files, writing them only if no + /// reusable set exists for the current format version. Data and delete + /// files go into separate manifests. + async fn added_manifests( + &mut self, + table: &Table, + snapshot_id: i64, + ) -> Result> { + let format_version = table.metadata().format_version(); + + if let Some(cached) = &self.added_manifests + && cached.format_version == format_version + { + return Ok(cached.manifests.clone()); + } + + self.validate_added_files(table)?; + + let mut manifests = Vec::new(); + for (content, files) in [ + (ManifestContentType::Data, &self.intent.added_data_files), + ( + ManifestContentType::Deletes, + &self.intent.added_delete_files, + ), + ] { + if files.is_empty() { + continue; + } + // Allocate the path before writing so a partial write is still owned. + // + // TODO(#3500): group by partition spec. TODO(#3499): roll at the + // target size and write concurrently. + let path = manifest_path(table, self.commit_uuid, self.manifest_counter)?; + self.manifest_counter += 1; + self.owned_artifacts.insert(path.clone()); + manifests.push( + write_added_manifest( + table, + path, + snapshot_id, + content, + table.metadata().default_partition_spec_id(), + files, + ) + .await?, + ); + } + + self.added_manifests = Some(AddedManifests { + manifests: manifests.clone(), + format_version, + }); + Ok(manifests) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::super::tests::{append_to, data_file}; + use super::{MergingIntent, MergingSnapshotProducer}; + use crate::spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, ManifestContentType, + ManifestStatus, Operation, SnapshotRef, Struct, + }; + use crate::table::Table; + use crate::transaction::tests::{make_v1_table, make_v2_minimal_table}; + use crate::{ErrorKind, TableUpdate}; + + fn delete_file(table: &Table, path: &str) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::PositionDeletes) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(50) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap() + } + + fn intent(operation: Operation) -> MergingIntent { + MergingIntent { + commit_uuid: None, + operation, + snapshot_properties: HashMap::new(), + added_data_files: vec![], + added_delete_files: vec![], + removed_data_files: vec![], + removed_delete_files: vec![], + truncate_full_table: false, + } + } + + fn added_snapshot(commit: &mut crate::transaction::ActionCommit) -> SnapshotRef { + commit + .take_updates() + .into_iter() + .find_map(|u| match u { + TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot)), + _ => None, + }) + .expect("apply should add a snapshot") + } + + /// Reads every (status, path) pair in the manifest at `manifest_path`. + async fn entries_of( + table: &Table, + manifest: &crate::spec::ManifestFile, + ) -> Vec<(ManifestStatus, String)> { + let mut entries: Vec<_> = table + .manifest_reader() + .read(manifest) + .await + .unwrap() + .entries() + .iter() + .map(|e| (e.status(), e.file_path().to_string())) + .collect(); + entries.sort_by(|a, b| a.1.cmp(&b.1)); + entries + } + + /// Removing one of two files in a manifest rewrites it with one `Deleted` + /// and one `Existing` entry; a manifest without removed files is carried + /// forward by path. + #[tokio::test] + async fn test_remove_data_file_rewrites_only_its_manifest() { + let base = make_v2_minimal_table(); + // Two commits so the current snapshot has two data manifests. + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + let table = append_to(&table, vec![data_file(&base, "test/c.parquet", 1)]).await; + let current = table.metadata().current_snapshot().unwrap(); + let before = table.manifest_list_reader(current).load().await.unwrap(); + assert_eq!(before.entries().len(), 2); + let untouched = before + .entries() + .iter() + .find(|m| m.added_files_count == Some(1)) + .unwrap() + .manifest_path + .clone(); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let after = table.manifest_list_reader(&snapshot).load().await.unwrap(); + assert_eq!(after.entries().len(), 2); + assert!(after.entries().iter().any(|m| m.manifest_path == untouched)); + let rewritten = after + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .expect("a rewritten manifest belongs to the new snapshot"); + assert_eq!(rewritten.deleted_files_count, Some(1)); + assert_eq!(rewritten.existing_files_count, Some(1)); + assert_eq!(entries_of(&table, rewritten).await, vec![ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + ]); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("deleted-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-data-files").unwrap(), "2"); + } + + /// Added data and delete files go into separate manifests of the right + /// content type, and the summary counts both. + #[tokio::test] + async fn test_add_data_and_delete_files() { + let table = make_v2_minimal_table(); + let mut i = intent(Operation::Overwrite); + i.added_data_files = vec![data_file(&table, "test/d.parquet", 3)]; + i.added_delete_files = vec![delete_file(&table, "test/d-deletes.parquet")]; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let mut contents: Vec<_> = list.entries().iter().map(|m| m.content).collect(); + contents.sort_by_key(|c| *c as i32); + assert_eq!(contents, vec![ + ManifestContentType::Data, + ManifestContentType::Deletes + ]); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("added-data-files").unwrap(), "1"); + assert_eq!(summary.get("added-delete-files").unwrap(), "1"); + assert_eq!(summary.get("added-records").unwrap(), "3"); + } + + /// A retry reuses both the rewritten and the added manifests; only the + /// manifest list is written again. + #[tokio::test] + async fn test_retry_reuses_rewritten_and_added_manifests() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + let mut i = intent(Operation::Overwrite); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + i.added_data_files = vec![data_file(&base, "test/x.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + + let mut first = producer.apply(&table).await.unwrap(); + let mut second = producer.apply(&table).await.unwrap(); + let first = added_snapshot(&mut first); + let second = added_snapshot(&mut second); + + assert_eq!(first.snapshot_id(), second.snapshot_id()); + assert_ne!(first.manifest_list(), second.manifest_list()); + + let paths = |list: crate::spec::ManifestList| { + let mut p: Vec<_> = list + .entries() + .iter() + .map(|m| m.manifest_path.clone()) + .collect(); + p.sort(); + p + }; + let first_paths = paths(table.manifest_list_reader(&first).load().await.unwrap()); + let second_paths = paths(table.manifest_list_reader(&second).load().await.unwrap()); + assert_eq!(first_paths, second_paths); + // One rewrite plus one added manifest. + assert_eq!(producer.manifest_counter, 2); + assert_eq!(producer.attempt, 2); + } + + /// Removing a file that is not live anywhere fails and names the file. + #[tokio::test] + async fn test_remove_missing_file_fails() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/nope.parquet", 1)]; + let err = MergingSnapshotProducer::new(i) + .apply(&table) + .await + .err() + .expect("removing a file that is not live must fail"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("test/nope.parquet")); + } + + /// A file removed by a concurrent commit is no longer live on the rebased + /// table, so the retry fails rather than silently succeeding. + #[tokio::test] + async fn test_rebase_onto_table_without_removed_file_fails() { + let base = make_v2_minimal_table(); + let s1 = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + producer.apply(&s1).await.unwrap(); + + // Concurrent writer removed `a` first; our retry rebases onto that. + let mut other = intent(Operation::Delete); + other.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut other_commit = MergingSnapshotProducer::new(other) + .apply(&s1) + .await + .unwrap(); + let s2_snapshot = added_snapshot(&mut other_commit); + let s2 = { + use crate::spec::{MAIN_BRANCH, SnapshotReference, SnapshotRetention}; + let metadata = s1 + .metadata() + .clone() + .into_builder(None) + .add_snapshot(s2_snapshot.as_ref().clone()) + .unwrap() + .set_ref( + MAIN_BRANCH, + SnapshotReference::new( + s2_snapshot.snapshot_id(), + SnapshotRetention::branch(None, None, None), + ), + ) + .unwrap() + .build() + .unwrap() + .metadata; + s1.clone().with_metadata(std::sync::Arc::new(metadata)) + }; + + let err = producer + .apply(&s2) + .await + .err() + .expect("file already removed concurrently must not be removed twice"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_delete_files_rejected_on_v1_table() { + let table = make_v1_table(); + let mut i = intent(Operation::Overwrite); + i.added_delete_files = vec![delete_file(&table, "test/d-deletes.parquet")]; + let err = MergingSnapshotProducer::new(i) + .apply(&table) + .await + .err() + .expect("v1 tables cannot hold delete files"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } +} diff --git a/crates/iceberg/src/transaction/snapshot/mod.rs b/crates/iceberg/src/transaction/snapshot/mod.rs index eb982e1e4e..6ccc90f3cf 100644 --- a/crates/iceberg/src/transaction/snapshot/mod.rs +++ b/crates/iceberg/src/transaction/snapshot/mod.rs @@ -23,6 +23,12 @@ //! Iceberg metadata and files, never select operation policy, and never cache //! across calls. +// TODO(#1607): remove the allows once `RowDeltaAction` consumes the merging +// producer. +#[allow(dead_code)] +pub(crate) mod manifest_filter; +#[allow(dead_code)] +pub(crate) mod merging; pub(crate) mod simple; use std::collections::{HashMap, HashSet}; @@ -33,10 +39,11 @@ use uuid::Uuid; use crate::error::{Result, invalid_data}; use crate::spec::{ - DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestFile, - ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, PartitionSpecRef, - SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, - StructType, Summary, TableProperties, set_iceberg_version, update_snapshot_summaries, + DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, + ManifestFile, ManifestListWriter, ManifestStatus, ManifestWriter, ManifestWriterBuilder, + Operation, PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, + SnapshotSummaryCollector, Struct, StructType, Summary, TableProperties, set_iceberg_version, + update_snapshot_summaries, }; use crate::table::Table; use crate::transaction::ActionCommit; @@ -173,6 +180,46 @@ pub(crate) async fn current_manifests(table: &Table) -> Result .collect()) } +/// Writes one manifest at `path` holding `files` as `Added` entries, all of +/// which must belong to `partition_spec_id`, and returns its manifest-list entry. +pub(crate) async fn write_added_manifest( + table: &Table, + path: String, + snapshot_id: i64, + content: ManifestContentType, + partition_spec_id: i32, + files: &[DataFile], +) -> Result { + let metadata = table.metadata(); + let partition_spec = metadata + .partition_spec_by_id(partition_spec_id) + .ok_or_else(|| invalid_data!("Unknown partition spec id {partition_spec_id}"))?; + + let mut writer = new_manifest_writer( + table, + path, + snapshot_id, + content, + metadata.current_schema().clone(), + partition_spec.clone(), + )?; + let format_version = metadata.format_version(); + for file in files { + let builder = ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(file.clone()); + let entry = if format_version == FormatVersion::V1 { + builder.snapshot_id(snapshot_id).build() + } else { + // For format version > 1, the snapshot id is inherited from the + // manifest list, so a rebase does not require rewriting the manifest. + builder.build() + }; + writer.add_entry(entry)?; + } + writer.write_manifest_file().await +} + /// Fails if any of `data_files` is already live in the table's current snapshot. /// /// Must run against each attempt's base, since a concurrent commit may have @@ -238,6 +285,7 @@ fn build_summary( mut properties: HashMap, added_files: &[DataFile], removed_files: &[DataFile], + removed_delete_files: &[DataFile], truncate_full_table: bool, ) -> Result { let metadata = table.metadata(); @@ -260,7 +308,7 @@ fn build_summary( } let mut schemas_by_spec = HashMap::new(); - for data_file in removed_files { + for data_file in removed_files.iter().chain(removed_delete_files) { let partition_spec = metadata .partition_spec_by_id(data_file.partition_spec_id) .ok_or_else(|| { @@ -309,9 +357,12 @@ pub(crate) struct SnapshotCommitInput<'a> { pub manifest_list_path: String, pub operation: Operation, pub properties: HashMap, + /// Data and delete files added by this commit. pub added_files: &'a [DataFile], /// Must match the data files marked Deleted by this commit, including their metrics. pub removed_files: &'a [DataFile], + /// Must match the delete files marked Deleted by this commit, including their metrics. + pub removed_delete_files: &'a [DataFile], pub manifests: Vec, /// Whether the summary resets table totals (a full-table overwrite). pub truncate_full_table: bool, @@ -330,20 +381,29 @@ pub(crate) async fn write_snapshot_commit( properties, added_files, removed_files, + removed_delete_files, manifests, truncate_full_table, } = input; - debug_assert_eq!( + let deleted_count = |content: ManifestContentType| { manifests .iter() - .filter(|m| m.added_snapshot_id == snapshot_id && m.content == ManifestContentType::Data) + .filter(|m| m.added_snapshot_id == snapshot_id && m.content == content) .filter_map(|m| m.deleted_files_count) .map(u64::from) - .sum::(), + .sum::() + }; + debug_assert_eq!( + deleted_count(ManifestContentType::Data), removed_files.len() as u64, "deleted files must match this commit's manifest entries" ); + debug_assert_eq!( + deleted_count(ManifestContentType::Deletes), + removed_delete_files.len() as u64, + "deleted delete files must match this commit's manifest entries" + ); let metadata = table.metadata(); let parent_snapshot_id = metadata.current_snapshot_id(); @@ -356,6 +416,7 @@ pub(crate) async fn write_snapshot_commit( properties, added_files, removed_files, + removed_delete_files, truncate_full_table, ) .map_err(|err| { @@ -458,7 +519,7 @@ pub(crate) async fn write_snapshot_commit( } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::collections::HashMap; use std::sync::Arc; @@ -473,7 +534,7 @@ mod tests { use crate::transaction::tests::make_v2_minimal_table; use crate::{ErrorKind, TableUpdate}; - fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { + pub(super) fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { DataFileBuilder::default() .content(DataContentType::Data) .file_path(path.to_string()) @@ -492,7 +553,7 @@ mod tests { /// Commits `data_files` to `table` through a fresh producer, returning the /// table with that snapshot as current. - async fn append_to(table: &Table, data_files: Vec) -> Table { + pub(super) async fn append_to(table: &Table, data_files: Vec) -> Table { let mut producer = SimpleSnapshotProducer::new(AppendIntent { commit_uuid: None, snapshot_properties: HashMap::new(), @@ -579,6 +640,7 @@ mod tests { HashMap::new(), &[], &[deleted], + &[], false, ) .unwrap(); @@ -675,6 +737,7 @@ mod tests { properties: HashMap::new(), added_files: &[], removed_files: &[deleted], + removed_delete_files: &[], manifests: vec![], truncate_full_table: false, }) diff --git a/crates/iceberg/src/transaction/snapshot/simple.rs b/crates/iceberg/src/transaction/snapshot/simple.rs index 7aaa505399..bd810b846f 100644 --- a/crates/iceberg/src/transaction/snapshot/simple.rs +++ b/crates/iceberg/src/transaction/snapshot/simple.rs @@ -23,13 +23,10 @@ use uuid::Uuid; use super::{ SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, - manifest_path, new_manifest_writer, validate_partition_value, write_snapshot_commit, + manifest_path, validate_partition_value, write_added_manifest, write_snapshot_commit, }; use crate::error::{Result, invalid_data}; -use crate::spec::{ - DataFile, FormatVersion, ManifestContentType, ManifestEntry, ManifestFile, ManifestStatus, - Operation, -}; +use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::ActionCommit; @@ -122,6 +119,7 @@ impl SimpleSnapshotProducer { properties: self.intent.snapshot_properties.clone(), added_files: &self.intent.data_files, removed_files: &[], + removed_delete_files: &[], manifests, truncate_full_table: false, }) @@ -197,6 +195,7 @@ impl SimpleSnapshotProducer { table, path, snapshot_id, + ManifestContentType::Data, table.metadata().default_partition_spec_id(), &self.intent.data_files, ) @@ -211,45 +210,6 @@ impl SimpleSnapshotProducer { } } -/// Writes one data manifest at `path` for `data_files`, all of which must -/// belong to `partition_spec_id`, and returns its manifest-list entry. -async fn write_added_manifest( - table: &Table, - path: String, - snapshot_id: i64, - partition_spec_id: i32, - data_files: &[DataFile], -) -> Result { - let metadata = table.metadata(); - let partition_spec = metadata - .partition_spec_by_id(partition_spec_id) - .ok_or_else(|| invalid_data!("Unknown partition spec id {partition_spec_id}"))?; - - let mut writer = new_manifest_writer( - table, - path, - snapshot_id, - ManifestContentType::Data, - metadata.current_schema().clone(), - partition_spec.clone(), - )?; - let format_version = metadata.format_version(); - for data_file in data_files { - let builder = ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file(data_file.clone()); - let entry = if format_version == FormatVersion::V1 { - builder.snapshot_id(snapshot_id).build() - } else { - // For format version > 1, the snapshot id is inherited from the - // manifest list, so a rebase does not require rewriting the manifest. - builder.build() - }; - writer.add_entry(entry)?; - } - writer.write_manifest_file().await -} - #[cfg(test)] mod tests { use std::collections::HashMap; From 0369c0b39303e2c64fe919e4bf2e0b1acac57f76 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 27 Oct 2026 17:49:51 -0700 Subject: [PATCH 45/50] robust caching --- crates/iceberg/src/io/object_cache.rs | 6 +- .../transaction/snapshot/manifest_filter.rs | 65 ++-- .../src/transaction/snapshot/merging.rs | 295 +++++++++++++++++- .../iceberg/src/transaction/snapshot/mod.rs | 33 +- .../src/transaction/snapshot/simple.rs | 21 +- 5 files changed, 350 insertions(+), 70 deletions(-) diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 874c9d3483..7ac2ecd3ca 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -52,6 +52,10 @@ pub(crate) enum CachedObjectKey { Manifest(ManifestKey), } +/// Identifies a parsed manifest: its path plus every manifest-list value the +/// reader inherits onto its entries. Two references to one physical manifest +/// with different inherited values parse to different entries, so anything +/// derived from parsed entries must be keyed by this, not by path alone. #[derive(Clone, Debug, Hash, Eq, PartialEq)] pub(crate) struct ManifestKey { path: String, @@ -61,7 +65,7 @@ pub(crate) struct ManifestKey { } impl ManifestKey { - fn new(manifest_file: &ManifestFile) -> Self { + pub(crate) fn new(manifest_file: &ManifestFile) -> Self { Self { path: manifest_file.manifest_path.clone(), added_snapshot_id: manifest_file.added_snapshot_id, diff --git a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs index 1debd09553..11eccfe393 100644 --- a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs +++ b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs @@ -23,6 +23,7 @@ use uuid::Uuid; use super::{manifest_path, new_manifest_writer}; use crate::error::{Result, invalid_data}; +use crate::io::object_cache::ManifestKey; use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile}; use crate::table::Table; @@ -30,21 +31,24 @@ use crate::table::Table; /// /// One manager lives for one logical execution of an action and is driven /// once per attempt via [`ManifestFilterManager::filter`]. A manifest that -/// references none of the requested files passes through untouched. One that -/// does is rewritten with those entries marked `Deleted` and the rest -/// `Existing`; the rewrite is keyed by the source manifest's path, which is -/// immutable, so a rebase only rewrites manifests not seen before. +/// references none of the files passes through untouched. One that does is +/// rewritten with those entries marked `Deleted` and the rest `Existing`. +/// +/// A rewrite is reused across attempts while its inputs are unchanged. Its +/// entries carry the sequence numbers and row ids the source's entries +/// inherited from the manifest list when they were read, so the reuse key is +/// the source's [`ManifestKey`], not its path alone: a transaction-local +/// source manifest keeps its path across attempts but inherits new values +/// after a rebase. pub(crate) struct ManifestFilterManager { content: ManifestContentType, /// Paths of the files to remove. Fixed for the execution. - requested_paths: HashSet, - /// Source manifest path → rewritten result. - filtered: HashMap, + files_to_remove: HashSet, + /// Source manifest → rewritten result. + filtered: HashMap, /// Format version the rewrites were written under. Per spec, v1 entries /// cannot inherit sequence numbers, so an upgrade invalidates them. format_version: Option, - /// Metadata files written by this manager. - owned_artifacts: HashSet, } /// One rewritten manifest and the files it marked as deleted. @@ -64,32 +68,27 @@ pub(crate) struct Filtered { } impl ManifestFilterManager { - pub(crate) fn new(content: ManifestContentType, removed_files: &[DataFile]) -> Self { + pub(crate) fn new(content: ManifestContentType, files_to_remove: &[DataFile]) -> Self { Self { content, - requested_paths: removed_files + files_to_remove: files_to_remove .iter() .map(|f| f.file_path().to_string()) .collect(), filtered: HashMap::new(), format_version: None, - owned_artifacts: HashSet::new(), } } - /// Metadata files written by this manager so far. - pub(crate) fn owned_artifacts(&self) -> &HashSet { - &self.owned_artifacts - } - /// Applies the removals to `manifests`, rewriting those of this manager's - /// content type that reference a requested file. + /// content type that reference a file to remove. /// - /// Fails if a requested file is not live in any manifest: it was never in + /// Fails if a file to remove is not live in any manifest: it was never in /// the table, or a concurrent commit already removed it. /// - /// `next_manifest_number` allocates paths for rewritten manifests from the - /// producer's counter, so they never collide with other manifests it writes. + /// Rewritten manifests take their paths from `next_manifest_number` and are + /// recorded in `owned_artifacts` before being written, so a failed write + /// still leaves its path tracked for cleanup. pub(crate) async fn filter( &mut self, table: &Table, @@ -97,8 +96,9 @@ impl ManifestFilterManager { commit_uuid: Uuid, manifests: Vec, next_manifest_number: &mut u64, + owned_artifacts: &mut HashSet, ) -> Result { - if self.requested_paths.is_empty() { + if self.files_to_remove.is_empty() { return Ok(Filtered { manifests, removed_files: Vec::new(), @@ -120,7 +120,8 @@ impl ManifestFilterManager { continue; } - let result = match self.filtered.get(&source.manifest_path) { + let key = ManifestKey::new(&source); + let result = match self.filtered.get(&key) { Some(cached) => cached.clone(), None => { let result = self @@ -130,10 +131,10 @@ impl ManifestFilterManager { commit_uuid, &source, next_manifest_number, + owned_artifacts, ) .await?; - self.filtered - .insert(source.manifest_path.clone(), result.clone()); + self.filtered.insert(key, result.clone()); result } }; @@ -144,7 +145,7 @@ impl ManifestFilterManager { let found: HashSet<&str> = removed_files.iter().map(|f| f.file_path()).collect(); let mut missing: Vec<&str> = self - .requested_paths + .files_to_remove .iter() .map(String::as_str) .filter(|p| !found.contains(p)) @@ -163,22 +164,23 @@ impl ManifestFilterManager { }) } - /// Reads `source` and, if it references any requested file, writes a copy + /// Reads `source` and, if it references any file to remove, writes a copy /// with those entries marked `Deleted` and the rest `Existing`. async fn rewrite( - &mut self, + &self, table: &Table, snapshot_id: i64, commit_uuid: Uuid, source: &ManifestFile, next_manifest_number: &mut u64, + owned_artifacts: &mut HashSet, ) -> Result { let manifest = table.object_cache().get_manifest(source).await?; let removed_files: Vec = manifest .entries() .iter() - .filter(|e| e.is_alive() && self.requested_paths.contains(e.file_path())) + .filter(|e| e.is_alive() && self.files_to_remove.contains(e.file_path())) .map(|e| e.data_file().clone()) .collect(); if removed_files.is_empty() { @@ -188,10 +190,9 @@ impl ManifestFilterManager { }); } - // Allocate the path before writing so a partial write is still owned. let path = manifest_path(table, commit_uuid, *next_manifest_number)?; *next_manifest_number += 1; - self.owned_artifacts.insert(path.clone()); + owned_artifacts.insert(path.clone()); // Keep the source manifest's spec: its entries' partition values are // laid out for that spec, not the table's current default. @@ -209,7 +210,7 @@ impl ManifestFilterManager { continue; } let entry = (**entry).clone(); - if self.requested_paths.contains(entry.file_path()) { + if self.files_to_remove.contains(entry.file_path()) { writer.add_delete_entry(entry)?; } else { writer.add_existing_entry(entry)?; diff --git a/crates/iceberg/src/transaction/snapshot/merging.rs b/crates/iceberg/src/transaction/snapshot/merging.rs index 16e1ba85f4..8d370befe3 100644 --- a/crates/iceberg/src/transaction/snapshot/merging.rs +++ b/crates/iceberg/src/transaction/snapshot/merging.rs @@ -24,8 +24,9 @@ use uuid::Uuid; use super::manifest_filter::ManifestFilterManager; use super::{ - SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, - manifest_path, validate_partition_value, write_added_manifest, write_snapshot_commit, + AddedManifests, SnapshotCommitInput, current_manifests, generate_snapshot_id, + manifest_list_path, manifest_path, validate_partition_value, write_added_manifest, + write_snapshot_commit, }; use crate::error::{Result, invalid_data}; use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; @@ -81,15 +82,6 @@ pub(crate) struct MergingSnapshotProducer { owned_artifacts: HashSet, } -/// The manifests written for one execution's added data and delete files. -/// -/// Reusable while the table's format version is unchanged; see -/// `simple::AddedManifests` for why. -struct AddedManifests { - manifests: Vec, - format_version: FormatVersion, -} - impl MergingSnapshotProducer { pub(crate) fn new(intent: MergingIntent) -> Self { Self { @@ -128,6 +120,7 @@ impl MergingSnapshotProducer { self.commit_uuid, manifests, &mut self.manifest_counter, + &mut self.owned_artifacts, ) .await?; let deletes = self @@ -138,13 +131,10 @@ impl MergingSnapshotProducer { self.commit_uuid, data.manifests, &mut self.manifest_counter, + &mut self.owned_artifacts, ) .await?; let mut manifests = deletes.manifests; - self.owned_artifacts - .extend(self.data_filter.owned_artifacts().iter().cloned()); - self.owned_artifacts - .extend(self.delete_filter.owned_artifacts().iter().cloned()); manifests.extend(self.added_manifests(table, snapshot_id).await?); @@ -529,6 +519,281 @@ mod tests { assert_eq!(err.kind(), ErrorKind::DataInvalid); } + /// Multi-action replay (RFC §5.2): action A's manifest keeps its path across + /// attempts, but after a rebase the manifest list assigns it a new sequence + /// number. Action B's rewrite of that manifest bakes the inherited sequence + /// numbers into its `Existing`/`Deleted` entries, so B must not reuse a + /// rewrite made under the previous sequence number. + #[tokio::test] + async fn test_rewrite_is_not_reused_when_source_manifest_sequence_number_changes() { + use super::super::simple::{AppendIntent, SimpleSnapshotProducer}; + use super::super::tests::with_snapshot; + + let s0 = make_v2_minimal_table(); + + // Action A appends x and y; action B removes x from A's manifest. + let mut a = SimpleSnapshotProducer::new(AppendIntent { + commit_uuid: None, + snapshot_properties: HashMap::new(), + data_files: vec![ + data_file(&s0, "test/x.parquet", 1), + data_file(&s0, "test/y.parquet", 1), + ], + }); + let mut b_intent = intent(Operation::Delete); + b_intent.removed_data_files = vec![data_file(&s0, "test/x.parquet", 1)]; + let mut b = MergingSnapshotProducer::new(b_intent); + + // Attempt 0 on S0. + let sa0 = added_snapshot(&mut a.apply(&s0).await.unwrap()); + let after_a0 = with_snapshot(&s0, sa0.clone()); + let sb0 = added_snapshot(&mut b.apply(&after_a0).await.unwrap()); + + // A concurrent commit lands on S0, taking the next sequence number. The + // retry rebases onto it: A reuses its manifest but it now inherits a new + // sequence number, and B sees the same manifest path under it. + let s1 = append_to(&s0, vec![data_file(&s0, "test/other.parquet", 1)]).await; + let sa1 = added_snapshot(&mut a.apply(&s1).await.unwrap()); + let after_a1 = with_snapshot(&s1, sa1.clone()); + let sb1 = added_snapshot(&mut b.apply(&after_a1).await.unwrap()); + assert_eq!(sa1.sequence_number(), sa0.sequence_number() + 1); + + // B's rewritten manifest in each attempt. + let rewritten_in = |table: Table, snapshot: SnapshotRef| async move { + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .cloned() + .unwrap(); + let manifest = table.manifest_reader().read(&rewritten).await.unwrap(); + let y = manifest + .entries() + .iter() + .find(|e| e.file_path() == "test/y.parquet") + .unwrap(); + (rewritten.manifest_path.clone(), y.sequence_number()) + }; + let (path0, y_seq0) = rewritten_in(after_a0, sb0).await; + let (path1, y_seq1) = rewritten_in(after_a1, sb1).await; + + // `y` stays live under A's snapshot, whose sequence number moved; the + // rewrite that keeps it as `Existing` must carry the new one. + assert_eq!(y_seq0, Some(sa0.sequence_number())); + assert_eq!(y_seq1, Some(sa1.sequence_number())); + assert_ne!( + path0, path1, + "a rewrite under a different sequence number is a new file" + ); + } + + /// Removing a data file from a v1 table: v1 entries read with sequence + /// number 0, which satisfies the writer, and v1 serialization drops the + /// field again. The rewritten manifest is readable and `b` stays live. + #[tokio::test] + async fn test_remove_data_file_from_v1_table() { + let base = make_v1_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut commit = MergingSnapshotProducer::new(i).apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + let manifest = table.manifest_reader().read(rewritten).await.unwrap(); + let mut live: Vec<_> = manifest + .entries() + .iter() + .filter(|e| e.is_alive()) + .map(|e| e.file_path().to_string()) + .collect(); + live.sort(); + assert_eq!(live, vec!["test/b.parquet".to_string()]); + // Both the rewritten entries and the snapshot stay at v1's sequence number 0. + assert!( + manifest + .entries() + .iter() + .all(|e| e.sequence_number() == Some(0)) + ); + assert_eq!(snapshot.sequence_number(), 0); + assert_eq!( + snapshot + .summary() + .additional_properties + .get("deleted-data-files") + .unwrap(), + "1" + ); + } + + /// A manifest written under an earlier schema is rewritten under the + /// current one; its partition spec still binds because partition source + /// columns cannot be dropped. + #[tokio::test] + async fn test_rewrite_manifest_written_under_earlier_schema() { + use crate::spec::{NestedField, PrimitiveType, Type}; + + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + // Evolve the schema after the manifest was written. + let new_schema = table + .metadata() + .current_schema() + .as_ref() + .clone() + .into_builder() + .with_fields([ + NestedField::optional(100, "added", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .add_current_schema(new_schema) + .unwrap() + .build() + .unwrap() + .metadata; + let evolved = table.clone().with_metadata(std::sync::Arc::new(metadata)); + assert_ne!( + evolved.metadata().current_schema_id(), + table.metadata().current_schema_id() + ); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut commit = MergingSnapshotProducer::new(i) + .apply(&evolved) + .await + .unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = evolved + .manifest_list_reader(&snapshot) + .load() + .await + .unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + assert_eq!( + rewritten.partition_spec_id, + table.metadata().default_partition_spec_id() + ); + assert_eq!(entries_of(&evolved, rewritten).await, vec![ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + ]); + // Partition values survive the rewrite under the new schema. + let manifest = evolved.manifest_reader().read(rewritten).await.unwrap(); + for entry in manifest.entries() { + assert_eq!( + entry.data_file().partition(), + &Struct::from_iter([Some(Literal::long(300))]) + ); + } + } + + /// On v3, existing entries keep the row ids they were assigned when first + /// committed, and the rewritten manifest gets its own range from the list + /// writer. + #[tokio::test] + async fn test_rewrite_preserves_first_row_ids_on_v3() { + use crate::transaction::tests::make_v3_minimal_table; + + let base = make_v3_minimal_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 5), + data_file(&base, "test/b.parquet", 7), + ]) + .await; + let before = table.metadata().current_snapshot().unwrap(); + let before_list = table.manifest_list_reader(before).load().await.unwrap(); + let before_manifest = table + .manifest_reader() + .read(&before_list.entries()[0]) + .await + .unwrap(); + let b_row_id = before_manifest + .entries() + .iter() + .find(|e| e.file_path() == "test/b.parquet") + .unwrap() + .data_file() + .first_row_id(); + assert_eq!(b_row_id, Some(5)); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 5)]; + let mut commit = MergingSnapshotProducer::new(i).apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + let manifest = table.manifest_reader().read(rewritten).await.unwrap(); + let b = manifest + .entries() + .iter() + .find(|e| e.file_path() == "test/b.parquet") + .unwrap(); + assert_eq!(b.data_file().first_row_id(), b_row_id); + // The new snapshot starts its range where the table's next_row_id was. + assert_eq!( + snapshot.first_row_id(), + Some(table.metadata().next_row_id()) + ); + } + + /// Every manifest the producer writes, including rewrites, is recorded as + /// owned before the write starts. + #[tokio::test] + async fn test_owned_artifacts_cover_rewrites_and_added_manifests() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Overwrite); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + i.added_data_files = vec![data_file(&base, "test/x.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + for m in list.entries() { + if m.added_snapshot_id == snapshot.snapshot_id() { + assert!(producer.owned_artifacts.contains(&m.manifest_path)); + } + } + assert!(producer.owned_artifacts.contains(snapshot.manifest_list())); + // One rewrite, one added manifest, one manifest list. + assert_eq!(producer.owned_artifacts.len(), 3); + } + #[tokio::test] async fn test_delete_files_rejected_on_v1_table() { let table = make_v1_table(); diff --git a/crates/iceberg/src/transaction/snapshot/mod.rs b/crates/iceberg/src/transaction/snapshot/mod.rs index 6ccc90f3cf..6e347756c7 100644 --- a/crates/iceberg/src/transaction/snapshot/mod.rs +++ b/crates/iceberg/src/transaction/snapshot/mod.rs @@ -180,6 +180,21 @@ pub(crate) async fn current_manifests(table: &Table) -> Result .collect()) } +/// The manifests a producer wrote for one execution's added files. +/// +/// Reusable while the table's format version is unchanged. Per spec, sequence +/// number inheritance does not apply to v1 manifests, so one written before a +/// concurrent upgrade cannot carry the v2 sequence number a retry assigns and +/// must be rewritten. The schema is not a guard: the manifest's embedded schema +/// describes the files it was written for, which do not change. Nor is the +/// default partition spec: the files name their own spec, which stays valid. +pub(crate) struct AddedManifests { + /// One per content type today. Several once added files are grouped by + /// partition spec (#3500) or rolled over at a target size (#3499). + pub manifests: Vec, + pub format_version: FormatVersion, +} + /// Writes one manifest at `path` holding `files` as `Added` entries, all of /// which must belong to `partition_spec_id`, and returns its manifest-list entry. pub(crate) async fn write_added_manifest( @@ -559,17 +574,25 @@ pub(crate) mod tests { snapshot_properties: HashMap::new(), data_files, }); - let snapshot = producer - .apply(table) - .await - .unwrap() + let commit = producer.apply(table).await.unwrap(); + with_snapshot(table, added_snapshot(commit)) + } + + /// Extracts the snapshot an `ActionCommit` adds. + pub(super) fn added_snapshot(mut commit: ActionCommit) -> SnapshotRef { + commit .take_updates() .into_iter() .find_map(|u| match u { TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot)), _ => None, }) - .expect("append should add a snapshot"); + .expect("commit should add a snapshot") + } + + /// Returns `table` with `snapshot` added and set as current, as a local + /// `TableCommit::apply` would. + pub(super) fn with_snapshot(table: &Table, snapshot: SnapshotRef) -> Table { let metadata = table .metadata() .clone() diff --git a/crates/iceberg/src/transaction/snapshot/simple.rs b/crates/iceberg/src/transaction/snapshot/simple.rs index bd810b846f..4da190e2f6 100644 --- a/crates/iceberg/src/transaction/snapshot/simple.rs +++ b/crates/iceberg/src/transaction/snapshot/simple.rs @@ -22,11 +22,12 @@ use std::collections::{HashMap, HashSet}; use uuid::Uuid; use super::{ - SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, - manifest_path, validate_partition_value, write_added_manifest, write_snapshot_commit, + AddedManifests, SnapshotCommitInput, current_manifests, generate_snapshot_id, + manifest_list_path, manifest_path, validate_partition_value, write_added_manifest, + write_snapshot_commit, }; use crate::error::{Result, invalid_data}; -use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; +use crate::spec::{DataFile, ManifestContentType, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::ActionCommit; @@ -73,20 +74,6 @@ pub(crate) struct SimpleSnapshotProducer { owned_artifacts: HashSet, } -/// The manifests written for one append's added files. -/// -/// Reusable while the table's format version is unchanged. Per spec, sequence -/// number inheritance does not apply to v1 manifests, so one written before a -/// concurrent upgrade cannot carry the v2 sequence number a retry assigns and -/// must be rewritten. The schema is not a guard: the manifest's embedded schema -/// describes the files it was written for, which do not change. -struct AddedManifests { - /// One manifest today. Several once added files are grouped by partition - /// spec (#3500) or rolled over at a target size (#3499). - manifests: Vec, - format_version: FormatVersion, -} - impl SimpleSnapshotProducer { pub(crate) fn new(intent: AppendIntent) -> Self { Self { From 8efe3742611c0df8e3ef07fa002c0d952845eff7 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Wed, 28 Oct 2026 13:40:31 -0700 Subject: [PATCH 46/50] use manifest's schema --- crates/iceberg/src/spec/transform.rs | 18 +++ .../transaction/snapshot/manifest_filter.rs | 11 +- .../src/transaction/snapshot/merging.rs | 114 ++++++++++++++++++ 3 files changed, 138 insertions(+), 5 deletions(-) diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index f52202b78b..576b6553a9 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -221,6 +221,9 @@ impl Transform { (Self::Hour, PrimitiveLiteral::Int(ordinal)) => Self::human_hour(ordinal), (_, value) => { let field_type = field_type.as_primitive_type().unwrap(); + // A value written before its source column was promoted (`int` + // to `long`, `float` to `double`) still carries the old type. + let value = value.promote_to(field_type).unwrap_or(value); Datum::new(field_type.clone(), value).to_human_string() } } @@ -1322,6 +1325,21 @@ mod tests { ); } + /// A partition value written before its source column was promoted keeps + /// the old literal type; rendering it under the promoted type must widen it + /// rather than panic. + #[test] + fn test_to_human_string_promotes_value_to_declared_type() { + assert_eq!(human(Transform::Identity, PrimitiveType::Long, 7), "7"); + assert_eq!( + Transform::Identity.to_human_string( + &Type::Primitive(PrimitiveType::Double), + Some(&Literal::float(1.5_f32)) + ), + "1.5" + ); + } + #[test] fn test_to_human_string_null_cases() { assert_eq!(human(Transform::Void, PrimitiveType::Int, 47), "null"); diff --git a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs index 11eccfe393..e977adb89b 100644 --- a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs +++ b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs @@ -194,16 +194,17 @@ impl ManifestFilterManager { *next_manifest_number += 1; owned_artifacts.insert(path.clone()); - // Keep the source manifest's spec: its entries' partition values are - // laid out for that spec, not the table's current default. - let spec = manifest.metadata().partition_spec().clone(); + // Write under the source manifest's own schema and spec: its entries' + // partition values and column bounds were encoded for that schema's + // types, which the table's current schema may since have promoted. + let source_metadata = manifest.metadata(); let mut writer = new_manifest_writer( table, path, snapshot_id, self.content, - table.metadata().current_schema().clone(), - spec.into(), + source_metadata.schema().clone(), + source_metadata.partition_spec().clone().into(), )?; for entry in manifest.entries() { if !entry.is_alive() { diff --git a/crates/iceberg/src/transaction/snapshot/merging.rs b/crates/iceberg/src/transaction/snapshot/merging.rs index 8d370befe3..3e93f8aba2 100644 --- a/crates/iceberg/src/transaction/snapshot/merging.rs +++ b/crates/iceberg/src/transaction/snapshot/merging.rs @@ -715,6 +715,120 @@ mod tests { } } + /// A manifest's partition values are encoded for the partition type its + /// spec had when it was written. Promoting the source column (`int` to + /// `long`) changes that type under the current schema; the rewrite must + /// keep the entries readable and their partition values intact. + #[tokio::test] + async fn test_rewrite_manifest_after_partition_source_type_promotion() { + use crate::TableIdent; + use crate::io::FileIO; + use crate::spec::{ + FormatVersion, NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, + TableMetadataBuilder, Transform, Type, + }; + use crate::test_utils::test_runtime; + + let int_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let spec = PartitionSpec::builder(int_schema.clone()) + .add_partition_field("x", "x", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let metadata = TableMetadataBuilder::new( + int_schema, + spec, + SortOrder::unsorted_order(), + "memory:///table".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + let base = Table::builder() + .metadata(metadata) + .metadata_location("memory:///table/metadata/v1.json") + .identifier(TableIdent::from_strs(["ns", "t"]).unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) + .build() + .unwrap(); + + let int_file = |path: &str| { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(base.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::int(7))])) + .build() + .unwrap() + }; + let table = append_to(&base, vec![ + int_file("test/a.parquet"), + int_file("test/b.parquet"), + ]) + .await; + + // Promote x: int -> long. + let long_schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .add_current_schema(long_schema) + .unwrap() + .build() + .unwrap() + .metadata; + let promoted = table.clone().with_metadata(std::sync::Arc::new(metadata)); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![int_file("test/a.parquet")]; + let mut commit = MergingSnapshotProducer::new(i) + .apply(&promoted) + .await + .unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = promoted + .manifest_list_reader(&snapshot) + .load() + .await + .unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + assert_eq!(entries_of(&promoted, rewritten).await, vec![ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + ]); + let manifest = promoted.manifest_reader().read(rewritten).await.unwrap(); + for entry in manifest.entries() { + assert_eq!( + entry.data_file().partition(), + &Struct::from_iter([Some(Literal::int(7))]) + ); + } + } + /// On v3, existing entries keep the row ids they were assigned when first /// committed, and the rewritten manifest gets its own range from the list /// writer. From 0d80948a19acc62a106e46b78131475330b796ae Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 3 Nov 2026 14:35:30 -0800 Subject: [PATCH 47/50] add fail_missing_delete_paths, validate delete files identify and version comb better, specify comments --- .../transaction/snapshot/manifest_filter.rs | 96 +++++-- .../src/transaction/snapshot/merging.rs | 267 +++++++++++++++++- .../iceberg/src/transaction/snapshot/mod.rs | 1 + 3 files changed, 330 insertions(+), 34 deletions(-) diff --git a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs index e977adb89b..378148bae2 100644 --- a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs +++ b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs @@ -27,6 +27,39 @@ use crate::io::object_cache::ManifestKey; use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile}; use crate::table::Table; +/// Identifies one entry to mark as deleted. +/// +/// A path alone is not enough: a Puffin file may hold several deletion +/// vectors, each its own entry at a distinct blob offset. Data files and +/// non-DV delete files have no offset, so for them this is the path. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct DeleteFileIdentity { + /// The absolute path, as stored in the manifest entry. Comparison is + /// textual, so a relative path would not match the entry it names. + path: String, + content_offset: Option, + content_size_in_bytes: Option, +} + +impl DeleteFileIdentity { + fn of(file: &DataFile) -> Self { + Self { + path: file.file_path().to_string(), + content_offset: file.content_offset(), + content_size_in_bytes: file.content_size_in_bytes(), + } + } +} + +impl std::fmt::Display for DeleteFileIdentity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match (self.content_offset, self.content_size_in_bytes) { + (Some(offset), Some(size)) => write!(f, "{}@{offset}+{size}", self.path), + _ => f.write_str(&self.path), + } + } +} + /// Removes a fixed set of files from the manifests of one content type. /// /// One manager lives for one logical execution of an action and is driven @@ -42,8 +75,13 @@ use crate::table::Table; /// after a rebase. pub(crate) struct ManifestFilterManager { content: ManifestContentType, - /// Paths of the files to remove. Fixed for the execution. - files_to_remove: HashSet, + /// The files to remove, by absolute path (and blob range for deletion + /// vectors). Fixed for the execution. + files_to_remove: HashSet, + /// Whether a file to remove that is not live in any manifest fails the + /// attempt. Off lets a retry succeed after a concurrent commit removed the + /// same file; the action decides which it needs. + fail_missing_delete_paths: bool, /// Source manifest → rewritten result. filtered: HashMap, /// Format version the rewrites were written under. Per spec, v1 entries @@ -68,13 +106,15 @@ pub(crate) struct Filtered { } impl ManifestFilterManager { - pub(crate) fn new(content: ManifestContentType, files_to_remove: &[DataFile]) -> Self { + pub(crate) fn new( + content: ManifestContentType, + files_to_remove: &[DataFile], + fail_missing_delete_paths: bool, + ) -> Self { Self { content, - files_to_remove: files_to_remove - .iter() - .map(|f| f.file_path().to_string()) - .collect(), + files_to_remove: files_to_remove.iter().map(DeleteFileIdentity::of).collect(), + fail_missing_delete_paths, filtered: HashMap::new(), format_version: None, } @@ -83,8 +123,9 @@ impl ManifestFilterManager { /// Applies the removals to `manifests`, rewriting those of this manager's /// content type that reference a file to remove. /// - /// Fails if a file to remove is not live in any manifest: it was never in - /// the table, or a concurrent commit already removed it. + /// With `fail_missing_delete_paths`, fails if a file to remove is not live + /// in any manifest: it was never in the table, or a concurrent commit + /// already removed it. Otherwise such files are skipped. /// /// Rewritten manifests take their paths from `next_manifest_number` and are /// recorded in `owned_artifacts` before being written, so a failed write @@ -143,19 +184,22 @@ impl ManifestFilterManager { out.push(result.manifest); } - let found: HashSet<&str> = removed_files.iter().map(|f| f.file_path()).collect(); - let mut missing: Vec<&str> = self - .files_to_remove - .iter() - .map(String::as_str) - .filter(|p| !found.contains(p)) - .collect(); - if !missing.is_empty() { - missing.sort_unstable(); - return Err(invalid_data!( - "Cannot remove files that are not live in the table: {}", - missing.join(", ") - )); + if self.fail_missing_delete_paths { + let found: HashSet = + removed_files.iter().map(DeleteFileIdentity::of).collect(); + let mut missing: Vec = self + .files_to_remove + .iter() + .filter(|f| !found.contains(f)) + .map(ToString::to_string) + .collect(); + if !missing.is_empty() { + missing.sort_unstable(); + return Err(invalid_data!( + "Cannot remove files that are not live in the table: {}", + missing.join(", ") + )); + } } Ok(Filtered { @@ -180,7 +224,7 @@ impl ManifestFilterManager { let removed_files: Vec = manifest .entries() .iter() - .filter(|e| e.is_alive() && self.files_to_remove.contains(e.file_path())) + .filter(|e| e.is_alive() && self.removes(e.data_file())) .map(|e| e.data_file().clone()) .collect(); if removed_files.is_empty() { @@ -211,7 +255,7 @@ impl ManifestFilterManager { continue; } let entry = (**entry).clone(); - if self.files_to_remove.contains(entry.file_path()) { + if self.removes(entry.data_file()) { writer.add_delete_entry(entry)?; } else { writer.add_existing_entry(entry)?; @@ -224,4 +268,8 @@ impl ManifestFilterManager { removed_files, }) } + + fn removes(&self, file: &DataFile) -> bool { + self.files_to_remove.contains(&DeleteFileIdentity::of(file)) + } } diff --git a/crates/iceberg/src/transaction/snapshot/merging.rs b/crates/iceberg/src/transaction/snapshot/merging.rs index 3e93f8aba2..683ad0d0bb 100644 --- a/crates/iceberg/src/transaction/snapshot/merging.rs +++ b/crates/iceberg/src/transaction/snapshot/merging.rs @@ -29,7 +29,10 @@ use super::{ write_snapshot_commit, }; use crate::error::{Result, invalid_data}; -use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; +use crate::spec::{ + DataContentType, DataFile, DataFileFormat, FormatVersion, ManifestContentType, ManifestFile, + Operation, +}; use crate::table::Table; use crate::transaction::ActionCommit; @@ -48,6 +51,14 @@ pub(crate) struct MergingIntent { pub added_delete_files: Vec, pub removed_data_files: Vec, pub removed_delete_files: Vec, + /// Whether a removed file that is no longer live fails the commit. + /// + /// Required by actions that must not double-remove (a rewrite whose + /// replacement files would otherwise duplicate rows). Actions whose + /// removal is harmless when already done, such as a merge-on-read delete + /// replacing a position delete file, leave it off so a retry after a + /// concurrent removal still commits. + pub fail_missing_delete_paths: bool, /// Whether the summary resets table totals (a full-table overwrite). pub truncate_full_table: bool, } @@ -89,10 +100,12 @@ impl MergingSnapshotProducer { data_filter: ManifestFilterManager::new( ManifestContentType::Data, &intent.removed_data_files, + intent.fail_missing_delete_paths, ), delete_filter: ManifestFilterManager::new( ManifestContentType::Deletes, &intent.removed_delete_files, + intent.fail_missing_delete_paths, ), intent, snapshot_id: None, @@ -134,7 +147,17 @@ impl MergingSnapshotProducer { &mut self.owned_artifacts, ) .await?; - let mut manifests = deletes.manifests; + // A manifest from an earlier snapshot whose entries are all `Deleted` + // has nothing left to contribute; dropping it keeps the manifest list + // from growing with every commit. This commit's own rewrites stay even + // when delete-only, since they record what it removed. + let mut manifests: Vec = deletes + .manifests + .into_iter() + .filter(|m| { + m.has_added_files() || m.has_existing_files() || m.added_snapshot_id == snapshot_id + }) + .collect(); manifests.extend(self.added_manifests(table, snapshot_id).await?); @@ -163,20 +186,17 @@ impl MergingSnapshotProducer { } /// Checks that every added file is written under the table's default - /// partition spec with a compatible partition value, and that delete files - /// are only added to tables whose format supports them. + /// partition spec with a compatible partition value, and that every added + /// delete file is in a format the table's version allows. /// /// Runs against the table the manifests are written for; a later rebase /// does not invalidate manifests already written under a spec the table /// still has. fn validate_added_files(&self, table: &Table) -> Result<()> { let metadata = table.metadata(); - if !self.intent.added_delete_files.is_empty() - && metadata.format_version() == FormatVersion::V1 - { - return Err(invalid_data!( - "Cannot add delete files to a format version 1 table" - )); + let format_version = metadata.format_version(); + for file in &self.intent.added_delete_files { + validate_delete_file_format(file, format_version)?; } for file in self .intent @@ -266,6 +286,29 @@ impl MergingSnapshotProducer { } } +/// Checks that a delete file's format is allowed by the table's version: v1 +/// has no deletes, v2 has no deletion vectors, and v3 position deletes must be +/// deletion vectors. +fn validate_delete_file_format(file: &DataFile, format_version: FormatVersion) -> Result<()> { + let is_dv = file.file_format() == DataFileFormat::Puffin; + match format_version { + FormatVersion::V1 => Err(invalid_data!( + "Cannot add delete files to a format version 1 table" + )), + FormatVersion::V2 if is_dv => Err(invalid_data!( + "Cannot add deletion vector {} to a format version 2 table", + file.file_path() + )), + FormatVersion::V3 if file.content_type() == DataContentType::PositionDeletes && !is_dv => { + Err(invalid_data!( + "Position deletes must be deletion vectors in a format version 3 table: {}", + file.file_path() + )) + } + _ => Ok(()), + } +} + #[cfg(test)] mod tests { use std::collections::HashMap; @@ -302,6 +345,7 @@ mod tests { added_delete_files: vec![], removed_data_files: vec![], removed_delete_files: vec![], + fail_missing_delete_paths: true, truncate_full_table: false, } } @@ -920,4 +964,207 @@ mod tests { .expect("v1 tables cannot hold delete files"); assert_eq!(err.kind(), ErrorKind::DataInvalid); } + + /// A deletion vector: one blob of a Puffin file, located by offset and size. + fn deletion_vector(table: &Table, path: &str, offset: i64, size: i64) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::PositionDeletes) + .file_path(path.to_string()) + .file_format(DataFileFormat::Puffin) + .file_size_in_bytes(200) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .referenced_data_file(Some(format!("test/{offset}.parquet"))) + .content_offset(Some(offset)) + .content_size_in_bytes(Some(size)) + .build() + .unwrap() + } + + fn equality_delete_file(table: &Table, path: &str) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::EqualityDeletes) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(50) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .equality_ids(Some(vec![1])) + .build() + .unwrap() + } + + #[tokio::test] + async fn test_deletion_vector_rejected_on_v2_table() { + let table = make_v2_minimal_table(); + let mut i = intent(Operation::Overwrite); + i.added_delete_files = vec![deletion_vector(&table, "test/dv.puffin", 4, 10)]; + let err = MergingSnapshotProducer::new(i) + .apply(&table) + .await + .err() + .expect("v2 tables cannot hold deletion vectors"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("test/dv.puffin")); + } + + #[tokio::test] + async fn test_position_delete_file_rejected_on_v3_table() { + use crate::transaction::tests::make_v3_minimal_table; + + let table = make_v3_minimal_table(); + let mut i = intent(Operation::Overwrite); + i.added_delete_files = vec![delete_file(&table, "test/d-deletes.parquet")]; + let err = MergingSnapshotProducer::new(i) + .apply(&table) + .await + .err() + .expect("v3 position deletes must be deletion vectors"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("test/d-deletes.parquet")); + } + + /// v3 still accepts equality delete files and deletion vectors. + #[tokio::test] + async fn test_v3_accepts_equality_deletes_and_deletion_vectors() { + use crate::transaction::tests::make_v3_minimal_table; + + let table = make_v3_minimal_table(); + let mut i = intent(Operation::Overwrite); + i.added_delete_files = vec![ + equality_delete_file(&table, "test/eq.parquet"), + deletion_vector(&table, "test/dv.puffin", 4, 10), + ]; + let mut commit = MergingSnapshotProducer::new(i).apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + assert_eq!( + snapshot + .summary() + .additional_properties + .get("added-delete-files") + .unwrap(), + "2" + ); + } + + /// Two deletion vectors share one Puffin file. Removing one marks only + /// that blob's entry `Deleted`; the other stays `Existing`. + #[tokio::test] + async fn test_remove_one_deletion_vector_keeps_others_in_same_puffin_file() { + use crate::transaction::tests::make_v3_minimal_table; + + let base = make_v3_minimal_table(); + let dv_a = deletion_vector(&base, "test/dvs.puffin", 4, 10); + let dv_b = deletion_vector(&base, "test/dvs.puffin", 14, 12); + + let mut add = intent(Operation::Overwrite); + add.added_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + add.added_delete_files = vec![dv_a.clone(), dv_b.clone()]; + let mut commit = MergingSnapshotProducer::new(add) + .apply(&base) + .await + .unwrap(); + let table = super::super::tests::with_snapshot(&base, added_snapshot(&mut commit)); + + let mut remove = intent(Operation::Delete); + remove.removed_delete_files = vec![dv_a.clone()]; + let mut commit = MergingSnapshotProducer::new(remove) + .apply(&table) + .await + .unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| { + m.added_snapshot_id == snapshot.snapshot_id() + && m.content == ManifestContentType::Deletes + }) + .expect("the delete manifest is rewritten"); + let manifest = table.manifest_reader().read(rewritten).await.unwrap(); + let mut entries: Vec<_> = manifest + .entries() + .iter() + .map(|e| (e.status(), e.data_file().content_offset())) + .collect(); + entries.sort_by_key(|(_, offset)| *offset); + assert_eq!(entries, vec![ + (ManifestStatus::Deleted, dv_a.content_offset()), + (ManifestStatus::Existing, dv_b.content_offset()), + ]); + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("removed-delete-files").unwrap(), "1"); + assert_eq!(summary.get("total-delete-files").unwrap(), "1"); + } + + /// With missing-file failures off, a file removed by a concurrent commit + /// is skipped and the attempt commits; nothing is rewritten for it. + #[tokio::test] + async fn test_missing_removed_file_is_skipped_when_not_failing() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/nope.parquet", 1)]; + i.fail_missing_delete_paths = false; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + assert!( + list.entries() + .iter() + .all(|m| m.added_snapshot_id != snapshot.snapshot_id()) + ); + assert_eq!(producer.manifest_counter, 0); + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("total-data-files").unwrap(), "1"); + assert!(!summary.contains_key("deleted-data-files")); + } + + /// A manifest left with only `Deleted` entries by an earlier snapshot is + /// dropped by the next commit; the one this commit writes is kept. + #[tokio::test] + async fn test_delete_only_manifest_from_earlier_snapshot_is_dropped() { + use super::super::tests::with_snapshot; + + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut remove = intent(Operation::Delete); + remove.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut commit = MergingSnapshotProducer::new(remove) + .apply(&table) + .await + .unwrap(); + let s1 = added_snapshot(&mut commit); + let s1_list = table.manifest_list_reader(&s1).load().await.unwrap(); + assert_eq!(s1_list.entries().len(), 1); + assert_eq!(s1_list.entries()[0].deleted_files_count, Some(1)); + assert_eq!(s1_list.entries()[0].existing_files_count, Some(0)); + let table = with_snapshot(&table, s1); + + let mut add = intent(Operation::Append); + add.added_data_files = vec![data_file(&base, "test/b.parquet", 1)]; + let mut commit = MergingSnapshotProducer::new(add) + .apply(&table) + .await + .unwrap(); + let s2 = added_snapshot(&mut commit); + let s2_list = table.manifest_list_reader(&s2).load().await.unwrap(); + assert_eq!(s2_list.entries().len(), 1); + assert_eq!(s2_list.entries()[0].added_snapshot_id, s2.snapshot_id()); + assert_eq!( + s2.summary() + .additional_properties + .get("total-data-files") + .unwrap(), + "1" + ); + } } diff --git a/crates/iceberg/src/transaction/snapshot/mod.rs b/crates/iceberg/src/transaction/snapshot/mod.rs index 6e347756c7..8a11da3854 100644 --- a/crates/iceberg/src/transaction/snapshot/mod.rs +++ b/crates/iceberg/src/transaction/snapshot/mod.rs @@ -734,6 +734,7 @@ pub(crate) mod tests { HashMap::new(), &[], &[deleted], + &[], false, ) .unwrap(); From 58004f61f55a1e0bc31726f3d1f129208689fe45 Mon Sep 17 00:00:00 2001 From: Shawn Chang Date: Tue, 3 Nov 2026 14:42:40 -0800 Subject: [PATCH 48/50] fix tests, add new tests --- crates/iceberg/src/spec/transform.rs | 8 ++ .../src/transaction/snapshot/merging.rs | 79 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index 576b6553a9..00f5b0cfcb 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -1338,6 +1338,14 @@ mod tests { ), "1.5" ); + // Not exact in f32: the widened double is rendered, as Java does. + assert_eq!( + Transform::Identity.to_human_string( + &Type::Primitive(PrimitiveType::Double), + Some(&Literal::float(0.1_f32)) + ), + "0.10000000149011612" + ); } #[test] diff --git a/crates/iceberg/src/transaction/snapshot/merging.rs b/crates/iceberg/src/transaction/snapshot/merging.rs index 683ad0d0bb..4b39f4469c 100644 --- a/crates/iceberg/src/transaction/snapshot/merging.rs +++ b/crates/iceberg/src/transaction/snapshot/merging.rs @@ -1127,6 +1127,85 @@ mod tests { assert!(!summary.contains_key("deleted-data-files")); } + /// A concurrent format upgrade invalidates both caches: the rewrite, whose + /// entries were written without v2 sequence numbers, and the added + /// manifest, whose entries must now inherit one. + #[tokio::test] + async fn test_rewrite_and_added_manifest_are_redone_when_format_version_changes() { + use crate::spec::FormatVersion; + + let base = make_v1_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + let mut i = intent(Operation::Overwrite); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + i.added_data_files = vec![data_file(&base, "test/x.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + let mut first = producer.apply(&table).await.unwrap(); + let first = added_snapshot(&mut first); + assert_eq!(producer.manifest_counter, 2); + + let metadata = table + .metadata() + .clone() + .into_builder(None) + .upgrade_format_version(FormatVersion::V2) + .unwrap() + .build() + .unwrap() + .metadata; + let upgraded = table.clone().with_metadata(std::sync::Arc::new(metadata)); + + let mut second = producer.apply(&upgraded).await.unwrap(); + let second = added_snapshot(&mut second); + assert_eq!(producer.manifest_counter, 4); + + let own_paths = |list: crate::spec::ManifestList, snapshot_id: i64| { + let mut p: Vec<_> = list + .entries() + .iter() + .filter(|m| m.added_snapshot_id == snapshot_id) + .map(|m| m.manifest_path.clone()) + .collect(); + p.sort(); + p + }; + let first_list = table.manifest_list_reader(&first).load().await.unwrap(); + let second_list = upgraded.manifest_list_reader(&second).load().await.unwrap(); + let first_paths = own_paths(first_list, first.snapshot_id()); + let second_paths = own_paths(second_list.clone(), second.snapshot_id()); + assert_eq!(first_paths.len(), 2); + assert_eq!(second_paths.len(), 2); + assert!(first_paths.iter().all(|p| !second_paths.contains(p))); + + // Under v2 the added entry inherits the new sequence number, while the + // rewritten entries keep the v1 data's sequence number 0. + for m in second_list + .entries() + .iter() + .filter(|m| m.added_snapshot_id == second.snapshot_id()) + { + let manifest = upgraded.manifest_reader().read(m).await.unwrap(); + for entry in manifest.entries() { + let expected = if entry.file_path() == "test/x.parquet" { + second.sequence_number() + } else { + 0 + }; + assert_eq!( + entry.sequence_number(), + Some(expected), + "{}", + entry.file_path() + ); + } + } + } + /// A manifest left with only `Deleted` entries by an earlier snapshot is /// dropped by the next commit; the one this commit writes is kept. #[tokio::test] From a8b6d7c978443c1dff9f1b92ec1ca6de75144533 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 6 Nov 2026 11:54:49 +0100 Subject: [PATCH 49/50] Build OverwriteAction on the MergingSnapshotProducer The producer's ManifestFilter now does the manifest rewrite, path matching and dedupe, so the action only states its intent. --- crates/iceberg/src/transaction/append.rs | 3 +- crates/iceberg/src/transaction/overwrite.rs | 968 ++------------------ 2 files changed, 87 insertions(+), 884 deletions(-) diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index a01a229bdb..08ac900adf 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -130,8 +130,7 @@ impl TransactionAction for FastAppendAction { ) -> Result { self.validate()?; if self.check_duplicate { - validate_files_not_live(table, self.added_data_files.iter().map(DataFile::file_path)) - .await?; + validate_files_not_live(table, &self.added_data_files).await?; } state.apply(table).await } diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 0e26d543e0..5c07a736d2 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -16,21 +16,16 @@ // under the License. use std::collections::{HashMap, HashSet}; -use std::sync::Arc; use async_trait::async_trait; use uuid::Uuid; use crate::error::{Result, invalid_data}; -use crate::spec::{DataContentType, DataFile, ManifestContentType, Operation}; +use crate::spec::{DataContentType, DataFile, Operation}; use crate::table::Table; use crate::transaction::action::CommitStatus; -use crate::transaction::snapshot::simple::write_added_manifest; -use crate::transaction::snapshot::{ - SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, - manifest_path, new_manifest_writer, validate_files_not_live, validate_partition_value, - write_snapshot_commit, -}; +use crate::transaction::snapshot::merging::{MergingIntent, MergingSnapshotProducer}; +use crate::transaction::snapshot::validate_files_not_live; use crate::transaction::{ActionCommit, TransactionAction}; use crate::{Error, ErrorKind}; @@ -79,14 +74,13 @@ impl OverwriteAction { /// Specify data files to be removed from the table in this overwrite. /// - /// Files are matched by path against the current snapshot's live data files. - /// Their `record_count`, `file_size_in_bytes`, `partition_spec_id` and `partition` - /// must match the committed entry. Duplicate paths keep the first occurrence. + /// Files are matched by path against the current snapshot's live data files, and the + /// summary counts the committed entries' metrics. /// /// # Errors /// - /// The commit fails with [`crate::ErrorKind::DataInvalid`] for non-data files, - /// paths that are not live in the current snapshot, or mismatched metadata. + /// The commit fails with [`crate::ErrorKind::DataInvalid`] for non-data files and for + /// paths that are not live in the current snapshot. pub fn delete_data_files(mut self, data_files: impl IntoIterator) -> Self { self.deleted_data_files.extend(data_files); self @@ -104,7 +98,7 @@ impl OverwriteAction { self } - fn validate(&self, table: &Table) -> Result<()> { + fn validate(&self) -> Result<()> { // TODO: Allowing snapshot property setup with no data files is a workaround. // For details, please refer to https://github.com/apache/iceberg-rust/issues/1548 if self.added_data_files.is_empty() @@ -128,220 +122,78 @@ impl OverwriteAction { data_file.file_path() )); } - let metadata = table.metadata(); - for data_file in &self.added_data_files { - if metadata.default_partition_spec_id() != data_file.partition_spec_id { - return Err(invalid_data!( - "Data file partition spec id does not match table default partition spec id" - )); - } - validate_partition_value(data_file.partition(), metadata.default_partition_type())?; - } Ok(()) } } #[async_trait] impl TransactionAction for OverwriteAction { - type State = OverwriteState; - - fn new_state(&self) -> OverwriteState { - OverwriteState { - commit_uuid: self.commit_uuid.unwrap_or_else(Uuid::now_v7), - snapshot_id: None, - attempt: 0, - manifest_counter: 0, - } - } - - async fn commit(&self, state: &mut OverwriteState, table: &Table) -> Result { - self.validate(table)?; - let mut deleted_files = HashMap::new(); - for file in &self.deleted_data_files { - deleted_files.entry(file.file_path()).or_insert(file); - } - if self.check_duplicate { - let added = self.added_data_files.iter().map(DataFile::file_path); - validate_files_not_live(table, added.filter(|p| !deleted_files.contains_key(p))) - .await?; - } - - // Every requested delete is matched before any manifest is written, so a - // missing path or mismatched metadata fails without leaving files behind. - let mut matched = HashSet::new(); - let mut planned = vec![]; - for manifest_file in current_manifests(table).await? { - if manifest_file.content == ManifestContentType::Deletes || deleted_files.is_empty() { - planned.push((manifest_file, None)); - continue; - } - - let manifest = table.object_cache().get_manifest(&manifest_file).await?; - let mut has_deletes = false; - for entry in manifest.entries().iter().filter(|e| e.is_alive()) { - let Some((path, requested)) = deleted_files.get_key_value(entry.file_path()) else { - continue; - }; - if let Some(mismatch) = metadata_mismatch(requested, entry.data_file()) { - return Err(invalid_data!( - "Delete file metadata does not match the table: {} ({mismatch})", - entry.file_path() - )); - } - matched.insert(*path); - has_deletes = true; - } - planned.push((manifest_file, has_deletes.then_some(manifest))); - } - - let mut missing: Vec<&str> = deleted_files - .keys() - .copied() - .filter(|path| !matched.contains(path)) - .collect(); - if !missing.is_empty() { - missing.sort_unstable(); - return Err(invalid_data!( - "Missing required files to delete: {}", - missing.join(", ") - )); - } - - let snapshot_id = *state - .snapshot_id - .get_or_insert_with(|| generate_snapshot_id(table)); - let mut manifests = vec![]; - // One per tombstone written, so a path live in several manifests counts each copy. - let mut removed_files = vec![]; - for (manifest_file, rewrite) in planned { - let Some(manifest) = rewrite else { - manifests.push(manifest_file); - continue; - }; - let mut writer = new_manifest_writer( - table, - state.next_manifest_path(table)?, - snapshot_id, - manifest_file.content, - manifest.metadata().schema.clone(), - Arc::new(manifest.metadata().partition_spec.clone()), - )?; - // Like Java, only live entries are carried into the rewrite. - for entry in manifest.entries().iter().filter(|e| e.is_alive()) { - if deleted_files.contains_key(entry.file_path()) { - removed_files.push(entry.data_file().clone()); - writer.add_delete_entry(entry.as_ref().clone())?; - } else { - writer.add_existing_entry(entry.as_ref().clone())?; - } - } - manifests.push(writer.write_manifest_file().await?); - } - - if !self.added_data_files.is_empty() { - manifests.push( - write_added_manifest( - table, - state.next_manifest_path(table)?, - snapshot_id, - table.metadata().default_partition_spec_id(), - &self.added_data_files, - ) - .await?, - ); - } - - let operation = match (!self.added_data_files.is_empty(), !removed_files.is_empty()) { - (true, true) => Operation::Overwrite, - (false, true) => Operation::Delete, + type State = MergingSnapshotProducer; + + fn new_state(&self) -> MergingSnapshotProducer { + let operation = match ( + self.added_data_files.is_empty(), + self.deleted_data_files.is_empty(), + ) { + (false, false) => Operation::Overwrite, + (true, false) => Operation::Delete, // Also a properties-only commit, the workaround from #1548. _ => Operation::Append, }; - let manifest_list_path = - manifest_list_path(table, snapshot_id, state.attempt, state.commit_uuid)?; - state.attempt += 1; - write_snapshot_commit(table, SnapshotCommitInput { - snapshot_id, - manifest_list_path, + MergingSnapshotProducer::new(MergingIntent { + commit_uuid: self.commit_uuid, operation, - properties: self.snapshot_properties.clone(), - added_files: &self.added_data_files, - removed_files: &removed_files, - manifests, - // Only the listed files are replaced, so the table totals carry over. + snapshot_properties: self.snapshot_properties.clone(), + added_data_files: self.added_data_files.clone(), + added_delete_files: vec![], + removed_data_files: self.deleted_data_files.clone(), + removed_delete_files: vec![], + fail_missing_delete_paths: true, truncate_full_table: false, }) - .await } - async fn cleanup(self, _state: OverwriteState, _table: &Table, _status: CommitStatus) { - // TODO(#3417): remove this attempt's manifests once terminal cleanup is - // wired up. - } -} - -/// Kept across retries, so no attempt overwrites a file an earlier attempt wrote. -pub(crate) struct OverwriteState { - commit_uuid: Uuid, - snapshot_id: Option, - attempt: u64, - manifest_counter: u64, -} - -impl OverwriteState { - fn next_manifest_path(&mut self, table: &Table) -> Result { - let path = manifest_path(table, self.commit_uuid, self.manifest_counter); - self.manifest_counter += 1; - path + async fn commit( + &self, + state: &mut MergingSnapshotProducer, + table: &Table, + ) -> Result { + self.validate()?; + if self.check_duplicate { + let deleted: HashSet<&str> = self + .deleted_data_files + .iter() + .map(DataFile::file_path) + .collect(); + let added: Vec = self + .added_data_files + .iter() + .filter(|f| !deleted.contains(f.file_path())) + .cloned() + .collect(); + validate_files_not_live(table, &added).await?; + } + state.apply(table).await } -} -/// Describes the first field where a requested delete differs from the table's entry. -fn metadata_mismatch(requested: &DataFile, actual: &DataFile) -> Option { - if requested.record_count != actual.record_count { - Some(format!( - "record_count: requested {}, table has {}", - requested.record_count, actual.record_count - )) - } else if requested.file_size_in_bytes != actual.file_size_in_bytes { - Some(format!( - "file_size_in_bytes: requested {}, table has {}", - requested.file_size_in_bytes, actual.file_size_in_bytes - )) - } else if requested.partition_spec_id != actual.partition_spec_id { - Some(format!( - "partition_spec_id: requested {}, table has {}", - requested.partition_spec_id, actual.partition_spec_id - )) - } else if requested.partition != actual.partition { - Some(format!( - "partition: requested {:?}, table has {:?}", - requested.partition, actual.partition - )) - } else { - None + async fn cleanup(self, _state: MergingSnapshotProducer, _table: &Table, _status: CommitStatus) { + // TODO(#3417): remove the producer's uncommitted artifacts once terminal + // cleanup is wired up. } } #[cfg(test)] mod tests { - use std::sync::Arc; - - use uuid::Uuid; - use crate::memory::tests::new_memory_catalog; use crate::spec::{ - DataContentType, DataFile, DataFileBuilder, DataFileFormat, FormatVersion, Literal, - ManifestListWriter, ManifestStatus, ManifestWriterBuilder, NestedField, Operation, - PrimitiveType, Schema, SnapshotRef, Struct, Transform, Type, UnboundPartitionField, - UnboundPartitionSpec, + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, ManifestStatus, + Operation, Struct, }; use crate::table::Table; use crate::test_utils::commit_with_fresh_state; - use crate::transaction::snapshot::manifest_path; - use crate::transaction::tests::{make_v2_minimal_table, make_v3_minimal_table_in_catalog}; - use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction, TransactionAction}; - use crate::{Catalog, ErrorKind, TableCommit, TableCreation, TableUpdate}; + use crate::transaction::tests::make_v3_minimal_table_in_catalog; + use crate::transaction::{ApplyTransactionAction, Transaction}; + use crate::{Catalog, ErrorKind}; fn test_data_file(path: &str, partition_spec_id: i32) -> DataFile { DataFileBuilder::default() @@ -367,330 +219,54 @@ mod tests { entries.push((entry.status(), entry.file_path().to_string())); } } + entries.sort_by(|a, b| a.1.cmp(&b.1)); entries } - #[tokio::test] - async fn test_overwrite_with_deleted_files() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let spec_id = table.metadata().default_partition_spec_id(); - - let original_file1 = test_data_file("test/original1.parquet", spec_id); - let original_file2 = test_data_file("test/original2.parquet", spec_id); - let tx = Transaction::new(&table); - let action = tx - .fast_append() - .add_data_files(vec![original_file1.clone(), original_file2.clone()]); - let tx = action.apply(tx).unwrap(); - let table = tx.commit(&catalog).await.unwrap(); - - let replacement_file = test_data_file("test/replacement.parquet", spec_id); - let tx = Transaction::new(&table); - let action = tx - .overwrite() - .add_data_files(vec![replacement_file.clone()]) - .delete_data_files(vec![original_file1.clone(), original_file2.clone()]); - let tx = action.apply(tx).unwrap(); - let table = tx.commit(&catalog).await.unwrap(); - - let snapshot = table.metadata().current_snapshot().unwrap(); - assert_eq!(snapshot.summary().operation, Operation::Overwrite); - let summary = &snapshot.summary().additional_properties; - assert_eq!(summary.get("deleted-data-files").unwrap(), "2"); - assert_eq!(summary.get("deleted-records").unwrap(), "2"); - - let entries = current_manifest_entries(&table).await; - assert_eq!(3, entries.len(), "{entries:?}"); - for entry in [ - (ManifestStatus::Deleted, "test/original1.parquet"), - (ManifestStatus::Deleted, "test/original2.parquet"), - (ManifestStatus::Added, "test/replacement.parquet"), - ] { - assert!( - entries.contains(&(entry.0, entry.1.to_string())), - "{entries:?}" - ); - } - } - - #[tokio::test] - async fn test_second_overwrite_does_not_resurrect_a_deleted_file() { - let catalog = new_memory_catalog().await; - let base = make_v3_minimal_table_in_catalog(&catalog).await; - let v2 = make_v2_minimal_table(); - let table = catalog - .create_table( - base.identifier().namespace(), - TableCreation::builder() - .name("v2".to_string()) - .schema(v2.metadata().current_schema().as_ref().clone()) - .partition_spec(v2.metadata().default_partition_spec().as_ref().clone()) - .format_version(FormatVersion::V2) - .build(), - ) - .await - .unwrap(); - let spec_id = table.metadata().default_partition_spec_id(); - - let file_a = test_data_file("test/a.parquet", spec_id); - let file_b = test_data_file("test/b.parquet", spec_id); - let tx = Transaction::new(&table); - let action = tx - .fast_append() - .add_data_files(vec![file_a.clone(), file_b.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let tx = Transaction::new(&table); - let action = tx - .overwrite() - .add_data_files(vec![test_data_file("test/c.parquet", spec_id)]) - .delete_data_files(vec![file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let tx = Transaction::new(&table); - let action = tx - .overwrite() - .add_data_files(vec![test_data_file("test/d.parquet", spec_id)]) - .delete_data_files(vec![file_b.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - assert_eq!(current_manifest_entries(&table).await, vec![ - (ManifestStatus::Deleted, "test/b.parquet".to_string()), - (ManifestStatus::Added, "test/c.parquet".to_string()), - (ManifestStatus::Added, "test/d.parquet".to_string()), - ]); - } - - #[tokio::test] - async fn test_overwrite_retry_keeps_its_snapshot_id_and_never_reuses_a_manifest_path() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let spec_id = table.metadata().default_partition_spec_id(); - let file_a = test_data_file("test/a.parquet", spec_id); - let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(vec![file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let action = Transaction::new(&table) - .overwrite() - .set_commit_uuid(Uuid::nil()) - .add_data_files(vec![test_data_file("test/b.parquet", spec_id)]) - .delete_data_files(vec![file_a]); - let mut state = action.new_state(); - let mut attempts = vec![]; - for _ in 0..2 { - let mut action_commit = action.commit(&mut state, &table).await.unwrap(); - let Some(TableUpdate::AddSnapshot { snapshot }) = - action_commit.take_updates().into_iter().next() - else { - unreachable!() - }; - let snapshot = SnapshotRef::new(snapshot); - let manifest_list = table.manifest_list_reader(&snapshot).load().await.unwrap(); - let mut paths: Vec<_> = manifest_list - .entries() - .iter() - .map(|m| m.manifest_path.clone()) - .collect(); - paths.push(snapshot.manifest_list().to_string()); - attempts.push((snapshot.snapshot_id(), paths)); - } - - assert_eq!(attempts[0].0, attempts[1].0); - assert!(attempts[1].1.iter().all(|p| !attempts[0].1.contains(p))); - } - - #[tokio::test] - async fn test_rewritten_manifest_keeps_the_schema_and_spec_it_was_written_with() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let original_spec_id = table.metadata().default_partition_spec_id(); - let original_schema_id = table.metadata().current_schema_id(); - - let file_a = test_data_file("test/a.parquet", original_spec_id); - let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(vec![file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let tx = Transaction::new(&table); - let action = tx.update_schema().add_column(AddColumn::optional( - "extra", - Type::Primitive(PrimitiveType::Int), - )); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - // There is no transaction action for partition evolution yet, so the spec is replaced - // on the metadata directly. - let evolved = table - .metadata() - .clone() - .into_builder(None) - .add_default_partition_spec( - UnboundPartitionSpec::builder() - .add_partition_field( - UnboundPartitionField::builder() - .source_ids(vec![2]) - .name("y") - .transform(Transform::Identity) - .build() - .unwrap(), - ) - .unwrap() - .build(), - ) - .unwrap() - .build() - .unwrap() - .metadata; - let table = table.with_metadata(Arc::new(evolved)); - assert_ne!(original_schema_id, table.metadata().current_schema_id()); - assert_ne!( - original_spec_id, - table.metadata().default_partition_spec_id() - ); - - let tx = Transaction::new(&table); - let action = tx - .overwrite() - .add_data_files(vec![test_data_file( - "test/b.parquet", - table.metadata().default_partition_spec_id(), - )]) - .delete_data_files(vec![file_a.clone()]); - let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap(); - let updates = action_commit.take_updates(); - let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] { - SnapshotRef::new(snapshot.clone()) - } else { - unreachable!() - }; - - let manifest_list = table - .manifest_list_reader(&new_snapshot) - .load() - .await - .unwrap(); - let mut rewritten = None; - for manifest_file in manifest_list.entries() { - let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); - if manifest - .entries() - .iter() - .any(|entry| entry.status() == ManifestStatus::Deleted) - { - rewritten = Some((manifest_file.clone(), manifest)); - } - } - let (rewritten_file, rewritten_manifest) = - rewritten.expect("the manifest holding test/a.parquet must have been rewritten"); - - assert_eq!(original_schema_id, rewritten_manifest.metadata().schema_id); - assert_eq!(original_spec_id, rewritten_file.partition_spec_id); + async fn append(catalog: &impl Catalog, table: &Table, files: Vec) -> Table { + let tx = Transaction::new(table); + let action = tx.fast_append().add_data_files(files); + action.apply(tx).unwrap().commit(catalog).await.unwrap() } #[tokio::test] - async fn test_overwrite_counts_a_duplicated_delete_path_once() { + async fn test_partial_overwrite_then_delete() { let catalog = new_memory_catalog().await; let table = make_v3_minimal_table_in_catalog(&catalog).await; let spec_id = table.metadata().default_partition_spec_id(); - let file_a = test_data_file("test/a.parquet", spec_id); let file_b = test_data_file("test/b.parquet", spec_id); - let tx = Transaction::new(&table); - let action = tx - .fast_append() - .add_data_files(vec![file_a.clone(), file_b.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + let table = append(&catalog, &table, vec![file_a.clone(), file_b.clone()]).await; let tx = Transaction::new(&table); let action = tx .overwrite() .add_data_files(vec![test_data_file("test/c.parquet", spec_id)]) - .delete_data_files(vec![file_a.clone(), file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let summary = table.metadata().current_snapshot().unwrap().summary(); - assert_eq!( - summary.additional_properties.get("deleted-data-files"), - Some(&"1".to_string()) - ); - assert_eq!( - summary.additional_properties.get("deleted-records"), - Some(&"1".to_string()) - ); - assert_eq!( - summary.additional_properties.get("total-data-files"), - Some(&"2".to_string()) - ); - } - - #[tokio::test] - async fn test_overwrite_deletes_every_live_copy_of_a_path() { - let catalog = new_memory_catalog().await; - let mut table = make_v3_minimal_table_in_catalog(&catalog).await; - let file_a = test_data_file( - "test/a.parquet", - table.metadata().default_partition_spec_id(), - ); - for _ in 0..2 { - let tx = Transaction::new(&table); - let action = tx - .fast_append() - .with_check_duplicate(false) - .add_data_files(vec![file_a.clone()]); - table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - } - - let tx = Transaction::new(&table); - let action = tx.overwrite().delete_data_files(vec![file_a]); + .delete_data_files(vec![file_a]); let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - let summary = table.metadata().current_snapshot().unwrap().summary(); - assert_eq!(summary.additional_properties["deleted-data-files"], "2"); - assert_eq!(summary.additional_properties["total-data-files"], "0"); + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.summary().operation, Operation::Overwrite); + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary["added-data-files"], "1"); + assert_eq!(summary["deleted-data-files"], "1"); + assert_eq!(summary["deleted-records"], "1"); + assert_eq!(summary["total-data-files"], "2"); + assert_eq!(summary["total-records"], "2"); assert_eq!(current_manifest_entries(&table).await, [ (ManifestStatus::Deleted, "test/a.parquet".to_string()), - (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + (ManifestStatus::Added, "test/c.parquet".to_string()), ]); - } - #[tokio::test] - async fn test_delete_only_overwrite_reports_a_delete_operation() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let spec_id = table.metadata().default_partition_spec_id(); - - let file_a = test_data_file("test/a.parquet", spec_id); - let file_b = test_data_file("test/b.parquet", spec_id); let tx = Transaction::new(&table); - let action = tx - .fast_append() - .add_data_files(vec![file_a.clone(), file_b.clone()]); + let action = tx.overwrite().delete_data_files(vec![file_b]); let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - - let tx = Transaction::new(&table); - let action = tx.overwrite().delete_data_files(vec![file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); - let snapshot = table.metadata().current_snapshot().unwrap(); assert_eq!(snapshot.summary().operation, Operation::Delete); assert_eq!( - snapshot - .summary() - .additional_properties - .get("deleted-data-files"), - Some(&"1".to_string()) - ); - - let entries = current_manifest_entries(&table).await; - assert!( - entries.contains(&(ManifestStatus::Deleted, "test/a.parquet".to_string())), - "the deleted file must be a tombstone, entries: {entries:?}" - ); - assert!( - entries.contains(&(ManifestStatus::Existing, "test/b.parquet".to_string())), - "the untouched file must stay live, entries: {entries:?}" + snapshot.summary().additional_properties["total-data-files"], + "1" ); } @@ -699,117 +275,19 @@ mod tests { let catalog = new_memory_catalog().await; let table = make_v3_minimal_table_in_catalog(&catalog).await; let spec_id = table.metadata().default_partition_spec_id(); - let file_a = test_data_file("test/a.parquet", spec_id); - let tx = Transaction::new(&table); - let action = tx.fast_append().add_data_files(vec![file_a.clone()]); - let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + let table = append(&catalog, &table, vec![file_a.clone()]).await; - let tx = Transaction::new(&table); - let action = tx - .overwrite() - .set_commit_uuid(Uuid::nil()) - .add_data_files(vec![test_data_file("test/b.parquet", spec_id)]) - .delete_data_files(vec![ - file_a.clone(), - test_data_file("test/never-committed.parquet", spec_id), - ]); + let action = Transaction::new(&table).overwrite().delete_data_files(vec![ + file_a, + test_data_file("test/never-committed.parquet", spec_id), + ]); let err = commit_with_fresh_state(action, &table).await.err().unwrap(); assert_eq!(err.kind(), ErrorKind::DataInvalid); assert!( - err.to_string() - .contains("Missing required files to delete: test/never-committed.parquet"), + err.message().contains("test/never-committed.parquet"), "{err}" ); - let first_manifest = manifest_path(&table, Uuid::nil(), 0).unwrap(); - assert!(!table.file_io().exists(&first_manifest).await.unwrap()); - } - - #[tokio::test] - async fn test_overwrite_deletes_files_after_partition_source_promotion() { - let catalog = new_memory_catalog().await; - let base = make_v3_minimal_table_in_catalog(&catalog).await; - for transform in [Transform::Identity, Transform::Truncate(10)] { - let schema = Schema::builder() - .with_fields([NestedField::required(1, "x", PrimitiveType::Int).into()]) - .build() - .unwrap(); - let spec = UnboundPartitionSpec::builder() - .add_partition_field( - UnboundPartitionField::builder() - .source_ids(vec![1]) - .name("p") - .transform(transform) - .build() - .unwrap(), - ) - .unwrap() - .build(); - let table = catalog - .create_table( - base.identifier().namespace(), - TableCreation::builder() - .name(format!("promoted-{transform}")) - .schema(schema) - .partition_spec(spec) - .format_version(FormatVersion::V3) - .build(), - ) - .await - .unwrap(); - let file = DataFile { - partition: Struct::from_iter([Some(Literal::int(300))]), - ..test_data_file( - "test/a.parquet", - table.metadata().default_partition_spec_id(), - ) - }; - let tx = Transaction::new(&table); - let table = tx - .fast_append() - .add_data_files([file.clone()]) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - let schema = Schema::builder() - .with_fields([NestedField::required(1, "x", PrimitiveType::Long).into()]) - .build() - .unwrap(); - let table = catalog - .update_table( - TableCommit::builder() - .ident(table.identifier().clone()) - .requirements(vec![]) - .updates(vec![ - TableUpdate::AddSchema { schema }, - TableUpdate::SetCurrentSchema { schema_id: -1 }, - ]) - .build(), - ) - .await - .unwrap(); - let tx = Transaction::new(&table); - let table = tx - .overwrite() - .delete_data_files([file]) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - let snapshot = table.metadata().current_snapshot().unwrap(); - assert_eq!(snapshot.summary().operation, Operation::Delete); - assert_eq!( - snapshot.summary().additional_properties["deleted-records"], - "1" - ); - assert_eq!(current_manifest_entries(&table).await, [( - ManifestStatus::Deleted, - "test/a.parquet".to_string() - )]); - } } #[tokio::test] @@ -820,291 +298,17 @@ mod tests { "test/a.parquet", table.metadata().default_partition_spec_id(), ); + let table = append(&catalog, &table, vec![file.clone()]).await; + let tx = Transaction::new(&table); - let table = tx - .fast_append() - .add_data_files([file.clone()]) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - let tx = Transaction::new(&table); - let table = tx + let action = tx .overwrite() .delete_data_files([file.clone()]) - .add_data_files([file]) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); + .add_data_files([file]); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); assert_eq!(current_manifest_entries(&table).await, [ (ManifestStatus::Deleted, "test/a.parquet".to_string()), (ManifestStatus::Added, "test/a.parquet".to_string()), ]); } - - #[tokio::test] - async fn test_overwrite_rejects_incorrect_delete_metrics() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let spec_id = table.metadata().default_partition_spec_id(); - let file = test_data_file("test/a.parquet", spec_id); - let tx = Transaction::new(&table); - let table = tx - .fast_append() - .add_data_files(vec![file.clone()]) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - let partition_mismatch = |requested: &Struct| { - format!( - "partition: requested {requested:?}, table has {:?}", - file.partition - ) - }; - let long_partition = Struct::from_iter([Some(Literal::long(999))]); - let string_partition = Struct::from_iter([Some(Literal::string("invalid"))]); - for (incorrect, mismatch) in [ - ( - DataFile { - record_count: 99, - ..file.clone() - }, - "record_count: requested 99, table has 1".to_string(), - ), - ( - DataFile { - file_size_in_bytes: 999, - ..file.clone() - }, - "file_size_in_bytes: requested 999, table has 100".to_string(), - ), - ( - DataFile { - partition: long_partition.clone(), - ..file.clone() - }, - partition_mismatch(&long_partition), - ), - ( - DataFile { - partition_spec_id: 99, - ..file.clone() - }, - "partition_spec_id: requested 99, table has 0".to_string(), - ), - ( - DataFile { - partition: string_partition.clone(), - ..file.clone() - }, - partition_mismatch(&string_partition), - ), - ] { - let action = Transaction::new(&table) - .overwrite() - .delete_data_files(vec![incorrect]); - let err = commit_with_fresh_state(action, &table).await.err().unwrap(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains(&mismatch), "{err}"); - } - } - - #[tokio::test] - async fn test_overwrite_rejects_non_data_deletions() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let spec_id = table.metadata().default_partition_spec_id(); - for content in [ - DataContentType::PositionDeletes, - DataContentType::EqualityDeletes, - ] { - let file = DataFile { - content, - ..test_data_file("test/deletes.parquet", spec_id) - }; - let action = Transaction::new(&table) - .overwrite() - .delete_data_files(vec![file]); - let err = commit_with_fresh_state(action, &table).await.err().unwrap(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.message().contains("Only data content type is allowed")); - } - } - - #[tokio::test] - async fn test_overwrite_preserves_v3_row_lineage() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let spec_id = table.metadata().default_partition_spec_id(); - let a = test_data_file("test/a.parquet", spec_id); - let b = test_data_file("test/b.parquet", spec_id); - let c = test_data_file("test/c.parquet", spec_id); - let tx = Transaction::new(&table); - let table = tx - .fast_append() - .add_data_files(vec![a, b.clone(), c]) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - let snapshot = table.metadata().current_snapshot().unwrap(); - let manifests = table.manifest_list_reader(snapshot).load().await.unwrap(); - let manifest = table - .manifest_reader() - .read(&manifests.entries()[0]) - .await - .unwrap(); - let original_ids: Vec<_> = manifest - .entries() - .iter() - .map(|e| (e.file_path().to_string(), e.data_file().first_row_id)) - .collect(); - assert!(original_ids.iter().all(|(_, id)| id.is_some())); - let next_row_id = table.metadata().next_row_id(); - let replacement = test_data_file("test/d.parquet", spec_id); - let tx = Transaction::new(&table); - let table = tx - .overwrite() - .delete_data_files(vec![b]) - .add_data_files(vec![replacement]) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - let manifests = table - .manifest_list_reader(table.metadata().current_snapshot().unwrap()) - .load() - .await - .unwrap(); - let mut actual_ids = vec![]; - for file in manifests.entries() { - let manifest = table.manifest_reader().read(file).await.unwrap(); - actual_ids.extend( - manifest - .entries() - .iter() - .filter(|e| e.is_alive()) - .map(|e| (e.file_path().to_string(), e.data_file().first_row_id)), - ); - } - for (path, id) in original_ids.iter().filter(|(p, _)| p != "test/b.parquet") { - assert!(actual_ids.contains(&(path.clone(), *id))); - } - // The rewritten manifest reserves IDs for its existing rows; their explicit IDs stay unchanged. - assert!(actual_ids.contains(&( - "test/d.parquet".to_string(), - Some((next_row_id + 2).try_into().unwrap()) - ))); - assert_eq!(table.metadata().next_row_id(), next_row_id + 3); - } - - #[tokio::test] - async fn test_overwrite_preserves_delete_manifests_and_rejects_their_paths() { - let catalog = new_memory_catalog().await; - let table = make_v3_minimal_table_in_catalog(&catalog).await; - let spec_id = table.metadata().default_partition_spec_id(); - let tx = Transaction::new(&table); - let table = tx - .fast_append() - .add_data_files(vec![test_data_file("test/a.parquet", spec_id)]) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - let snapshot = table.metadata().current_snapshot().unwrap(); - let file = test_data_file("test/deletes.parquet", spec_id); - let output = table - .file_io() - .new_output("test/delete-manifest.avro") - .unwrap(); - let mut writer = ManifestWriterBuilder::new( - output, - Some(snapshot.snapshot_id()), - table.metadata().current_schema().clone(), - table.metadata().default_partition_spec().as_ref().clone(), - ) - .build_v3_deletes(); - writer - .add_file( - DataFile { - content: DataContentType::PositionDeletes, - ..file.clone() - }, - snapshot.sequence_number(), - ) - .unwrap(); - let delete_manifest = writer.write_manifest_file().await.unwrap(); - let mut manifests: Vec<_> = table - .manifest_list_reader(snapshot) - .load() - .await - .unwrap() - .consume_entries() - .into_iter() - .collect(); - manifests.push(delete_manifest); - let mut writer = ManifestListWriter::v3( - table - .file_io() - .new_output(snapshot.manifest_list()) - .unwrap() - .writer() - .await - .unwrap(), - snapshot.snapshot_id(), - snapshot.parent_snapshot_id(), - snapshot.sequence_number(), - Some(table.metadata().next_row_id()), - ); - writer.add_manifests(manifests.into_iter()).unwrap(); - writer.close().await.unwrap(); - let action = Transaction::new(&table) - .overwrite() - .delete_data_files(vec![file]); - let err = commit_with_fresh_state(action, &table).await.err().unwrap(); - assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!( - err.message() - .contains("Missing required files to delete: test/deletes.parquet") - ); - - let original = table - .manifest_list_reader(table.metadata().current_snapshot().unwrap()) - .load() - .await - .unwrap() - .entries() - .iter() - .find(|m| m.manifest_path == "test/delete-manifest.avro") - .unwrap() - .clone(); - let tx = Transaction::new(&table); - let table = tx - .overwrite() - .delete_data_files([test_data_file("test/a.parquet", spec_id)]) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - let manifests = table - .manifest_list_reader(table.metadata().current_snapshot().unwrap()) - .load() - .await - .unwrap(); - let preserved = manifests - .entries() - .iter() - .find(|m| m.manifest_path == original.manifest_path) - .unwrap(); - assert_eq!(preserved, &original); - } } From 4aa7e9e48d861e51f74cb3db04c6c7bac174e2e4 Mon Sep 17 00:00:00 2001 From: Jegor van Opdorp Date: Fri, 6 Nov 2026 12:01:41 +0100 Subject: [PATCH 50/50] Test that overwrite rejects adding a live path it does not delete --- crates/iceberg/src/transaction/overwrite.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/iceberg/src/transaction/overwrite.rs b/crates/iceberg/src/transaction/overwrite.rs index 5c07a736d2..060935a07b 100644 --- a/crates/iceberg/src/transaction/overwrite.rs +++ b/crates/iceberg/src/transaction/overwrite.rs @@ -290,6 +290,20 @@ mod tests { ); } + #[tokio::test] + async fn test_overwrite_rejects_adding_a_live_path_it_does_not_delete() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let spec_id = table.metadata().default_partition_spec_id(); + let file = test_data_file("test/a.parquet", spec_id); + let table = append(&catalog, &table, vec![file.clone()]).await; + + let action = Transaction::new(&table).overwrite().add_data_files([file]); + let err = commit_with_fresh_state(action, &table).await.err().unwrap(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("test/a.parquet"), "{err}"); + } + #[tokio::test] async fn test_overwrite_readds_a_deleted_path_by_default() { let catalog = new_memory_catalog().await;