diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 92267f0a99..baa2e1c852 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -3465,6 +3465,12 @@ pub fn iceberg::transaction::ReplaceSortOrderAction::desc(self, name: &str, null pub fn iceberg::transaction::ReplaceSortOrderAction::desc_with_transform(self, name: &str, transform: iceberg::spec::Transform, null_order: iceberg::spec::NullOrder) -> Self impl core::clone::Clone for iceberg::transaction::ReplaceSortOrderAction pub fn iceberg::transaction::ReplaceSortOrderAction::clone(&self) -> iceberg::transaction::ReplaceSortOrderAction +pub struct iceberg::transaction::RewriteFilesAction +impl iceberg::transaction::RewriteFilesAction +pub fn iceberg::transaction::RewriteFilesAction::add_file(self, file: iceberg::spec::DataFile) -> Self +pub fn iceberg::transaction::RewriteFilesAction::delete_file(self, file: iceberg::spec::DataFile) -> Self +impl core::clone::Clone for iceberg::transaction::RewriteFilesAction +pub fn iceberg::transaction::RewriteFilesAction::clone(&self) -> iceberg::transaction::RewriteFilesAction pub struct iceberg::transaction::Transaction impl iceberg::transaction::Transaction pub async fn iceberg::transaction::Transaction::commit(self, catalog: &dyn iceberg::Catalog) -> iceberg::Result @@ -3472,6 +3478,7 @@ pub fn iceberg::transaction::Transaction::expire_snapshots(&self) -> iceberg::tr pub fn iceberg::transaction::Transaction::fast_append(&self) -> iceberg::transaction::FastAppendAction pub fn iceberg::transaction::Transaction::new(table: &iceberg::table::Table) -> Self pub fn iceberg::transaction::Transaction::replace_sort_order(&self) -> iceberg::transaction::ReplaceSortOrderAction +pub fn iceberg::transaction::Transaction::rewrite_files(&self) -> iceberg::transaction::RewriteFilesAction pub fn iceberg::transaction::Transaction::update_location(&self) -> iceberg::transaction::UpdateLocationAction pub fn iceberg::transaction::Transaction::update_schema(&self) -> iceberg::transaction::UpdateSchemaAction pub fn iceberg::transaction::Transaction::update_statistics(&self) -> iceberg::transaction::UpdateStatisticsAction diff --git a/crates/iceberg/src/io/object_cache.rs b/crates/iceberg/src/io/object_cache.rs index 874c9d3483..7ac2ecd3ca 100644 --- a/crates/iceberg/src/io/object_cache.rs +++ b/crates/iceberg/src/io/object_cache.rs @@ -52,6 +52,10 @@ pub(crate) enum CachedObjectKey { Manifest(ManifestKey), } +/// Identifies a parsed manifest: its path plus every manifest-list value the +/// reader inherits onto its entries. Two references to one physical manifest +/// with different inherited values parse to different entries, so anything +/// derived from parsed entries must be keyed by this, not by path alone. #[derive(Clone, Debug, Hash, Eq, PartialEq)] pub(crate) struct ManifestKey { path: String, @@ -61,7 +65,7 @@ pub(crate) struct ManifestKey { } impl ManifestKey { - fn new(manifest_file: &ManifestFile) -> Self { + pub(crate) fn new(manifest_file: &ManifestFile) -> Self { Self { path: manifest_file.manifest_path.clone(), added_snapshot_id: manifest_file.added_snapshot_id, diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index a79312dfab..41a3d88e44 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -315,10 +315,6 @@ impl ManifestWriter { /// Add a delete manifest entry. This method will update following status of the entry: /// - Update the entry status to `Deleted` /// - Set the snapshot id to the current snapshot id - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_delete_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Deleted; @@ -350,10 +346,6 @@ impl ManifestWriter { /// Add an existing manifest entry. This method will update following status of the entry: /// - Update the entry status to `Existing` - /// - /// # TODO - /// Remove this allow later - #[allow(dead_code)] pub(crate) fn add_existing_entry(&mut self, mut entry: ManifestEntry) -> Result<()> { self.check_data_file(&entry.data_file)?; entry.status = ManifestStatus::Existing; diff --git a/crates/iceberg/src/spec/snapshot_summary.rs b/crates/iceberg/src/spec/snapshot_summary.rs index fdaf277043..c629444216 100644 --- a/crates/iceberg/src/spec/snapshot_summary.rs +++ b/crates/iceberg/src/spec/snapshot_summary.rs @@ -348,6 +348,7 @@ pub(crate) fn update_snapshot_summaries( if summary.operation != Operation::Append && summary.operation != Operation::Overwrite && summary.operation != Operation::Delete + && summary.operation != Operation::Replace { return Err(invalid_data!("Operation is not supported.")); } diff --git a/crates/iceberg/src/spec/transform.rs b/crates/iceberg/src/spec/transform.rs index f52202b78b..576b6553a9 100644 --- a/crates/iceberg/src/spec/transform.rs +++ b/crates/iceberg/src/spec/transform.rs @@ -221,6 +221,9 @@ impl Transform { (Self::Hour, PrimitiveLiteral::Int(ordinal)) => Self::human_hour(ordinal), (_, value) => { let field_type = field_type.as_primitive_type().unwrap(); + // A value written before its source column was promoted (`int` + // to `long`, `float` to `double`) still carries the old type. + let value = value.promote_to(field_type).unwrap_or(value); Datum::new(field_type.clone(), value).to_human_string() } } @@ -1322,6 +1325,21 @@ mod tests { ); } + /// A partition value written before its source column was promoted keeps + /// the old literal type; rendering it under the promoted type must widen it + /// rather than panic. + #[test] + fn test_to_human_string_promotes_value_to_declared_type() { + assert_eq!(human(Transform::Identity, PrimitiveType::Long, 7), "7"); + assert_eq!( + Transform::Identity.to_human_string( + &Type::Primitive(PrimitiveType::Double), + Some(&Literal::float(1.5_f32)) + ), + "1.5" + ); + } + #[test] fn test_to_human_string_null_cases() { assert_eq!(human(Transform::Void, PrimitiveType::Int, 47), "null"); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index cd31ca0a2e..6bdab7135a 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -55,6 +55,7 @@ mod action; pub use action::*; mod append; mod expire_snapshots; +mod rewrite; mod snapshot; mod sort_order; mod update_location; @@ -75,6 +76,7 @@ use crate::table::Table; use crate::transaction::action::TransactionActionEntry; pub use crate::transaction::append::FastAppendAction; pub use crate::transaction::expire_snapshots::ExpireSnapshotsAction; +pub use crate::transaction::rewrite::RewriteFilesAction; pub use crate::transaction::sort_order::ReplaceSortOrderAction; pub use crate::transaction::update_location::UpdateLocationAction; pub use crate::transaction::update_properties::UpdatePropertiesAction; @@ -150,6 +152,15 @@ impl Transaction { FastAppendAction::new() } + /// Creates a rewrite files action for compaction. + /// + /// This action replaces a set of data files with a new set while keeping + /// the logical table contents unchanged. The resulting snapshot uses + /// [`Operation::Replace`](crate::spec::Operation::Replace). + pub fn rewrite_files(&self) -> RewriteFilesAction { + RewriteFilesAction::new(self.table.metadata().current_snapshot_id()) + } + /// Creates replace sort order action. pub fn replace_sort_order(&self) -> ReplaceSortOrderAction { ReplaceSortOrderAction::new() @@ -264,7 +275,7 @@ mod tests { use crate::io::FileIO; use crate::memory::tests::new_memory_catalog; use crate::spec::{ - DataContentType, DataFileBuilder, DataFileFormat, Literal, Struct, TableMetadata, + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, Struct, TableMetadata, TableProperties, }; use crate::table::{StaticTable, Table}; @@ -272,6 +283,31 @@ mod tests { use crate::transaction::{ApplyTransactionAction, Transaction}; use crate::{Catalog, Error, ErrorKind, TableCreation, TableIdent}; + /// Create a test data file with the given path, record count, and file size. + pub(crate) fn make_data_file(table: &Table, path: &str, records: u64, size: u64) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(size) + .record_count(records) + .partition(Struct::from_iter([Some(Literal::long(1))])) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .build() + .unwrap() + } + + /// Append data files to a table via fast_append and return the updated table. + pub(crate) async fn append_files( + catalog: &impl Catalog, + table: &Table, + files: Vec, + ) -> Table { + let tx = Transaction::new(table); + let tx = tx.fast_append().add_data_files(files).apply(tx).unwrap(); + tx.commit(catalog).await.unwrap() + } + pub fn make_v1_table() -> Table { let file = File::open(format!( "{}/testdata/table_metadata/{}", @@ -332,6 +368,24 @@ mod tests { .unwrap() } + pub(crate) async fn make_v1_minimal_table_in_catalog(catalog: &impl Catalog) -> Table { + make_minimal_table_in_catalog( + catalog, + "TableMetadataV1Valid.json", + crate::spec::FormatVersion::V1, + ) + .await + } + + pub(crate) async fn make_v2_minimal_table_in_catalog(catalog: &impl Catalog) -> Table { + make_minimal_table_in_catalog( + catalog, + "TableMetadataV2ValidMinimal.json", + crate::spec::FormatVersion::V2, + ) + .await + } + pub(crate) fn make_v3_minimal_table() -> Table { let file = File::open(format!( "{}/testdata/table_metadata/{}", @@ -365,6 +419,19 @@ mod tests { } pub(crate) async fn make_v3_minimal_table_in_catalog(catalog: &impl Catalog) -> Table { + make_minimal_table_in_catalog( + catalog, + "TableMetadataV3ValidMinimal.json", + crate::spec::FormatVersion::V3, + ) + .await + } + + async fn make_minimal_table_in_catalog( + catalog: &impl Catalog, + metadata_file: &str, + format_version: crate::spec::FormatVersion, + ) -> Table { let table_ident = TableIdent::from_strs([format!("ns1-{}", uuid::Uuid::new_v4()), "test1".to_string()]) .unwrap(); @@ -377,7 +444,7 @@ mod tests { let file = File::open(format!( "{}/testdata/table_metadata/{}", env!("CARGO_MANIFEST_DIR"), - "TableMetadataV3ValidMinimal.json" + metadata_file )) .unwrap(); let reader = BufReader::new(file); @@ -388,7 +455,7 @@ mod tests { .partition_spec((**base_metadata.default_partition_spec()).clone()) .sort_order((**base_metadata.default_sort_order()).clone()) .name(table_ident.name().to_string()) - .format_version(crate::spec::FormatVersion::V3) + .format_version(format_version) .build(); catalog diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs new file mode 100644 index 0000000000..7cc9b68e2c --- /dev/null +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -0,0 +1,1056 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Transaction action for rewriting data files (compaction). +//! +//! [`RewriteFilesAction`] replaces a set of data files with a new set while +//! keeping the logical table contents unchanged. This is used for compaction — +//! merging many small files into fewer large ones. +//! +//! The resulting snapshot uses [`Operation::Replace`] to indicate that files +//! were reorganised without changing the data. + +use std::collections::HashMap; + +use async_trait::async_trait; + +use crate::error::{Result, invalid_data}; +use crate::spec::{DataContentType, DataFile, ManifestContentType, Operation}; +use crate::table::Table; +use crate::transaction::action::CommitStatus; +use crate::transaction::snapshot::merging::{MergingIntent, MergingSnapshotProducer}; +use crate::transaction::snapshot::validate_files_not_live; +use crate::transaction::{ActionCommit, TransactionAction}; + +/// A transaction action that rewrites (replaces) data files. +/// +/// This is the Rust equivalent of Java's `BaseRewriteFiles`. It uses +/// `MergingSnapshotProducer` to handle manifest filtering and creation, +/// and commits a snapshot with [`Operation::Replace`]. +/// +/// # Example +/// +/// ```ignore +/// let tx = Transaction::new(&table); +/// let action = tx.rewrite_files() +/// .delete_file(old_file_1) +/// .delete_file(old_file_2) +/// .add_file(merged_file); +/// let tx = action.apply(tx)?; +/// let table = tx.commit(&catalog).await?; +/// ``` +/// +/// # Concurrent deletes +/// +/// The action records the table's current snapshot when it is created, and +/// refuses to commit if the table gained a delete manifest after it. Java +/// narrows that to the files being replaced in +/// `validateNoNewDeletesForDataFiles`; until that exists here the check is +/// table-wide, so a delete committed to an unrelated partition while the +/// rewrite ran also fails the commit with +/// [`ErrorKind::DataInvalid`](crate::ErrorKind::DataInvalid). So do a +/// rewrite planned against a table that had no snapshot yet, and one whose +/// starting snapshot has since been expired: neither can rule out a delete. +/// Replan the rewrite against the current table and run it again. +/// +/// Deletes that were already committed when the rewrite was planned are not +/// covered by this check, so apply them while rewriting. +#[derive(Clone)] +pub struct RewriteFilesAction { + added_data_files: Vec, + deleted_data_files: Vec, + /// The snapshot the rewrite was planned against, if the table had one. + /// Deletes newer than it fail the commit. + starting_snapshot_id: Option, +} + +impl RewriteFilesAction { + pub(crate) fn new(starting_snapshot_id: Option) -> Self { + Self { + added_data_files: Vec::new(), + deleted_data_files: Vec::new(), + starting_snapshot_id, + } + } + + /// Register a data file to be removed from the table. + /// + /// The file must exist in the current snapshot; otherwise the commit + /// will fail with a validation error. + pub fn delete_file(mut self, file: DataFile) -> Self { + self.deleted_data_files.push(file); + self + } + + /// Register a data file to be added to the table. + /// + /// Typically this is the merged output of the files being deleted. + pub fn add_file(mut self, file: DataFile) -> Self { + self.added_data_files.push(file); + self + } + + fn validate(&self) -> Result<()> { + if self.deleted_data_files.is_empty() { + return Err(invalid_data!( + "Rewrite files requires at least one file to delete" + )); + } + if let Some(file) = self + .added_data_files + .iter() + .find(|f| f.content_type() != DataContentType::Data) + { + return Err(invalid_data!( + "Only data content type is allowed for rewrite files, got {:?} for {}", + file.content_type(), + file.file_path() + )); + } + Ok(()) + } + + /// Reject the rewrite when a delete manifest was added after the snapshot + /// it was planned against. + /// + /// Java re-checks those deletes file by file in + /// `validateNoNewDeletesForDataFiles`. Until that exists here the commit + /// fails closed, because a rewritten file carries the deletes of the files + /// it replaces only if nothing was deleted from them in the meantime. + async fn validate_no_new_deletes(&self, table: &Table) -> Result<()> { + // Nothing has been committed, so there is nothing to conflict with. + // The filter rejects the rewrite for its missing sources instead. + let Some(current_snapshot) = table.metadata().current_snapshot() else { + return Ok(()); + }; + // The rewrite was planned against a table without a snapshot, so every + // snapshot it has now, deletes included, landed after that. + let Some(starting_snapshot_id) = self.starting_snapshot_id else { + return Err(invalid_data!( + "Cannot rewrite files: the rewrite was planned against a table with no snapshot, and the table has been committed to since" + )); + }; + let Some(starting_snapshot) = table.metadata().snapshot_by_id(starting_snapshot_id) else { + // Without the starting snapshot there is no sequence number to + // compare against, so no delete can be ruled out. + return Err(invalid_data!( + "Cannot rewrite files: the starting snapshot {starting_snapshot_id} is no longer in the table, so deletes added since it cannot be ruled out" + )); + }; + let starting_sequence_number = starting_snapshot.sequence_number(); + + let manifest_list = table.manifest_list_reader(current_snapshot).load().await?; + let conflict = manifest_list.entries().iter().find(|entry| { + entry.content == ManifestContentType::Deletes + && entry.sequence_number > starting_sequence_number + }); + + match conflict { + Some(entry) => Err(invalid_data!( + "Cannot rewrite files: delete manifest {} was added after the starting snapshot {starting_snapshot_id}", + entry.manifest_path, + )), + None => Ok(()), + } + } +} + +#[async_trait] +impl TransactionAction for RewriteFilesAction { + type State = MergingSnapshotProducer; + + fn new_state(&self) -> MergingSnapshotProducer { + MergingSnapshotProducer::new(MergingIntent { + commit_uuid: None, + operation: Operation::Replace, + snapshot_properties: HashMap::new(), + added_data_files: self.added_data_files.clone(), + added_delete_files: Vec::new(), + removed_data_files: self.deleted_data_files.clone(), + removed_delete_files: Vec::new(), + truncate_full_table: false, + }) + } + + async fn commit( + &self, + state: &mut MergingSnapshotProducer, + table: &Table, + ) -> Result { + self.validate()?; + self.validate_no_new_deletes(table).await?; + validate_files_not_live(table, &self.added_data_files).await?; + state.apply(table).await + } + + async fn cleanup(self, _state: MergingSnapshotProducer, _table: &Table, _status: CommitStatus) { + // TODO(#3417): remove the producer's uncommitted artifacts once terminal + // cleanup is wired up. + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use uuid::Uuid; + + use super::*; + use crate::memory::tests::new_memory_catalog; + use crate::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, Datum, FormatVersion, Literal, + ManifestEntryRef, ManifestFile, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, + NestedField, Operation, PrimitiveType, Schema, SnapshotRef, Struct, Transform, Type, + UnboundPartitionField, UnboundPartitionSpec, + }; + use crate::table::Table; + use crate::test_utils::commit_with_fresh_state; + use crate::transaction::tests::{ + append_files, make_data_file, make_v1_minimal_table_in_catalog, + make_v2_minimal_table_in_catalog, make_v3_minimal_table_in_catalog, + }; + use crate::transaction::{ApplyTransactionAction, Transaction}; + use crate::{Catalog, ErrorKind, NamespaceIdent, TableCommit, TableCreation, TableUpdate}; + + /// Read back the manifest entry for `path` from the manifests of `snapshot`. + async fn find_entry( + table: &Table, + snapshot: &SnapshotRef, + path: &str, + ) -> Option { + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + for manifest_file in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + if let Some(entry) = manifest.entries().iter().find(|e| e.file_path() == path) { + return Some(entry.clone()); + } + } + None + } + + /// E2E: Compact 3 small files into 1 merged file. + /// Verify: operation=Replace, file counts, record counts. + #[tokio::test] + async fn test_rewrite_files_compaction() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + rewrite_files_compaction(&catalog, table).await; + } + + #[tokio::test] + async fn test_rewrite_files_compaction_on_v1_table() { + let catalog = new_memory_catalog().await; + let table = make_v1_minimal_table_in_catalog(&catalog).await; + rewrite_files_compaction(&catalog, table).await; + } + + #[tokio::test] + async fn test_rewrite_files_compaction_on_v2_table() { + let catalog = new_memory_catalog().await; + let table = make_v2_minimal_table_in_catalog(&catalog).await; + rewrite_files_compaction(&catalog, table).await; + } + + async fn rewrite_files_compaction(catalog: &impl Catalog, table: Table) { + // Append 3 small files (10 records each). + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let f3 = make_data_file(&table, "test/3.parquet", 10, 100); + let table = append_files(catalog, &table, vec![f1.clone(), f2.clone(), f3.clone()]).await; + + // Verify pre-compaction state. + let summary = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + assert_eq!(summary.get("total-data-files").unwrap(), "3"); + assert_eq!(summary.get("total-records").unwrap(), "30"); + + // Rewrite: delete 3 files, add 1 merged file (30 records). + let merged = make_data_file(&table, "test/merged.parquet", 30, 300); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(f1) + .delete_file(f2) + .delete_file(f3) + .add_file(merged); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(catalog).await.unwrap(); + + // Verify post-compaction snapshot. + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.summary().operation, Operation::Replace); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("total-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-records").unwrap(), "30"); + assert_eq!(summary.get("added-data-files").unwrap(), "1"); + assert_eq!(summary.get("deleted-data-files").unwrap(), "3"); + + // Verify manifest list: merged file is the only live file. + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let mut live_files: Vec = Vec::new(); + for manifest_entry in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_entry).await.unwrap(); + for entry in manifest.entries() { + if entry.is_alive() { + live_files.push(entry.file_path().to_string()); + } + } + } + assert_eq!(live_files, vec!["test/merged.parquet"]); + } + + /// Rewrite with a non-existent delete target should fail. + #[tokio::test] + async fn test_rewrite_files_missing_delete_target() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + // Append 1 file. + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1]).await; + + // Try to delete a file that doesn't exist. + let ghost = make_data_file(&table, "test/ghost.parquet", 10, 100); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(ghost).add_file(merged); + let tx = action.apply(tx).unwrap(); + let result = tx.commit(&catalog).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.message() + .contains("Cannot remove files that are not live in the table"), + "unexpected error: {}", + err.message() + ); + } + + /// Rewrite with no deletes should fail validation. + #[tokio::test] + async fn test_rewrite_files_no_deletes() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().add_file(merged); + let tx = action.apply(tx).unwrap(); + let result = tx.commit(&catalog).await; + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .message() + .contains("at least one file to delete") + ); + } + + /// Partial rewrite: delete 2 of 3 files, keeping one. + /// Verifies that the rewritten manifest for the surviving file + /// has the correct snapshot_id for sequence number assignment. + #[tokio::test] + async fn test_rewrite_files_partial() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + rewrite_files_partial(&catalog, table).await; + } + + #[tokio::test] + async fn test_rewrite_files_partial_on_v1_table() { + let catalog = new_memory_catalog().await; + let table = make_v1_minimal_table_in_catalog(&catalog).await; + rewrite_files_partial(&catalog, table).await; + } + + #[tokio::test] + async fn test_rewrite_files_partial_on_v2_table() { + let catalog = new_memory_catalog().await; + let table = make_v2_minimal_table_in_catalog(&catalog).await; + rewrite_files_partial(&catalog, table).await; + } + + async fn rewrite_files_partial(catalog: &impl Catalog, table: Table) { + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let f3 = make_data_file(&table, "test/3.parquet", 10, 100); + let table = append_files(catalog, &table, vec![f1.clone(), f2.clone(), f3.clone()]).await; + let append_snapshot = table.metadata().current_snapshot().unwrap().clone(); + + // Rewrite only f1 and f2, keep f3. + let merged = make_data_file(&table, "test/merged.parquet", 20, 200); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(f1) + .delete_file(f2) + .add_file(merged); + let tx = action.apply(tx).unwrap(); + let table = tx.commit(catalog).await.unwrap(); + + let snapshot = table.metadata().current_snapshot().unwrap(); + assert_eq!(snapshot.summary().operation, Operation::Replace); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("total-data-files").unwrap(), "2"); + assert_eq!(summary.get("total-records").unwrap(), "30"); + assert_eq!(summary.get("deleted-data-files").unwrap(), "2"); + assert_eq!(summary.get("added-data-files").unwrap(), "1"); + + // Verify live files: f3 (surviving) + merged (new). + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let mut live_files: Vec = Vec::new(); + for manifest_entry in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_entry).await.unwrap(); + for entry in manifest.entries() { + if entry.is_alive() { + live_files.push(entry.file_path().to_string()); + } + } + } + live_files.sort(); + assert_eq!(live_files, vec!["test/3.parquet", "test/merged.parquet"]); + + let survivor = find_entry(&table, snapshot, "test/3.parquet") + .await + .expect("surviving file should still be in a manifest"); + assert_eq!(survivor.status(), ManifestStatus::Existing); + assert_eq!(survivor.snapshot_id(), Some(append_snapshot.snapshot_id())); + assert_eq!( + survivor.sequence_number(), + Some(append_snapshot.sequence_number()) + ); + assert_eq!( + survivor.file_sequence_number, + Some(append_snapshot.sequence_number()) + ); + } + + /// Rewrite on an empty table (no snapshot) should fail because + /// the delete target doesn't exist. + #[tokio::test] + async fn test_rewrite_files_empty_table() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(merged); + let tx = action.apply(tx).unwrap(); + let result = tx.commit(&catalog).await; + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .message() + .contains("Cannot remove files that are not live in the table"), + ); + } + + /// Add a delete manifest, attributed to the table's current snapshot, to + /// that snapshot's manifest list. + async fn add_delete_manifest(table: &Table) -> String { + let snapshot = table.metadata().current_snapshot().unwrap(); + let output = table + .file_io() + .new_output(format!( + "{}/deletes-{}.avro", + table.metadata().metadata_location().unwrap(), + Uuid::new_v4() + )) + .unwrap(); + let mut writer = ManifestWriterBuilder::new( + output, + Some(snapshot.snapshot_id()), + table.metadata().current_schema().clone(), + table.metadata().default_partition_spec().as_ref().clone(), + ) + .build_v3_deletes(); + let mut delete_file = make_data_file(table, "test/1-deletes.parquet", 1, 100); + delete_file.content = DataContentType::PositionDeletes; + writer + .add_file(delete_file, snapshot.sequence_number()) + .unwrap(); + let delete_manifest = writer.write_manifest_file().await.unwrap(); + let delete_manifest_path = delete_manifest.manifest_path.clone(); + + append_to_manifest_list(table, delete_manifest).await; + delete_manifest_path + } + + /// Rewrite the manifest list of the table's current snapshot so it also + /// contains `manifest`. + async fn append_to_manifest_list(table: &Table, manifest: ManifestFile) { + let snapshot = table.metadata().current_snapshot().unwrap(); + let mut entries: Vec = table + .manifest_list_reader(snapshot) + .load() + .await + .unwrap() + .consume_entries() + .into_iter() + .collect(); + entries.push(manifest); + + let mut manifest_list_writer = ManifestListWriter::v3( + table + .file_io() + .new_output(snapshot.manifest_list()) + .unwrap() + .writer() + .await + .unwrap(), + snapshot.snapshot_id(), + snapshot.parent_snapshot_id(), + snapshot.sequence_number(), + Some(table.metadata().next_row_id()), + ); + manifest_list_writer + .add_manifests(entries.into_iter()) + .unwrap(); + manifest_list_writer.close().await.unwrap(); + } + + /// A delete manifest added after the snapshot the rewrite was planned + /// against must stop the commit. + #[tokio::test] + async fn test_rewrite_files_rejects_deletes_added_after_starting_snapshot() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + + // Plan the rewrite against this snapshot, then let another commit land. + let tx = Transaction::new(&table); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let action = tx.rewrite_files().delete_file(f1).add_file(merged); + + let table = append_files(&catalog, &table, vec![f2]).await; + let delete_manifest_path = add_delete_manifest(&table).await; + + let Err(err) = commit_with_fresh_state(action, &table).await else { + panic!("a delete manifest newer than the starting snapshot must be rejected"); + }; + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains(&format!( + "delete manifest {delete_manifest_path} was added after" + )), + "{err}" + ); + } + + /// A rewrite planned against a table with no snapshot knows nothing about + /// what the table holds once something is committed to it. + #[tokio::test] + async fn test_rewrite_files_rejects_commit_after_empty_starting_table() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + // Planned before the table had any snapshot. + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1.clone()).add_file(merged); + + // The file it wants to replace only exists because of a later commit. + let table = append_files(&catalog, &table, vec![f1]).await; + + let Err(err) = commit_with_fresh_state(action, &table).await else { + panic!("a rewrite planned against an empty table must be rejected"); + }; + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("planned against a table with no snapshot"), + "{err}" + ); + } + + /// A rewrite whose starting snapshot has been expired cannot rule out + /// deletes added since, and says so rather than comparing against zero. + #[tokio::test] + async fn test_rewrite_files_rejects_expired_starting_snapshot() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + let starting_snapshot_id = table.metadata().current_snapshot().unwrap().snapshot_id(); + + // Plan the rewrite against this snapshot, then expire it behind us. + let tx = Transaction::new(&table); + let merged = make_data_file(&table, "test/merged.parquet", 10, 100); + let action = tx.rewrite_files().delete_file(f1).add_file(merged); + + let table = append_files(&catalog, &table, vec![f2]).await; + let tx = Transaction::new(&table); + let tx = tx + .expire_snapshots() + .expire_snapshot_ids([starting_snapshot_id]) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + assert!( + table + .metadata() + .snapshot_by_id(starting_snapshot_id) + .is_none(), + "the starting snapshot should have been expired" + ); + + let Err(err) = commit_with_fresh_state(action, &table).await else { + panic!("an expired starting snapshot must be rejected"); + }; + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains(&format!( + "the starting snapshot {starting_snapshot_id} is no longer in the table" + )), + "{err}" + ); + } + + /// An added file whose partition value does not fit the default spec is + /// rejected before anything is written. + #[tokio::test] + async fn test_rewrite_files_rejects_incompatible_partition_value() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + + let mut merged = make_data_file(&table, "test/merged.parquet", 10, 100); + merged.partition = Struct::from_iter([Some(Literal::string("not-a-long"))]); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(merged); + let tx = action.apply(tx).unwrap(); + let err = tx.commit(&catalog).await.unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string() + .contains("Partition value is not compatible partition type"), + "{err}" + ); + } + + /// A delete file handed to `add_file` is rejected: a rewrite replaces data + /// files, and the action's validation names the offending file. + #[tokio::test] + async fn test_rewrite_files_rejects_non_data_content_type() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone()]).await; + + let mut deletes = make_data_file(&table, "test/1-deletes.parquet", 1, 100); + deletes.content = DataContentType::PositionDeletes; + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(deletes); + let tx = action.apply(tx).unwrap(); + let err = tx.commit(&catalog).await.unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains( + "Only data content type is allowed for rewrite files, got PositionDeletes for test/1-deletes.parquet" + ), + "{err}" + ); + } + + /// An added file that is already live in the current snapshot is rejected. + #[tokio::test] + async fn test_rewrite_files_rejects_already_referenced_file() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + + let f1 = make_data_file(&table, "test/1.parquet", 10, 100); + let f2 = make_data_file(&table, "test/2.parquet", 10, 100); + let table = append_files(&catalog, &table, vec![f1.clone(), f2.clone()]).await; + + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(f1).add_file(f2); + let tx = action.apply(tx).unwrap(); + let err = tx.commit(&catalog).await.unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!( + err.to_string().contains( + "Cannot add files that are already referenced by table, files: test/2.parquet" + ), + "{err}" + ); + } + + #[tokio::test] + async fn test_rewrite_files_without_output_files() { + let catalog = new_memory_catalog().await; + let table = make_v3_minimal_table_in_catalog(&catalog).await; + let removed = make_data_file(&table, "test/removed.parquet", 10, 100); + let kept = make_data_file(&table, "test/kept.parquet", 5, 50); + let table = append_files(&catalog, &table, vec![removed.clone(), kept]).await; + let tx = Transaction::new(&table); + let table = tx + .rewrite_files() + .delete_file(removed) + .apply(tx) + .unwrap() + .commit(&catalog) + .await + .unwrap(); + let snapshot = table.metadata().current_snapshot().unwrap(); + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("deleted-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-records").unwrap(), "5"); + assert_eq!(summary.get("total-files-size").unwrap(), "50"); + assert_eq!( + find_entry(&table, snapshot, "test/removed.parquet") + .await + .unwrap() + .status(), + ManifestStatus::Deleted + ); + assert_eq!( + find_entry(&table, snapshot, "test/kept.parquet") + .await + .unwrap() + .status(), + ManifestStatus::Existing + ); + } + + // Partition-spec evolution: `ts` is identity-partitioned under spec 0, then + // `day(ts)` becomes the default spec. Old files keep spec 0 and a + // timestamptz partition value; new files carry spec 1 and a date. + + /// 2026-08-24T09:30:00Z and 2026-08-24T17:45:00Z, in microseconds. + const TS_MORNING: i64 = 1_787_563_800_000_000; + const TS_EVENING: i64 = 1_787_593_500_000_000; + /// 2026-08-24, in days since the epoch. + const DAY: i32 = 20_689; + + /// A table partitioned by `identity(ts)`, still on its first spec. + async fn make_identity_ts_table( + catalog: &impl Catalog, + format_version: FormatVersion, + ) -> Table { + let namespace = NamespaceIdent::new(format!("ns1-{}", Uuid::new_v4())); + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap(); + + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "ts", Type::Primitive(PrimitiveType::Timestamptz)).into(), + ]) + .build() + .unwrap(); + let spec = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("ts") + .transform(Transform::Identity) + .build() + .unwrap(), + ) + .unwrap() + .build(); + let creation = TableCreation::builder() + .name("events".to_string()) + .schema(schema) + .partition_spec(spec) + .format_version(format_version) + .build(); + catalog.create_table(&namespace, creation).await.unwrap() + } + + /// Replace `identity(ts)` with `day(ts)` as the default spec. + async fn evolve_to_day_ts(catalog: &impl Catalog, table: &Table) -> Table { + let spec = UnboundPartitionSpec::builder() + .add_partition_field( + UnboundPartitionField::builder() + .source_ids(vec![2]) + .name("ts_day") + .transform(Transform::Day) + .build() + .unwrap(), + ) + .unwrap() + .build(); + let commit = TableCommit::builder() + .ident(table.identifier().clone()) + .requirements(vec![]) + .updates(vec![ + TableUpdate::AddSpec { spec }, + TableUpdate::SetDefaultSpec { spec_id: -1 }, + ]) + .build(); + let table = catalog.update_table(commit).await.unwrap(); + assert_eq!(table.metadata().default_partition_spec_id(), 1); + table + } + + fn make_file_in_spec( + path: &str, + records: u64, + spec_id: i32, + partition_value: Literal, + ) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(records * 10) + .record_count(records) + .partition(Struct::from_iter([Some(partition_value)])) + .partition_spec_id(spec_id) + .build() + .unwrap() + } + + /// Live files of the current snapshot, keyed by path, with the spec id of + /// the manifest that lists them. + async fn live_files_by_manifest_spec(table: &Table) -> Vec<(String, i32)> { + let snapshot = table.metadata().current_snapshot().unwrap(); + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let mut live = Vec::new(); + for manifest_file in manifest_list.entries() { + let manifest = table.manifest_reader().read(manifest_file).await.unwrap(); + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { + assert_eq!( + entry.data_file().partition_spec_id, + manifest_file.partition_spec_id, + "{} is listed by a manifest of another spec", + entry.file_path() + ); + live.push(( + entry.file_path().to_string(), + manifest_file.partition_spec_id, + )); + } + } + live.sort(); + live + } + + /// Two old-spec files and one new-spec file; returns the table and the + /// snapshot that appended the old-spec files. + async fn spec_evolved_table_with_files( + catalog: &impl Catalog, + format_version: FormatVersion, + ) -> (Table, SnapshotRef, [DataFile; 3]) { + let table = make_identity_ts_table(catalog, format_version).await; + let old_1 = make_file_in_spec( + "test/old-1.parquet", + 10, + 0, + Literal::timestamptz(TS_MORNING), + ); + let old_2 = make_file_in_spec( + "test/old-2.parquet", + 10, + 0, + Literal::timestamptz(TS_EVENING), + ); + let table = append_files(catalog, &table, vec![old_1.clone(), old_2.clone()]).await; + let old_snapshot = table.metadata().current_snapshot().unwrap().clone(); + + let table = evolve_to_day_ts(catalog, &table).await; + let new_1 = make_file_in_spec("test/new-1.parquet", 10, 1, Literal::date(DAY)); + let table = append_files(catalog, &table, vec![new_1.clone()]).await; + (table, old_snapshot, [old_1, old_2, new_1]) + } + + /// Compacting one old-spec file keeps the other in a manifest written under + /// spec 0: its entry, partition value and the manifest's partition summary + /// are still those of `identity(ts)`, not re-encoded under `day(ts)`. + #[tokio::test] + async fn test_rewrite_files_keeps_survivors_on_their_partition_spec() { + for format_version in [FormatVersion::V2, FormatVersion::V3] { + let catalog = new_memory_catalog().await; + let (table, old_snapshot, [old_1, _old_2, _new_1]) = + spec_evolved_table_with_files(&catalog, format_version).await; + + let merged = make_file_in_spec("test/merged.parquet", 10, 1, Literal::date(DAY)); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(old_1).add_file(merged); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + assert_eq!( + live_files_by_manifest_spec(&table).await, + vec![ + ("test/merged.parquet".to_string(), 1), + ("test/new-1.parquet".to_string(), 1), + ("test/old-2.parquet".to_string(), 0), + ], + "{format_version}" + ); + + let snapshot = table.metadata().current_snapshot().unwrap(); + let survivor = find_entry(&table, snapshot, "test/old-2.parquet") + .await + .unwrap(); + assert_eq!(survivor.status(), ManifestStatus::Existing); + assert_eq!(survivor.snapshot_id(), Some(old_snapshot.snapshot_id())); + assert_eq!( + survivor.sequence_number(), + Some(old_snapshot.sequence_number()) + ); + assert_eq!( + survivor.data_file().partition, + Struct::from_iter([Some(Literal::timestamptz(TS_EVENING))]) + ); + + let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let old_spec_manifest = manifest_list + .entries() + .iter() + .find(|m| m.partition_spec_id == 0) + .unwrap(); + // The summary spans the deleted entry as well as the survivor, + // as it does in Java, where every entry updates it. + let summary = &old_spec_manifest.partitions.as_ref().unwrap()[0]; + let morning = Datum::timestamptz_micros(TS_MORNING).to_bytes().unwrap(); + let evening = Datum::timestamptz_micros(TS_EVENING).to_bytes().unwrap(); + assert_eq!(summary.lower_bound.as_ref(), Some(&morning)); + assert_eq!(summary.upper_bound.as_ref(), Some(&evening)); + + let totals = &snapshot.summary().additional_properties; + assert_eq!(totals.get("total-data-files").unwrap(), "3"); + assert_eq!(totals.get("total-records").unwrap(), "30"); + } + } + + /// One rewrite removing a file from each spec: both manifests are filtered + /// under their own spec and the merged file lands under the default spec. + #[tokio::test] + async fn test_rewrite_files_across_partition_specs() { + for format_version in [FormatVersion::V2, FormatVersion::V3] { + let catalog = new_memory_catalog().await; + let (table, _, [old_1, _old_2, new_1]) = + spec_evolved_table_with_files(&catalog, format_version).await; + + let merged = make_file_in_spec("test/merged.parquet", 20, 1, Literal::date(DAY)); + let tx = Transaction::new(&table); + let action = tx + .rewrite_files() + .delete_file(old_1) + .delete_file(new_1) + .add_file(merged); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + assert_eq!( + live_files_by_manifest_spec(&table).await, + vec![ + ("test/merged.parquet".to_string(), 1), + ("test/old-2.parquet".to_string(), 0), + ], + "{format_version}" + ); + let totals = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + assert_eq!(totals.get("deleted-data-files").unwrap(), "2"); + assert_eq!(totals.get("total-data-files").unwrap(), "2"); + assert_eq!(totals.get("total-records").unwrap(), "30"); + } + } + + /// The summary counts and names the partitions a rewrite changed, each + /// under its file's own spec: one `identity(ts)` partition lost a file, one + /// `day(ts)` partition gained one. + #[tokio::test] + async fn test_rewrite_files_summarizes_partitions_of_both_specs() { + let catalog = new_memory_catalog().await; + let (table, _, [old_1, _old_2, _new_1]) = + spec_evolved_table_with_files(&catalog, FormatVersion::V2).await; + // Fast append reports the partition it changed. + let appended = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + assert_eq!( + appended.get("changed-partition-count").map(String::as_str), + Some("1") + ); + + let tx = Transaction::new(&table); + let tx = tx + .update_table_properties() + .set( + "write.summary.partition-limit".to_string(), + "10".to_string(), + ) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let merged = make_file_in_spec("test/merged.parquet", 10, 1, Literal::date(DAY)); + let tx = Transaction::new(&table); + let action = tx.rewrite_files().delete_file(old_1).add_file(merged); + let table = action.apply(tx).unwrap().commit(&catalog).await.unwrap(); + + let summary = &table + .metadata() + .current_snapshot() + .unwrap() + .summary() + .additional_properties; + assert_eq!( + summary.get("changed-partition-count").map(String::as_str), + Some("2"), + "{summary:?}" + ); + // The removed file is named under `identity(ts)`, the added one under + // `day(ts)`. + let mut partitions: Vec<&str> = summary + .keys() + .filter_map(|k| k.strip_prefix("partitions.")) + .collect(); + partitions.sort(); + assert_eq!( + partitions, + vec!["ts=2026-08-24+09%3A30%3A00+UTC", "ts_day=2026-08-24"], + "{summary:?}" + ); + } +} diff --git a/crates/iceberg/src/transaction/snapshot/manifest_filter.rs b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs new file mode 100644 index 0000000000..e977adb89b --- /dev/null +++ b/crates/iceberg/src/transaction/snapshot/manifest_filter.rs @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Rewrites existing manifests to mark files as deleted. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use super::{manifest_path, new_manifest_writer}; +use crate::error::{Result, invalid_data}; +use crate::io::object_cache::ManifestKey; +use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile}; +use crate::table::Table; + +/// Removes a fixed set of files from the manifests of one content type. +/// +/// One manager lives for one logical execution of an action and is driven +/// once per attempt via [`ManifestFilterManager::filter`]. A manifest that +/// references none of the files passes through untouched. One that does is +/// rewritten with those entries marked `Deleted` and the rest `Existing`. +/// +/// A rewrite is reused across attempts while its inputs are unchanged. Its +/// entries carry the sequence numbers and row ids the source's entries +/// inherited from the manifest list when they were read, so the reuse key is +/// the source's [`ManifestKey`], not its path alone: a transaction-local +/// source manifest keeps its path across attempts but inherits new values +/// after a rebase. +pub(crate) struct ManifestFilterManager { + content: ManifestContentType, + /// Paths of the files to remove. Fixed for the execution. + files_to_remove: HashSet, + /// Source manifest → rewritten result. + filtered: HashMap, + /// Format version the rewrites were written under. Per spec, v1 entries + /// cannot inherit sequence numbers, so an upgrade invalidates them. + format_version: Option, +} + +/// One rewritten manifest and the files it marked as deleted. +#[derive(Clone)] +struct FilteredManifest { + manifest: ManifestFile, + removed_files: Vec, +} + +/// The result of one filter pass. +pub(crate) struct Filtered { + /// The resulting manifests, in input order. + pub manifests: Vec, + /// The files that were live and are now marked deleted, including their + /// metrics as read from the source manifests. + pub removed_files: Vec, +} + +impl ManifestFilterManager { + pub(crate) fn new(content: ManifestContentType, files_to_remove: &[DataFile]) -> Self { + Self { + content, + files_to_remove: files_to_remove + .iter() + .map(|f| f.file_path().to_string()) + .collect(), + filtered: HashMap::new(), + format_version: None, + } + } + + /// Applies the removals to `manifests`, rewriting those of this manager's + /// content type that reference a file to remove. + /// + /// Fails if a file to remove is not live in any manifest: it was never in + /// the table, or a concurrent commit already removed it. + /// + /// Rewritten manifests take their paths from `next_manifest_number` and are + /// recorded in `owned_artifacts` before being written, so a failed write + /// still leaves its path tracked for cleanup. + pub(crate) async fn filter( + &mut self, + table: &Table, + snapshot_id: i64, + commit_uuid: Uuid, + manifests: Vec, + next_manifest_number: &mut u64, + owned_artifacts: &mut HashSet, + ) -> Result { + if self.files_to_remove.is_empty() { + return Ok(Filtered { + manifests, + removed_files: Vec::new(), + }); + } + + let format_version = table.metadata().format_version(); + if self.format_version != Some(format_version) { + self.filtered.clear(); + self.format_version = Some(format_version); + } + + let mut out = Vec::with_capacity(manifests.len()); + let mut removed_files: Vec = Vec::new(); + + for source in manifests { + if source.content != self.content { + out.push(source); + continue; + } + + let key = ManifestKey::new(&source); + let result = match self.filtered.get(&key) { + Some(cached) => cached.clone(), + None => { + let result = self + .rewrite( + table, + snapshot_id, + commit_uuid, + &source, + next_manifest_number, + owned_artifacts, + ) + .await?; + self.filtered.insert(key, result.clone()); + result + } + }; + + removed_files.extend(result.removed_files); + out.push(result.manifest); + } + + let found: HashSet<&str> = removed_files.iter().map(|f| f.file_path()).collect(); + let mut missing: Vec<&str> = self + .files_to_remove + .iter() + .map(String::as_str) + .filter(|p| !found.contains(p)) + .collect(); + if !missing.is_empty() { + missing.sort_unstable(); + return Err(invalid_data!( + "Cannot remove files that are not live in the table: {}", + missing.join(", ") + )); + } + + Ok(Filtered { + manifests: out, + removed_files, + }) + } + + /// Reads `source` and, if it references any file to remove, writes a copy + /// with those entries marked `Deleted` and the rest `Existing`. + async fn rewrite( + &self, + table: &Table, + snapshot_id: i64, + commit_uuid: Uuid, + source: &ManifestFile, + next_manifest_number: &mut u64, + owned_artifacts: &mut HashSet, + ) -> Result { + let manifest = table.object_cache().get_manifest(source).await?; + + let removed_files: Vec = manifest + .entries() + .iter() + .filter(|e| e.is_alive() && self.files_to_remove.contains(e.file_path())) + .map(|e| e.data_file().clone()) + .collect(); + if removed_files.is_empty() { + return Ok(FilteredManifest { + manifest: source.clone(), + removed_files, + }); + } + + let path = manifest_path(table, commit_uuid, *next_manifest_number)?; + *next_manifest_number += 1; + owned_artifacts.insert(path.clone()); + + // Write under the source manifest's own schema and spec: its entries' + // partition values and column bounds were encoded for that schema's + // types, which the table's current schema may since have promoted. + let source_metadata = manifest.metadata(); + let mut writer = new_manifest_writer( + table, + path, + snapshot_id, + self.content, + source_metadata.schema().clone(), + source_metadata.partition_spec().clone().into(), + )?; + for entry in manifest.entries() { + if !entry.is_alive() { + continue; + } + let entry = (**entry).clone(); + if self.files_to_remove.contains(entry.file_path()) { + writer.add_delete_entry(entry)?; + } else { + writer.add_existing_entry(entry)?; + } + } + let manifest = writer.write_manifest_file().await?; + + Ok(FilteredManifest { + manifest, + removed_files, + }) + } +} diff --git a/crates/iceberg/src/transaction/snapshot/merging.rs b/crates/iceberg/src/transaction/snapshot/merging.rs new file mode 100644 index 0000000000..3e93f8aba2 --- /dev/null +++ b/crates/iceberg/src/transaction/snapshot/merging.rs @@ -0,0 +1,923 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Persistent producer for snapshot actions that add and remove data and +//! delete files. + +use std::collections::{HashMap, HashSet}; + +use uuid::Uuid; + +use super::manifest_filter::ManifestFilterManager; +use super::{ + AddedManifests, SnapshotCommitInput, current_manifests, generate_snapshot_id, + manifest_list_path, manifest_path, validate_partition_value, write_added_manifest, + write_snapshot_commit, +}; +use crate::error::{Result, invalid_data}; +use crate::spec::{DataFile, FormatVersion, ManifestContentType, ManifestFile, Operation}; +use crate::table::Table; +use crate::transaction::ActionCommit; + +/// What a merging action asks the producer to do. +/// +/// Fixed for one logical execution; the producer derives everything else from +/// it and the table of the current attempt. The action validates the +/// table-independent parts (content types, disjoint add/remove sets) before +/// each attempt. +pub(crate) struct MergingIntent { + /// Overrides the generated commit UUID. + pub commit_uuid: Option, + pub operation: Operation, + pub snapshot_properties: HashMap, + pub added_data_files: Vec, + pub added_delete_files: Vec, + pub removed_data_files: Vec, + pub removed_delete_files: Vec, + /// Whether the summary resets table totals (a full-table overwrite). + pub truncate_full_table: bool, +} + +/// Retry-persistent state for an action that adds and removes data and delete +/// files. +/// +/// One producer lives for one logical execution of an action and is driven once +/// per attempt via [`MergingSnapshotProducer::apply`]. Per attempt it takes the +/// current snapshot's manifests, rewrites those that reference a removed file, +/// appends manifests for the added files, and writes a manifest list. Identity +/// (commit UUID, snapshot id) is fixed on first use; rewritten and added +/// manifests are reused across attempts while their inputs are unchanged. +pub(crate) struct MergingSnapshotProducer { + intent: MergingIntent, + /// Fixed when the producer is created, so every attempt shares one UUID. + commit_uuid: Uuid, + /// Resolved against the table on the first attempt, then stable. + snapshot_id: Option, + /// Incremented per attempt; distinguishes manifest-list paths. + attempt: u64, + /// Numbers every manifest this producer writes, including rewrites. Never + /// reset, so paths never collide across attempts. + manifest_counter: u64, + data_filter: ManifestFilterManager, + delete_filter: ManifestFilterManager, + /// Manifests written for the added files, reused across attempts. + added_manifests: Option, + /// Metadata files written by this producer (manifests and every attempt's + /// manifest list). + // TODO(#3417): consumed by terminal cleanup once it is wired up. + owned_artifacts: HashSet, +} + +impl MergingSnapshotProducer { + pub(crate) fn new(intent: MergingIntent) -> Self { + Self { + commit_uuid: intent.commit_uuid.unwrap_or_else(Uuid::now_v7), + data_filter: ManifestFilterManager::new( + ManifestContentType::Data, + &intent.removed_data_files, + ), + delete_filter: ManifestFilterManager::new( + ManifestContentType::Deletes, + &intent.removed_delete_files, + ), + intent, + snapshot_id: None, + attempt: 0, + manifest_counter: 0, + added_manifests: None, + owned_artifacts: HashSet::new(), + } + } + + /// Produces the resulting manifest set for the current table and writes + /// this attempt's snapshot. + pub(crate) async fn apply(&mut self, table: &Table) -> Result { + let snapshot_id = self.snapshot_id(table); + + let manifests = current_manifests(table).await?; + + // Removals first: filtering only touches the current snapshot's + // manifests, never the ones this producer adds. + let data = self + .data_filter + .filter( + table, + snapshot_id, + self.commit_uuid, + manifests, + &mut self.manifest_counter, + &mut self.owned_artifacts, + ) + .await?; + let deletes = self + .delete_filter + .filter( + table, + snapshot_id, + self.commit_uuid, + data.manifests, + &mut self.manifest_counter, + &mut self.owned_artifacts, + ) + .await?; + let mut manifests = deletes.manifests; + + manifests.extend(self.added_manifests(table, snapshot_id).await?); + + let manifest_list_path = self.next_manifest_list_path(table, snapshot_id)?; + + let added_files: Vec = self + .intent + .added_data_files + .iter() + .chain(&self.intent.added_delete_files) + .cloned() + .collect(); + + write_snapshot_commit(table, SnapshotCommitInput { + snapshot_id, + manifest_list_path, + operation: self.intent.operation.clone(), + properties: self.intent.snapshot_properties.clone(), + added_files: &added_files, + removed_files: &data.removed_files, + removed_delete_files: &deletes.removed_files, + manifests, + truncate_full_table: self.intent.truncate_full_table, + }) + .await + } + + /// Checks that every added file is written under the table's default + /// partition spec with a compatible partition value, and that delete files + /// are only added to tables whose format supports them. + /// + /// Runs against the table the manifests are written for; a later rebase + /// does not invalidate manifests already written under a spec the table + /// still has. + fn validate_added_files(&self, table: &Table) -> Result<()> { + let metadata = table.metadata(); + if !self.intent.added_delete_files.is_empty() + && metadata.format_version() == FormatVersion::V1 + { + return Err(invalid_data!( + "Cannot add delete files to a format version 1 table" + )); + } + for file in self + .intent + .added_data_files + .iter() + .chain(&self.intent.added_delete_files) + { + // TODO(#3500): accept any spec known to the table and group the + // files by spec when writing manifests. + if metadata.default_partition_spec_id() != file.partition_spec_id { + return Err(invalid_data!( + "File partition spec id does not match table default partition spec id" + )); + } + validate_partition_value(file.partition(), metadata.default_partition_type())?; + } + Ok(()) + } + + fn snapshot_id(&mut self, table: &Table) -> i64 { + *self + .snapshot_id + .get_or_insert_with(|| generate_snapshot_id(table)) + } + + fn next_manifest_list_path(&mut self, table: &Table, snapshot_id: i64) -> Result { + let path = manifest_list_path(table, snapshot_id, self.attempt, self.commit_uuid)?; + self.attempt += 1; + self.owned_artifacts.insert(path.clone()); + Ok(path) + } + + /// Returns the manifests for the added files, writing them only if no + /// reusable set exists for the current format version. Data and delete + /// files go into separate manifests. + async fn added_manifests( + &mut self, + table: &Table, + snapshot_id: i64, + ) -> Result> { + let format_version = table.metadata().format_version(); + + if let Some(cached) = &self.added_manifests + && cached.format_version == format_version + { + return Ok(cached.manifests.clone()); + } + + self.validate_added_files(table)?; + + let mut manifests = Vec::new(); + for (content, files) in [ + (ManifestContentType::Data, &self.intent.added_data_files), + ( + ManifestContentType::Deletes, + &self.intent.added_delete_files, + ), + ] { + if files.is_empty() { + continue; + } + // Allocate the path before writing so a partial write is still owned. + // + // TODO(#3500): group by partition spec. TODO(#3499): roll at the + // target size and write concurrently. + let path = manifest_path(table, self.commit_uuid, self.manifest_counter)?; + self.manifest_counter += 1; + self.owned_artifacts.insert(path.clone()); + manifests.push( + write_added_manifest( + table, + path, + snapshot_id, + content, + table.metadata().default_partition_spec_id(), + files, + ) + .await?, + ); + } + + self.added_manifests = Some(AddedManifests { + manifests: manifests.clone(), + format_version, + }); + Ok(manifests) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::super::tests::{append_to, data_file}; + use super::{MergingIntent, MergingSnapshotProducer}; + use crate::spec::{ + DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, ManifestContentType, + ManifestStatus, Operation, SnapshotRef, Struct, + }; + use crate::table::Table; + use crate::transaction::tests::{make_v1_table, make_v2_minimal_table}; + use crate::{ErrorKind, TableUpdate}; + + fn delete_file(table: &Table, path: &str) -> DataFile { + DataFileBuilder::default() + .content(DataContentType::PositionDeletes) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(50) + .record_count(1) + .partition_spec_id(table.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::long(300))])) + .build() + .unwrap() + } + + fn intent(operation: Operation) -> MergingIntent { + MergingIntent { + commit_uuid: None, + operation, + snapshot_properties: HashMap::new(), + added_data_files: vec![], + added_delete_files: vec![], + removed_data_files: vec![], + removed_delete_files: vec![], + truncate_full_table: false, + } + } + + fn added_snapshot(commit: &mut crate::transaction::ActionCommit) -> SnapshotRef { + commit + .take_updates() + .into_iter() + .find_map(|u| match u { + TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot)), + _ => None, + }) + .expect("apply should add a snapshot") + } + + /// Reads every (status, path) pair in the manifest at `manifest_path`. + async fn entries_of( + table: &Table, + manifest: &crate::spec::ManifestFile, + ) -> Vec<(ManifestStatus, String)> { + let mut entries: Vec<_> = table + .manifest_reader() + .read(manifest) + .await + .unwrap() + .entries() + .iter() + .map(|e| (e.status(), e.file_path().to_string())) + .collect(); + entries.sort_by(|a, b| a.1.cmp(&b.1)); + entries + } + + /// Removing one of two files in a manifest rewrites it with one `Deleted` + /// and one `Existing` entry; a manifest without removed files is carried + /// forward by path. + #[tokio::test] + async fn test_remove_data_file_rewrites_only_its_manifest() { + let base = make_v2_minimal_table(); + // Two commits so the current snapshot has two data manifests. + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + let table = append_to(&table, vec![data_file(&base, "test/c.parquet", 1)]).await; + let current = table.metadata().current_snapshot().unwrap(); + let before = table.manifest_list_reader(current).load().await.unwrap(); + assert_eq!(before.entries().len(), 2); + let untouched = before + .entries() + .iter() + .find(|m| m.added_files_count == Some(1)) + .unwrap() + .manifest_path + .clone(); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let after = table.manifest_list_reader(&snapshot).load().await.unwrap(); + assert_eq!(after.entries().len(), 2); + assert!(after.entries().iter().any(|m| m.manifest_path == untouched)); + let rewritten = after + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .expect("a rewritten manifest belongs to the new snapshot"); + assert_eq!(rewritten.deleted_files_count, Some(1)); + assert_eq!(rewritten.existing_files_count, Some(1)); + assert_eq!(entries_of(&table, rewritten).await, vec![ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + ]); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("deleted-data-files").unwrap(), "1"); + assert_eq!(summary.get("total-data-files").unwrap(), "2"); + } + + /// Added data and delete files go into separate manifests of the right + /// content type, and the summary counts both. + #[tokio::test] + async fn test_add_data_and_delete_files() { + let table = make_v2_minimal_table(); + let mut i = intent(Operation::Overwrite); + i.added_data_files = vec![data_file(&table, "test/d.parquet", 3)]; + i.added_delete_files = vec![delete_file(&table, "test/d-deletes.parquet")]; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let mut contents: Vec<_> = list.entries().iter().map(|m| m.content).collect(); + contents.sort_by_key(|c| *c as i32); + assert_eq!(contents, vec![ + ManifestContentType::Data, + ManifestContentType::Deletes + ]); + + let summary = &snapshot.summary().additional_properties; + assert_eq!(summary.get("added-data-files").unwrap(), "1"); + assert_eq!(summary.get("added-delete-files").unwrap(), "1"); + assert_eq!(summary.get("added-records").unwrap(), "3"); + } + + /// A retry reuses both the rewritten and the added manifests; only the + /// manifest list is written again. + #[tokio::test] + async fn test_retry_reuses_rewritten_and_added_manifests() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + let mut i = intent(Operation::Overwrite); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + i.added_data_files = vec![data_file(&base, "test/x.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + + let mut first = producer.apply(&table).await.unwrap(); + let mut second = producer.apply(&table).await.unwrap(); + let first = added_snapshot(&mut first); + let second = added_snapshot(&mut second); + + assert_eq!(first.snapshot_id(), second.snapshot_id()); + assert_ne!(first.manifest_list(), second.manifest_list()); + + let paths = |list: crate::spec::ManifestList| { + let mut p: Vec<_> = list + .entries() + .iter() + .map(|m| m.manifest_path.clone()) + .collect(); + p.sort(); + p + }; + let first_paths = paths(table.manifest_list_reader(&first).load().await.unwrap()); + let second_paths = paths(table.manifest_list_reader(&second).load().await.unwrap()); + assert_eq!(first_paths, second_paths); + // One rewrite plus one added manifest. + assert_eq!(producer.manifest_counter, 2); + assert_eq!(producer.attempt, 2); + } + + /// Removing a file that is not live anywhere fails and names the file. + #[tokio::test] + async fn test_remove_missing_file_fails() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/nope.parquet", 1)]; + let err = MergingSnapshotProducer::new(i) + .apply(&table) + .await + .err() + .expect("removing a file that is not live must fail"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(err.message().contains("test/nope.parquet")); + } + + /// A file removed by a concurrent commit is no longer live on the rebased + /// table, so the retry fails rather than silently succeeding. + #[tokio::test] + async fn test_rebase_onto_table_without_removed_file_fails() { + let base = make_v2_minimal_table(); + let s1 = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + producer.apply(&s1).await.unwrap(); + + // Concurrent writer removed `a` first; our retry rebases onto that. + let mut other = intent(Operation::Delete); + other.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut other_commit = MergingSnapshotProducer::new(other) + .apply(&s1) + .await + .unwrap(); + let s2_snapshot = added_snapshot(&mut other_commit); + let s2 = { + use crate::spec::{MAIN_BRANCH, SnapshotReference, SnapshotRetention}; + let metadata = s1 + .metadata() + .clone() + .into_builder(None) + .add_snapshot(s2_snapshot.as_ref().clone()) + .unwrap() + .set_ref( + MAIN_BRANCH, + SnapshotReference::new( + s2_snapshot.snapshot_id(), + SnapshotRetention::branch(None, None, None), + ), + ) + .unwrap() + .build() + .unwrap() + .metadata; + s1.clone().with_metadata(std::sync::Arc::new(metadata)) + }; + + let err = producer + .apply(&s2) + .await + .err() + .expect("file already removed concurrently must not be removed twice"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } + + /// Multi-action replay (RFC §5.2): action A's manifest keeps its path across + /// attempts, but after a rebase the manifest list assigns it a new sequence + /// number. Action B's rewrite of that manifest bakes the inherited sequence + /// numbers into its `Existing`/`Deleted` entries, so B must not reuse a + /// rewrite made under the previous sequence number. + #[tokio::test] + async fn test_rewrite_is_not_reused_when_source_manifest_sequence_number_changes() { + use super::super::simple::{AppendIntent, SimpleSnapshotProducer}; + use super::super::tests::with_snapshot; + + let s0 = make_v2_minimal_table(); + + // Action A appends x and y; action B removes x from A's manifest. + let mut a = SimpleSnapshotProducer::new(AppendIntent { + commit_uuid: None, + snapshot_properties: HashMap::new(), + data_files: vec![ + data_file(&s0, "test/x.parquet", 1), + data_file(&s0, "test/y.parquet", 1), + ], + }); + let mut b_intent = intent(Operation::Delete); + b_intent.removed_data_files = vec![data_file(&s0, "test/x.parquet", 1)]; + let mut b = MergingSnapshotProducer::new(b_intent); + + // Attempt 0 on S0. + let sa0 = added_snapshot(&mut a.apply(&s0).await.unwrap()); + let after_a0 = with_snapshot(&s0, sa0.clone()); + let sb0 = added_snapshot(&mut b.apply(&after_a0).await.unwrap()); + + // A concurrent commit lands on S0, taking the next sequence number. The + // retry rebases onto it: A reuses its manifest but it now inherits a new + // sequence number, and B sees the same manifest path under it. + let s1 = append_to(&s0, vec![data_file(&s0, "test/other.parquet", 1)]).await; + let sa1 = added_snapshot(&mut a.apply(&s1).await.unwrap()); + let after_a1 = with_snapshot(&s1, sa1.clone()); + let sb1 = added_snapshot(&mut b.apply(&after_a1).await.unwrap()); + assert_eq!(sa1.sequence_number(), sa0.sequence_number() + 1); + + // B's rewritten manifest in each attempt. + let rewritten_in = |table: Table, snapshot: SnapshotRef| async move { + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .cloned() + .unwrap(); + let manifest = table.manifest_reader().read(&rewritten).await.unwrap(); + let y = manifest + .entries() + .iter() + .find(|e| e.file_path() == "test/y.parquet") + .unwrap(); + (rewritten.manifest_path.clone(), y.sequence_number()) + }; + let (path0, y_seq0) = rewritten_in(after_a0, sb0).await; + let (path1, y_seq1) = rewritten_in(after_a1, sb1).await; + + // `y` stays live under A's snapshot, whose sequence number moved; the + // rewrite that keeps it as `Existing` must carry the new one. + assert_eq!(y_seq0, Some(sa0.sequence_number())); + assert_eq!(y_seq1, Some(sa1.sequence_number())); + assert_ne!( + path0, path1, + "a rewrite under a different sequence number is a new file" + ); + } + + /// Removing a data file from a v1 table: v1 entries read with sequence + /// number 0, which satisfies the writer, and v1 serialization drops the + /// field again. The rewritten manifest is readable and `b` stays live. + #[tokio::test] + async fn test_remove_data_file_from_v1_table() { + let base = make_v1_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut commit = MergingSnapshotProducer::new(i).apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + let manifest = table.manifest_reader().read(rewritten).await.unwrap(); + let mut live: Vec<_> = manifest + .entries() + .iter() + .filter(|e| e.is_alive()) + .map(|e| e.file_path().to_string()) + .collect(); + live.sort(); + assert_eq!(live, vec!["test/b.parquet".to_string()]); + // Both the rewritten entries and the snapshot stay at v1's sequence number 0. + assert!( + manifest + .entries() + .iter() + .all(|e| e.sequence_number() == Some(0)) + ); + assert_eq!(snapshot.sequence_number(), 0); + assert_eq!( + snapshot + .summary() + .additional_properties + .get("deleted-data-files") + .unwrap(), + "1" + ); + } + + /// A manifest written under an earlier schema is rewritten under the + /// current one; its partition spec still binds because partition source + /// columns cannot be dropped. + #[tokio::test] + async fn test_rewrite_manifest_written_under_earlier_schema() { + use crate::spec::{NestedField, PrimitiveType, Type}; + + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 1), + data_file(&base, "test/b.parquet", 1), + ]) + .await; + + // Evolve the schema after the manifest was written. + let new_schema = table + .metadata() + .current_schema() + .as_ref() + .clone() + .into_builder() + .with_fields([ + NestedField::optional(100, "added", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .add_current_schema(new_schema) + .unwrap() + .build() + .unwrap() + .metadata; + let evolved = table.clone().with_metadata(std::sync::Arc::new(metadata)); + assert_ne!( + evolved.metadata().current_schema_id(), + table.metadata().current_schema_id() + ); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + let mut commit = MergingSnapshotProducer::new(i) + .apply(&evolved) + .await + .unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = evolved + .manifest_list_reader(&snapshot) + .load() + .await + .unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + assert_eq!( + rewritten.partition_spec_id, + table.metadata().default_partition_spec_id() + ); + assert_eq!(entries_of(&evolved, rewritten).await, vec![ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + ]); + // Partition values survive the rewrite under the new schema. + let manifest = evolved.manifest_reader().read(rewritten).await.unwrap(); + for entry in manifest.entries() { + assert_eq!( + entry.data_file().partition(), + &Struct::from_iter([Some(Literal::long(300))]) + ); + } + } + + /// A manifest's partition values are encoded for the partition type its + /// spec had when it was written. Promoting the source column (`int` to + /// `long`) changes that type under the current schema; the rewrite must + /// keep the entries readable and their partition values intact. + #[tokio::test] + async fn test_rewrite_manifest_after_partition_source_type_promotion() { + use crate::TableIdent; + use crate::io::FileIO; + use crate::spec::{ + FormatVersion, NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, + TableMetadataBuilder, Transform, Type, + }; + use crate::test_utils::test_runtime; + + let int_schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let spec = PartitionSpec::builder(int_schema.clone()) + .add_partition_field("x", "x", Transform::Identity) + .unwrap() + .build() + .unwrap(); + let metadata = TableMetadataBuilder::new( + int_schema, + spec, + SortOrder::unsorted_order(), + "memory:///table".to_string(), + FormatVersion::V2, + HashMap::new(), + ) + .unwrap() + .build() + .unwrap() + .metadata; + let base = Table::builder() + .metadata(metadata) + .metadata_location("memory:///table/metadata/v1.json") + .identifier(TableIdent::from_strs(["ns", "t"]).unwrap()) + .file_io(FileIO::new_with_memory()) + .runtime(test_runtime()) + .build() + .unwrap(); + + let int_file = |path: &str| { + DataFileBuilder::default() + .content(DataContentType::Data) + .file_path(path.to_string()) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(1) + .partition_spec_id(base.metadata().default_partition_spec_id()) + .partition(Struct::from_iter([Some(Literal::int(7))])) + .build() + .unwrap() + }; + let table = append_to(&base, vec![ + int_file("test/a.parquet"), + int_file("test/b.parquet"), + ]) + .await; + + // Promote x: int -> long. + let long_schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![ + NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + let metadata = table + .metadata() + .clone() + .into_builder(None) + .add_current_schema(long_schema) + .unwrap() + .build() + .unwrap() + .metadata; + let promoted = table.clone().with_metadata(std::sync::Arc::new(metadata)); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![int_file("test/a.parquet")]; + let mut commit = MergingSnapshotProducer::new(i) + .apply(&promoted) + .await + .unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = promoted + .manifest_list_reader(&snapshot) + .load() + .await + .unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + assert_eq!(entries_of(&promoted, rewritten).await, vec![ + (ManifestStatus::Deleted, "test/a.parquet".to_string()), + (ManifestStatus::Existing, "test/b.parquet".to_string()), + ]); + let manifest = promoted.manifest_reader().read(rewritten).await.unwrap(); + for entry in manifest.entries() { + assert_eq!( + entry.data_file().partition(), + &Struct::from_iter([Some(Literal::int(7))]) + ); + } + } + + /// On v3, existing entries keep the row ids they were assigned when first + /// committed, and the rewritten manifest gets its own range from the list + /// writer. + #[tokio::test] + async fn test_rewrite_preserves_first_row_ids_on_v3() { + use crate::transaction::tests::make_v3_minimal_table; + + let base = make_v3_minimal_table(); + let table = append_to(&base, vec![ + data_file(&base, "test/a.parquet", 5), + data_file(&base, "test/b.parquet", 7), + ]) + .await; + let before = table.metadata().current_snapshot().unwrap(); + let before_list = table.manifest_list_reader(before).load().await.unwrap(); + let before_manifest = table + .manifest_reader() + .read(&before_list.entries()[0]) + .await + .unwrap(); + let b_row_id = before_manifest + .entries() + .iter() + .find(|e| e.file_path() == "test/b.parquet") + .unwrap() + .data_file() + .first_row_id(); + assert_eq!(b_row_id, Some(5)); + + let mut i = intent(Operation::Delete); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 5)]; + let mut commit = MergingSnapshotProducer::new(i).apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + let rewritten = list + .entries() + .iter() + .find(|m| m.added_snapshot_id == snapshot.snapshot_id()) + .unwrap(); + let manifest = table.manifest_reader().read(rewritten).await.unwrap(); + let b = manifest + .entries() + .iter() + .find(|e| e.file_path() == "test/b.parquet") + .unwrap(); + assert_eq!(b.data_file().first_row_id(), b_row_id); + // The new snapshot starts its range where the table's next_row_id was. + assert_eq!( + snapshot.first_row_id(), + Some(table.metadata().next_row_id()) + ); + } + + /// Every manifest the producer writes, including rewrites, is recorded as + /// owned before the write starts. + #[tokio::test] + async fn test_owned_artifacts_cover_rewrites_and_added_manifests() { + let base = make_v2_minimal_table(); + let table = append_to(&base, vec![data_file(&base, "test/a.parquet", 1)]).await; + + let mut i = intent(Operation::Overwrite); + i.removed_data_files = vec![data_file(&base, "test/a.parquet", 1)]; + i.added_data_files = vec![data_file(&base, "test/x.parquet", 1)]; + let mut producer = MergingSnapshotProducer::new(i); + let mut commit = producer.apply(&table).await.unwrap(); + let snapshot = added_snapshot(&mut commit); + + let list = table.manifest_list_reader(&snapshot).load().await.unwrap(); + for m in list.entries() { + if m.added_snapshot_id == snapshot.snapshot_id() { + assert!(producer.owned_artifacts.contains(&m.manifest_path)); + } + } + assert!(producer.owned_artifacts.contains(snapshot.manifest_list())); + // One rewrite, one added manifest, one manifest list. + assert_eq!(producer.owned_artifacts.len(), 3); + } + + #[tokio::test] + async fn test_delete_files_rejected_on_v1_table() { + let table = make_v1_table(); + let mut i = intent(Operation::Overwrite); + i.added_delete_files = vec![delete_file(&table, "test/d-deletes.parquet")]; + let err = MergingSnapshotProducer::new(i) + .apply(&table) + .await + .err() + .expect("v1 tables cannot hold delete files"); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + } +} diff --git a/crates/iceberg/src/transaction/snapshot/mod.rs b/crates/iceberg/src/transaction/snapshot/mod.rs index 36616d0125..7ac1e4c7a8 100644 --- a/crates/iceberg/src/transaction/snapshot/mod.rs +++ b/crates/iceberg/src/transaction/snapshot/mod.rs @@ -23,6 +23,12 @@ //! Iceberg metadata and files, never select operation policy, and never cache //! across calls. +// TODO(#1607): remove the allows once `RowDeltaAction` consumes the merging +// producer. +#[allow(dead_code)] +pub(crate) mod manifest_filter; +#[allow(dead_code)] +pub(crate) mod merging; pub(crate) mod simple; use std::collections::{HashMap, HashSet}; @@ -33,10 +39,11 @@ use uuid::Uuid; use crate::error::{Result, invalid_data}; use crate::spec::{ - DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestFile, - ManifestListWriter, ManifestWriter, ManifestWriterBuilder, Operation, PartitionSpecRef, - SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, SnapshotSummaryCollector, Struct, - StructType, Summary, TableProperties, set_iceberg_version, update_snapshot_summaries, + DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, ManifestEntry, + ManifestFile, ManifestListWriter, ManifestStatus, ManifestWriter, ManifestWriterBuilder, + Operation, PartitionSpecRef, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention, + SnapshotSummaryCollector, Struct, StructType, Summary, TableProperties, set_iceberg_version, + update_snapshot_summaries, }; use crate::table::Table; use crate::transaction::ActionCommit; @@ -173,6 +180,61 @@ pub(crate) async fn current_manifests(table: &Table) -> Result .collect()) } +/// The manifests a producer wrote for one execution's added files. +/// +/// Reusable while the table's format version is unchanged. Per spec, sequence +/// number inheritance does not apply to v1 manifests, so one written before a +/// concurrent upgrade cannot carry the v2 sequence number a retry assigns and +/// must be rewritten. The schema is not a guard: the manifest's embedded schema +/// describes the files it was written for, which do not change. Nor is the +/// default partition spec: the files name their own spec, which stays valid. +pub(crate) struct AddedManifests { + /// One per content type today. Several once added files are grouped by + /// partition spec (#3500) or rolled over at a target size (#3499). + pub manifests: Vec, + pub format_version: FormatVersion, +} + +/// Writes one manifest at `path` holding `files` as `Added` entries, all of +/// which must belong to `partition_spec_id`, and returns its manifest-list entry. +pub(crate) async fn write_added_manifest( + table: &Table, + path: String, + snapshot_id: i64, + content: ManifestContentType, + partition_spec_id: i32, + files: &[DataFile], +) -> Result { + let metadata = table.metadata(); + let partition_spec = metadata + .partition_spec_by_id(partition_spec_id) + .ok_or_else(|| invalid_data!("Unknown partition spec id {partition_spec_id}"))?; + + let mut writer = new_manifest_writer( + table, + path, + snapshot_id, + content, + metadata.current_schema().clone(), + partition_spec.clone(), + )?; + let format_version = metadata.format_version(); + for file in files { + let builder = ManifestEntry::builder() + .status(ManifestStatus::Added) + .data_file(file.clone()); + let entry = if format_version == FormatVersion::V1 { + builder.snapshot_id(snapshot_id).build() + } else { + // For format version > 1, the snapshot id is inherited from the + // manifest list, so a rebase does not require rewriting the manifest. + builder.build() + }; + writer.add_entry(entry)?; + } + writer.write_manifest_file().await +} + /// Fails if any of `data_files` is already live in the table's current snapshot. /// /// Must run against each attempt's base, since a concurrent commit may have @@ -238,6 +300,7 @@ fn build_summary( mut properties: HashMap, added_files: &[DataFile], removed_files: &[DataFile], + removed_delete_files: &[DataFile], truncate_full_table: bool, ) -> Result { let metadata = table.metadata(); @@ -259,7 +322,7 @@ fn build_summary( ); } - for data_file in removed_files { + for data_file in removed_files.iter().chain(removed_delete_files) { let partition_spec = metadata .partition_spec_by_id(data_file.partition_spec_id) .ok_or_else(|| { @@ -297,9 +360,12 @@ pub(crate) struct SnapshotCommitInput<'a> { pub manifest_list_path: String, pub operation: Operation, pub properties: HashMap, + /// Data and delete files added by this commit. pub added_files: &'a [DataFile], /// Must match the data files marked Deleted by this commit, including their metrics. pub removed_files: &'a [DataFile], + /// Must match the delete files marked Deleted by this commit, including their metrics. + pub removed_delete_files: &'a [DataFile], pub manifests: Vec, /// Whether the summary resets table totals (a full-table overwrite). pub truncate_full_table: bool, @@ -318,20 +384,29 @@ pub(crate) async fn write_snapshot_commit( properties, added_files, removed_files, + removed_delete_files, manifests, truncate_full_table, } = input; - debug_assert_eq!( + let deleted_count = |content: ManifestContentType| { manifests .iter() - .filter(|m| m.added_snapshot_id == snapshot_id && m.content == ManifestContentType::Data) + .filter(|m| m.added_snapshot_id == snapshot_id && m.content == content) .filter_map(|m| m.deleted_files_count) .map(u64::from) - .sum::(), + .sum::() + }; + debug_assert_eq!( + deleted_count(ManifestContentType::Data), removed_files.len() as u64, "deleted files must match this commit's manifest entries" ); + debug_assert_eq!( + deleted_count(ManifestContentType::Deletes), + removed_delete_files.len() as u64, + "deleted delete files must match this commit's manifest entries" + ); let metadata = table.metadata(); let parent_snapshot_id = metadata.current_snapshot_id(); @@ -344,6 +419,7 @@ pub(crate) async fn write_snapshot_commit( properties, added_files, removed_files, + removed_delete_files, truncate_full_table, ) .map_err(|err| { @@ -446,7 +522,7 @@ pub(crate) async fn write_snapshot_commit( } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::collections::HashMap; use std::sync::Arc; @@ -461,7 +537,7 @@ mod tests { use crate::transaction::tests::make_v2_minimal_table; use crate::{ErrorKind, TableUpdate}; - fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { + pub(super) fn data_file(table: &Table, path: &str, record_count: u64) -> DataFile { DataFileBuilder::default() .content(DataContentType::Data) .file_path(path.to_string()) @@ -480,23 +556,31 @@ mod tests { /// Commits `data_files` to `table` through a fresh producer, returning the /// table with that snapshot as current. - async fn append_to(table: &Table, data_files: Vec) -> Table { + pub(super) async fn append_to(table: &Table, data_files: Vec) -> Table { let mut producer = SimpleSnapshotProducer::new(AppendIntent { commit_uuid: None, snapshot_properties: HashMap::new(), data_files, }); - let snapshot = producer - .apply(table) - .await - .unwrap() + let commit = producer.apply(table).await.unwrap(); + with_snapshot(table, added_snapshot(commit)) + } + + /// Extracts the snapshot an `ActionCommit` adds. + pub(super) fn added_snapshot(mut commit: ActionCommit) -> SnapshotRef { + commit .take_updates() .into_iter() .find_map(|u| match u { TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot)), _ => None, }) - .expect("append should add a snapshot"); + .expect("commit should add a snapshot") + } + + /// Returns `table` with `snapshot` added and set as current, as a local + /// `TableCommit::apply` would. + pub(super) fn with_snapshot(table: &Table, snapshot: SnapshotRef) -> Table { let metadata = table .metadata() .clone() @@ -567,6 +651,7 @@ mod tests { HashMap::new(), &[], &[deleted], + &[], false, ) .unwrap(); @@ -592,6 +677,7 @@ mod tests { properties: HashMap::new(), added_files: &[], removed_files: &[deleted], + removed_delete_files: &[], manifests: vec![], truncate_full_table: false, }) diff --git a/crates/iceberg/src/transaction/snapshot/simple.rs b/crates/iceberg/src/transaction/snapshot/simple.rs index 7aaa505399..4da190e2f6 100644 --- a/crates/iceberg/src/transaction/snapshot/simple.rs +++ b/crates/iceberg/src/transaction/snapshot/simple.rs @@ -22,14 +22,12 @@ use std::collections::{HashMap, HashSet}; use uuid::Uuid; use super::{ - SnapshotCommitInput, current_manifests, generate_snapshot_id, manifest_list_path, - manifest_path, new_manifest_writer, validate_partition_value, write_snapshot_commit, + AddedManifests, SnapshotCommitInput, current_manifests, generate_snapshot_id, + manifest_list_path, manifest_path, validate_partition_value, write_added_manifest, + write_snapshot_commit, }; use crate::error::{Result, invalid_data}; -use crate::spec::{ - DataFile, FormatVersion, ManifestContentType, ManifestEntry, ManifestFile, ManifestStatus, - Operation, -}; +use crate::spec::{DataFile, ManifestContentType, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::ActionCommit; @@ -76,20 +74,6 @@ pub(crate) struct SimpleSnapshotProducer { owned_artifacts: HashSet, } -/// The manifests written for one append's added files. -/// -/// Reusable while the table's format version is unchanged. Per spec, sequence -/// number inheritance does not apply to v1 manifests, so one written before a -/// concurrent upgrade cannot carry the v2 sequence number a retry assigns and -/// must be rewritten. The schema is not a guard: the manifest's embedded schema -/// describes the files it was written for, which do not change. -struct AddedManifests { - /// One manifest today. Several once added files are grouped by partition - /// spec (#3500) or rolled over at a target size (#3499). - manifests: Vec, - format_version: FormatVersion, -} - impl SimpleSnapshotProducer { pub(crate) fn new(intent: AppendIntent) -> Self { Self { @@ -122,6 +106,7 @@ impl SimpleSnapshotProducer { properties: self.intent.snapshot_properties.clone(), added_files: &self.intent.data_files, removed_files: &[], + removed_delete_files: &[], manifests, truncate_full_table: false, }) @@ -197,6 +182,7 @@ impl SimpleSnapshotProducer { table, path, snapshot_id, + ManifestContentType::Data, table.metadata().default_partition_spec_id(), &self.intent.data_files, ) @@ -211,45 +197,6 @@ impl SimpleSnapshotProducer { } } -/// Writes one data manifest at `path` for `data_files`, all of which must -/// belong to `partition_spec_id`, and returns its manifest-list entry. -async fn write_added_manifest( - table: &Table, - path: String, - snapshot_id: i64, - partition_spec_id: i32, - data_files: &[DataFile], -) -> Result { - let metadata = table.metadata(); - let partition_spec = metadata - .partition_spec_by_id(partition_spec_id) - .ok_or_else(|| invalid_data!("Unknown partition spec id {partition_spec_id}"))?; - - let mut writer = new_manifest_writer( - table, - path, - snapshot_id, - ManifestContentType::Data, - metadata.current_schema().clone(), - partition_spec.clone(), - )?; - let format_version = metadata.format_version(); - for data_file in data_files { - let builder = ManifestEntry::builder() - .status(ManifestStatus::Added) - .data_file(data_file.clone()); - let entry = if format_version == FormatVersion::V1 { - builder.snapshot_id(snapshot_id).build() - } else { - // For format version > 1, the snapshot id is inherited from the - // manifest list, so a rebase does not require rewriting the manifest. - builder.build() - }; - writer.add_entry(entry)?; - } - writer.write_manifest_file().await -} - #[cfg(test)] mod tests { use std::collections::HashMap;