From f30da58615b8055c05f9dbe813868e8e87fb2fdb Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Mon, 19 Oct 2026 14:51:34 +0800 Subject: [PATCH] feat: stage a fast append, to write a chain of snapshots together MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A commit that chains several appends — each a snapshot on the last — wrote each one's manifests, then its manifest list, then read that list back for the next: four round trips to the object store a snapshot, one after another. `FastAppendAction::stage` plans the snapshot against the table without writing it: the commit that adds it, and a `StagedSnapshot` holding its files to write. `write_manifests` writes its own manifests together and returns them as its manifest list records them (with its sequence number); `write_manifest_list` writes the list naming the manifests the caller passes — its parent's, then its own. A chain's manifests can then all be written at once, and then all its lists. V1 and v2 tables only: v3's row lineage takes its range from the written list. `SnapshotProducer::commit` shares the list writer and the snapshot construction with `stage`. Co-Authored-By: Claude Opus 5.5 --- crates/iceberg/src/transaction/append.rs | 199 +++++++++++++++++++- crates/iceberg/src/transaction/mod.rs | 1 + crates/iceberg/src/transaction/snapshot.rs | 200 +++++++++++++++++---- 3 files changed, 366 insertions(+), 34 deletions(-) diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index 2a5f2bc6e8..d0a9c66eb8 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -25,7 +25,7 @@ use crate::error::Result; use crate::spec::{DataFile, ManifestEntry, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::snapshot::{ - DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, + DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, StagedSnapshot, }; use crate::transaction::{ActionCommit, TransactionAction}; @@ -91,6 +91,27 @@ impl FastAppendAction { } } +impl FastAppendAction { + /// Plan this append's snapshot against `table` without writing it: the + /// commit that adds it, and its files to write. For a chain of appends + /// one commit lands, whose files are then written together — their + /// manifests, then their manifest lists, each naming its parent's + /// manifests and its own. Doesn't check for duplicate files. V1 and v2 + /// tables only. + pub async fn stage(&self, table: &Table) -> Result<(ActionCommit, StagedSnapshot)> { + let snapshot_producer = SnapshotProducer::new( + table, + self.commit_uuid.unwrap_or_else(Uuid::now_v7), + self.snapshot_properties.clone(), + self.added_data_files.clone(), + self.added_delete_files.clone(), + ); + snapshot_producer.validate_added_files(&self.added_data_files)?; + snapshot_producer.validate_added_files(&self.added_delete_files)?; + snapshot_producer.stage(&FastAppendOperation).await + } +} + #[async_trait] impl TransactionAction for FastAppendAction { async fn commit(self: Arc, table: &Table) -> Result { @@ -622,6 +643,182 @@ mod tests { } } + /// A fresh in-memory table, and its catalog. + async fn memory_table() -> (crate::memory::MemoryCatalog, Table) { + use crate::spec::{NestedField, PrimitiveType, Schema, Type}; + use crate::{Catalog, CatalogBuilder}; + let schema = Schema::builder() + .with_schema_id(0) + .with_identifier_field_ids([1]) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(), + ]) + .build() + .unwrap(); + let memory: crate::memory::MemoryCatalog = crate::memory::MemoryCatalogBuilder::default() + .load( + "test", + HashMap::from([( + crate::memory::MEMORY_CATALOG_WAREHOUSE.to_string(), + "memory:///warehouse".to_string(), + )]), + ) + .await + .unwrap(); + let ns = crate::NamespaceIdent::from_strs(["public"]).unwrap(); + memory.create_namespace(&ns, HashMap::new()).await.unwrap(); + let creation = crate::TableCreation::builder() + .name("orders".to_string()) + .schema(schema) + .build(); + let table = memory.create_table(&ns, creation).await.unwrap(); + (memory, table) + } + + /// Step `n`'s files: a data file, and an equality-delete file. + fn step_files(table: &Table, n: usize) -> Vec { + let file = |content, kind: &str| { + let mut builder = DataFileBuilder::default(); + builder + .content(content) + .file_path(format!( + "{}/{kind}/{n}.parquet", + table.metadata().location() + )) + .file_format(DataFileFormat::Parquet) + .file_size_in_bytes(100) + .record_count(n as u64 + 1) + .partition(Struct::empty()) + .partition_spec_id(table.metadata().default_partition_spec_id()); + if content == DataContentType::EqualityDeletes { + builder.equality_ids(Some(vec![1])); + } + builder.build().unwrap() + }; + vec![ + file(DataContentType::Data, "data"), + file(DataContentType::EqualityDeletes, "deletes"), + ] + } + + /// Each snapshot as its manifest list has it: per manifest, its content, + /// sequence numbers and the files it holds, with their sequence numbers. + async fn snapshots_as_listed(table: &Table) -> Vec<(i64, Vec)> { + let mut out = Vec::new(); + for snapshot in table.metadata().snapshots() { + let list = table.manifest_list_reader(snapshot).load().await.unwrap(); + let mut manifests = Vec::new(); + for m in list.entries() { + let manifest = m.load_manifest(table.file_io()).await.unwrap(); + let files: Vec<_> = manifest + .entries() + .iter() + .map(|e| (e.file_path().to_string(), e.sequence_number())) + .collect(); + manifests.push(format!( + "{:?} seq {} min {} {:?}", + m.content, m.sequence_number, m.min_sequence_number, files + )); + } + manifests.sort(); + out.push((snapshot.sequence_number(), manifests)); + } + out.sort(); + out + } + + /// A chain of appends staged and written together — every snapshot's + /// manifests at once, then every manifest list, each naming its + /// parent's manifests and its own — lands as the same table as the + /// appends committed one after another. + #[tokio::test] + async fn a_staged_chain_of_appends_lands_as_appends_one_by_one() { + use crate::transaction::ApplyTransactionAction; + use crate::{Catalog, TableCommit}; + const STEPS: usize = 3; + + let (one_by_one_catalog, mut one_by_one) = memory_table().await; + for n in 0..STEPS { + let tx = Transaction::new(&one_by_one); + let files = step_files(&one_by_one, n); + let tx = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(files) + .apply(tx) + .unwrap(); + one_by_one = tx.commit(&one_by_one_catalog).await.unwrap(); + } + + let (staged_catalog, table) = memory_table().await; + let mut local = table.clone(); + let mut updates = Vec::new(); + let mut requirements = Vec::new(); + let mut staged = Vec::new(); + for n in 0..STEPS { + let files = step_files(&local, n); + let (commit, snapshot) = Transaction::new(&local) + .fast_append() + .with_check_duplicate(false) + .add_data_files(files) + .stage(&local) + .await + .unwrap(); + // The table as the commit finds it: the first step's. + let mut later = Vec::new(); + let step_requirements = if n == 0 { + &mut requirements + } else { + &mut later + }; + local = Transaction::apply(local, commit, &mut updates, step_requirements).unwrap(); + staged.push(snapshot); + } + let added = futures::future::try_join_all(staged.iter_mut().map(|s| s.write_manifests())) + .await + .unwrap(); + let mut parent = Vec::new(); + let mut lists = Vec::new(); + for (snapshot, added) in staged.into_iter().zip(added) { + let manifests: Vec = + parent.into_iter().chain(added).collect(); + parent = manifests.clone(); + lists.push(snapshot.write_manifest_list(manifests)); + } + futures::future::try_join_all(lists).await.unwrap(); + let staged_table = staged_catalog + .update_table( + TableCommit::builder() + .ident(table.identifier().clone()) + .updates(updates) + .requirements(requirements) + .build(), + ) + .await + .unwrap(); + + assert_eq!(staged_table.metadata().snapshots().count(), STEPS); + assert_eq!( + snapshots_as_listed(&staged_table).await, + snapshots_as_listed(&one_by_one).await + ); + let summaries = |t: &Table| { + let mut s: Vec<_> = t + .metadata() + .snapshots() + .map(|s| { + ( + s.sequence_number(), + s.summary().additional_properties.clone(), + ) + }) + .collect(); + s.sort_by_key(|(seq, _)| *seq); + s + }; + assert_eq!(summaries(&staged_table), summaries(&one_by_one)); + } + #[tokio::test] async fn test_fast_append_file_with_incompatible_partition_value() { let table = make_v2_minimal_table(); diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index c11de6f1c1..4bd20aafdc 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -53,6 +53,7 @@ mod action; pub use action::*; +pub use snapshot::StagedSnapshot; mod append; mod expire_snapshots; mod replace_schema; diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index d0811a5bcf..029a35fa47 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -386,6 +386,19 @@ impl<'a> SnapshotProducer<'a> { content: ManifestContentType, data_sequence_number: Option, ) -> Result { + self.added_manifest_writer(added_files, content, data_sequence_number)? + .write_manifest_file() + .await + } + + /// A writer holding the manifest for a batch of added files, not yet + /// written. + fn added_manifest_writer( + &mut self, + added_files: Vec, + content: ManifestContentType, + data_sequence_number: Option, + ) -> Result { if added_files.is_empty() { return Err(Error::new( ErrorKind::PreconditionFailed, @@ -411,7 +424,7 @@ impl<'a> SnapshotProducer<'a> { for entry in manifest_entries { writer.add_entry(entry)?; } - writer.write_manifest_file().await + Ok(writer) } async fn manifest_file( @@ -549,15 +562,97 @@ impl<'a> SnapshotProducer<'a> { process: MP, ) -> Result { let manifest_list_path = self.generate_manifest_list_file_path(0); - let next_seq_num = self.table.metadata().next_sequence_number(); let first_row_id = self.table.metadata().next_row_id(); - let writer = self - .table - .file_io() - .new_output(manifest_list_path.clone())? - .writer() + let mut manifest_list_writer = self.manifest_list_writer(&manifest_list_path).await?; + + // Calling self.summary() before self.manifest_file() is important because the + // added_*_files vecs are drained by self.manifest_file(), which would result in + // an empty summary being generated otherwise. + let summary = self.summary(&snapshot_produce_operation).map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) + })?; + + let new_manifests = self + .manifest_file(&snapshot_produce_operation, &process) .await?; - let mut manifest_list_writer = match self.table.metadata().format_version() { + + manifest_list_writer.add_manifests(new_manifests.into_iter())?; + let writer_next_row_id = manifest_list_writer.next_row_id(); + manifest_list_writer.close().await?; + + let row_range = writer_next_row_id.map(|writer_next_row_id| { + let assigned_rows = writer_next_row_id - self.table.metadata().next_row_id(); + (first_row_id, assigned_rows) + }); + Ok(self.snapshot_commit(manifest_list_path, summary, row_range)) + } + + /// Plan this snapshot without writing it: the [`ActionCommit`] that adds + /// it, and its files to write ([`StagedSnapshot`]). Its manifest list + /// carries the manifests the caller names when it writes it, rather + /// than reading the parent's: snapshots one commit chains can be + /// written together. V1 and v2 tables only — v3's row lineage takes + /// its range from the written manifest list. + pub(crate) async fn stage( + mut self, + snapshot_produce_operation: &OP, + ) -> Result<(ActionCommit, StagedSnapshot)> { + if self.table.metadata().format_version() == FormatVersion::V3 { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "staged snapshots don't support format version 3", + )); + } + if self.added_data_files.is_empty() + && self.added_delete_files.is_empty() + && self.snapshot_properties.is_empty() + { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "No added files or snapshot properties found when writing a manifest file", + )); + } + let manifest_list_path = self.generate_manifest_list_file_path(0); + let list = self.manifest_list_writer(&manifest_list_path).await?; + // Before the added files are taken below. + let summary = self.summary(snapshot_produce_operation).map_err(|err| { + Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) + })?; + let mut manifests = Vec::new(); + if !self.added_data_files.is_empty() { + let added_data_files = std::mem::take(&mut self.added_data_files); + let seq = self.added_data_sequence_number; + manifests.push(self.added_manifest_writer( + added_data_files, + ManifestContentType::Data, + seq, + )?); + } + if !self.added_delete_files.is_empty() { + let added_delete_files = std::mem::take(&mut self.added_delete_files); + manifests.push(self.added_manifest_writer( + added_delete_files, + ManifestContentType::Deletes, + None, + )?); + } + let staged = StagedSnapshot { + manifests, + list, + sequence_number: self.table.metadata().next_sequence_number(), + format_version: self.table.metadata().format_version(), + }; + Ok(( + self.snapshot_commit(manifest_list_path, summary, None), + staged, + )) + } + + /// A writer for this snapshot's manifest list at `path`. + async fn manifest_list_writer(&self, path: &str) -> Result { + let next_seq_num = self.table.metadata().next_sequence_number(); + let writer = self.table.file_io().new_output(path)?.writer().await?; + Ok(match self.table.metadata().format_version() { FormatVersion::V1 => ManifestListWriter::v1( writer, self.snapshot_id, @@ -574,42 +669,34 @@ impl<'a> SnapshotProducer<'a> { self.snapshot_id, self.table.metadata().current_snapshot_id(), next_seq_num, - Some(first_row_id), + Some(self.table.metadata().next_row_id()), ), - }; - - // Calling self.summary() before self.manifest_file() is important because the - // added_*_files vecs are drained by self.manifest_file(), which would result in - // an empty summary being generated otherwise. - let summary = self.summary(&snapshot_produce_operation).map_err(|err| { - Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) - })?; - - let new_manifests = self - .manifest_file(&snapshot_produce_operation, &process) - .await?; - - manifest_list_writer.add_manifests(new_manifests.into_iter())?; - let writer_next_row_id = manifest_list_writer.next_row_id(); - manifest_list_writer.close().await?; + }) + } + /// The commit adding this snapshot, its manifest list at + /// `manifest_list_path`, and making it the main branch's. + fn snapshot_commit( + &self, + manifest_list_path: String, + summary: Summary, + row_range: Option<(u64, u64)>, + ) -> ActionCommit { let commit_ts = chrono::Utc::now().timestamp_millis(); let new_snapshot = Snapshot::builder() .with_manifest_list(manifest_list_path) .with_snapshot_id(self.snapshot_id) .with_parent_snapshot_id(self.table.metadata().current_snapshot_id()) - .with_sequence_number(next_seq_num) + .with_sequence_number(self.table.metadata().next_sequence_number()) .with_summary(summary) .with_schema_id(self.table.metadata().current_schema_id()) .with_timestamp_ms(commit_ts); - let new_snapshot = if let Some(writer_next_row_id) = writer_next_row_id { - let assigned_rows = writer_next_row_id - self.table.metadata().next_row_id(); - new_snapshot + let new_snapshot = match row_range { + Some((first_row_id, assigned_rows)) => new_snapshot .with_row_range(first_row_id, assigned_rows) - .build() - } else { - new_snapshot.build() + .build(), + None => new_snapshot.build(), }; let updates = vec![ @@ -635,6 +722,53 @@ impl<'a> SnapshotProducer<'a> { }, ]; - Ok(ActionCommit::new(updates, requirements)) + ActionCommit::new(updates, requirements) + } +} + +/// A snapshot planned but not written ([`crate::transaction::FastAppendAction::stage`]): +/// its own manifests, written by [`Self::write_manifests`], then its +/// manifest list, by [`Self::write_manifest_list`]. Each is a round trip to +/// the object store; snapshots one commit chains can be written together, +/// their manifests first, then their lists. +pub struct StagedSnapshot { + manifests: Vec, + list: ManifestListWriter, + sequence_number: i64, + format_version: FormatVersion, +} + +impl StagedSnapshot { + /// Write the snapshot's own manifests, together. Returns them as its + /// manifest list records them — with its sequence number — so a later + /// snapshot's list can carry them. + pub async fn write_manifests(&mut self) -> Result> { + let written = std::mem::take(&mut self.manifests) + .into_iter() + .map(|writer| writer.write_manifest_file()) + .collect::>() + .try_collect::>() + .await?; + Ok(written + .into_iter() + .map(|mut manifest| { + if self.format_version != FormatVersion::V1 { + if manifest.sequence_number == crate::spec::UNASSIGNED_SEQUENCE_NUMBER { + manifest.sequence_number = self.sequence_number; + } + if manifest.min_sequence_number == crate::spec::UNASSIGNED_SEQUENCE_NUMBER { + manifest.min_sequence_number = self.sequence_number; + } + } + manifest + }) + .collect()) + } + + /// Write the snapshot's manifest list, naming `manifests`: the ones it + /// carries from its parent, then its own. + pub async fn write_manifest_list(mut self, manifests: Vec) -> Result<()> { + self.list.add_manifests(manifests.into_iter())?; + self.list.close().await } }