From 98a3eca034beb570e333270b8ec2982723d9a0b9 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 17 Nov 2026 03:31:59 +0800 Subject: [PATCH] feat(transaction): stage snapshots of v3 tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A staged snapshot now works on a v3 table: the rows it adds take the row ids after the table's last — its snapshot records the range (`first-row-id`, `added-rows`), its data manifest the first id as it's written, so a later snapshot's list carries the manifest with its ids rather than giving its rows new ones. The range counts the snapshot's own rows only, so every manifest its list carries must have its ids already; one that doesn't — a table's files from before it was upgraded to v3, which its next commit gives ids — makes `write_manifest_list` fail: that snapshot is committed on its own. Co-Authored-By: Claude Opus 5.5 --- crates/iceberg/src/transaction/append.rs | 96 +++++++++++++++++++--- crates/iceberg/src/transaction/snapshot.rs | 48 ++++++++--- 2 files changed, 120 insertions(+), 24 deletions(-) diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index a15c09fc6c..d05176115f 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -108,8 +108,9 @@ impl FastAppendAction { /// commit that adds it, and its files to write. For a chain of appends /// one commit lands, whose files are then written together — their /// manifests, then their manifest lists, each naming its parent's - /// manifests and its own. Doesn't check for duplicate files. V1 and v2 - /// tables only. + /// manifests and its own. Doesn't check for duplicate files. On a v3 + /// table, every manifest a list carries must have its row ids already + /// ([`StagedSnapshot::write_manifest_list`]). pub async fn stage(&self, table: &Table) -> Result<(ActionCommit, StagedSnapshot)> { let snapshot_producer = SnapshotProducer::new( table, @@ -952,7 +953,9 @@ mod tests { } /// A fresh in-memory table, and its catalog. - async fn memory_table() -> (crate::memory::MemoryCatalog, Table) { + async fn memory_table_in( + format_version: crate::spec::FormatVersion, + ) -> (crate::memory::MemoryCatalog, Table) { use crate::spec::{NestedField, PrimitiveType, Schema, Type}; use crate::{Catalog, CatalogBuilder}; let schema = Schema::builder() @@ -978,6 +981,7 @@ mod tests { let creation = crate::TableCreation::builder() .name("orders".to_string()) .schema(schema) + .format_version(format_version) .build(); let table = memory.create_table(&ns, creation).await.unwrap(); (memory, table) @@ -1010,8 +1014,9 @@ mod tests { } /// Each snapshot as its manifest list has it: per manifest, its content, - /// sequence numbers and the files it holds, with their sequence numbers. - async fn snapshots_as_listed(table: &Table) -> Vec<(i64, Vec)> { + /// sequence numbers, first row id and the files it holds, with their + /// sequence numbers and first row ids; and the snapshot's row range. + async fn snapshots_as_listed(table: &Table) -> Vec<(i64, Option<(u64, u64)>, Vec)> { let mut out = Vec::new(); for snapshot in table.metadata().snapshots() { let list = table.manifest_list_reader(snapshot).load().await.unwrap(); @@ -1021,15 +1026,21 @@ mod tests { let files: Vec<_> = manifest .entries() .iter() - .map(|e| (e.file_path().to_string(), e.sequence_number())) + .map(|e| { + ( + e.file_path().to_string(), + e.sequence_number(), + e.data_file().first_row_id(), + ) + }) .collect(); manifests.push(format!( - "{:?} seq {} min {} {:?}", - m.content, m.sequence_number, m.min_sequence_number, files + "{:?} seq {} min {} first row id {:?} {:?}", + m.content, m.sequence_number, m.min_sequence_number, m.first_row_id, files )); } manifests.sort(); - out.push((snapshot.sequence_number(), manifests)); + out.push((snapshot.sequence_number(), snapshot.row_range(), manifests)); } out.sort(); out @@ -1041,11 +1052,70 @@ mod tests { /// appends committed one after another. #[tokio::test] async fn a_staged_chain_of_appends_lands_as_appends_one_by_one() { + staged_chain_lands_as_appends_one_by_one(crate::spec::FormatVersion::V2).await; + } + + /// As above, on a v3 table: each snapshot's rows take the row ids after + /// the last's — its data manifest records the first, and keeps it in + /// the later snapshots' lists — and the table's next row id follows. + #[tokio::test] + async fn a_staged_chain_of_appends_lands_as_appends_one_by_one_on_v3() { + staged_chain_lands_as_appends_one_by_one(crate::spec::FormatVersion::V3).await; + } + + /// A v2 table's files, its first commit since an upgrade to v3 gives + /// row ids, in its manifest list: a staged snapshot, whose row range + /// counts its own rows only, refuses to carry them. + #[tokio::test] + async fn a_staged_snapshot_refuses_to_carry_files_without_row_ids() { + use crate::transaction::ApplyTransactionAction; + + let (catalog, table) = memory_table_in(crate::spec::FormatVersion::V2).await; + let tx = Transaction::new(&table); + let tx = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(step_files(&table, 0)) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + let tx = Transaction::new(&table); + let tx = tx + .upgrade_table_version() + .set_format_version(crate::spec::FormatVersion::V3) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let (_, mut staged) = Transaction::new(&table) + .fast_append() + .with_check_duplicate(false) + .add_data_files(step_files(&table, 1)) + .stage(&table) + .await + .unwrap(); + let parent = table + .manifest_list_reader(table.metadata().current_snapshot().unwrap()) + .load() + .await + .unwrap() + .entries() + .to_vec(); + assert!(parent.iter().any(|m| m.first_row_id.is_none())); + let added = staged.write_manifests().await.unwrap(); + let err = staged + .write_manifest_list(parent.into_iter().chain(added).collect()) + .await + .unwrap_err(); + assert!(err.message().contains("without row ids"), "{err}"); + } + + async fn staged_chain_lands_as_appends_one_by_one(format_version: crate::spec::FormatVersion) { use crate::transaction::ApplyTransactionAction; use crate::{Catalog, TableCommit}; const STEPS: usize = 3; - let (one_by_one_catalog, mut one_by_one) = memory_table().await; + let (one_by_one_catalog, mut one_by_one) = memory_table_in(format_version).await; for n in 0..STEPS { let tx = Transaction::new(&one_by_one); let files = step_files(&one_by_one, n); @@ -1058,7 +1128,7 @@ mod tests { one_by_one = tx.commit(&one_by_one_catalog).await.unwrap(); } - let (staged_catalog, table) = memory_table().await; + let (staged_catalog, table) = memory_table_in(format_version).await; let mut local = table.clone(); let mut updates = Vec::new(); let mut requirements = Vec::new(); @@ -1110,6 +1180,10 @@ mod tests { snapshots_as_listed(&staged_table).await, snapshots_as_listed(&one_by_one).await ); + assert_eq!( + staged_table.metadata().next_row_id(), + one_by_one.metadata().next_row_id() + ); let summaries = |t: &Table| { let mut s: Vec<_> = t .metadata() diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index f71501741c..3e175c3137 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -605,20 +605,16 @@ impl<'a> SnapshotProducer<'a> { /// it, and its files to write ([`StagedSnapshot`]). Its manifest list /// carries the manifests the caller names when it writes it, rather /// than reading the parent's: snapshots one commit chains can be - /// written together. V1 and v2 tables only — v3's row lineage takes - /// its range from the written manifest list — and unencrypted tables - /// only: an encrypted list's key, which the snapshot names, is only - /// known once the list is written. + /// written together. On a v3 table, its rows take the row ids after the + /// table's last, which its data manifest records as it's written: every + /// manifest its list carries must have its ids already (see + /// [`StagedSnapshot::write_manifest_list`]). Unencrypted tables only: + /// an encrypted list's key, which the snapshot names, is only known + /// once the list is written. pub(crate) async fn stage( mut self, snapshot_produce_operation: &OP, ) -> Result<(ActionCommit, StagedSnapshot)> { - if self.table.metadata().format_version() == FormatVersion::V3 { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "staged snapshots don't support format version 3", - )); - } if self.table.encryption_manager().is_some() { return Err(Error::new( ErrorKind::FeatureUnsupported, @@ -636,6 +632,11 @@ impl<'a> SnapshotProducer<'a> { } let manifest_list_path = self.generate_manifest_list_file_path(0)?; let (list, _) = self.manifest_list_writer(&manifest_list_path).await?; + // v3: the rows it adds take the row ids after the table's last. + let row_range = (self.table.metadata().format_version() == FormatVersion::V3).then(|| { + let added_rows = self.added_data_files.iter().map(|f| f.record_count()).sum(); + (self.table.metadata().next_row_id(), added_rows) + }); // Before the added files are taken below. let summary = self.summary(snapshot_produce_operation).map_err(|err| { Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) @@ -663,9 +664,10 @@ impl<'a> SnapshotProducer<'a> { list, sequence_number: self.table.metadata().next_sequence_number(), format_version: self.table.metadata().format_version(), + first_row_id: row_range.map(|(first_row_id, _)| first_row_id), }; Ok(( - self.snapshot_commit(manifest_list_path, summary, None, None), + self.snapshot_commit(manifest_list_path, summary, row_range, None), staged, )) } @@ -799,12 +801,15 @@ pub struct StagedSnapshot { list: ManifestListWriter, sequence_number: i64, format_version: FormatVersion, + /// v3: the first row id of the rows it adds. + first_row_id: Option, } impl StagedSnapshot { /// Write the snapshot's own manifests, together. Returns them as its - /// manifest list records them — with its sequence number — so a later - /// snapshot's list can carry them. + /// manifest list records them — with its sequence number and, on a v3 + /// table, the first row id of its rows — so a later snapshot's list + /// can carry them. pub async fn write_manifests(&mut self) -> Result> { let written = std::mem::take(&mut self.manifests) .into_iter() @@ -823,6 +828,10 @@ impl StagedSnapshot { manifest.min_sequence_number = self.sequence_number; } } + // Its one data manifest holds all the rows it adds. + if manifest.content == ManifestContentType::Data { + manifest.first_row_id = self.first_row_id; + } manifest }) .collect()) @@ -830,8 +839,21 @@ impl StagedSnapshot { /// Write the snapshot's manifest list, naming `manifests`: the ones it /// carries from its parent, then its own. + /// + /// On a v3 table, every data manifest it names must have its row ids: + /// its own takes them when it's written ([`Self::write_manifests`]), + /// its parent's had them from their own lists. One that didn't — a + /// table's files from before it was upgraded to v3 — would take ids its + /// snapshot's row range doesn't count: the snapshot has to be committed + /// on its own, its list assigning them. pub async fn write_manifest_list(mut self, manifests: Vec) -> Result<()> { self.list.add_manifests(manifests.into_iter())?; + if self.list.next_row_id() != self.first_row_id { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "a staged snapshot carries a data manifest without row ids; commit it on its own", + )); + } self.list.close().await.map(drop) } }