diff --git a/crates/iceberg/src/transaction/append.rs b/crates/iceberg/src/transaction/append.rs index a15c09fc6c..d05176115f 100644 --- a/crates/iceberg/src/transaction/append.rs +++ b/crates/iceberg/src/transaction/append.rs @@ -108,8 +108,9 @@ impl FastAppendAction { /// commit that adds it, and its files to write. For a chain of appends /// one commit lands, whose files are then written together — their /// manifests, then their manifest lists, each naming its parent's - /// manifests and its own. Doesn't check for duplicate files. V1 and v2 - /// tables only. + /// manifests and its own. Doesn't check for duplicate files. On a v3 + /// table, every manifest a list carries must have its row ids already + /// ([`StagedSnapshot::write_manifest_list`]). pub async fn stage(&self, table: &Table) -> Result<(ActionCommit, StagedSnapshot)> { let snapshot_producer = SnapshotProducer::new( table, @@ -952,7 +953,9 @@ mod tests { } /// A fresh in-memory table, and its catalog. - async fn memory_table() -> (crate::memory::MemoryCatalog, Table) { + async fn memory_table_in( + format_version: crate::spec::FormatVersion, + ) -> (crate::memory::MemoryCatalog, Table) { use crate::spec::{NestedField, PrimitiveType, Schema, Type}; use crate::{Catalog, CatalogBuilder}; let schema = Schema::builder() @@ -978,6 +981,7 @@ mod tests { let creation = crate::TableCreation::builder() .name("orders".to_string()) .schema(schema) + .format_version(format_version) .build(); let table = memory.create_table(&ns, creation).await.unwrap(); (memory, table) @@ -1010,8 +1014,9 @@ mod tests { } /// Each snapshot as its manifest list has it: per manifest, its content, - /// sequence numbers and the files it holds, with their sequence numbers. - async fn snapshots_as_listed(table: &Table) -> Vec<(i64, Vec)> { + /// sequence numbers, first row id and the files it holds, with their + /// sequence numbers and first row ids; and the snapshot's row range. + async fn snapshots_as_listed(table: &Table) -> Vec<(i64, Option<(u64, u64)>, Vec)> { let mut out = Vec::new(); for snapshot in table.metadata().snapshots() { let list = table.manifest_list_reader(snapshot).load().await.unwrap(); @@ -1021,15 +1026,21 @@ mod tests { let files: Vec<_> = manifest .entries() .iter() - .map(|e| (e.file_path().to_string(), e.sequence_number())) + .map(|e| { + ( + e.file_path().to_string(), + e.sequence_number(), + e.data_file().first_row_id(), + ) + }) .collect(); manifests.push(format!( - "{:?} seq {} min {} {:?}", - m.content, m.sequence_number, m.min_sequence_number, files + "{:?} seq {} min {} first row id {:?} {:?}", + m.content, m.sequence_number, m.min_sequence_number, m.first_row_id, files )); } manifests.sort(); - out.push((snapshot.sequence_number(), manifests)); + out.push((snapshot.sequence_number(), snapshot.row_range(), manifests)); } out.sort(); out @@ -1041,11 +1052,70 @@ mod tests { /// appends committed one after another. #[tokio::test] async fn a_staged_chain_of_appends_lands_as_appends_one_by_one() { + staged_chain_lands_as_appends_one_by_one(crate::spec::FormatVersion::V2).await; + } + + /// As above, on a v3 table: each snapshot's rows take the row ids after + /// the last's — its data manifest records the first, and keeps it in + /// the later snapshots' lists — and the table's next row id follows. + #[tokio::test] + async fn a_staged_chain_of_appends_lands_as_appends_one_by_one_on_v3() { + staged_chain_lands_as_appends_one_by_one(crate::spec::FormatVersion::V3).await; + } + + /// A v2 table's files, its first commit since an upgrade to v3 gives + /// row ids, in its manifest list: a staged snapshot, whose row range + /// counts its own rows only, refuses to carry them. + #[tokio::test] + async fn a_staged_snapshot_refuses_to_carry_files_without_row_ids() { + use crate::transaction::ApplyTransactionAction; + + let (catalog, table) = memory_table_in(crate::spec::FormatVersion::V2).await; + let tx = Transaction::new(&table); + let tx = tx + .fast_append() + .with_check_duplicate(false) + .add_data_files(step_files(&table, 0)) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + let tx = Transaction::new(&table); + let tx = tx + .upgrade_table_version() + .set_format_version(crate::spec::FormatVersion::V3) + .apply(tx) + .unwrap(); + let table = tx.commit(&catalog).await.unwrap(); + + let (_, mut staged) = Transaction::new(&table) + .fast_append() + .with_check_duplicate(false) + .add_data_files(step_files(&table, 1)) + .stage(&table) + .await + .unwrap(); + let parent = table + .manifest_list_reader(table.metadata().current_snapshot().unwrap()) + .load() + .await + .unwrap() + .entries() + .to_vec(); + assert!(parent.iter().any(|m| m.first_row_id.is_none())); + let added = staged.write_manifests().await.unwrap(); + let err = staged + .write_manifest_list(parent.into_iter().chain(added).collect()) + .await + .unwrap_err(); + assert!(err.message().contains("without row ids"), "{err}"); + } + + async fn staged_chain_lands_as_appends_one_by_one(format_version: crate::spec::FormatVersion) { use crate::transaction::ApplyTransactionAction; use crate::{Catalog, TableCommit}; const STEPS: usize = 3; - let (one_by_one_catalog, mut one_by_one) = memory_table().await; + let (one_by_one_catalog, mut one_by_one) = memory_table_in(format_version).await; for n in 0..STEPS { let tx = Transaction::new(&one_by_one); let files = step_files(&one_by_one, n); @@ -1058,7 +1128,7 @@ mod tests { one_by_one = tx.commit(&one_by_one_catalog).await.unwrap(); } - let (staged_catalog, table) = memory_table().await; + let (staged_catalog, table) = memory_table_in(format_version).await; let mut local = table.clone(); let mut updates = Vec::new(); let mut requirements = Vec::new(); @@ -1110,6 +1180,10 @@ mod tests { snapshots_as_listed(&staged_table).await, snapshots_as_listed(&one_by_one).await ); + assert_eq!( + staged_table.metadata().next_row_id(), + one_by_one.metadata().next_row_id() + ); let summaries = |t: &Table| { let mut s: Vec<_> = t .metadata() diff --git a/crates/iceberg/src/transaction/snapshot.rs b/crates/iceberg/src/transaction/snapshot.rs index f71501741c..3e175c3137 100644 --- a/crates/iceberg/src/transaction/snapshot.rs +++ b/crates/iceberg/src/transaction/snapshot.rs @@ -605,20 +605,16 @@ impl<'a> SnapshotProducer<'a> { /// it, and its files to write ([`StagedSnapshot`]). Its manifest list /// carries the manifests the caller names when it writes it, rather /// than reading the parent's: snapshots one commit chains can be - /// written together. V1 and v2 tables only — v3's row lineage takes - /// its range from the written manifest list — and unencrypted tables - /// only: an encrypted list's key, which the snapshot names, is only - /// known once the list is written. + /// written together. On a v3 table, its rows take the row ids after the + /// table's last, which its data manifest records as it's written: every + /// manifest its list carries must have its ids already (see + /// [`StagedSnapshot::write_manifest_list`]). Unencrypted tables only: + /// an encrypted list's key, which the snapshot names, is only known + /// once the list is written. pub(crate) async fn stage( mut self, snapshot_produce_operation: &OP, ) -> Result<(ActionCommit, StagedSnapshot)> { - if self.table.metadata().format_version() == FormatVersion::V3 { - return Err(Error::new( - ErrorKind::FeatureUnsupported, - "staged snapshots don't support format version 3", - )); - } if self.table.encryption_manager().is_some() { return Err(Error::new( ErrorKind::FeatureUnsupported, @@ -636,6 +632,11 @@ impl<'a> SnapshotProducer<'a> { } let manifest_list_path = self.generate_manifest_list_file_path(0)?; let (list, _) = self.manifest_list_writer(&manifest_list_path).await?; + // v3: the rows it adds take the row ids after the table's last. + let row_range = (self.table.metadata().format_version() == FormatVersion::V3).then(|| { + let added_rows = self.added_data_files.iter().map(|f| f.record_count()).sum(); + (self.table.metadata().next_row_id(), added_rows) + }); // Before the added files are taken below. let summary = self.summary(snapshot_produce_operation).map_err(|err| { Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err) @@ -663,9 +664,10 @@ impl<'a> SnapshotProducer<'a> { list, sequence_number: self.table.metadata().next_sequence_number(), format_version: self.table.metadata().format_version(), + first_row_id: row_range.map(|(first_row_id, _)| first_row_id), }; Ok(( - self.snapshot_commit(manifest_list_path, summary, None, None), + self.snapshot_commit(manifest_list_path, summary, row_range, None), staged, )) } @@ -799,12 +801,15 @@ pub struct StagedSnapshot { list: ManifestListWriter, sequence_number: i64, format_version: FormatVersion, + /// v3: the first row id of the rows it adds. + first_row_id: Option, } impl StagedSnapshot { /// Write the snapshot's own manifests, together. Returns them as its - /// manifest list records them — with its sequence number — so a later - /// snapshot's list can carry them. + /// manifest list records them — with its sequence number and, on a v3 + /// table, the first row id of its rows — so a later snapshot's list + /// can carry them. pub async fn write_manifests(&mut self) -> Result> { let written = std::mem::take(&mut self.manifests) .into_iter() @@ -823,6 +828,10 @@ impl StagedSnapshot { manifest.min_sequence_number = self.sequence_number; } } + // Its one data manifest holds all the rows it adds. + if manifest.content == ManifestContentType::Data { + manifest.first_row_id = self.first_row_id; + } manifest }) .collect()) @@ -830,8 +839,21 @@ impl StagedSnapshot { /// Write the snapshot's manifest list, naming `manifests`: the ones it /// carries from its parent, then its own. + /// + /// On a v3 table, every data manifest it names must have its row ids: + /// its own takes them when it's written ([`Self::write_manifests`]), + /// its parent's had them from their own lists. One that didn't — a + /// table's files from before it was upgraded to v3 — would take ids its + /// snapshot's row range doesn't count: the snapshot has to be committed + /// on its own, its list assigning them. pub async fn write_manifest_list(mut self, manifests: Vec) -> Result<()> { self.list.add_manifests(manifests.into_iter())?; + if self.list.next_row_id() != self.first_row_id { + return Err(Error::new( + ErrorKind::PreconditionFailed, + "a staged snapshot carries a data manifest without row ids; commit it on its own", + )); + } self.list.close().await.map(drop) } }