diff --git a/Cargo.lock b/Cargo.lock index c6d440a..76d128e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2461,7 +2461,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#fae9f8fe687d85f38ada75426313902efc53d19e" dependencies = [ "aes-gcm", "anyhow", @@ -2520,7 +2520,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#fae9f8fe687d85f38ada75426313902efc53d19e" dependencies = [ "async-trait", "aws-config", @@ -2544,7 +2544,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#fae9f8fe687d85f38ada75426313902efc53d19e" dependencies = [ "proc-macro2", "quote", @@ -2554,7 +2554,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#fae9f8fe687d85f38ada75426313902efc53d19e" dependencies = [ "anyhow", "async-trait", diff --git a/crates/pg2iceberg-iceberg/src/prod/catalog.rs b/crates/pg2iceberg-iceberg/src/prod/catalog.rs index 9d086a4..6524467 100644 --- a/crates/pg2iceberg-iceberg/src/prod/catalog.rs +++ b/crates/pg2iceberg-iceberg/src/prod/catalog.rs @@ -32,8 +32,8 @@ use async_trait::async_trait; use futures::stream::{self, StreamExt, TryStreamExt}; use iceberg::spec::{ DataContentType, DataFile as IcebergDataFile, DataFileBuilder, DataFileFormat, FormatVersion, - Literal, ManifestFile, NestedField, PrimitiveLiteral, PrimitiveType, Schema as IcebergSchema, - Struct, Type, + Literal, ManifestContentType, ManifestFile, NestedField, PrimitiveLiteral, PrimitiveType, + Schema as IcebergSchema, Struct, Type, }; use iceberg::table::Table; use iceberg::transaction::{ @@ -756,11 +756,9 @@ impl TransactionAction for ChainedAppendAction { /// together: every step's manifests at once, then every step's manifest /// list, each naming its parent's manifests and its own. Written one /// step at a time, each took four round trips to object storage — its - /// manifests, its list, and reading that list back for the next. + /// manifests, its list, and reading that list back for the next. On a + /// v3 table, each step's rows take the row ids after the last's. async fn commit(self: Arc, table: &Table) -> iceberg::Result { - if table.metadata().format_version() == FormatVersion::V3 { - return self.commit_one_by_one(table).await; - } // The manifests the first step's list carries forward: as a fast // append keeps them. let mut parent: Vec = match table.metadata().current_snapshot() { @@ -775,6 +773,16 @@ impl TransactionAction for ChainedAppendAction { .collect(), None => Vec::new(), }; + // A v3 table's files from before it was upgraded have no row ids + // until its next commit gives them some, in the first step's list: + // a range a staged step, which counts its own rows, can't record. + let unassigned = table.metadata().format_version() == FormatVersion::V3 + && parent + .iter() + .any(|m| m.content == ManifestContentType::Data && m.first_row_id.is_none()); + if unassigned { + return self.commit_one_by_one(table).await; + } let mut local = table.clone(); let mut updates = Vec::new(); let mut requirements = None; @@ -814,9 +822,9 @@ impl TransactionAction for ChainedAppendAction { } impl ChainedAppendAction { - /// One step after another, each a fast append on the last: for v3 - /// tables, whose snapshots take their row range from their written - /// manifest list. + /// One step after another, each a fast append on the last: for a v3 + /// table's first commit since its upgrade, whose first step's list + /// gives the files before it row ids. async fn commit_one_by_one(&self, table: &Table) -> iceberg::Result { let mut local = table.clone(); let mut updates = Vec::new(); @@ -2400,7 +2408,7 @@ mod tests { fn step(i: usize) -> PreparedCommit { let file = |kind: &str, equality_field_ids: Vec| DataFile { path: format!("memory:///warehouse/public/orders/step-{kind}-{i}.parquet"), - record_count: 1, + record_count: if equality_field_ids.is_empty() { 2 } else { 1 }, byte_size: 100, equality_field_ids, partition_values: Vec::new(), @@ -2421,13 +2429,25 @@ mod tests { /// snapshot on the last. #[tokio::test(start_paused = true)] async fn a_commit_of_several_steps_writes_them_together() { + several_steps_written_together(crate::FormatVersion::V2).await; + } + + /// As above, on a v3 table — each step's rows taking the row ids after + /// the last's, and a manifest carried into a later step's list keeping + /// the ones it took. + #[tokio::test(start_paused = true)] + async fn a_commit_of_several_steps_writes_them_together_on_v3() { + several_steps_written_together(crate::FormatVersion::V3).await; + } + + async fn several_steps_written_together(version: crate::FormatVersion) { const LATENCY: std::time::Duration = std::time::Duration::from_millis(100); const STEPS: usize = 4; let storage = CountingStorage { latency: LATENCY, ..Default::default() }; - let c = fresh_on(storage.clone()).await; + let c = fresh_on(storage.clone()).await.with_format_version(version); c.ensure_namespace(&ident().namespace).await.unwrap(); c.create_table(&schema()).await.unwrap(); append(&c, 0).await; @@ -2456,6 +2476,30 @@ mod tests { .collect(); assert_eq!(deletes.len(), STEPS); assert!(deletes.windows(2).all(|w| w[0] < w[1]), "{deletes:?}"); + + if version == crate::FormatVersion::V3 { + // Every data file's rows have ids of their own: one run from 0, + // in commit order, up to the table's next row id. + let mut ranges: Vec<(i64, i64)> = snapshots + .iter() + .flat_map(|s| &s.data_files) + .map(|f| { + let first = f + .first_row_id + .expect("a data file of a v3 table has row ids"); + (first, first + f.record_count as i64) + }) + .collect(); + ranges.sort(); + let it = to_iceberg_table_ident(&ident()).unwrap(); + let table = c.inner.load_table(&it).await.unwrap(); + let next_row_id = table.metadata().next_row_id() as i64; + let expected: Vec<(i64, i64)> = std::iter::once((0, 1)) + .chain((0..STEPS as i64).map(|i| (1 + 2 * i, 3 + 2 * i))) + .collect(); + assert_eq!(ranges, expected); + assert_eq!(next_row_id, 1 + 2 * STEPS as i64); + } } #[tokio::test] diff --git a/crates/pg2iceberg-tests/tests/dst.rs b/crates/pg2iceberg-tests/tests/dst.rs index 8a27002..3f53514 100644 --- a/crates/pg2iceberg-tests/tests/dst.rs +++ b/crates/pg2iceberg-tests/tests/dst.rs @@ -6680,6 +6680,53 @@ fn upgraded_while_a_pass_is_in_flight(prod: bool) { v3_case(prod, true, &steps); } +/// A transaction too big for one step — every row updated, more added — +/// commits its steps as one update, written together: on a v3 table, each +/// step's rows take the row ids after the last's (invariant 20). +#[test] +fn a_commit_of_several_steps_gives_each_row_an_id() { + several_steps_on_v3(false, false); +} + +#[cfg(feature = "integration")] +#[test] +fn a_commit_of_several_steps_gives_each_row_an_id_on_iceberg() { + several_steps_on_v3(true, false); +} + +/// So does the first such commit since the table was upgraded to v3, +/// whose first step gives the files from before the upgrade ids too. +#[test] +fn a_commit_of_several_steps_after_an_upgrade_gives_each_row_an_id() { + several_steps_on_v3(false, true); +} + +#[cfg(feature = "integration")] +#[test] +fn a_commit_of_several_steps_after_an_upgrade_gives_each_row_an_id_on_iceberg() { + several_steps_on_v3(true, true); +} + +fn several_steps_on_v3(prod: bool, upgraded: bool) { + V3_UPGRADED.set(upgraded); + let mut steps = vec![ + Step::Insert { id: 1, qty: 10 }, + Step::Insert { id: 2, qty: 20 }, + Step::Insert { id: 3, qty: 30 }, + Step::DriveFlush, + Step::MaterializerCycle, + ]; + if upgraded { + steps.push(Step::UpgradeToV3); + } + steps.extend([ + Step::BigTx { inserts: 5, qty: 1 }, + Step::DriveFlush, + Step::MaterializerCycle, + ]); + v3_case(prod, false, &steps); +} + /// So does a `compact` job's pass, planned before the upgrade and /// committed after pg2iceberg's next commit gave the rows ids. #[test]