From 2e5eccd5801e9f4e2f598fa1c15bcc7acdccdef3 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 17 Nov 2026 03:45:34 +0800 Subject: [PATCH 1/2] perf: write a v3 table's multi-step commits together MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ChainedAppendAction` wrote a v3 table's steps one by one — four sequential round trips to object storage each — since the fork's staged snapshots couldn't record v3 row ranges. polynya-dev/iceberg-rust#8 lets them: each step's rows take the row ids after the last's. A v3 table's first such commit since an upgrade with data still goes one by one: its first list gives the files from before the upgrade row ids too, a range a staged step can't record. Pins it: the catalog test of a chain's round trips on v3 (and its row ids), and DST cases of a several-step commit on v3 and right after an upgrade, on the sim and iceberg-rust. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 14 ++-- Cargo.toml | 6 +- crates/pg2iceberg-iceberg/src/prod/catalog.rs | 66 +++++++++++++++---- crates/pg2iceberg-tests/tests/dst.rs | 47 +++++++++++++ 4 files changed, 112 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c6d440a..32a41e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2413,7 +2413,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2461,7 +2461,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=staged-v3-row-lineage#98a3eca034beb570e333270b8ec2982723d9a0b9" dependencies = [ "aes-gcm", "anyhow", @@ -2520,7 +2520,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=staged-v3-row-lineage#98a3eca034beb570e333270b8ec2982723d9a0b9" dependencies = [ "async-trait", "aws-config", @@ -2544,7 +2544,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=staged-v3-row-lineage#98a3eca034beb570e333270b8ec2982723d9a0b9" dependencies = [ "proc-macro2", "quote", @@ -2554,7 +2554,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=staged-v3-row-lineage#98a3eca034beb570e333270b8ec2982723d9a0b9" dependencies = [ "anyhow", "async-trait", @@ -4270,7 +4270,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror", "tokio", "tracing", @@ -4308,7 +4308,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.52.0", ] diff --git a/Cargo.toml b/Cargo.toml index c8c6359..805682f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ all = { level = "warn", priority = -1 } # (Supabase's etl maintainer) rather than rolling our own to avoid the # maintenance burden of two forks. [patch.crates-io] -iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" } +iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "staged-v3-row-lineage" } # `iceberg-catalog-rest` also has to come from the fork: the # polynya-patches branch threads the REST `loadTable` response's # `config` map through `Table::builder().properties(response_config)`, @@ -68,7 +68,7 @@ iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polyn # servers (Polaris, Lakekeeper, ...) return per-table S3 STS tokens # in `config`, but our `extract_creds` sees an empty map because the # `Table::properties()` accessor surfaces nothing. -iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" } -iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" } +iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "staged-v3-row-lineage" } +iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "staged-v3-row-lineage" } tokio-postgres = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" } postgres-replication = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" } diff --git a/crates/pg2iceberg-iceberg/src/prod/catalog.rs b/crates/pg2iceberg-iceberg/src/prod/catalog.rs index 9d086a4..6524467 100644 --- a/crates/pg2iceberg-iceberg/src/prod/catalog.rs +++ b/crates/pg2iceberg-iceberg/src/prod/catalog.rs @@ -32,8 +32,8 @@ use async_trait::async_trait; use futures::stream::{self, StreamExt, TryStreamExt}; use iceberg::spec::{ DataContentType, DataFile as IcebergDataFile, DataFileBuilder, DataFileFormat, FormatVersion, - Literal, ManifestFile, NestedField, PrimitiveLiteral, PrimitiveType, Schema as IcebergSchema, - Struct, Type, + Literal, ManifestContentType, ManifestFile, NestedField, PrimitiveLiteral, PrimitiveType, + Schema as IcebergSchema, Struct, Type, }; use iceberg::table::Table; use iceberg::transaction::{ @@ -756,11 +756,9 @@ impl TransactionAction for ChainedAppendAction { /// together: every step's manifests at once, then every step's manifest /// list, each naming its parent's manifests and its own. Written one /// step at a time, each took four round trips to object storage — its - /// manifests, its list, and reading that list back for the next. + /// manifests, its list, and reading that list back for the next. On a + /// v3 table, each step's rows take the row ids after the last's. async fn commit(self: Arc, table: &Table) -> iceberg::Result { - if table.metadata().format_version() == FormatVersion::V3 { - return self.commit_one_by_one(table).await; - } // The manifests the first step's list carries forward: as a fast // append keeps them. let mut parent: Vec = match table.metadata().current_snapshot() { @@ -775,6 +773,16 @@ impl TransactionAction for ChainedAppendAction { .collect(), None => Vec::new(), }; + // A v3 table's files from before it was upgraded have no row ids + // until its next commit gives them some, in the first step's list: + // a range a staged step, which counts its own rows, can't record. + let unassigned = table.metadata().format_version() == FormatVersion::V3 + && parent + .iter() + .any(|m| m.content == ManifestContentType::Data && m.first_row_id.is_none()); + if unassigned { + return self.commit_one_by_one(table).await; + } let mut local = table.clone(); let mut updates = Vec::new(); let mut requirements = None; @@ -814,9 +822,9 @@ impl TransactionAction for ChainedAppendAction { } impl ChainedAppendAction { - /// One step after another, each a fast append on the last: for v3 - /// tables, whose snapshots take their row range from their written - /// manifest list. + /// One step after another, each a fast append on the last: for a v3 + /// table's first commit since its upgrade, whose first step's list + /// gives the files before it row ids. async fn commit_one_by_one(&self, table: &Table) -> iceberg::Result { let mut local = table.clone(); let mut updates = Vec::new(); @@ -2400,7 +2408,7 @@ mod tests { fn step(i: usize) -> PreparedCommit { let file = |kind: &str, equality_field_ids: Vec| DataFile { path: format!("memory:///warehouse/public/orders/step-{kind}-{i}.parquet"), - record_count: 1, + record_count: if equality_field_ids.is_empty() { 2 } else { 1 }, byte_size: 100, equality_field_ids, partition_values: Vec::new(), @@ -2421,13 +2429,25 @@ mod tests { /// snapshot on the last. #[tokio::test(start_paused = true)] async fn a_commit_of_several_steps_writes_them_together() { + several_steps_written_together(crate::FormatVersion::V2).await; + } + + /// As above, on a v3 table — each step's rows taking the row ids after + /// the last's, and a manifest carried into a later step's list keeping + /// the ones it took. + #[tokio::test(start_paused = true)] + async fn a_commit_of_several_steps_writes_them_together_on_v3() { + several_steps_written_together(crate::FormatVersion::V3).await; + } + + async fn several_steps_written_together(version: crate::FormatVersion) { const LATENCY: std::time::Duration = std::time::Duration::from_millis(100); const STEPS: usize = 4; let storage = CountingStorage { latency: LATENCY, ..Default::default() }; - let c = fresh_on(storage.clone()).await; + let c = fresh_on(storage.clone()).await.with_format_version(version); c.ensure_namespace(&ident().namespace).await.unwrap(); c.create_table(&schema()).await.unwrap(); append(&c, 0).await; @@ -2456,6 +2476,30 @@ mod tests { .collect(); assert_eq!(deletes.len(), STEPS); assert!(deletes.windows(2).all(|w| w[0] < w[1]), "{deletes:?}"); + + if version == crate::FormatVersion::V3 { + // Every data file's rows have ids of their own: one run from 0, + // in commit order, up to the table's next row id. + let mut ranges: Vec<(i64, i64)> = snapshots + .iter() + .flat_map(|s| &s.data_files) + .map(|f| { + let first = f + .first_row_id + .expect("a data file of a v3 table has row ids"); + (first, first + f.record_count as i64) + }) + .collect(); + ranges.sort(); + let it = to_iceberg_table_ident(&ident()).unwrap(); + let table = c.inner.load_table(&it).await.unwrap(); + let next_row_id = table.metadata().next_row_id() as i64; + let expected: Vec<(i64, i64)> = std::iter::once((0, 1)) + .chain((0..STEPS as i64).map(|i| (1 + 2 * i, 3 + 2 * i))) + .collect(); + assert_eq!(ranges, expected); + assert_eq!(next_row_id, 1 + 2 * STEPS as i64); + } } #[tokio::test] diff --git a/crates/pg2iceberg-tests/tests/dst.rs b/crates/pg2iceberg-tests/tests/dst.rs index 8a27002..3f53514 100644 --- a/crates/pg2iceberg-tests/tests/dst.rs +++ b/crates/pg2iceberg-tests/tests/dst.rs @@ -6680,6 +6680,53 @@ fn upgraded_while_a_pass_is_in_flight(prod: bool) { v3_case(prod, true, &steps); } +/// A transaction too big for one step — every row updated, more added — +/// commits its steps as one update, written together: on a v3 table, each +/// step's rows take the row ids after the last's (invariant 20). +#[test] +fn a_commit_of_several_steps_gives_each_row_an_id() { + several_steps_on_v3(false, false); +} + +#[cfg(feature = "integration")] +#[test] +fn a_commit_of_several_steps_gives_each_row_an_id_on_iceberg() { + several_steps_on_v3(true, false); +} + +/// So does the first such commit since the table was upgraded to v3, +/// whose first step gives the files from before the upgrade ids too. +#[test] +fn a_commit_of_several_steps_after_an_upgrade_gives_each_row_an_id() { + several_steps_on_v3(false, true); +} + +#[cfg(feature = "integration")] +#[test] +fn a_commit_of_several_steps_after_an_upgrade_gives_each_row_an_id_on_iceberg() { + several_steps_on_v3(true, true); +} + +fn several_steps_on_v3(prod: bool, upgraded: bool) { + V3_UPGRADED.set(upgraded); + let mut steps = vec![ + Step::Insert { id: 1, qty: 10 }, + Step::Insert { id: 2, qty: 20 }, + Step::Insert { id: 3, qty: 30 }, + Step::DriveFlush, + Step::MaterializerCycle, + ]; + if upgraded { + steps.push(Step::UpgradeToV3); + } + steps.extend([ + Step::BigTx { inserts: 5, qty: 1 }, + Step::DriveFlush, + Step::MaterializerCycle, + ]); + v3_case(prod, false, &steps); +} + /// So does a `compact` job's pass, planned before the upgrade and /// committed after pg2iceberg's next commit gave the rows ids. #[test] From ca3b6d580dde513e457a97423a0de32d038b229d Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 17 Nov 2026 14:03:40 +0800 Subject: [PATCH 2/2] chore: iceberg-rust from polynya-patches-v0.11.0 polynya-dev/iceberg-rust#8 merged. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 14 +++++++------- Cargo.toml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32a41e4..76d128e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2413,7 +2413,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -2461,7 +2461,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=staged-v3-row-lineage#98a3eca034beb570e333270b8ec2982723d9a0b9" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#fae9f8fe687d85f38ada75426313902efc53d19e" dependencies = [ "aes-gcm", "anyhow", @@ -2520,7 +2520,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=staged-v3-row-lineage#98a3eca034beb570e333270b8ec2982723d9a0b9" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#fae9f8fe687d85f38ada75426313902efc53d19e" dependencies = [ "async-trait", "aws-config", @@ -2544,7 +2544,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=staged-v3-row-lineage#98a3eca034beb570e333270b8ec2982723d9a0b9" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#fae9f8fe687d85f38ada75426313902efc53d19e" dependencies = [ "proc-macro2", "quote", @@ -2554,7 +2554,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=staged-v3-row-lineage#98a3eca034beb570e333270b8ec2982723d9a0b9" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#fae9f8fe687d85f38ada75426313902efc53d19e" dependencies = [ "anyhow", "async-trait", @@ -4270,7 +4270,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror", "tokio", "tracing", @@ -4308,7 +4308,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.52.0", ] diff --git a/Cargo.toml b/Cargo.toml index 805682f..c8c6359 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ all = { level = "warn", priority = -1 } # (Supabase's etl maintainer) rather than rolling our own to avoid the # maintenance burden of two forks. [patch.crates-io] -iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "staged-v3-row-lineage" } +iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" } # `iceberg-catalog-rest` also has to come from the fork: the # polynya-patches branch threads the REST `loadTable` response's # `config` map through `Table::builder().properties(response_config)`, @@ -68,7 +68,7 @@ iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "stage # servers (Polaris, Lakekeeper, ...) return per-table S3 STS tokens # in `config`, but our `extract_creds` sees an empty map because the # `Table::properties()` accessor surfaces nothing. -iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "staged-v3-row-lineage" } -iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "staged-v3-row-lineage" } +iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" } +iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" } tokio-postgres = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" } postgres-replication = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" }