From 558975aeb756b4d0879b1f0e4299be2874f9163c Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Mon, 19 Oct 2026 14:58:54 +0800 Subject: [PATCH] perf: a commit writes its steps' manifests together MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A commit of several steps — a large transaction, or a materializer catching up — chained fast appends, one step after another: each wrote its manifests, then its manifest list, then read that list back for the next step. Four round trips to object storage a step, while committing was most of what the materializer did. Now each step's snapshot is planned in memory (the fork's `FastAppendAction::stage`), then every step's manifests are written at once, then every step's manifest list, each naming its parent's manifests and its own: a read of the table's manifest list and two rounds of writes, however many steps. V3 tables, whose snapshots take their row range from the written list, still commit step by step. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 6 +- crates/pg2iceberg-iceberg/Cargo.toml | 3 +- crates/pg2iceberg-iceberg/src/prod/catalog.rs | 210 ++++++++++++++++-- 3 files changed, 193 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d32b7d..3a3d786 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2450,7 +2450,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#55d6a3c7a5c462b7aba3867b6626ded24ded2935" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#0ded76c9e9f627822b7a7a4d032771c410fc7a48" dependencies = [ "aes-gcm", "anyhow", @@ -2506,7 +2506,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.10.1" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#55d6a3c7a5c462b7aba3867b6626ded24ded2935" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#0ded76c9e9f627822b7a7a4d032771c410fc7a48" dependencies = [ "async-trait", "aws-config", @@ -2529,7 +2529,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#55d6a3c7a5c462b7aba3867b6626ded24ded2935" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#0ded76c9e9f627822b7a7a4d032771c410fc7a48" dependencies = [ "anyhow", "async-trait", diff --git a/crates/pg2iceberg-iceberg/Cargo.toml b/crates/pg2iceberg-iceberg/Cargo.toml index 5442f6b..6d86922 100644 --- a/crates/pg2iceberg-iceberg/Cargo.toml +++ b/crates/pg2iceberg-iceberg/Cargo.toml @@ -51,7 +51,8 @@ pollster = "0.4" # Iceberg's Transaction::commit uses tokio::time::sleep in its retry backoff. # Smoke tests under `--features prod` therefore need a tokio runtime; the # non-prod tests still run on pollster. -tokio = { version = "1", features = ["rt", "macros", "time"] } +# `test-util`: paused time, for a storage a round trip away. +tokio = { version = "1", features = ["rt", "macros", "time", "test-util"] } # A counting `iceberg::io::Storage` in the prod catalog's tests: iceberg's # storage traits are `#[typetag::serde]`. typetag = "0.2" diff --git a/crates/pg2iceberg-iceberg/src/prod/catalog.rs b/crates/pg2iceberg-iceberg/src/prod/catalog.rs index 54776fa..6903fc1 100644 --- a/crates/pg2iceberg-iceberg/src/prod/catalog.rs +++ b/crates/pg2iceberg-iceberg/src/prod/catalog.rs @@ -29,12 +29,16 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; use async_trait::async_trait; +use futures::stream::{self, StreamExt, TryStreamExt}; use iceberg::spec::{ DataContentType, DataFile as IcebergDataFile, DataFileBuilder, DataFileFormat, FormatVersion, - Literal, NestedField, PrimitiveLiteral, PrimitiveType, Schema as IcebergSchema, Struct, Type, + Literal, ManifestFile, NestedField, PrimitiveLiteral, PrimitiveType, Schema as IcebergSchema, + Struct, Type, }; use iceberg::table::Table; -use iceberg::transaction::{ActionCommit, ApplyTransactionAction, Transaction, TransactionAction}; +use iceberg::transaction::{ + ActionCommit, ApplyTransactionAction, StagedSnapshot, Transaction, TransactionAction, +}; use iceberg::TableUpdate; use iceberg::{ Catalog as IcebergCatalogTrait, ErrorKind, NamespaceIdent, TableCreation, @@ -700,12 +704,38 @@ struct ChainedAppendAction { properties: HashMap, } +/// Most snapshots of a chain written at once. +const CHAIN_WRITES: usize = 16; + #[async_trait] impl TransactionAction for ChainedAppendAction { + /// Each step a snapshot on the last, planned in memory, then written + /// together: every step's manifests at once, then every step's manifest + /// list, each naming its parent's manifests and its own. Written one + /// step at a time, each took four round trips to object storage — its + /// manifests, its list, and reading that list back for the next. async fn commit(self: Arc, table: &Table) -> iceberg::Result { + if table.metadata().format_version() == FormatVersion::V3 { + return self.commit_one_by_one(table).await; + } + // The manifests the first step's list carries forward: as a fast + // append keeps them. + let mut parent: Vec = match table.metadata().current_snapshot() { + Some(snapshot) => table + .manifest_list_reader(snapshot) + .load() + .await? + .entries() + .iter() + .filter(|m| m.has_added_files() || m.has_existing_files() || m.has_deleted_files()) + .cloned() + .collect(), + None => Vec::new(), + }; let mut local = table.clone(); let mut updates = Vec::new(); let mut requirements = None; + let mut staged = Vec::with_capacity(self.steps.len()); for (i, files) in self.steps.iter().enumerate() { let mut append = Transaction::new(&local) .fast_append() @@ -714,33 +744,80 @@ impl TransactionAction for ChainedAppendAction { if i + 1 == self.steps.len() { append = append.set_snapshot_properties(self.properties.clone()); } - let mut step = Arc::new(append).commit(&local).await?; + let (mut step, snapshot) = append.stage(&local).await?; + staged.push(snapshot); let step_updates = step.take_updates(); requirements.get_or_insert(step.take_requirements()); - let mut builder = local - .metadata() - .clone() - .into_builder(local.metadata_location().map(str::to_string)); - for update in &step_updates { - builder = update.clone().apply(builder)?; - } - // `local`'s runtime isn't public; pg2iceberg always runs - // inside the tokio runtime iceberg-rust was given. - let mut next = Table::builder() - .identifier(local.identifier().clone()) - .file_io(local.file_io().clone()) - .metadata(builder.build()?.metadata) - .runtime(iceberg::Runtime::try_current()?); - if let Some(location) = local.metadata_location() { - next = next.metadata_location(location); + local = with_updates(&local, &step_updates)?; + updates.extend(step_updates); + } + let added: Vec> = stream::iter(staged.iter_mut()) + .map(StagedSnapshot::write_manifests) + .buffered(CHAIN_WRITES) + .try_collect() + .await?; + let mut lists = Vec::with_capacity(staged.len()); + for (snapshot, added) in staged.into_iter().zip(added) { + let manifests: Vec = parent.into_iter().chain(added).collect(); + parent = manifests.clone(); + lists.push(snapshot.write_manifest_list(manifests)); + } + stream::iter(lists) + .buffered(CHAIN_WRITES) + .try_collect::>() + .await?; + Ok(ActionCommit::new(updates, requirements.unwrap_or_default())) + } +} + +impl ChainedAppendAction { + /// One step after another, each a fast append on the last: for v3 + /// tables, whose snapshots take their row range from their written + /// manifest list. + async fn commit_one_by_one(&self, table: &Table) -> iceberg::Result { + let mut local = table.clone(); + let mut updates = Vec::new(); + let mut requirements = None; + for (i, files) in self.steps.iter().enumerate() { + let mut append = Transaction::new(&local) + .fast_append() + .with_check_duplicate(false) + .add_data_files(files.clone()); + if i + 1 == self.steps.len() { + append = append.set_snapshot_properties(self.properties.clone()); } - local = next.build()?; + let mut step = Arc::new(append).commit(&local).await?; + let step_updates = step.take_updates(); + requirements.get_or_insert(step.take_requirements()); + local = with_updates(&local, &step_updates)?; updates.extend(step_updates); } Ok(ActionCommit::new(updates, requirements.unwrap_or_default())) } } +/// `table` with `updates` applied: the next step's base. +fn with_updates(table: &Table, updates: &[TableUpdate]) -> iceberg::Result { + let mut builder = table + .metadata() + .clone() + .into_builder(table.metadata_location().map(str::to_string)); + for update in updates { + builder = update.clone().apply(builder)?; + } + // `table`'s runtime isn't public; pg2iceberg always runs inside the + // tokio runtime iceberg-rust was given. + let mut next = Table::builder() + .identifier(table.identifier().clone()) + .file_io(table.file_io().clone()) + .metadata(builder.build()?.metadata) + .runtime(iceberg::Runtime::try_current()?); + if let Some(location) = table.metadata_location() { + next = next.metadata_location(location); + } + next.build() +} + /// Sets the table's `pg2iceberg.log-ends` property to each group's end: /// what the table already records, with `range`'s. Taken from the table /// the commit lands on — after the appends before it in the transaction, @@ -1988,13 +2065,35 @@ mod tests { assert!(snaps.is_empty()); } - /// In-memory storage that records the path of every read. + /// In-memory storage that records the path of every read — and with a + /// `latency`, is a round trip away, in tokio's (paused) time: each read, + /// and each write as it completes. #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] struct CountingStorage { #[serde(skip)] inner: iceberg::io::MemoryStorage, #[serde(skip)] reads: Arc>>, + #[serde(skip)] + latency: std::time::Duration, + } + + /// A write that takes `latency` to complete. + struct SlowWrite { + inner: Box, + latency: std::time::Duration, + } + + #[async_trait] + impl iceberg::io::FileWrite for SlowWrite { + async fn write(&mut self, bs: bytes::Bytes) -> iceberg::Result<()> { + self.inner.write(bs).await + } + + async fn close(&mut self) -> iceberg::Result<()> { + tokio::time::sleep(self.latency).await; + self.inner.close().await + } } impl CountingStorage { @@ -2017,20 +2116,26 @@ mod tests { async fn read(&self, path: &str) -> iceberg::Result { self.reads.lock().unwrap().push(path.to_string()); + tokio::time::sleep(self.latency).await; self.inner.read(path).await } async fn reader(&self, path: &str) -> iceberg::Result> { self.reads.lock().unwrap().push(path.to_string()); + tokio::time::sleep(self.latency).await; self.inner.reader(path).await } async fn write(&self, path: &str, bs: bytes::Bytes) -> iceberg::Result<()> { + tokio::time::sleep(self.latency).await; self.inner.write(path, bs).await } async fn writer(&self, path: &str) -> iceberg::Result> { - self.inner.writer(path).await + Ok(Box::new(SlowWrite { + inner: self.inner.writer(path).await?, + latency: self.latency, + })) } async fn delete(&self, path: &str) -> iceberg::Result<()> { @@ -2227,6 +2332,67 @@ mod tests { assert_eq!(second_reads.len(), 2, "{second_reads:?}"); } + /// Step `i` of a commit: a data file and an equality-delete file. + fn step(i: usize) -> PreparedCommit { + let file = |kind: &str, equality_field_ids: Vec| DataFile { + path: format!("memory:///warehouse/public/orders/step-{kind}-{i}.parquet"), + record_count: 1, + byte_size: 100, + equality_field_ids, + partition_values: Vec::new(), + sequence_number: None, + }; + PreparedCommit { + ident: ident(), + data_files: vec![file("data", vec![])], + equality_deletes: vec![file("eq-delete", vec![1])], + } + } + + /// A commit of several steps — a large transaction, or a materializer + /// catching up — writes every step's manifests at once, then every + /// step's manifest list, and reads back nothing it wrote: a few round + /// trips to object storage, however many steps. Each step is still a + /// snapshot on the last. + #[tokio::test(start_paused = true)] + async fn a_commit_of_several_steps_writes_them_together() { + const LATENCY: std::time::Duration = std::time::Duration::from_millis(100); + const STEPS: usize = 4; + let storage = CountingStorage { + latency: LATENCY, + ..Default::default() + }; + let c = fresh_on(storage.clone()).await; + c.ensure_namespace(&ident().namespace).await.unwrap(); + c.create_table(&schema()).await.unwrap(); + append(&c, 0).await; + storage.take_reads(); + + let start = tokio::time::Instant::now(); + c.commit_snapshots((0..STEPS).map(step).collect(), None, BTreeSet::new()) + .await + .unwrap(); + let took = start.elapsed(); + + let reads = storage.take_reads(); + let lists: Vec<&String> = reads.iter().filter(|p| p.ends_with(".avro")).collect(); + // The table's manifest list, which the first step carries forward. + assert_eq!(lists.len(), 1, "{lists:?}"); + // That read, every manifest, every list — and the table's metadata, + // read and written. + assert!(took <= 6 * LATENCY, "{STEPS} steps took {took:?}"); + let snapshots = c.snapshots(&ident()).await.unwrap(); + assert_eq!(snapshots.len(), 1 + STEPS); + // The first commit's file, and each step's data and delete files. + assert_eq!(live_files(&snapshots).len(), 1 + 2 * STEPS); + let deletes: Vec = snapshots + .iter() + .flat_map(|s| s.delete_files.iter().map(move |_| s.id)) + .collect(); + assert_eq!(deletes.len(), STEPS); + assert!(deletes.windows(2).all(|w| w[0] < w[1]), "{deletes:?}"); + } + #[tokio::test] async fn evolve_schema_add_column_appends_to_schema_with_fresh_field_id() { let c = fresh().await;