From 606dd48c6dba5e7ff4434ce15cf758c93a1b49e8 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Mon, 19 Oct 2026 10:42:03 +0800 Subject: [PATCH] perf: a commit loads its table once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every commit — the materializer's, compaction's, expiry, a schema change — loaded the table to build the transaction, then iceberg-rust loaded it again before applying it: two catalog round trips. The transaction now declares the table it was built on fresh (`with_fresh_base`), so the first attempt commits on it; a conflict still reloads and retries. Also takes the fork's single concurrent manifest scan for rewrites: a compaction's commit read the table's every manifest twice, one at a time — seconds once a backlog's commits have piled manifests up. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 6 +++--- crates/pg2iceberg-iceberg/src/prod/catalog.rs | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4de6c8d..4d32b7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2450,7 +2450,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#cc60df5c899beac69a19145c7f18eac86972e316" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#55d6a3c7a5c462b7aba3867b6626ded24ded2935" dependencies = [ "aes-gcm", "anyhow", @@ -2506,7 +2506,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.10.1" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#cc60df5c899beac69a19145c7f18eac86972e316" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#55d6a3c7a5c462b7aba3867b6626ded24ded2935" dependencies = [ "async-trait", "aws-config", @@ -2529,7 +2529,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#cc60df5c899beac69a19145c7f18eac86972e316" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#55d6a3c7a5c462b7aba3867b6626ded24ded2935" dependencies = [ "anyhow", "async-trait", diff --git a/crates/pg2iceberg-iceberg/src/prod/catalog.rs b/crates/pg2iceberg-iceberg/src/prod/catalog.rs index e6dc06e..54776fa 100644 --- a/crates/pg2iceberg-iceberg/src/prod/catalog.rs +++ b/crates/pg2iceberg-iceberg/src/prod/catalog.rs @@ -235,7 +235,8 @@ impl Catalog for IcebergRustCata .chain(log_ends_property(&ends)) .collect(); let appends = !files.is_empty(); - let tx = Transaction::new(&table); + // Loaded just above: no reload before the first attempt. + let tx = Transaction::new(&table).with_fresh_base(); let tx = match files.len() { // No work — match the sim-catalog noop semantics so the // materializer can flush "no data, no deletes" without a @@ -322,7 +323,8 @@ impl Catalog for IcebergRustCata ); } - let tx = Transaction::new(&table); + // Loaded just above: no reload before the first attempt. + let tx = Transaction::new(&table).with_fresh_base(); let mut action = tx .rewrite_files() .add_data_files(iceberg_added) @@ -389,7 +391,8 @@ impl Catalog for IcebergRustCata let action = ExpireSnapshotsAction { snapshot_ids: to_remove, }; - let tx = Transaction::new(&table); + // Loaded just above: no reload before the first attempt. + let tx = Transaction::new(&table).with_fresh_base(); let tx = action.apply(tx).map_err(map_iceberg_err)?; tx.commit(self.inner.as_ref()) .await @@ -424,7 +427,8 @@ impl Catalog for IcebergRustCata apply_schema_changes(&mut our_schema, &changes)?; let new_iceberg_schema = to_iceberg_schema(&our_schema)?; - let mut tx = Transaction::new(&table); + // Loaded just above: no reload before the first attempt. + let mut tx = Transaction::new(&table).with_fresh_base(); if !changes.is_empty() { let action = tx.replace_schema().set_schema(new_iceberg_schema); tx = action.apply(tx).map_err(map_iceberg_err)?;