diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs index 226be3501f..c11de6f1c1 100644 --- a/crates/iceberg/src/transaction/mod.rs +++ b/crates/iceberg/src/transaction/mod.rs @@ -91,6 +91,9 @@ use crate::{Catalog, Error, ErrorKind, TableCommit, TableRequirement, TableUpdat pub struct Transaction { table: Table, actions: Vec, + /// Whether `table` was loaded just before: the first attempt commits + /// on it rather than reloading it. + fresh_base: bool, } impl Transaction { @@ -99,9 +102,21 @@ impl Transaction { Self { table: table.clone(), actions: vec![], + fresh_base: false, } } + /// Declares the table this transaction was created with as current — + /// loaded just before — so its first commit attempt doesn't load it + /// again: a round trip to the catalog saved. If the table changed + /// meanwhile, that attempt fails with a conflict, and the retry + /// reloads it and applies the actions again, as every attempt does + /// otherwise. + pub fn with_fresh_base(mut self) -> Self { + self.fresh_base = true; + self + } + fn update_table_metadata(table: Table, updates: &[TableUpdate]) -> Result { let mut metadata_builder = table.metadata().clone().into_builder(None); for update in updates { @@ -235,13 +250,15 @@ impl Transaction { } async fn do_commit(&mut self, catalog: &dyn Catalog) -> Result
{ - let refreshed = catalog.load_table(self.table.identifier()).await?; - - if self.table.metadata() != refreshed.metadata() - || self.table.metadata_location() != refreshed.metadata_location() - { - // current base is stale, use refreshed as base and re-apply transaction actions - self.table = refreshed.clone(); + if !std::mem::take(&mut self.fresh_base) { + let refreshed = catalog.load_table(self.table.identifier()).await?; + + if self.table.metadata() != refreshed.metadata() + || self.table.metadata_location() != refreshed.metadata_location() + { + // current base is stale, use refreshed as base and re-apply transaction actions + self.table = refreshed.clone(); + } } let mut current_table = self.table.clone(); @@ -486,6 +503,63 @@ mod tests { mock_catalog } + /// A mock catalog whose first `conflicts` commits fail with a + /// retryable conflict, and that expects `loads` table loads. + fn mock_catalog_counting_loads(conflicts: u32, loads: usize) -> MockCatalog { + let mut mock_catalog = MockCatalog::new(); + mock_catalog + .expect_load_table() + .times(loads) + .returning_st(|_| Box::pin(async move { Ok(make_v2_table()) })); + let attempts = AtomicU32::new(0); + mock_catalog + .expect_update_table() + .times(conflicts as usize + 1) + .returning_st(move |_| { + if attempts.fetch_add(1, Ordering::SeqCst) < conflicts { + Box::pin(async move { + Err( + Error::new(ErrorKind::CatalogCommitConflicts, "Commit conflict") + .with_retryable(true), + ) + }) + } else { + Box::pin(async move { Ok(make_v2_table()) }) + } + }); + mock_catalog + } + + #[tokio::test] + async fn a_fresh_base_commits_without_reloading_the_table() { + let table = setup_test_table("3"); + let tx = create_test_transaction(&table).with_fresh_base(); + + let mock_catalog = mock_catalog_counting_loads(0, 0); + + assert!(tx.commit(&mock_catalog).await.is_ok()); + } + + #[tokio::test] + async fn a_fresh_base_reloads_the_table_to_retry_a_conflict() { + let table = setup_test_table("3"); + let tx = create_test_transaction(&table).with_fresh_base(); + + let mock_catalog = mock_catalog_counting_loads(1, 1); + + assert!(tx.commit(&mock_catalog).await.is_ok()); + } + + #[tokio::test] + async fn without_a_fresh_base_every_attempt_reloads_the_table() { + let table = setup_test_table("3"); + let tx = create_test_transaction(&table); + + let mock_catalog = mock_catalog_counting_loads(1, 2); + + assert!(tx.commit(&mock_catalog).await.is_ok()); + } + #[tokio::test] async fn test_commit_retryable_error() { // Create a test table with retry properties diff --git a/crates/iceberg/src/transaction/rewrite.rs b/crates/iceberg/src/transaction/rewrite.rs index c0fb6682ba..1c81b8c560 100644 --- a/crates/iceberg/src/transaction/rewrite.rs +++ b/crates/iceberg/src/transaction/rewrite.rs @@ -19,10 +19,11 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use async_trait::async_trait; +use futures::{StreamExt, TryStreamExt}; use uuid::Uuid; use crate::error::{Error, ErrorKind, Result}; -use crate::spec::{DataContentType, DataFile, ManifestEntry, ManifestFile, Operation}; +use crate::spec::{DataContentType, DataFile, Manifest, ManifestEntry, ManifestFile, Operation}; use crate::table::Table; use crate::transaction::snapshot::{ DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer, @@ -128,7 +129,10 @@ impl RewriteFilesAction { #[async_trait] impl TransactionAction for RewriteFilesAction { async fn commit(self: Arc, table: &Table) -> Result { - let removed_files = self.removed_files(table).await?; + // One read of the table this attempt would land on serves both + // what the rewrite removes and which manifests it keeps. + let manifests = current_manifests(table).await?; + let removed_files = self.removed_files(&manifests)?; let snapshot_producer = SnapshotProducer::new( table, self.commit_uuid.unwrap_or_else(Uuid::now_v7), @@ -146,6 +150,7 @@ impl TransactionAction for RewriteFilesAction { .commit( RewriteFilesOperation { removed_paths: self.removed_paths.clone(), + manifests, }, DefaultManifestProcess, ) @@ -153,24 +158,49 @@ impl TransactionAction for RewriteFilesAction { } } +/// Most manifests a rewrite reads at once. +const MANIFEST_READ_CONCURRENCY: usize = 16; + +/// `table`'s current manifests, each with its manifest-list entry, in the +/// manifest list's order: read [`MANIFEST_READ_CONCURRENCY`] at a time, as +/// each is a round trip to the object store. +async fn current_manifests(table: &Table) -> Result> { + let Some(snapshot) = table.metadata().current_snapshot() else { + return Ok(Vec::new()); + }; + let manifest_list = table.manifest_list_reader(snapshot).load().await?; + let entries: Vec = manifest_list.consume_entries().into_iter().collect(); + let runtime = table.runtime(); + let file_io = table.file_io(); + futures::stream::iter(entries) + .map(|entry| { + let file_io = file_io.clone(); + runtime.io().spawn(async move { + let manifest = entry.load_manifest(&file_io).await?; + Ok::<_, Error>((entry, manifest)) + }) + }) + .buffered(MANIFEST_READ_CONCURRENCY) + // The task's own result, inside its join's. + .and_then(futures::future::ready) + .try_collect() + .await +} + impl RewriteFilesAction { - /// The live files of `table` that `removed_paths` names — the snapshot - /// summary subtracts them. Fails if one isn't live: another rewrite - /// replaced it first, and committing this one's files as well would - /// duplicate their rows. Runs on every commit attempt, against the - /// table the commit would land on. - async fn removed_files(&self, table: &Table) -> Result> { + /// The live files in `manifests` that `removed_paths` names — the + /// snapshot summary subtracts them. Fails if one isn't live: another + /// rewrite replaced it first, and committing this one's files as well + /// would duplicate their rows. Runs on every commit attempt, against + /// the manifests of the table the commit would land on. + fn removed_files(&self, manifests: &[(ManifestFile, Manifest)]) -> Result> { let mut removed = Vec::new(); let mut found: HashSet<&str> = HashSet::new(); - if let Some(snapshot) = table.metadata().current_snapshot() { - let manifest_list = table.manifest_list_reader(snapshot).load().await?; - for mfile in manifest_list.entries() { - let manifest = mfile.load_manifest(table.file_io()).await?; - for entry in manifest.entries().iter().filter(|e| e.is_alive()) { - if let Some(path) = self.removed_paths.get(entry.data_file().file_path()) { - found.insert(path); - removed.push(entry.data_file().clone()); - } + for (_, manifest) in manifests { + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { + if let Some(path) = self.removed_paths.get(entry.data_file().file_path()) { + found.insert(path); + removed.push(entry.data_file().clone()); } } } @@ -193,6 +223,8 @@ impl RewriteFilesAction { struct RewriteFilesOperation { removed_paths: HashSet, + /// The current snapshot's manifests, as the commit read them. + manifests: Vec<(ManifestFile, Manifest)>, } impl SnapshotProduceOperation for RewriteFilesOperation { @@ -216,16 +248,6 @@ impl SnapshotProduceOperation for RewriteFilesOperation { &self, snapshot_produce: &mut SnapshotProducer<'_>, ) -> Result> { - let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else { - return Ok(vec![]); - }; - - let manifest_list = snapshot_produce - .table - .manifest_list_reader(snapshot) - .load() - .await?; - // Walk each prior manifest, decide carry-forward vs. partial-rewrite // vs. drop. // @@ -233,8 +255,6 @@ impl SnapshotProduceOperation for RewriteFilesOperation { // can't write rewritten manifests yet because that needs `&mut // snapshot_produce`, and we also need the producer to compute // file-IO paths during this loop. - let manifest_list_entries = manifest_list.entries().to_vec(); - let file_io = snapshot_produce.table.file_io().clone(); enum Plan { CarryForward(ManifestFile), @@ -244,9 +264,8 @@ impl SnapshotProduceOperation for RewriteFilesOperation { content: crate::spec::ManifestContentType, }, } - let mut plans: Vec = Vec::with_capacity(manifest_list_entries.len()); - for mfile in &manifest_list_entries { - let manifest = mfile.load_manifest(&file_io).await?; + let mut plans: Vec = Vec::with_capacity(self.manifests.len()); + for (mfile, manifest) in &self.manifests { let total = manifest.entries().len(); let survivors: Vec = manifest .entries()