Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 81 additions & 7 deletions crates/iceberg/src/transaction/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ use crate::{Catalog, Error, ErrorKind, TableCommit, TableRequirement, TableUpdat
pub struct Transaction {
table: Table,
actions: Vec<BoxedTransactionAction>,
/// Whether `table` was loaded just before: the first attempt commits
/// on it rather than reloading it.
fresh_base: bool,
}

impl Transaction {
Expand All @@ -99,9 +102,21 @@ impl Transaction {
Self {
table: table.clone(),
actions: vec![],
fresh_base: false,
}
}

/// Declares the table this transaction was created with as current —
/// loaded just before — so its first commit attempt doesn't load it
/// again: a round trip to the catalog saved. If the table changed
/// meanwhile, that attempt fails with a conflict, and the retry
/// reloads it and applies the actions again, as every attempt does
/// otherwise.
pub fn with_fresh_base(mut self) -> Self {
self.fresh_base = true;
self
}

fn update_table_metadata(table: Table, updates: &[TableUpdate]) -> Result<Table> {
let mut metadata_builder = table.metadata().clone().into_builder(None);
for update in updates {
Expand Down Expand Up @@ -235,13 +250,15 @@ impl Transaction {
}

async fn do_commit(&mut self, catalog: &dyn Catalog) -> Result<Table> {
let refreshed = catalog.load_table(self.table.identifier()).await?;

if self.table.metadata() != refreshed.metadata()
|| self.table.metadata_location() != refreshed.metadata_location()
{
// current base is stale, use refreshed as base and re-apply transaction actions
self.table = refreshed.clone();
if !std::mem::take(&mut self.fresh_base) {
let refreshed = catalog.load_table(self.table.identifier()).await?;

if self.table.metadata() != refreshed.metadata()
|| self.table.metadata_location() != refreshed.metadata_location()
{
// current base is stale, use refreshed as base and re-apply transaction actions
self.table = refreshed.clone();
}
}

let mut current_table = self.table.clone();
Expand Down Expand Up @@ -486,6 +503,63 @@ mod tests {
mock_catalog
}

/// A mock catalog whose first `conflicts` commits fail with a
/// retryable conflict, and that expects `loads` table loads.
fn mock_catalog_counting_loads(conflicts: u32, loads: usize) -> MockCatalog {
let mut mock_catalog = MockCatalog::new();
mock_catalog
.expect_load_table()
.times(loads)
.returning_st(|_| Box::pin(async move { Ok(make_v2_table()) }));
let attempts = AtomicU32::new(0);
mock_catalog
.expect_update_table()
.times(conflicts as usize + 1)
.returning_st(move |_| {
if attempts.fetch_add(1, Ordering::SeqCst) < conflicts {
Box::pin(async move {
Err(
Error::new(ErrorKind::CatalogCommitConflicts, "Commit conflict")
.with_retryable(true),
)
})
} else {
Box::pin(async move { Ok(make_v2_table()) })
}
});
mock_catalog
}

#[tokio::test]
async fn a_fresh_base_commits_without_reloading_the_table() {
let table = setup_test_table("3");
let tx = create_test_transaction(&table).with_fresh_base();

let mock_catalog = mock_catalog_counting_loads(0, 0);

assert!(tx.commit(&mock_catalog).await.is_ok());
}

#[tokio::test]
async fn a_fresh_base_reloads_the_table_to_retry_a_conflict() {
let table = setup_test_table("3");
let tx = create_test_transaction(&table).with_fresh_base();

let mock_catalog = mock_catalog_counting_loads(1, 1);

assert!(tx.commit(&mock_catalog).await.is_ok());
}

#[tokio::test]
async fn without_a_fresh_base_every_attempt_reloads_the_table() {
let table = setup_test_table("3");
let tx = create_test_transaction(&table);

let mock_catalog = mock_catalog_counting_loads(1, 2);

assert!(tx.commit(&mock_catalog).await.is_ok());
}

#[tokio::test]
async fn test_commit_retryable_error() {
// Create a test table with retry properties
Expand Down
83 changes: 51 additions & 32 deletions crates/iceberg/src/transaction/rewrite.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ use std::collections::{HashMap, HashSet};
use std::sync::Arc;

use async_trait::async_trait;
use futures::{StreamExt, TryStreamExt};
use uuid::Uuid;

use crate::error::{Error, ErrorKind, Result};
use crate::spec::{DataContentType, DataFile, ManifestEntry, ManifestFile, Operation};
use crate::spec::{DataContentType, DataFile, Manifest, ManifestEntry, ManifestFile, Operation};
use crate::table::Table;
use crate::transaction::snapshot::{
DefaultManifestProcess, SnapshotProduceOperation, SnapshotProducer,
Expand Down Expand Up @@ -128,7 +129,10 @@ impl RewriteFilesAction {
#[async_trait]
impl TransactionAction for RewriteFilesAction {
async fn commit(self: Arc<Self>, table: &Table) -> Result<ActionCommit> {
let removed_files = self.removed_files(table).await?;
// One read of the table this attempt would land on serves both
// what the rewrite removes and which manifests it keeps.
let manifests = current_manifests(table).await?;
let removed_files = self.removed_files(&manifests)?;
let snapshot_producer = SnapshotProducer::new(
table,
self.commit_uuid.unwrap_or_else(Uuid::now_v7),
Expand All @@ -146,31 +150,57 @@ impl TransactionAction for RewriteFilesAction {
.commit(
RewriteFilesOperation {
removed_paths: self.removed_paths.clone(),
manifests,
},
DefaultManifestProcess,
)
.await
}
}

/// Most manifests a rewrite reads at once.
const MANIFEST_READ_CONCURRENCY: usize = 16;

/// `table`'s current manifests, each with its manifest-list entry, in the
/// manifest list's order: read [`MANIFEST_READ_CONCURRENCY`] at a time, as
/// each is a round trip to the object store.
async fn current_manifests(table: &Table) -> Result<Vec<(ManifestFile, Manifest)>> {
let Some(snapshot) = table.metadata().current_snapshot() else {
return Ok(Vec::new());
};
let manifest_list = table.manifest_list_reader(snapshot).load().await?;
let entries: Vec<ManifestFile> = manifest_list.consume_entries().into_iter().collect();
let runtime = table.runtime();
let file_io = table.file_io();
futures::stream::iter(entries)
.map(|entry| {
let file_io = file_io.clone();
runtime.io().spawn(async move {
let manifest = entry.load_manifest(&file_io).await?;
Ok::<_, Error>((entry, manifest))
})
})
.buffered(MANIFEST_READ_CONCURRENCY)
// The task's own result, inside its join's.
.and_then(futures::future::ready)
.try_collect()
.await
}

impl RewriteFilesAction {
/// The live files of `table` that `removed_paths` names — the snapshot
/// summary subtracts them. Fails if one isn't live: another rewrite
/// replaced it first, and committing this one's files as well would
/// duplicate their rows. Runs on every commit attempt, against the
/// table the commit would land on.
async fn removed_files(&self, table: &Table) -> Result<Vec<DataFile>> {
/// The live files in `manifests` that `removed_paths` names — the
/// snapshot summary subtracts them. Fails if one isn't live: another
/// rewrite replaced it first, and committing this one's files as well
/// would duplicate their rows. Runs on every commit attempt, against
/// the manifests of the table the commit would land on.
fn removed_files(&self, manifests: &[(ManifestFile, Manifest)]) -> Result<Vec<DataFile>> {
let mut removed = Vec::new();
let mut found: HashSet<&str> = HashSet::new();
if let Some(snapshot) = table.metadata().current_snapshot() {
let manifest_list = table.manifest_list_reader(snapshot).load().await?;
for mfile in manifest_list.entries() {
let manifest = mfile.load_manifest(table.file_io()).await?;
for entry in manifest.entries().iter().filter(|e| e.is_alive()) {
if let Some(path) = self.removed_paths.get(entry.data_file().file_path()) {
found.insert(path);
removed.push(entry.data_file().clone());
}
for (_, manifest) in manifests {
for entry in manifest.entries().iter().filter(|e| e.is_alive()) {
if let Some(path) = self.removed_paths.get(entry.data_file().file_path()) {
found.insert(path);
removed.push(entry.data_file().clone());
}
}
}
Expand All @@ -193,6 +223,8 @@ impl RewriteFilesAction {

struct RewriteFilesOperation {
removed_paths: HashSet<String>,
/// The current snapshot's manifests, as the commit read them.
manifests: Vec<(ManifestFile, Manifest)>,
}

impl SnapshotProduceOperation for RewriteFilesOperation {
Expand All @@ -216,25 +248,13 @@ impl SnapshotProduceOperation for RewriteFilesOperation {
&self,
snapshot_produce: &mut SnapshotProducer<'_>,
) -> Result<Vec<ManifestFile>> {
let Some(snapshot) = snapshot_produce.table.metadata().current_snapshot() else {
return Ok(vec![]);
};

let manifest_list = snapshot_produce
.table
.manifest_list_reader(snapshot)
.load()
.await?;

// Walk each prior manifest, decide carry-forward vs. partial-rewrite
// vs. drop.
//
// Pass 1: classify each manifest based on its loaded entries. We
// can't write rewritten manifests yet because that needs `&mut
// snapshot_produce`, and we also need the producer to compute
// file-IO paths during this loop.
let manifest_list_entries = manifest_list.entries().to_vec();
let file_io = snapshot_produce.table.file_io().clone();

enum Plan {
CarryForward(ManifestFile),
Expand All @@ -244,9 +264,8 @@ impl SnapshotProduceOperation for RewriteFilesOperation {
content: crate::spec::ManifestContentType,
},
}
let mut plans: Vec<Plan> = Vec::with_capacity(manifest_list_entries.len());
for mfile in &manifest_list_entries {
let manifest = mfile.load_manifest(&file_io).await?;
let mut plans: Vec<Plan> = Vec::with_capacity(self.manifests.len());
for (mfile, manifest) in &self.manifests {
let total = manifest.entries().len();
let survivors: Vec<ManifestEntry> = manifest
.entries()
Expand Down
Loading