Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 85 additions & 11 deletions crates/iceberg/src/transaction/append.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,9 @@ impl FastAppendAction {
/// commit that adds it, and its files to write. For a chain of appends
/// one commit lands, whose files are then written together — their
/// manifests, then their manifest lists, each naming its parent's
/// manifests and its own. Doesn't check for duplicate files. V1 and v2
/// tables only.
/// manifests and its own. Doesn't check for duplicate files. On a v3
/// table, every manifest a list carries must have its row ids already
/// ([`StagedSnapshot::write_manifest_list`]).
pub async fn stage(&self, table: &Table) -> Result<(ActionCommit, StagedSnapshot)> {
let snapshot_producer = SnapshotProducer::new(
table,
Expand Down Expand Up @@ -952,7 +953,9 @@ mod tests {
}

/// A fresh in-memory table, and its catalog.
async fn memory_table() -> (crate::memory::MemoryCatalog, Table) {
async fn memory_table_in(
format_version: crate::spec::FormatVersion,
) -> (crate::memory::MemoryCatalog, Table) {
use crate::spec::{NestedField, PrimitiveType, Schema, Type};
use crate::{Catalog, CatalogBuilder};
let schema = Schema::builder()
Expand All @@ -978,6 +981,7 @@ mod tests {
let creation = crate::TableCreation::builder()
.name("orders".to_string())
.schema(schema)
.format_version(format_version)
.build();
let table = memory.create_table(&ns, creation).await.unwrap();
(memory, table)
Expand Down Expand Up @@ -1010,8 +1014,9 @@ mod tests {
}

/// Each snapshot as its manifest list has it: per manifest, its content,
/// sequence numbers and the files it holds, with their sequence numbers.
async fn snapshots_as_listed(table: &Table) -> Vec<(i64, Vec<String>)> {
/// sequence numbers, first row id and the files it holds, with their
/// sequence numbers and first row ids; and the snapshot's row range.
async fn snapshots_as_listed(table: &Table) -> Vec<(i64, Option<(u64, u64)>, Vec<String>)> {
let mut out = Vec::new();
for snapshot in table.metadata().snapshots() {
let list = table.manifest_list_reader(snapshot).load().await.unwrap();
Expand All @@ -1021,15 +1026,21 @@ mod tests {
let files: Vec<_> = manifest
.entries()
.iter()
.map(|e| (e.file_path().to_string(), e.sequence_number()))
.map(|e| {
(
e.file_path().to_string(),
e.sequence_number(),
e.data_file().first_row_id(),
)
})
.collect();
manifests.push(format!(
"{:?} seq {} min {} {:?}",
m.content, m.sequence_number, m.min_sequence_number, files
"{:?} seq {} min {} first row id {:?} {:?}",
m.content, m.sequence_number, m.min_sequence_number, m.first_row_id, files
));
}
manifests.sort();
out.push((snapshot.sequence_number(), manifests));
out.push((snapshot.sequence_number(), snapshot.row_range(), manifests));
}
out.sort();
out
Expand All @@ -1041,11 +1052,70 @@ mod tests {
/// appends committed one after another.
#[tokio::test]
async fn a_staged_chain_of_appends_lands_as_appends_one_by_one() {
staged_chain_lands_as_appends_one_by_one(crate::spec::FormatVersion::V2).await;
}

/// As above, on a v3 table: each snapshot's rows take the row ids after
/// the last's — its data manifest records the first, and keeps it in
/// the later snapshots' lists — and the table's next row id follows.
#[tokio::test]
async fn a_staged_chain_of_appends_lands_as_appends_one_by_one_on_v3() {
staged_chain_lands_as_appends_one_by_one(crate::spec::FormatVersion::V3).await;
}

/// A v2 table's files, its first commit since an upgrade to v3 gives
/// row ids, in its manifest list: a staged snapshot, whose row range
/// counts its own rows only, refuses to carry them.
#[tokio::test]
async fn a_staged_snapshot_refuses_to_carry_files_without_row_ids() {
use crate::transaction::ApplyTransactionAction;

let (catalog, table) = memory_table_in(crate::spec::FormatVersion::V2).await;
let tx = Transaction::new(&table);
let tx = tx
.fast_append()
.with_check_duplicate(false)
.add_data_files(step_files(&table, 0))
.apply(tx)
.unwrap();
let table = tx.commit(&catalog).await.unwrap();
let tx = Transaction::new(&table);
let tx = tx
.upgrade_table_version()
.set_format_version(crate::spec::FormatVersion::V3)
.apply(tx)
.unwrap();
let table = tx.commit(&catalog).await.unwrap();

let (_, mut staged) = Transaction::new(&table)
.fast_append()
.with_check_duplicate(false)
.add_data_files(step_files(&table, 1))
.stage(&table)
.await
.unwrap();
let parent = table
.manifest_list_reader(table.metadata().current_snapshot().unwrap())
.load()
.await
.unwrap()
.entries()
.to_vec();
assert!(parent.iter().any(|m| m.first_row_id.is_none()));
let added = staged.write_manifests().await.unwrap();
let err = staged
.write_manifest_list(parent.into_iter().chain(added).collect())
.await
.unwrap_err();
assert!(err.message().contains("without row ids"), "{err}");
}

async fn staged_chain_lands_as_appends_one_by_one(format_version: crate::spec::FormatVersion) {
use crate::transaction::ApplyTransactionAction;
use crate::{Catalog, TableCommit};
const STEPS: usize = 3;

let (one_by_one_catalog, mut one_by_one) = memory_table().await;
let (one_by_one_catalog, mut one_by_one) = memory_table_in(format_version).await;
for n in 0..STEPS {
let tx = Transaction::new(&one_by_one);
let files = step_files(&one_by_one, n);
Expand All @@ -1058,7 +1128,7 @@ mod tests {
one_by_one = tx.commit(&one_by_one_catalog).await.unwrap();
}

let (staged_catalog, table) = memory_table().await;
let (staged_catalog, table) = memory_table_in(format_version).await;
let mut local = table.clone();
let mut updates = Vec::new();
let mut requirements = Vec::new();
Expand Down Expand Up @@ -1110,6 +1180,10 @@ mod tests {
snapshots_as_listed(&staged_table).await,
snapshots_as_listed(&one_by_one).await
);
assert_eq!(
staged_table.metadata().next_row_id(),
one_by_one.metadata().next_row_id()
);
let summaries = |t: &Table| {
let mut s: Vec<_> = t
.metadata()
Expand Down
48 changes: 35 additions & 13 deletions crates/iceberg/src/transaction/snapshot.rs
Original file line number Diff line number Diff line change
Expand Up @@ -605,20 +605,16 @@ impl<'a> SnapshotProducer<'a> {
/// it, and its files to write ([`StagedSnapshot`]). Its manifest list
/// carries the manifests the caller names when it writes it, rather
/// than reading the parent's: snapshots one commit chains can be
/// written together. V1 and v2 tables only — v3's row lineage takes
/// its range from the written manifest list — and unencrypted tables
/// only: an encrypted list's key, which the snapshot names, is only
/// known once the list is written.
/// written together. On a v3 table, its rows take the row ids after the
/// table's last, which its data manifest records as it's written: every
/// manifest its list carries must have its ids already (see
/// [`StagedSnapshot::write_manifest_list`]). Unencrypted tables only:
/// an encrypted list's key, which the snapshot names, is only known
/// once the list is written.
pub(crate) async fn stage<OP: SnapshotProduceOperation>(
mut self,
snapshot_produce_operation: &OP,
) -> Result<(ActionCommit, StagedSnapshot)> {
if self.table.metadata().format_version() == FormatVersion::V3 {
return Err(Error::new(
ErrorKind::FeatureUnsupported,
"staged snapshots don't support format version 3",
));
}
if self.table.encryption_manager().is_some() {
return Err(Error::new(
ErrorKind::FeatureUnsupported,
Expand All @@ -636,6 +632,11 @@ impl<'a> SnapshotProducer<'a> {
}
let manifest_list_path = self.generate_manifest_list_file_path(0)?;
let (list, _) = self.manifest_list_writer(&manifest_list_path).await?;
// v3: the rows it adds take the row ids after the table's last.
let row_range = (self.table.metadata().format_version() == FormatVersion::V3).then(|| {
let added_rows = self.added_data_files.iter().map(|f| f.record_count()).sum();
(self.table.metadata().next_row_id(), added_rows)
});
// Before the added files are taken below.
let summary = self.summary(snapshot_produce_operation).map_err(|err| {
Error::new(ErrorKind::Unexpected, "Failed to create snapshot summary.").with_source(err)
Expand Down Expand Up @@ -663,9 +664,10 @@ impl<'a> SnapshotProducer<'a> {
list,
sequence_number: self.table.metadata().next_sequence_number(),
format_version: self.table.metadata().format_version(),
first_row_id: row_range.map(|(first_row_id, _)| first_row_id),
};
Ok((
self.snapshot_commit(manifest_list_path, summary, None, None),
self.snapshot_commit(manifest_list_path, summary, row_range, None),
staged,
))
}
Expand Down Expand Up @@ -799,12 +801,15 @@ pub struct StagedSnapshot {
list: ManifestListWriter,
sequence_number: i64,
format_version: FormatVersion,
/// v3: the first row id of the rows it adds.
first_row_id: Option<u64>,
}

impl StagedSnapshot {
/// Write the snapshot's own manifests, together. Returns them as its
/// manifest list records them — with its sequence number — so a later
/// snapshot's list can carry them.
/// manifest list records them — with its sequence number and, on a v3
/// table, the first row id of its rows — so a later snapshot's list
/// can carry them.
pub async fn write_manifests(&mut self) -> Result<Vec<ManifestFile>> {
let written = std::mem::take(&mut self.manifests)
.into_iter()
Expand All @@ -823,15 +828,32 @@ impl StagedSnapshot {
manifest.min_sequence_number = self.sequence_number;
}
}
// Its one data manifest holds all the rows it adds.
if manifest.content == ManifestContentType::Data {
manifest.first_row_id = self.first_row_id;
}
manifest
})
.collect())
}

/// Write the snapshot's manifest list, naming `manifests`: the ones it
/// carries from its parent, then its own.
///
/// On a v3 table, every data manifest it names must have its row ids:
/// its own takes them when it's written ([`Self::write_manifests`]),
/// its parent's had them from their own lists. One that didn't — a
/// table's files from before it was upgraded to v3 — would take ids its
/// snapshot's row range doesn't count: the snapshot has to be committed
/// on its own, its list assigning them.
pub async fn write_manifest_list(mut self, manifests: Vec<ManifestFile>) -> Result<()> {
self.list.add_manifests(manifests.into_iter())?;
if self.list.next_row_id() != self.first_row_id {
return Err(Error::new(
ErrorKind::PreconditionFailed,
"a staged snapshot carries a data manifest without row ids; commit it on its own",
));
}
self.list.close().await.map(drop)
}
}