diff --git a/crates/storage/src/append.rs b/crates/storage/src/append.rs new file mode 100644 index 0000000..4fc2dea --- /dev/null +++ b/crates/storage/src/append.rs @@ -0,0 +1,406 @@ +//! Appending rows in one snapshot, either unconditionally or only while +//! the table head is the one the caller read, with Infostash properties +//! set in the same commit. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; + +use arrow_array::RecordBatch; +use async_trait::async_trait; +use iceberg::table::Table; +use iceberg::transaction::{ApplyTransactionAction, Transaction}; +use iceberg::{ + Catalog, ErrorKind, Namespace, NamespaceIdent, TableCommit, TableCreation, TableIdent, +}; +use iceberg_catalog_sql::SqlCatalog; +use infostash_model::{Error, Result, SnapshotId, TableRef}; + +use crate::write::write_files; +use crate::{Commit, Warehouse, check_properties, ident, moved, storage}; + +impl Warehouse { + /// Write `batches` as new data files and commit them in one snapshot. + pub async fn append(&self, table: &TableRef, batches: &[RecordBatch]) -> Result { + let current = self.load(table).await?; + let (files, rows) = write_files(¤t, table, batches).await?; + let tx = Transaction::new(¤t); + let tx = tx + .fast_append() + .add_data_files(files) + .apply(tx) + .map_err(storage)?; + let committed = tx.commit(&self.catalog).await.map_err(storage)?; + commit_result(table, &committed, rows) + } + + /// Like [`Warehouse::append`], but commits only if the table head is + /// still `expected_head`, and sets `properties` in the same commit, both + /// as table properties and in the new snapshot's summary. + /// + /// Fails with [`Error::Conflict`] if another commit landed after the + /// caller read `expected_head`; nothing is committed. iceberg-rust would + /// otherwise rebase the commit onto the new head and duplicate rows. + /// + /// With no rows, the commit still makes a snapshot carrying + /// `properties`. Every key must start with [`crate::PROPERTY_PREFIX`]. + pub async fn append_if_head( + &self, + table: &TableRef, + batches: &[RecordBatch], + expected_head: Option, + properties: &HashMap, + ) -> Result { + check_properties(table, properties)?; + if properties.is_empty() && batches.iter().all(|b| b.num_rows() == 0) { + return Err(Error::Storage(format!( + "{table}: nothing to commit (no rows and no properties)" + ))); + } + let current = self.load(table).await?; + let found = current.metadata().current_snapshot_id(); + if found != expected_head { + return Err(moved(table, expected_head, found)); + } + let (files, rows) = write_files(¤t, table, batches).await?; + + let tx = Transaction::new(¤t); + let mut tx = tx + .fast_append() + .add_data_files(files) + .set_snapshot_properties(properties.clone()) + .apply(tx) + .map_err(storage)?; + if !properties.is_empty() { + let mut update = tx.update_table_properties(); + for (key, value) in properties { + update = update.set(key.clone(), value.clone()); + } + tx = update.apply(tx).map_err(storage)?; + } + let committed = commit_if_head(&self.catalog, table, expected_head, tx).await?; + commit_result(table, &committed, rows) + } +} + +fn commit_result(table: &TableRef, committed: &Table, rows: u64) -> Result { + let snapshot = committed + .metadata() + .current_snapshot() + .ok_or_else(|| Error::Storage(format!("{table}: commit produced no snapshot")))?; + Ok(Commit { + snapshot: snapshot.snapshot_id(), + metadata_location: committed + .metadata_location_result() + .map_err(storage)? + .into(), + rows, + }) +} + +/// The catalog, except that loading `table` fails unless its head is still +/// `expected`. `Transaction::commit` reloads the table before it applies its +/// actions, and would otherwise rebase a stale commit onto the new head. +/// +/// A commit that lands between this load and the catalog update is caught +/// too: the SQL catalog's compare-and-swap rejects the update with a +/// retryable conflict, and the retry reloads through this guard. +struct HeadGuard<'a> { + inner: &'a SqlCatalog, + table: TableIdent, + expected: Option, + /// Set when the guard refused a load, so the caller can report a + /// conflict rather than a storage error. + tripped: AtomicBool, +} + +impl std::fmt::Debug for HeadGuard<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HeadGuard") + .field("table", &self.table) + .field("expected", &self.expected) + .finish_non_exhaustive() + } +} + +#[async_trait] +impl Catalog for HeadGuard<'_> { + async fn load_table(&self, table: &TableIdent) -> iceberg::Result { + let loaded = self.inner.load_table(table).await?; + let found = loaded.metadata().current_snapshot_id(); + if *table == self.table && found != self.expected { + self.tripped.store(true, Ordering::SeqCst); + return Err(iceberg::Error::new( + ErrorKind::PreconditionFailed, + format!( + "{table}: expected head {:?}, found {found:?}", + self.expected + ), + )); + } + Ok(loaded) + } + + async fn update_table(&self, commit: TableCommit) -> iceberg::Result
{ + self.inner.update_table(commit).await + } + + async fn list_namespaces( + &self, + parent: Option<&NamespaceIdent>, + ) -> iceberg::Result> { + self.inner.list_namespaces(parent).await + } + + async fn create_namespace( + &self, + namespace: &NamespaceIdent, + properties: HashMap, + ) -> iceberg::Result { + self.inner.create_namespace(namespace, properties).await + } + + async fn get_namespace(&self, namespace: &NamespaceIdent) -> iceberg::Result { + self.inner.get_namespace(namespace).await + } + + async fn namespace_exists(&self, namespace: &NamespaceIdent) -> iceberg::Result { + self.inner.namespace_exists(namespace).await + } + + async fn update_namespace( + &self, + namespace: &NamespaceIdent, + properties: HashMap, + ) -> iceberg::Result<()> { + self.inner.update_namespace(namespace, properties).await + } + + async fn drop_namespace(&self, namespace: &NamespaceIdent) -> iceberg::Result<()> { + self.inner.drop_namespace(namespace).await + } + + async fn list_tables(&self, namespace: &NamespaceIdent) -> iceberg::Result> { + self.inner.list_tables(namespace).await + } + + async fn create_table( + &self, + namespace: &NamespaceIdent, + creation: TableCreation, + ) -> iceberg::Result
{ + self.inner.create_table(namespace, creation).await + } + + async fn drop_table(&self, table: &TableIdent) -> iceberg::Result<()> { + self.inner.drop_table(table).await + } + + async fn purge_table(&self, table: &TableIdent) -> iceberg::Result<()> { + self.inner.purge_table(table).await + } + + async fn table_exists(&self, table: &TableIdent) -> iceberg::Result { + self.inner.table_exists(table).await + } + + async fn rename_table(&self, src: &TableIdent, dest: &TableIdent) -> iceberg::Result<()> { + self.inner.rename_table(src, dest).await + } + + async fn register_table( + &self, + table: &TableIdent, + metadata_location: String, + ) -> iceberg::Result
{ + self.inner.register_table(table, metadata_location).await + } +} + +/// Commit `tx`, or fail with [`Error::Conflict`] if the head is not +/// `expected` whenever iceberg loads the table, retries included. +async fn commit_if_head( + catalog: &SqlCatalog, + table: &TableRef, + expected: Option, + tx: Transaction, +) -> Result
{ + let guard = HeadGuard { + inner: catalog, + table: ident(table), + expected, + tripped: AtomicBool::new(false), + }; + match tx.commit(&guard).await { + Ok(committed) => Ok(committed), + Err(_) if guard.tripped.load(Ordering::SeqCst) => { + let found = catalog.load_table(&ident(table)).await.map_err(storage)?; + Err(moved( + table, + expected, + found.metadata().current_snapshot_id(), + )) + } + Err(e) => Err(storage(e)), + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::atomic::{AtomicBool, Ordering}; + + use iceberg::arrow::arrow_schema_to_schema_auto_assign_ids; + use iceberg::transaction::{ApplyTransactionAction, Transaction}; + use iceberg::{Catalog, NamespaceIdent, TableCreation, TableIdent}; + use infostash_model::Error; + + use super::{HeadGuard, commit_if_head}; + use crate::ident; + use crate::test_support::{ids, ids_schema, orders, position}; + use crate::write::write_files; + + #[tokio::test] + async fn rows_and_properties_land_in_one_commit() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let before = wh + .load(&orders) + .await + .unwrap() + .metadata() + .metadata_log() + .len(); + + let commit = wh + .append_if_head(&orders, &[ids(&[1, 2])], None, &position("p1")) + .await + .unwrap(); + + let loaded = wh.load(&orders).await.unwrap(); + let metadata = loaded.metadata(); + // One new metadata file holds the rows and both copies of the property. + assert_eq!(metadata.metadata_log().len(), before + 1); + assert_eq!(metadata.current_snapshot_id(), Some(commit.snapshot)); + assert_eq!(commit.rows, 2); + let summary = &metadata.current_snapshot().unwrap().summary(); + assert_eq!(summary.additional_properties["infostash.position"], "p1"); + let state = wh.state(&orders).await.unwrap(); + assert_eq!(state.head, Some(commit.snapshot)); + assert_eq!(state.properties["infostash.position"], "p1"); + } + + #[tokio::test] + async fn a_moved_head_is_a_conflict_and_commits_nothing() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let first = wh + .append_if_head(&orders, &[ids(&[1])], None, &position("p1")) + .await + .unwrap(); + + // A second writer that also read the empty table. + let stale = wh + .append_if_head(&orders, &[ids(&[1])], None, &position("p1")) + .await; + assert!(matches!(stale, Err(Error::Conflict(_))), "{stale:?}"); + assert_eq!(wh.snapshots(&orders).await.unwrap(), [first.snapshot]); + assert_eq!(wh.read_all(&orders).await.unwrap()[0].num_rows(), 1); + } + + #[tokio::test] + async fn the_guard_stops_a_commit_that_lands_after_the_writer_loaded() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + // The writer loads the empty table, then another commit lands. + let stale = wh.load(&orders).await.unwrap(); + wh.append(&orders, &[ids(&[1])]).await.unwrap(); + + let (files, _) = write_files(&stale, &orders, &[ids(&[2])]).await.unwrap(); + let tx = Transaction::new(&stale); + let tx = tx.fast_append().add_data_files(files).apply(tx).unwrap(); + let lost = commit_if_head(&wh.catalog, &orders, None, tx).await; + assert!(matches!(&lost, Err(Error::Conflict(_))), "{lost:?}"); + // Without the guard, iceberg-rust would have rebased and committed. + assert_eq!(wh.snapshots(&orders).await.unwrap().len(), 1); + } + + #[tokio::test] + async fn the_guard_passes_everything_else_through() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let guard = HeadGuard { + inner: &wh.catalog, + table: ident(&orders), + expected: None, + tripped: AtomicBool::new(false), + }; + assert!(format!("{guard:?}").contains("HeadGuard")); + let raw = ident(&orders).namespace().clone(); + let other = NamespaceIdent::new("other".into()); + assert!(guard.list_namespaces(None).await.unwrap().contains(&raw)); + guard + .create_namespace(&other, HashMap::new()) + .await + .unwrap(); + assert!(guard.namespace_exists(&other).await.unwrap()); + assert_eq!(guard.get_namespace(&other).await.unwrap().name(), &other); + guard + .update_namespace(&other, HashMap::from([("k".into(), "v".into())])) + .await + .unwrap(); + guard.drop_namespace(&other).await.unwrap(); + + assert_eq!(guard.list_tables(&raw).await.unwrap(), [ident(&orders)]); + let schema = arrow_schema_to_schema_auto_assign_ids(&ids_schema()).unwrap(); + let creation = TableCreation::builder() + .name("scratch".into()) + .schema(schema) + .build(); + let created = guard.create_table(&raw, creation).await.unwrap(); + let scratch = created.identifier().clone(); + let moved_to = TableIdent::new(raw.clone(), "moved".into()); + guard.rename_table(&scratch, &moved_to).await.unwrap(); + assert!(guard.table_exists(&moved_to).await.unwrap()); + guard.drop_table(&moved_to).await.unwrap(); + let location = created.metadata_location().unwrap().to_string(); + guard.register_table(&scratch, location).await.unwrap(); + guard.purge_table(&scratch).await.unwrap(); + assert!(!guard.table_exists(&scratch).await.unwrap()); + assert!(!guard.tripped.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn properties_alone_make_a_snapshot_with_no_rows() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let marker = wh + .append_if_head(&orders, &[], None, &position("p1")) + .await + .unwrap(); + assert_eq!(marker.rows, 0); + assert_eq!(wh.snapshots(&orders).await.unwrap(), [marker.snapshot]); + + let nothing = wh + .append_if_head(&orders, &[], Some(marker.snapshot), &HashMap::new()) + .await; + assert!(matches!(nothing, Err(Error::Storage(_))), "{nothing:?}"); + let foreign = HashMap::from([("total-records".to_string(), "9".to_string())]); + let foreign = wh + .append_if_head(&orders, &[ids(&[1])], Some(marker.snapshot), &foreign) + .await; + assert!(matches!(&foreign, Err(Error::Storage(m)) if m.contains("total-records"))); + assert_eq!(wh.snapshots(&orders).await.unwrap(), [marker.snapshot]); + + // Rows need no properties. + let plain = wh + .append_if_head( + &orders, + &[ids(&[1])], + Some(marker.snapshot), + &HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(plain.rows, 1); + } +} diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 5d23d13..4b635b4 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -6,44 +6,32 @@ //! tables. Every table's data and metadata are real Iceberg files on disk: //! [`read_metadata_file`] reads a table back from its metadata file alone. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::path::Path; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; -use async_trait::async_trait; use futures::stream::BoxStream; -use futures::{StreamExt, TryStreamExt}; -use iceberg::arrow::{arrow_schema_to_schema_auto_assign_ids, schema_to_arrow_schema}; -use iceberg::io::{FileIO, LocalFsStorageFactory}; -use iceberg::spec::{ - DataFile, DataFileFormat, ManifestStatus, Operation, TableMetadata, TableProperties, -}; -use iceberg::table::{StaticTable, Table}; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; -use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; -use iceberg::writer::file_writer::ParquetWriterBuilder; -use iceberg::writer::file_writer::location_generator::{ - DefaultFileNameGenerator, DefaultLocationGenerator, -}; -use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; -use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{ - Catalog, CatalogBuilder, ErrorKind, Namespace, NamespaceIdent, TableCommit, TableCreation, - TableIdent, -}; +use iceberg::arrow::arrow_schema_to_schema_auto_assign_ids; +use iceberg::io::LocalFsStorageFactory; +use iceberg::spec::TableProperties; +use iceberg::table::Table; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; use iceberg_catalog_sql::{ SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE, SqlCatalog, SqlCatalogBuilder, }; -use infostash_model::{Error, Layer, Result, SnapshotId, SnapshotRange, TableRef}; +use infostash_model::{Error, Layer, Result, SnapshotId, TableRef}; +mod append; +mod read; mod replace; mod swap; #[cfg(test)] mod test_support; +mod write; +pub use read::read_metadata_file; pub use replace::Replacement; use swap::CatalogSwap; @@ -150,20 +138,6 @@ impl Warehouse { Ok(()) } - /// Write `batches` as new data files and commit them in one snapshot. - pub async fn append(&self, table: &TableRef, batches: &[RecordBatch]) -> Result { - let current = self.load(table).await?; - let (files, rows) = write_files(¤t, table, batches).await?; - let tx = Transaction::new(¤t); - let tx = tx - .fast_append() - .add_data_files(files) - .apply(tx) - .map_err(storage)?; - let committed = tx.commit(&self.catalog).await.map_err(storage)?; - commit_result(table, &committed, rows) - } - /// The table's head and properties, from one load. pub async fn state(&self, table: &TableRef) -> Result { let loaded = self.load(table).await?; @@ -182,115 +156,6 @@ impl Warehouse { .map_err(storage) } - /// Like [`Warehouse::append`], but commits only if the table head is - /// still `expected_head`, and sets `properties` in the same commit, both - /// as table properties and in the new snapshot's summary. - /// - /// Fails with [`Error::Conflict`] if another commit landed after the - /// caller read `expected_head`; nothing is committed. iceberg-rust would - /// otherwise rebase the commit onto the new head and duplicate rows. - /// - /// With no rows, the commit still makes a snapshot carrying - /// `properties`. Every key must start with [`PROPERTY_PREFIX`]. - pub async fn append_if_head( - &self, - table: &TableRef, - batches: &[RecordBatch], - expected_head: Option, - properties: &HashMap, - ) -> Result { - check_properties(table, properties)?; - if properties.is_empty() && batches.iter().all(|b| b.num_rows() == 0) { - return Err(Error::Storage(format!( - "{table}: nothing to commit (no rows and no properties)" - ))); - } - let current = self.load(table).await?; - let found = current.metadata().current_snapshot_id(); - if found != expected_head { - return Err(moved(table, expected_head, found)); - } - let (files, rows) = write_files(¤t, table, batches).await?; - - let tx = Transaction::new(¤t); - let mut tx = tx - .fast_append() - .add_data_files(files) - .set_snapshot_properties(properties.clone()) - .apply(tx) - .map_err(storage)?; - if !properties.is_empty() { - let mut update = tx.update_table_properties(); - for (key, value) in properties { - update = update.set(key.clone(), value.clone()); - } - tx = update.apply(tx).map_err(storage)?; - } - let committed = commit_if_head(&self.catalog, table, expected_head, tx).await?; - commit_result(table, &committed, rows) - } - - /// Every row in the table's current snapshot. - pub async fn read_all(&self, table: &TableRef) -> Result> { - scan_at(&self.load(table).await?, None, None) - .await? - .try_collect() - .await - } - - /// Rows committed after `range.from` up to and including `range.to`, as - /// a stream. `from == None` reads the whole table as of `to`. This is - /// what incremental runs consume. - /// - /// Only appends can be read this way. It is an error if a snapshot in - /// the range replaced or deleted files (a merge or a compaction), because - /// its added files would hold rows that are not new. It is also an error - /// if `from` is not an ancestor of `to`, rather than reading everything. - pub async fn scan_range( - &self, - table: &TableRef, - range: SnapshotRange, - ) -> Result { - let loaded = self.load(table).await?; - let metadata = loaded.metadata(); - let only = match range.from { - None => { - if metadata.snapshot_by_id(range.to).is_none() { - return Err(Error::NotFound(format!("{table}: snapshot {}", range.to))); - } - None - } - Some(from) => { - let snapshots = appends_between(metadata, table, from, range.to)?; - if snapshots.is_empty() { - return Ok(futures::stream::empty().boxed()); - } - Some(files_added_by(&loaded, range.to, &snapshots).await?) - } - }; - scan_at(&loaded, Some(range.to), only).await - } - - /// Rows committed after snapshot `after`, up to the current snapshot. - /// `None` reads everything. See [`Warehouse::scan_range`] for the errors. - pub async fn read_since( - &self, - table: &TableRef, - after: Option, - ) -> Result> { - let current = self.load(table).await?.metadata().current_snapshot_id(); - let range = match (current, after) { - (Some(to), from) => SnapshotRange { from, to }, - (None, None) => return Ok(Vec::new()), - (None, Some(from)) => { - return Err(Error::NotFound(format!( - "{table}: snapshot {from} (the table has no snapshots)" - ))); - } - }; - self.scan_range(table, range).await?.try_collect().await - } - /// The table's snapshot ids, oldest first. pub async fn snapshots(&self, table: &TableRef) -> Result> { let table = self.load(table).await?; @@ -329,17 +194,6 @@ impl Warehouse { } } -/// Read a table straight from an Iceberg metadata file, with no catalog. -/// Proves the data is durable on disk, and lets other engines read our output. -pub async fn read_metadata_file(table: &TableRef, location: &str) -> Result> { - let file_io = FileIO::new_with_fs(); - let table = StaticTable::from_metadata_file(location, ident(table), file_io) - .await - .map_err(storage)? - .into_table(); - scan_at(&table, None, None).await?.try_collect().await -} - fn ident(table: &TableRef) -> TableIdent { TableIdent::new( NamespaceIdent::new(table.layer.namespace().to_string()), @@ -369,607 +223,3 @@ fn moved(table: &TableRef, expected: Option, found: Option Result<(Vec, u64)> { - let metadata = current.metadata(); - let schema = metadata.current_schema().clone(); - // Batches must carry Iceberg field ids so columns map by id, not name. - let arrow_schema = Arc::new(schema_to_arrow_schema(&schema).map_err(storage)?); - - let files = RollingFileWriterBuilder::new_with_default_file_size( - // Codec, row group and page sizes come from the table's - // `write.parquet.*` properties. - ParquetWriterBuilder::from_table_properties(&metadata.table_properties(), schema) - .map_err(storage)?, - current.file_io().clone(), - DefaultLocationGenerator::new(metadata).map_err(storage)?, - // A fresh prefix per write: the generator only counts up from 0. - DefaultFileNameGenerator::new( - uuid::Uuid::new_v4().to_string(), - None, - DataFileFormat::Parquet, - ), - ); - let mut writer = DataFileWriterBuilder::new(files) - .build(None) - .await - .map_err(storage)?; - let mut rows = 0; - for batch in batches { - let names = |s: &ArrowSchema| { - s.fields() - .iter() - .map(|f| f.name().clone()) - .collect::>() - }; - if names(&batch.schema()) != names(&arrow_schema) { - return Err(Error::Storage(format!( - "batch does not match {table}: columns {:?}, expected {:?}", - names(&batch.schema()), - names(&arrow_schema) - ))); - } - let batch = RecordBatch::try_new(arrow_schema.clone(), batch.columns().to_vec()) - .map_err(|e| Error::Storage(format!("batch does not match {table}: {e}")))?; - rows += batch.num_rows() as u64; - writer.write(batch).await.map_err(storage)?; - } - let files = writer.close().await.map_err(storage)?; - Ok((files, rows)) -} - -fn commit_result(table: &TableRef, committed: &Table, rows: u64) -> Result { - let snapshot = committed - .metadata() - .current_snapshot() - .ok_or_else(|| Error::Storage(format!("{table}: commit produced no snapshot")))?; - Ok(Commit { - snapshot: snapshot.snapshot_id(), - metadata_location: committed - .metadata_location_result() - .map_err(storage)? - .into(), - rows, - }) -} - -/// The catalog, except that loading `table` fails unless its head is still -/// `expected`. `Transaction::commit` reloads the table before it applies its -/// actions, and would otherwise rebase a stale commit onto the new head. -/// -/// A commit that lands between this load and the catalog update is caught -/// too: the SQL catalog's compare-and-swap rejects the update with a -/// retryable conflict, and the retry reloads through this guard. -struct HeadGuard<'a> { - inner: &'a SqlCatalog, - table: TableIdent, - expected: Option, - /// Set when the guard refused a load, so the caller can report a - /// conflict rather than a storage error. - tripped: AtomicBool, -} - -impl std::fmt::Debug for HeadGuard<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("HeadGuard") - .field("table", &self.table) - .field("expected", &self.expected) - .finish_non_exhaustive() - } -} - -#[async_trait] -impl Catalog for HeadGuard<'_> { - async fn load_table(&self, table: &TableIdent) -> iceberg::Result
{ - let loaded = self.inner.load_table(table).await?; - let found = loaded.metadata().current_snapshot_id(); - if *table == self.table && found != self.expected { - self.tripped.store(true, Ordering::SeqCst); - return Err(iceberg::Error::new( - ErrorKind::PreconditionFailed, - format!( - "{table}: expected head {:?}, found {found:?}", - self.expected - ), - )); - } - Ok(loaded) - } - - async fn update_table(&self, commit: TableCommit) -> iceberg::Result
{ - self.inner.update_table(commit).await - } - - async fn list_namespaces( - &self, - parent: Option<&NamespaceIdent>, - ) -> iceberg::Result> { - self.inner.list_namespaces(parent).await - } - - async fn create_namespace( - &self, - namespace: &NamespaceIdent, - properties: HashMap, - ) -> iceberg::Result { - self.inner.create_namespace(namespace, properties).await - } - - async fn get_namespace(&self, namespace: &NamespaceIdent) -> iceberg::Result { - self.inner.get_namespace(namespace).await - } - - async fn namespace_exists(&self, namespace: &NamespaceIdent) -> iceberg::Result { - self.inner.namespace_exists(namespace).await - } - - async fn update_namespace( - &self, - namespace: &NamespaceIdent, - properties: HashMap, - ) -> iceberg::Result<()> { - self.inner.update_namespace(namespace, properties).await - } - - async fn drop_namespace(&self, namespace: &NamespaceIdent) -> iceberg::Result<()> { - self.inner.drop_namespace(namespace).await - } - - async fn list_tables(&self, namespace: &NamespaceIdent) -> iceberg::Result> { - self.inner.list_tables(namespace).await - } - - async fn create_table( - &self, - namespace: &NamespaceIdent, - creation: TableCreation, - ) -> iceberg::Result
{ - self.inner.create_table(namespace, creation).await - } - - async fn drop_table(&self, table: &TableIdent) -> iceberg::Result<()> { - self.inner.drop_table(table).await - } - - async fn purge_table(&self, table: &TableIdent) -> iceberg::Result<()> { - self.inner.purge_table(table).await - } - - async fn table_exists(&self, table: &TableIdent) -> iceberg::Result { - self.inner.table_exists(table).await - } - - async fn rename_table(&self, src: &TableIdent, dest: &TableIdent) -> iceberg::Result<()> { - self.inner.rename_table(src, dest).await - } - - async fn register_table( - &self, - table: &TableIdent, - metadata_location: String, - ) -> iceberg::Result
{ - self.inner.register_table(table, metadata_location).await - } -} - -/// Commit `tx`, or fail with [`Error::Conflict`] if the head is not -/// `expected` whenever iceberg loads the table, retries included. -async fn commit_if_head( - catalog: &SqlCatalog, - table: &TableRef, - expected: Option, - tx: Transaction, -) -> Result
{ - let guard = HeadGuard { - inner: catalog, - table: ident(table), - expected, - tripped: AtomicBool::new(false), - }; - match tx.commit(&guard).await { - Ok(committed) => Ok(committed), - Err(_) if guard.tripped.load(Ordering::SeqCst) => { - let found = catalog.load_table(&ident(table)).await.map_err(storage)?; - Err(moved( - table, - expected, - found.metadata().current_snapshot_id(), - )) - } - Err(e) => Err(storage(e)), - } -} - -/// The snapshots after `from` up to and including `to`, walking back from -/// `to`. Fails if one of them is not an append, or if the walk never reaches -/// `from`. -fn appends_between( - metadata: &TableMetadata, - table: &TableRef, - from: SnapshotId, - to: SnapshotId, -) -> Result> { - let mut snapshots = HashSet::new(); - let mut cursor = Some(to); - while let Some(id) = cursor { - // Look the snapshot up first, so a made-up `from == to` is not found. - let snapshot = metadata - .snapshot_by_id(id) - .ok_or_else(|| Error::NotFound(format!("{table}: snapshot {id}")))?; - if id == from { - return Ok(snapshots); - } - let operation = &snapshot.summary().operation; - if *operation != Operation::Append { - return Err(Error::Storage(format!( - "{table}: snapshot {id} is a `{}`, not an append, so it can't be read \ - incrementally", - operation.as_str() - ))); - } - snapshots.insert(id); - cursor = snapshot.parent_snapshot_id(); - } - Err(Error::NotFound(format!( - "{table}: snapshot {from} is not an ancestor of snapshot {to}" - ))) -} - -/// Paths of the data files that `snapshots` added, as listed by snapshot `to`. -async fn files_added_by( - table: &Table, - to: SnapshotId, - snapshots: &HashSet, -) -> Result> { - let metadata = table.metadata(); - let to = metadata - .snapshot_by_id(to) - .ok_or_else(|| Error::NotFound(format!("snapshot {to}")))?; - let list = table - .manifest_list_reader(to) - .load() - .await - .map_err(storage)?; - let mut files = HashSet::new(); - for manifest in list.entries() { - if !snapshots.contains(&manifest.added_snapshot_id) { - continue; - } - let manifest = table - .manifest_reader() - .read(manifest) - .await - .map_err(storage)?; - for entry in manifest.entries() { - let added_here = entry.snapshot_id().is_some_and(|s| snapshots.contains(&s)); - if added_here && entry.status() == ManifestStatus::Added { - files.insert(entry.file_path().to_string()); - } - } - } - Ok(files) -} - -/// Scan the table as of snapshot `at` (the current one if `None`), -/// optionally limited to some data files. -async fn scan_at( - table: &Table, - at: Option, - only: Option>, -) -> Result { - let mut scan = table.scan().select_all(); - if let Some(at) = at { - scan = scan.snapshot_id(at); - } - let tasks = scan - .build() - .map_err(storage)? - .plan_files() - .await - .map_err(storage)?; - let tasks = match only { - None => tasks, - Some(only) => tasks - .try_filter(move |t| std::future::ready(only.contains(t.data_file_path()))) - .boxed(), - }; - Ok(table - .reader_builder() - .build() - .read(tasks) - .map_err(storage)? - .stream() - .map_err(storage) - .boxed()) -} - -#[cfg(test)] -mod tests { - #![allow(clippy::unwrap_used)] - - use std::collections::HashMap; - use std::sync::Arc; - - use arrow_array::Int64Array; - use arrow_schema::{DataType, Field}; - - use super::*; - use crate::test_support::{ - all_ids, files_of, ids, ids_schema, job, orders, position, two_appends, - }; - - #[tokio::test] - async fn an_overwrite_in_the_range_is_refused() { - let dir = tempfile::tempdir().unwrap(); - let (wh, orders) = orders(dir.path()).await; - let (first, second) = two_appends(&wh, &orders).await; - let replacement = Replacement { - remove: files_of(&wh, &orders, &second).await, - batches: vec![ids(&[30])], - ..Replacement::default() - }; - let replaced = wh - .replace_if_head(&orders, second.snapshot, replacement, &job("j")) - .await - .unwrap(); - - let metadata = wh.load(&orders).await.unwrap().metadata().clone(); - let refused = |r: &Result<()>| matches!(r, Err(Error::Storage(m)) if m.contains("`overwrite`, not an append")); - let err = - appends_between(&metadata, &orders, second.snapshot, replaced.snapshot).map(|_| ()); - assert!(refused(&err), "{err:?}"); - let since_first = wh - .read_since(&orders, Some(first.snapshot)) - .await - .map(|_| ()); - assert!(refused(&since_first), "{since_first:?}"); - let across = SnapshotRange { - from: Some(second.snapshot), - to: replaced.snapshot, - }; - let across = wh.scan_range(&orders, across).await.map(|_| ()); - assert!(refused(&across), "{across:?}"); - - // An overwrite before the range doesn't matter, and reading - // everything still works. - let from_overwrite = - appends_between(&metadata, &orders, replaced.snapshot, replaced.snapshot); - assert!(from_overwrite.unwrap().is_empty()); - let since = wh.read_since(&orders, Some(replaced.snapshot)).await; - assert!(since.unwrap().is_empty()); - let everything = wh.read_since(&orders, None).await.unwrap(); - assert_eq!(all_ids(&everything), [1, 2, 30]); - - // iceberg-rust's own append builds on the hand-made snapshot, and - // reads incrementally from it. - let after = Some(replaced.snapshot); - wh.append_if_head(&orders, &[ids(&[6])], after, &position("p3")) - .await - .unwrap(); - let since = wh.read_since(&orders, after).await.unwrap(); - assert_eq!(all_ids(&since), [6]); - assert_eq!(all_ids(&wh.read_all(&orders).await.unwrap()), [1, 2, 6, 30]); - } - - /// The codec of every column chunk in the Parquet files under `dir`. - fn codecs(dir: &Path) -> HashSet { - use parquet::file::reader::{FileReader, SerializedFileReader}; - let mut found = HashSet::new(); - for entry in std::fs::read_dir(dir).unwrap() { - let path = entry.unwrap().path(); - if path.is_dir() { - found.extend(codecs(&path)); - } else if path.extension().is_some_and(|e| e == "parquet") { - let reader = SerializedFileReader::new(std::fs::File::open(&path).unwrap()); - for group in reader.unwrap().metadata().row_groups() { - for column in group.columns() { - let codec = format!("{:?}", column.compression()); - found.insert(codec.split('(').next().unwrap().to_string()); - } - } - } - } - found - } - - #[tokio::test] - async fn new_tables_write_zstd() { - let dir = tempfile::tempdir().unwrap(); - let (wh, orders) = orders(dir.path()).await; - let codec = TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC; - let state = wh.state(&orders).await.unwrap(); - assert_eq!(state.properties.get(codec).unwrap(), "zstd"); - - wh.append(&orders, &[ids(&[1, 2, 3])]).await.unwrap(); - assert_eq!(codecs(dir.path()), HashSet::from(["ZSTD".to_string()])); - } - - #[tokio::test] - async fn the_table_codec_is_honoured() { - let dir = tempfile::tempdir().unwrap(); - let (wh, orders) = orders(dir.path()).await; - let table = wh.load(&orders).await.unwrap(); - let tx = Transaction::new(&table); - let tx = tx - .update_table_properties() - .set( - TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), - "snappy".to_string(), - ) - .apply(tx) - .unwrap(); - tx.commit(&wh.catalog).await.unwrap(); - - wh.append(&orders, &[ids(&[1, 2, 3])]).await.unwrap(); - assert_eq!(codecs(dir.path()), HashSet::from(["SNAPPY".to_string()])); - } - - #[tokio::test] - async fn rows_and_properties_land_in_one_commit() { - let dir = tempfile::tempdir().unwrap(); - let (wh, orders) = orders(dir.path()).await; - let before = wh - .load(&orders) - .await - .unwrap() - .metadata() - .metadata_log() - .len(); - - let commit = wh - .append_if_head(&orders, &[ids(&[1, 2])], None, &position("p1")) - .await - .unwrap(); - - let loaded = wh.load(&orders).await.unwrap(); - let metadata = loaded.metadata(); - // One new metadata file holds the rows and both copies of the property. - assert_eq!(metadata.metadata_log().len(), before + 1); - assert_eq!(metadata.current_snapshot_id(), Some(commit.snapshot)); - assert_eq!(commit.rows, 2); - let summary = &metadata.current_snapshot().unwrap().summary(); - assert_eq!(summary.additional_properties["infostash.position"], "p1"); - let state = wh.state(&orders).await.unwrap(); - assert_eq!(state.head, Some(commit.snapshot)); - assert_eq!(state.properties["infostash.position"], "p1"); - } - - #[tokio::test] - async fn a_moved_head_is_a_conflict_and_commits_nothing() { - let dir = tempfile::tempdir().unwrap(); - let (wh, orders) = orders(dir.path()).await; - let first = wh - .append_if_head(&orders, &[ids(&[1])], None, &position("p1")) - .await - .unwrap(); - - // A second writer that also read the empty table. - let stale = wh - .append_if_head(&orders, &[ids(&[1])], None, &position("p1")) - .await; - assert!(matches!(stale, Err(Error::Conflict(_))), "{stale:?}"); - assert_eq!(wh.snapshots(&orders).await.unwrap(), [first.snapshot]); - assert_eq!(wh.read_all(&orders).await.unwrap()[0].num_rows(), 1); - } - - #[tokio::test] - async fn the_guard_stops_a_commit_that_lands_after_the_writer_loaded() { - let dir = tempfile::tempdir().unwrap(); - let (wh, orders) = orders(dir.path()).await; - // The writer loads the empty table, then another commit lands. - let stale = wh.load(&orders).await.unwrap(); - wh.append(&orders, &[ids(&[1])]).await.unwrap(); - - let (files, _) = write_files(&stale, &orders, &[ids(&[2])]).await.unwrap(); - let tx = Transaction::new(&stale); - let tx = tx.fast_append().add_data_files(files).apply(tx).unwrap(); - let lost = commit_if_head(&wh.catalog, &orders, None, tx).await; - assert!(matches!(&lost, Err(Error::Conflict(_))), "{lost:?}"); - // Without the guard, iceberg-rust would have rebased and committed. - assert_eq!(wh.snapshots(&orders).await.unwrap().len(), 1); - } - - #[tokio::test] - async fn the_guard_passes_everything_else_through() { - let dir = tempfile::tempdir().unwrap(); - let (wh, orders) = orders(dir.path()).await; - let guard = HeadGuard { - inner: &wh.catalog, - table: ident(&orders), - expected: None, - tripped: AtomicBool::new(false), - }; - assert!(format!("{guard:?}").contains("HeadGuard")); - let raw = ident(&orders).namespace().clone(); - let other = NamespaceIdent::new("other".into()); - assert!(guard.list_namespaces(None).await.unwrap().contains(&raw)); - guard - .create_namespace(&other, HashMap::new()) - .await - .unwrap(); - assert!(guard.namespace_exists(&other).await.unwrap()); - assert_eq!(guard.get_namespace(&other).await.unwrap().name(), &other); - guard - .update_namespace(&other, HashMap::from([("k".into(), "v".into())])) - .await - .unwrap(); - guard.drop_namespace(&other).await.unwrap(); - - assert_eq!(guard.list_tables(&raw).await.unwrap(), [ident(&orders)]); - let schema = arrow_schema_to_schema_auto_assign_ids(&ids_schema()).unwrap(); - let creation = TableCreation::builder() - .name("scratch".into()) - .schema(schema) - .build(); - let created = guard.create_table(&raw, creation).await.unwrap(); - let scratch = created.identifier().clone(); - let moved_to = TableIdent::new(raw.clone(), "moved".into()); - guard.rename_table(&scratch, &moved_to).await.unwrap(); - assert!(guard.table_exists(&moved_to).await.unwrap()); - guard.drop_table(&moved_to).await.unwrap(); - let location = created.metadata_location().unwrap().to_string(); - guard.register_table(&scratch, location).await.unwrap(); - guard.purge_table(&scratch).await.unwrap(); - assert!(!guard.table_exists(&scratch).await.unwrap()); - assert!(!guard.tripped.load(Ordering::SeqCst)); - } - - #[tokio::test] - async fn properties_alone_make_a_snapshot_with_no_rows() { - let dir = tempfile::tempdir().unwrap(); - let (wh, orders) = orders(dir.path()).await; - let marker = wh - .append_if_head(&orders, &[], None, &position("p1")) - .await - .unwrap(); - assert_eq!(marker.rows, 0); - assert_eq!(wh.snapshots(&orders).await.unwrap(), [marker.snapshot]); - - let nothing = wh - .append_if_head(&orders, &[], Some(marker.snapshot), &HashMap::new()) - .await; - assert!(matches!(nothing, Err(Error::Storage(_))), "{nothing:?}"); - let foreign = HashMap::from([("total-records".to_string(), "9".to_string())]); - let foreign = wh - .append_if_head(&orders, &[ids(&[1])], Some(marker.snapshot), &foreign) - .await; - assert!(matches!(&foreign, Err(Error::Storage(m)) if m.contains("total-records"))); - assert_eq!(wh.snapshots(&orders).await.unwrap(), [marker.snapshot]); - - // Rows need no properties. - let plain = wh - .append_if_head( - &orders, - &[ids(&[1])], - Some(marker.snapshot), - &HashMap::new(), - ) - .await - .unwrap(); - assert_eq!(plain.rows, 1); - } - - #[tokio::test] - async fn a_batch_with_other_column_names_is_refused() { - let dir = tempfile::tempdir().unwrap(); - let (wh, orders) = orders(dir.path()).await; - let schema = Arc::new(ArrowSchema::new(vec![Field::new( - "amount", - DataType::Int64, - false, - )])); - let batch = - RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![1]))]).unwrap(); - let err = wh.append(&orders, &[batch]).await; - assert!( - matches!(&err, Err(Error::Storage(m)) if m.contains("amount")), - "{err:?}" - ); - } -} diff --git a/crates/storage/src/read.rs b/crates/storage/src/read.rs new file mode 100644 index 0000000..caf5ffd --- /dev/null +++ b/crates/storage/src/read.rs @@ -0,0 +1,253 @@ +//! Reading a table: everything in a snapshot, or only the rows that +//! appends added after an earlier snapshot, which is what incremental +//! runs consume. + +use std::collections::HashSet; + +use arrow_array::RecordBatch; +use futures::{StreamExt, TryStreamExt}; +use iceberg::io::FileIO; +use iceberg::spec::{ManifestStatus, Operation, TableMetadata}; +use iceberg::table::{StaticTable, Table}; +use infostash_model::{Error, Result, SnapshotId, SnapshotRange, TableRef}; + +use crate::{RecordBatchStream, Warehouse, ident, storage}; + +impl Warehouse { + /// Every row in the table's current snapshot. + pub async fn read_all(&self, table: &TableRef) -> Result> { + scan_at(&self.load(table).await?, None, None) + .await? + .try_collect() + .await + } + + /// Rows committed after `range.from` up to and including `range.to`, as + /// a stream. `from == None` reads the whole table as of `to`. This is + /// what incremental runs consume. + /// + /// Only appends can be read this way. It is an error if a snapshot in + /// the range replaced or deleted files (a merge or a compaction), because + /// its added files would hold rows that are not new. It is also an error + /// if `from` is not an ancestor of `to`, rather than reading everything. + pub async fn scan_range( + &self, + table: &TableRef, + range: SnapshotRange, + ) -> Result { + let loaded = self.load(table).await?; + let metadata = loaded.metadata(); + let only = match range.from { + None => { + if metadata.snapshot_by_id(range.to).is_none() { + return Err(Error::NotFound(format!("{table}: snapshot {}", range.to))); + } + None + } + Some(from) => { + let snapshots = appends_between(metadata, table, from, range.to)?; + if snapshots.is_empty() { + return Ok(futures::stream::empty().boxed()); + } + Some(files_added_by(&loaded, range.to, &snapshots).await?) + } + }; + scan_at(&loaded, Some(range.to), only).await + } + + /// Rows committed after snapshot `after`, up to the current snapshot. + /// `None` reads everything. See [`Warehouse::scan_range`] for the errors. + pub async fn read_since( + &self, + table: &TableRef, + after: Option, + ) -> Result> { + let current = self.load(table).await?.metadata().current_snapshot_id(); + let range = match (current, after) { + (Some(to), from) => SnapshotRange { from, to }, + (None, None) => return Ok(Vec::new()), + (None, Some(from)) => { + return Err(Error::NotFound(format!( + "{table}: snapshot {from} (the table has no snapshots)" + ))); + } + }; + self.scan_range(table, range).await?.try_collect().await + } +} + +/// Read a table straight from an Iceberg metadata file, with no catalog. +/// Proves the data is durable on disk, and lets other engines read our output. +pub async fn read_metadata_file(table: &TableRef, location: &str) -> Result> { + let file_io = FileIO::new_with_fs(); + let table = StaticTable::from_metadata_file(location, ident(table), file_io) + .await + .map_err(storage)? + .into_table(); + scan_at(&table, None, None).await?.try_collect().await +} + +/// The snapshots after `from` up to and including `to`, walking back from +/// `to`. Fails if one of them is not an append, or if the walk never reaches +/// `from`. +fn appends_between( + metadata: &TableMetadata, + table: &TableRef, + from: SnapshotId, + to: SnapshotId, +) -> Result> { + let mut snapshots = HashSet::new(); + let mut cursor = Some(to); + while let Some(id) = cursor { + // Look the snapshot up first, so a made-up `from == to` is not found. + let snapshot = metadata + .snapshot_by_id(id) + .ok_or_else(|| Error::NotFound(format!("{table}: snapshot {id}")))?; + if id == from { + return Ok(snapshots); + } + let operation = &snapshot.summary().operation; + if *operation != Operation::Append { + return Err(Error::Storage(format!( + "{table}: snapshot {id} is a `{}`, not an append, so it can't be read \ + incrementally", + operation.as_str() + ))); + } + snapshots.insert(id); + cursor = snapshot.parent_snapshot_id(); + } + Err(Error::NotFound(format!( + "{table}: snapshot {from} is not an ancestor of snapshot {to}" + ))) +} + +/// Paths of the data files that `snapshots` added, as listed by snapshot `to`. +pub(crate) async fn files_added_by( + table: &Table, + to: SnapshotId, + snapshots: &HashSet, +) -> Result> { + let metadata = table.metadata(); + let to = metadata + .snapshot_by_id(to) + .ok_or_else(|| Error::NotFound(format!("snapshot {to}")))?; + let list = table + .manifest_list_reader(to) + .load() + .await + .map_err(storage)?; + let mut files = HashSet::new(); + for manifest in list.entries() { + if !snapshots.contains(&manifest.added_snapshot_id) { + continue; + } + let manifest = table + .manifest_reader() + .read(manifest) + .await + .map_err(storage)?; + for entry in manifest.entries() { + let added_here = entry.snapshot_id().is_some_and(|s| snapshots.contains(&s)); + if added_here && entry.status() == ManifestStatus::Added { + files.insert(entry.file_path().to_string()); + } + } + } + Ok(files) +} + +/// Scan the table as of snapshot `at` (the current one if `None`), +/// optionally limited to some data files. +pub(crate) async fn scan_at( + table: &Table, + at: Option, + only: Option>, +) -> Result { + let mut scan = table.scan().select_all(); + if let Some(at) = at { + scan = scan.snapshot_id(at); + } + let tasks = scan + .build() + .map_err(storage)? + .plan_files() + .await + .map_err(storage)?; + let tasks = match only { + None => tasks, + Some(only) => tasks + .try_filter(move |t| std::future::ready(only.contains(t.data_file_path()))) + .boxed(), + }; + Ok(table + .reader_builder() + .build() + .read(tasks) + .map_err(storage)? + .stream() + .map_err(storage) + .boxed()) +} + +#[cfg(test)] +mod tests { + use infostash_model::{Error, Result, SnapshotRange}; + + use super::appends_between; + use crate::Replacement; + use crate::test_support::{all_ids, files_of, ids, job, orders, position, two_appends}; + + #[tokio::test] + async fn an_overwrite_in_the_range_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (first, second) = two_appends(&wh, &orders).await; + let replacement = Replacement { + remove: files_of(&wh, &orders, &second).await, + batches: vec![ids(&[30])], + ..Replacement::default() + }; + let replaced = wh + .replace_if_head(&orders, second.snapshot, replacement, &job("j")) + .await + .unwrap(); + + let metadata = wh.load(&orders).await.unwrap().metadata().clone(); + let refused = |r: &Result<()>| matches!(r, Err(Error::Storage(m)) if m.contains("`overwrite`, not an append")); + let err = + appends_between(&metadata, &orders, second.snapshot, replaced.snapshot).map(|_| ()); + assert!(refused(&err), "{err:?}"); + let since_first = wh + .read_since(&orders, Some(first.snapshot)) + .await + .map(|_| ()); + assert!(refused(&since_first), "{since_first:?}"); + let across = SnapshotRange { + from: Some(second.snapshot), + to: replaced.snapshot, + }; + let across = wh.scan_range(&orders, across).await.map(|_| ()); + assert!(refused(&across), "{across:?}"); + + // An overwrite before the range doesn't matter, and reading + // everything still works. + let from_overwrite = + appends_between(&metadata, &orders, replaced.snapshot, replaced.snapshot); + assert!(from_overwrite.unwrap().is_empty()); + let since = wh.read_since(&orders, Some(replaced.snapshot)).await; + assert!(since.unwrap().is_empty()); + let everything = wh.read_since(&orders, None).await.unwrap(); + assert_eq!(all_ids(&everything), [1, 2, 30]); + + // iceberg-rust's own append builds on the hand-made snapshot, and + // reads incrementally from it. + let after = Some(replaced.snapshot); + wh.append_if_head(&orders, &[ids(&[6])], after, &position("p3")) + .await + .unwrap(); + let since = wh.read_since(&orders, after).await.unwrap(); + assert_eq!(all_ids(&since), [6]); + assert_eq!(all_ids(&wh.read_all(&orders).await.unwrap()), [1, 2, 6, 30]); + } +} diff --git a/crates/storage/src/replace.rs b/crates/storage/src/replace.rs index 1a61b4d..d467224 100644 --- a/crates/storage/src/replace.rs +++ b/crates/storage/src/replace.rs @@ -22,7 +22,8 @@ use iceberg::table::Table; use iceberg::{MetadataLocation, TableRequirement}; use infostash_model::{Error, Result, SnapshotId, TableRef}; -use crate::{Commit, Warehouse, check_properties, ident, moved, storage, write_files}; +use crate::write::write_files; +use crate::{Commit, Warehouse, check_properties, ident, moved, storage}; /// What a replace takes out of a table and puts in. #[derive(Debug, Clone, Default)] @@ -548,12 +549,14 @@ mod tests { use infostash_model::{Error, TableRef}; use super::{PreparedReplace, Replacement}; + use crate::read::scan_at; use crate::swap::CatalogSwap; use crate::test_support::{ all_ids, entries_of, files_of, ids, ids_schema, job, live_files, local_path, manifests_of, orders, position, referenced_files, two_appends, }; - use crate::{Warehouse, ident, read_metadata_file, scan_at, write_files}; + use crate::write::write_files; + use crate::{Warehouse, ident, read_metadata_file}; fn swap_in(remove: Vec, values: &[i64]) -> Replacement { Replacement { diff --git a/crates/storage/src/test_support.rs b/crates/storage/src/test_support.rs index 1d14212..7f9161f 100644 --- a/crates/storage/src/test_support.rs +++ b/crates/storage/src/test_support.rs @@ -11,7 +11,8 @@ use futures::TryStreamExt; use iceberg::spec::ManifestStatus; use infostash_model::{Layer, SnapshotId, TableRef}; -use crate::{Commit, Warehouse, files_added_by}; +use crate::read::files_added_by; +use crate::{Commit, Warehouse}; pub(crate) fn ids_schema() -> Arc { Arc::new(ArrowSchema::new(vec![Field::new( diff --git a/crates/storage/src/write.rs b/crates/storage/src/write.rs new file mode 100644 index 0000000..66f067a --- /dev/null +++ b/crates/storage/src/write.rs @@ -0,0 +1,162 @@ +//! Writing record batches as Parquet data files for a table, with the +//! codec and sizes its `write.parquet.*` properties set. Nothing here +//! commits: [`crate::append`] and [`crate::replace`] add the files to a +//! snapshot. + +use std::sync::Arc; + +use arrow_array::RecordBatch; +use arrow_schema::Schema as ArrowSchema; +use iceberg::arrow::schema_to_arrow_schema; +use iceberg::spec::{DataFile, DataFileFormat}; +use iceberg::table::Table; +use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; +use iceberg::writer::file_writer::ParquetWriterBuilder; +use iceberg::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, +}; +use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; +use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; +use infostash_model::{Error, Result, TableRef}; + +use crate::storage; + +/// Write `batches` as new Parquet data files for `table`. Returns the files +/// and the number of rows in them. Nothing is committed. +pub(crate) async fn write_files( + current: &Table, + table: &TableRef, + batches: &[RecordBatch], +) -> Result<(Vec, u64)> { + let metadata = current.metadata(); + let schema = metadata.current_schema().clone(); + // Batches must carry Iceberg field ids so columns map by id, not name. + let arrow_schema = Arc::new(schema_to_arrow_schema(&schema).map_err(storage)?); + + let files = RollingFileWriterBuilder::new_with_default_file_size( + // Codec, row group and page sizes come from the table's + // `write.parquet.*` properties. + ParquetWriterBuilder::from_table_properties(&metadata.table_properties(), schema) + .map_err(storage)?, + current.file_io().clone(), + DefaultLocationGenerator::new(metadata).map_err(storage)?, + // A fresh prefix per write: the generator only counts up from 0. + DefaultFileNameGenerator::new( + uuid::Uuid::new_v4().to_string(), + None, + DataFileFormat::Parquet, + ), + ); + let mut writer = DataFileWriterBuilder::new(files) + .build(None) + .await + .map_err(storage)?; + let mut rows = 0; + for batch in batches { + let names = |s: &ArrowSchema| { + s.fields() + .iter() + .map(|f| f.name().clone()) + .collect::>() + }; + if names(&batch.schema()) != names(&arrow_schema) { + return Err(Error::Storage(format!( + "batch does not match {table}: columns {:?}, expected {:?}", + names(&batch.schema()), + names(&arrow_schema) + ))); + } + let batch = RecordBatch::try_new(arrow_schema.clone(), batch.columns().to_vec()) + .map_err(|e| Error::Storage(format!("batch does not match {table}: {e}")))?; + rows += batch.num_rows() as u64; + writer.write(batch).await.map_err(storage)?; + } + let files = writer.close().await.map_err(storage)?; + Ok((files, rows)) +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::path::Path; + use std::sync::Arc; + + use arrow_array::{Int64Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use iceberg::spec::TableProperties; + use iceberg::transaction::{ApplyTransactionAction, Transaction}; + use infostash_model::Error; + + use crate::test_support::{ids, orders}; + + /// The codec of every column chunk in the Parquet files under `dir`. + fn codecs(dir: &Path) -> HashSet { + use parquet::file::reader::{FileReader, SerializedFileReader}; + let mut found = HashSet::new(); + for entry in std::fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + found.extend(codecs(&path)); + } else if path.extension().is_some_and(|e| e == "parquet") { + let reader = SerializedFileReader::new(std::fs::File::open(&path).unwrap()); + for group in reader.unwrap().metadata().row_groups() { + for column in group.columns() { + let codec = format!("{:?}", column.compression()); + found.insert(codec.split('(').next().unwrap().to_string()); + } + } + } + } + found + } + + #[tokio::test] + async fn new_tables_write_zstd() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let codec = TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC; + let state = wh.state(&orders).await.unwrap(); + assert_eq!(state.properties.get(codec).unwrap(), "zstd"); + + wh.append(&orders, &[ids(&[1, 2, 3])]).await.unwrap(); + assert_eq!(codecs(dir.path()), HashSet::from(["ZSTD".to_string()])); + } + + #[tokio::test] + async fn the_table_codec_is_honoured() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let table = wh.load(&orders).await.unwrap(); + let tx = Transaction::new(&table); + let tx = tx + .update_table_properties() + .set( + TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(), + "snappy".to_string(), + ) + .apply(tx) + .unwrap(); + tx.commit(&wh.catalog).await.unwrap(); + + wh.append(&orders, &[ids(&[1, 2, 3])]).await.unwrap(); + assert_eq!(codecs(dir.path()), HashSet::from(["SNAPPY".to_string()])); + } + + #[tokio::test] + async fn a_batch_with_other_column_names_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "amount", + DataType::Int64, + false, + )])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![1]))]).unwrap(); + let err = wh.append(&orders, &[batch]).await; + assert!( + matches!(&err, Err(Error::Storage(m)) if m.contains("amount")), + "{err:?}" + ); + } +}