diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index 9e8ba6f..8772f73 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -16,7 +16,8 @@ iceberg-catalog-sql.workspace = true infostash-model.workspace = true parquet.workspace = true # The SQL catalog reaches databases through sqlx; this turns on its SQLite -# driver so the catalog can live in one file next to the tables. +# driver so the catalog can live in one file next to the tables. A replace +# also commits through it, with its own compare-and-swap (ADR 0007). sqlx = { version = "0.8.6", default-features = false, features = ["runtime-tokio", "sqlite"] } uuid = { version = "1.29.0", features = ["v4"] } @@ -26,7 +27,3 @@ tokio = { version = "1.54.1", features = ["macros", "rt-multi-thread"] } [lints] workspace = true - -# sqlx is never imported; it is listed only to turn on its SQLite driver. -[package.metadata.cargo-machete] -ignored = ["sqlx"] diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index 755b546..5d23d13 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -39,7 +39,15 @@ use iceberg_catalog_sql::{ }; use infostash_model::{Error, Layer, Result, SnapshotId, SnapshotRange, TableRef}; -/// The result of one atomic append. +mod replace; +mod swap; +#[cfg(test)] +mod test_support; + +pub use replace::Replacement; +use swap::CatalogSwap; + +/// The result of one atomic commit. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Commit { pub snapshot: SnapshotId, @@ -68,6 +76,9 @@ pub type RecordBatchStream = BoxStream<'static, Result>; /// File in the warehouse root that holds the catalog. const CATALOG_FILE: &str = "catalog.db"; +/// Name the warehouse registers under in its catalog. +const CATALOG_NAME: &str = "infostash"; + /// Escape a file path for a SQLite URL. sqlx splits the URL at the first `?` /// and percent-decodes the path, so `%`, `?` and `#` in directory names must /// be encoded or the catalog opens the wrong file. @@ -82,6 +93,8 @@ fn sqlite_path(path: &Path) -> String { /// A set of Iceberg tables under one storage root. pub struct Warehouse { catalog: SqlCatalog, + /// Commits replaces, which the catalog's own commit cannot express. + swap: CatalogSwap, } impl Warehouse { @@ -96,9 +109,9 @@ impl Warehouse { let catalog = SqlCatalogBuilder::default() .with_storage_factory(Arc::new(LocalFsStorageFactory)) .load( - "infostash", + CATALOG_NAME, HashMap::from([ - (SQL_CATALOG_PROP_URI.to_string(), uri), + (SQL_CATALOG_PROP_URI.to_string(), uri.clone()), ( SQL_CATALOG_PROP_WAREHOUSE.to_string(), format!("file://{}", root.display()), @@ -107,7 +120,8 @@ impl Warehouse { ) .await .map_err(storage)?; - Ok(Self { catalog }) + let swap = CatalogSwap::connect(&uri, CATALOG_NAME).await?; + Ok(Self { catalog, swap }) } /// Create an empty table with the given schema. @@ -185,11 +199,7 @@ impl Warehouse { expected_head: Option, properties: &HashMap, ) -> Result { - if let Some(key) = properties.keys().find(|k| !k.starts_with(PROPERTY_PREFIX)) { - return Err(Error::Storage(format!( - "{table}: property `{key}` must start with `{PROPERTY_PREFIX}`" - ))); - } + check_properties(table, properties)?; if properties.is_empty() && batches.iter().all(|b| b.num_rows() == 0) { return Err(Error::Storage(format!( "{table}: nothing to commit (no rows and no properties)" @@ -341,6 +351,16 @@ fn storage(e: impl std::fmt::Display) -> Error { Error::Storage(e.to_string()) } +/// Refuse a property key Infostash does not own. +fn check_properties(table: &TableRef, properties: &HashMap) -> Result<()> { + match properties.keys().find(|k| !k.starts_with(PROPERTY_PREFIX)) { + Some(key) => Err(Error::Storage(format!( + "{table}: property `{key}` must start with `{PROPERTY_PREFIX}`" + ))), + None => Ok(()), + } +} + fn moved(table: &TableRef, expected: Option, found: Option) -> Error { let show = |id: Option| id.map_or("no snapshot".to_string(), |id| id.to_string()); Error::Conflict(format!( @@ -678,84 +698,63 @@ mod tests { use arrow_array::Int64Array; use arrow_schema::{DataType, Field}; - use iceberg::spec::{MAIN_BRANCH, Snapshot, Summary}; - use infostash_model::Layer; use super::*; + use crate::test_support::{ + all_ids, files_of, ids, ids_schema, job, orders, position, two_appends, + }; #[tokio::test] async fn an_overwrite_in_the_range_is_refused() { let dir = tempfile::tempdir().unwrap(); - let wh = Warehouse::open_local(dir.path()).await.unwrap(); - let orders = TableRef::new(Layer::Raw, "orders"); - let schema = Arc::new(ArrowSchema::new(vec![Field::new( - "id", - DataType::Int64, - false, - )])); - wh.create_table(&orders, &schema).await.unwrap(); - let batch = RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![1]))]); - let append = wh.append(&orders, &[batch.unwrap()]).await.unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (first, second) = two_appends(&wh, &orders).await; + let replacement = Replacement { + remove: files_of(&wh, &orders, &second).await, + batches: vec![ids(&[30])], + ..Replacement::default() + }; + let replaced = wh + .replace_if_head(&orders, second.snapshot, replacement, &job("j")) + .await + .unwrap(); - // Nothing public in the pinned iceberg-rust makes an overwrite, so - // add one to the metadata by hand. Its manifest list is never read. let metadata = wh.load(&orders).await.unwrap().metadata().clone(); - let overwrite = Snapshot::builder() - .with_snapshot_id(append.snapshot + 1) - .with_parent_snapshot_id(Some(append.snapshot)) - .with_sequence_number(metadata.next_sequence_number()) - .with_timestamp_ms(metadata.last_updated_ms() + 1) - .with_manifest_list("unused.avro") - .with_summary(Summary { - operation: Operation::Overwrite, - additional_properties: HashMap::new(), - }) - .with_schema_id(metadata.current_schema_id()) - .build(); - let metadata = metadata - .into_builder(None) - .set_branch_snapshot(overwrite, MAIN_BRANCH) - .unwrap() - .build() - .unwrap() - .metadata; + let refused = |r: &Result<()>| matches!(r, Err(Error::Storage(m)) if m.contains("`overwrite`, not an append")); + let err = + appends_between(&metadata, &orders, second.snapshot, replaced.snapshot).map(|_| ()); + assert!(refused(&err), "{err:?}"); + let since_first = wh + .read_since(&orders, Some(first.snapshot)) + .await + .map(|_| ()); + assert!(refused(&since_first), "{since_first:?}"); + let across = SnapshotRange { + from: Some(second.snapshot), + to: replaced.snapshot, + }; + let across = wh.scan_range(&orders, across).await.map(|_| ()); + assert!(refused(&across), "{across:?}"); - let err = appends_between(&metadata, &orders, append.snapshot, append.snapshot + 1); - assert!( - matches!(&err, Err(Error::Storage(m)) if m.contains("`overwrite`, not an append")), - "{err:?}" - ); - // An overwrite before the range doesn't matter. + // An overwrite before the range doesn't matter, and reading + // everything still works. let from_overwrite = - appends_between(&metadata, &orders, append.snapshot + 1, append.snapshot + 1); + appends_between(&metadata, &orders, replaced.snapshot, replaced.snapshot); assert!(from_overwrite.unwrap().is_empty()); - } - - fn ids_schema() -> Arc { - Arc::new(ArrowSchema::new(vec![Field::new( - "id", - DataType::Int64, - false, - )])) - } - - fn ids(values: &[i64]) -> RecordBatch { - RecordBatch::try_new( - ids_schema(), - vec![Arc::new(Int64Array::from(values.to_vec()))], - ) - .unwrap() - } - - fn position(value: &str) -> HashMap { - HashMap::from([("infostash.position".to_string(), value.to_string())]) - } - - async fn orders(dir: &Path) -> (Warehouse, TableRef) { - let wh = Warehouse::open_local(dir).await.unwrap(); - let orders = TableRef::new(Layer::Raw, "orders"); - wh.create_table(&orders, &ids_schema()).await.unwrap(); - (wh, orders) + let since = wh.read_since(&orders, Some(replaced.snapshot)).await; + assert!(since.unwrap().is_empty()); + let everything = wh.read_since(&orders, None).await.unwrap(); + assert_eq!(all_ids(&everything), [1, 2, 30]); + + // iceberg-rust's own append builds on the hand-made snapshot, and + // reads incrementally from it. + let after = Some(replaced.snapshot); + wh.append_if_head(&orders, &[ids(&[6])], after, &position("p3")) + .await + .unwrap(); + let since = wh.read_since(&orders, after).await.unwrap(); + assert_eq!(all_ids(&since), [6]); + assert_eq!(all_ids(&wh.read_all(&orders).await.unwrap()), [1, 2, 6, 30]); } /// The codec of every column chunk in the Parquet files under `dir`. diff --git a/crates/storage/src/replace.rs b/crates/storage/src/replace.rs new file mode 100644 index 0000000..1a61b4d --- /dev/null +++ b/crates/storage/src/replace.rs @@ -0,0 +1,1325 @@ +//! Replacing data files in one `overwrite` snapshot, built from public +//! iceberg-rust spec writers and committed with the catalog swap in +//! [`crate::swap`]. ADR 0007 records the route and its costs. +//! +//! A replace runs in three steps that each do one thing: [`plan`] reads +//! the base snapshot and checks the files to remove, writing nothing; +//! [`write_snapshot`] writes the data files' manifests, the manifest list +//! and the snapshot; and [`Warehouse::commit_replace`] swaps the catalog +//! pointer. + +use std::collections::{HashMap, HashSet}; +use std::str::FromStr; +use std::time::{SystemTime, UNIX_EPOCH}; + +use arrow_array::RecordBatch; +use iceberg::spec::{ + DataContentType, DataFile, DataFileFormat, FormatVersion, MAIN_BRANCH, ManifestContentType, + ManifestEntryRef, ManifestFile, ManifestListWriter, ManifestWriter, ManifestWriterBuilder, + Operation, PartitionSpecRef, Snapshot, SnapshotSummaryCollector, Summary, TableMetadata, +}; +use iceberg::table::Table; +use iceberg::{MetadataLocation, TableRequirement}; +use infostash_model::{Error, Result, SnapshotId, TableRef}; + +use crate::{Commit, Warehouse, check_properties, ident, moved, storage, write_files}; + +/// What a replace takes out of a table and puts in. +#[derive(Debug, Clone, Default)] +pub struct Replacement { + /// Paths of data files live at the expected head, to mark deleted. + pub remove: Vec, + /// Rows to write as new data files. + pub batches: Vec, + /// Data files already written for this table, such as rewritten + /// copies of removed files, to add as they are. + pub files: Vec, +} + +impl Replacement { + fn is_empty(&self) -> bool { + self.remove.is_empty() + && self.files.is_empty() + && self.batches.iter().all(|b| b.num_rows() == 0) + } +} + +/// A replace whose files and next metadata file are written but which the +/// catalog does not point at yet, so no reader sees it. +#[derive(Debug, Clone)] +pub(crate) struct PreparedReplace { + table: TableRef, + uuid: uuid::Uuid, + expected_head: SnapshotId, + base_location: String, + new_location: String, + snapshot: SnapshotId, + rows: u64, + /// The data files it adds, which tests check are orphaned on a loss. + #[cfg(test)] + added: Vec, +} + +impl Warehouse { + /// Replace the live data files `replacement.remove` with the rows and + /// files it adds, in one `overwrite` snapshot, if the table head is + /// still `expected_head`. Removing every live file with nothing to add + /// empties the table, which is how a full refresh starts over. + /// + /// `properties` land in the new snapshot's summary and in the table + /// properties, in the same commit, as for + /// [`Warehouse::append_if_head`], which is also the way to commit + /// properties alone: an overwrite would stop incremental reads across + /// it. A key outside [`crate::PROPERTY_PREFIX`], a replacement that + /// changes nothing, a moved head, or a file to remove that is not live + /// at the head is refused before any file is written. A head that moves after that is + /// [`Error::Conflict`], with no retry and no rebase; the files the + /// losing replace wrote stay on disk, referenced by no snapshot. + /// + /// Limits, from ADR 0007: it works only on the SQLite catalog that + /// [`Warehouse::open_local`] opens, commits outside the `Catalog` trait + /// through the SQL catalog's private `iceberg_tables` layout, and must + /// be redone for the Phase 1 REST catalog. It replaces whole data files + /// in unpartitioned format v2 tables and writes no delete files. + pub async fn replace_if_head( + &self, + table: &TableRef, + expected_head: SnapshotId, + replacement: Replacement, + properties: &HashMap, + ) -> Result { + let prepared = self + .prepare_replace(table, expected_head, replacement, properties) + .await?; + self.commit_replace(prepared).await + } + + /// Write everything a replace needs except the catalog pointer. + pub(crate) async fn prepare_replace( + &self, + table: &TableRef, + expected_head: SnapshotId, + replacement: Replacement, + properties: &HashMap, + ) -> Result { + check_properties(table, properties)?; + if replacement.is_empty() { + return Err(Error::Storage(format!( + "{table}: nothing to replace (no files to remove, no rows and no files); \ + append_if_head commits properties alone" + ))); + } + if let Some(file) = replacement + .files + .iter() + .find(|f| f.content_type() != DataContentType::Data) + { + return Err(Error::Storage(format!( + "{table}: {} is a delete file, not a data file", + file.file_path() + ))); + } + let current = self.load(table).await?; + let metadata = current.metadata(); + if metadata.format_version() != FormatVersion::V2 { + return Err(Error::Storage(format!( + "{table}: replacing files needs format version 2, the table is {}", + metadata.format_version() + ))); + } + let found = metadata.current_snapshot_id(); + if found != Some(expected_head) { + return Err(moved(table, Some(expected_head), found)); + } + let base_location = current.metadata_location_result().map_err(storage)?; + let plan = plan(¤t, table, expected_head, &replacement.remove).await?; + check_added(table, &plan, &replacement.files)?; + + let (mut added, written) = write_files(¤t, table, &replacement.batches).await?; + let prebuilt: u64 = replacement.files.iter().map(DataFile::record_count).sum(); + let rows = written + prebuilt; + added.extend(replacement.files); + let snapshot = write_snapshot(¤t, plan, &added, properties).await?; + let snapshot_id = snapshot.snapshot_id(); + let next = next_metadata(metadata, base_location, snapshot, properties)?; + let new_location = MetadataLocation::from_str(base_location) + .map_err(storage)? + .with_next_version() + .try_with_new_metadata(&next) + .map_err(storage)?; + next.write_to(current.file_io(), &new_location) + .await + .map_err(storage)?; + Ok(PreparedReplace { + table: table.clone(), + uuid: metadata.uuid(), + expected_head, + base_location: base_location.to_string(), + new_location: new_location.to_string(), + snapshot: snapshot_id, + rows, + #[cfg(test)] + added: added.iter().map(|f| f.file_path().to_string()).collect(), + }) + } + + /// Point the catalog at a prepared replace's metadata file, if the table + /// is still the one it was prepared on and its head has not moved. + /// + /// The requirement check gives the clearer error and catches a table + /// dropped and created again. The swap is what makes the commit safe: + /// it only lands while the catalog row still holds the metadata the + /// replace was built on, so it also stops a commit that lands between + /// the check and the swap. + pub(crate) async fn commit_replace(&self, prepared: PreparedReplace) -> Result { + let table = &prepared.table; + let current = self.load(table).await?; + let requirements = [ + TableRequirement::UuidMatch { + uuid: prepared.uuid, + }, + TableRequirement::RefSnapshotIdMatch { + r#ref: MAIN_BRANCH.to_string(), + snapshot_id: Some(prepared.expected_head), + }, + ]; + for requirement in requirements { + requirement + .check(Some(current.metadata())) + .map_err(|e| Error::Conflict(format!("{table}: {e}")))?; + } + let swapped = self + .swap + .swap( + &ident(table), + &prepared.base_location, + &prepared.new_location, + ) + .await?; + if !swapped { + let now = self.load(table).await?; + if now.metadata_location() == Some(prepared.base_location.as_str()) { + return Err(Error::Storage(format!( + "{table}: the catalog row still points at {} but the swap matched no \ + row; the catalog's layout no longer fits the swap", + prepared.base_location + ))); + } + let found = now.metadata().current_snapshot_id(); + return Err(moved(table, Some(prepared.expected_head), found)); + } + Ok(Commit { + snapshot: prepared.snapshot, + metadata_location: prepared.new_location, + rows: prepared.rows, + }) + } +} + +/// How a replace changes the base snapshot's manifests. Building it reads +/// the manifests and writes nothing. +struct Plan { + base: SnapshotId, + /// Manifests that list no removed file, reused as they are. + kept: Vec, + /// Manifests that list a removed file, with their live entries. + rewritten: Vec<(ManifestFile, Vec)>, + /// The data files removed, as their entries describe them. + removed: Vec, + /// Paths of the live data files the replace leaves in place. + staying: HashSet, +} + +/// Sort the base snapshot's manifests into those a replace keeps and those +/// it rewrites, and check that every path in `remove` is live there. +async fn plan( + current: &Table, + table: &TableRef, + base: SnapshotId, + remove: &[String], +) -> Result { + let metadata = current.metadata(); + let snapshot = metadata + .snapshot_by_id(base) + .ok_or_else(|| Error::NotFound(format!("{table}: snapshot {base}")))?; + let remove: HashSet<&str> = remove.iter().map(String::as_str).collect(); + let list = current + .manifest_list_reader(snapshot) + .load() + .await + .map_err(storage)?; + let mut plan = Plan { + base, + kept: Vec::new(), + rewritten: Vec::new(), + removed: Vec::new(), + staying: HashSet::new(), + }; + for manifest_file in list.entries() { + if manifest_file.content != ManifestContentType::Data { + plan.kept.push(manifest_file.clone()); + continue; + } + let manifest = current + .manifest_reader() + .read(manifest_file) + .await + .map_err(storage)?; + // Entries deleted by earlier snapshots are not carried: such an + // entry names the snapshot that deleted the file. + let live: Vec = manifest + .entries() + .iter() + .filter(|e| e.is_alive()) + .cloned() + .collect(); + let (gone, staying): (Vec<&ManifestEntryRef>, Vec<&ManifestEntryRef>) = + live.iter().partition(|e| remove.contains(e.file_path())); + plan.staying + .extend(staying.iter().map(|e| e.file_path().to_string())); + let gone: Vec = gone.iter().map(|e| e.data_file().clone()).collect(); + if gone.is_empty() { + plan.kept.push(manifest_file.clone()); + } else { + plan.removed.extend(gone); + plan.rewritten.push((manifest_file.clone(), live)); + } + } + let found: HashSet<&str> = plan.removed.iter().map(DataFile::file_path).collect(); + let mut missing: Vec<&str> = remove.difference(&found).copied().collect(); + if !missing.is_empty() { + missing.sort_unstable(); + return Err(Error::Storage(format!( + "{table}: not live in snapshot {base}, so not removed: {}", + missing.join(", ") + ))); + } + Ok(plan) +} + +/// Check that no prebuilt file is listed twice or is already live and +/// staying, either of which would make readers return its rows twice. +fn check_added(table: &TableRef, plan: &Plan, files: &[DataFile]) -> Result<()> { + let mut seen = HashSet::new(); + for path in files.iter().map(DataFile::file_path) { + if !seen.insert(path) { + return Err(Error::Storage(format!( + "{table}: {path} is added more than once" + ))); + } + if plan.staying.contains(path) { + return Err(Error::Storage(format!( + "{table}: {path} is already live and not removed" + ))); + } + } + Ok(()) +} + +/// Write the manifests and manifest list of an `overwrite` snapshot on top +/// of the plan's base, and return the snapshot. Each rewritten manifest +/// marks its removed files deleted and carries its other live files as +/// existing entries, keeping their snapshot ids and sequence numbers. The +/// added files go in one new manifest. +async fn write_snapshot( + current: &Table, + plan: Plan, + added: &[DataFile], + properties: &HashMap, +) -> Result { + let metadata = current.metadata(); + let snapshot_id = new_snapshot_id(metadata); + let sequence_number = metadata.next_sequence_number(); + let commit = uuid::Uuid::new_v4(); + let dir = metadata.metadata_location().map_err(storage)?; + let avro = DataFileFormat::Avro; + let manifest_path = |n: usize| format!("{dir}/{commit}-m{n}.{avro}"); + let removed: HashSet<&str> = plan.removed.iter().map(DataFile::file_path).collect(); + + let mut manifests = plan.kept; + for (n, (manifest_file, live)) in plan.rewritten.into_iter().enumerate() { + let spec = spec_of(metadata, &manifest_file)?; + let mut writer = manifest_writer(current, manifest_path(n), snapshot_id, spec)?; + for entry in live { + let file = entry.data_file().clone(); + // The reader fills these in from the manifest list entry; the + // fallbacks are the values it would use. + let seq = entry + .sequence_number() + .unwrap_or(manifest_file.sequence_number); + let file_seq = entry.file_sequence_number; + if removed.contains(entry.file_path()) { + writer.add_delete_file(file, seq, file_seq) + } else { + let by = entry + .snapshot_id() + .unwrap_or(manifest_file.added_snapshot_id); + writer.add_existing_file(file, by, seq, file_seq) + } + .map_err(storage)?; + } + manifests.push(writer.write_manifest_file().await.map_err(storage)?); + } + if !added.is_empty() { + let spec = metadata.default_partition_spec(); + let path = manifest_path(manifests.len()); + let mut writer = manifest_writer(current, path, snapshot_id, spec)?; + for file in added { + writer + .add_file(file.clone(), sequence_number) + .map_err(storage)?; + } + manifests.push(writer.write_manifest_file().await.map_err(storage)?); + } + + let list_path = format!("{dir}/snap-{snapshot_id}-0-{commit}.{avro}"); + let output = current.file_io().new_output(&list_path).map_err(storage)?; + let mut list = ManifestListWriter::v2( + output.writer().await.map_err(storage)?, + snapshot_id, + Some(plan.base), + sequence_number, + ); + list.add_manifests(manifests.into_iter()).map_err(storage)?; + list.close().await.map_err(storage)?; + + let mut summary = overwrite_summary(metadata, plan.base, added, &plan.removed); + summary.additional_properties.extend(properties.clone()); + Ok(Snapshot::builder() + .with_snapshot_id(snapshot_id) + .with_parent_snapshot_id(Some(plan.base)) + .with_sequence_number(sequence_number) + .with_timestamp_ms(now_ms().max(metadata.last_updated_ms())) + .with_manifest_list(list_path) + .with_summary(summary) + .with_schema_id(metadata.current_schema_id()) + .build()) +} + +/// The table metadata after the replace: the snapshot added, `main` +/// pointed at it with its retention kept, and the properties set. +fn next_metadata( + metadata: &TableMetadata, + base_location: &str, + snapshot: Snapshot, + properties: &HashMap, +) -> Result { + let mut builder = metadata + .clone() + .into_builder(Some(base_location.to_string())) + .set_branch_snapshot(snapshot, MAIN_BRANCH) + .map_err(storage)?; + if !properties.is_empty() { + builder = builder + .set_properties(properties.clone()) + .map_err(storage)?; + } + Ok(builder.build().map_err(storage)?.metadata) +} + +/// The partition spec a manifest was written with. +fn spec_of<'a>( + metadata: &'a TableMetadata, + manifest: &ManifestFile, +) -> Result<&'a PartitionSpecRef> { + let id = manifest.partition_spec_id; + metadata.partition_spec_by_id(id).ok_or_else(|| { + Error::Storage(format!( + "manifest {} has unknown partition spec {id}", + manifest.manifest_path + )) + }) +} + +/// A v2 data manifest writer for `path`, owned by `snapshot_id`. +fn manifest_writer( + current: &Table, + path: String, + snapshot_id: SnapshotId, + spec: &PartitionSpecRef, +) -> Result { + let output = current.file_io().new_output(path).map_err(storage)?; + Ok(ManifestWriterBuilder::new( + output, + Some(snapshot_id), + current.metadata().current_schema().clone(), + spec.as_ref().clone(), + ) + .build_v2_data()) +} + +/// Summary of an overwrite: `added-*` and `deleted-*` from iceberg's +/// collector, and `total-*` carried forward from the base by +/// [`carry_totals`]. iceberg-rust's helper that does this for its own +/// commits is crate-private at the pin. +fn overwrite_summary( + metadata: &TableMetadata, + base: SnapshotId, + added: &[DataFile], + removed: &[DataFile], +) -> Summary { + let schema = metadata.current_schema(); + let spec = metadata.default_partition_spec(); + let mut collector = SnapshotSummaryCollector::default(); + for file in added { + collector.add_file(file, schema.clone(), spec.clone()); + } + for file in removed { + collector.remove_file(file, schema.clone(), spec.clone()); + } + let mut fields = collector.build(); + if let Some(previous) = metadata.snapshot_by_id(base) { + carry_totals(&previous.summary().additional_properties, &mut fields); + } + Summary { + operation: Operation::Overwrite, + additional_properties: fields, + } +} + +/// Set each `total-*` in `fields` to the base's total plus what the +/// snapshot adds and minus what it removes. A total the base does not +/// hold as a number stays unset: carrying it from zero would publish a +/// total that leaves out the files the replace keeps. +fn carry_totals(previous: &HashMap, fields: &mut HashMap) { + let delta = |m: &HashMap, k: &str| -> i64 { + m.get(k).and_then(|v| v.parse().ok()).unwrap_or(0) + }; + for (total, plus, minus) in [ + ("total-records", "added-records", "deleted-records"), + ("total-data-files", "added-data-files", "deleted-data-files"), + ("total-files-size", "added-files-size", "removed-files-size"), + ( + "total-delete-files", + "added-delete-files", + "removed-delete-files", + ), + ( + "total-position-deletes", + "added-position-deletes", + "removed-position-deletes", + ), + ( + "total-equality-deletes", + "added-equality-deletes", + "removed-equality-deletes", + ), + ] { + let Some(base) = previous.get(total).and_then(|v| v.parse::().ok()) else { + continue; + }; + let value = base + delta(fields, plus) - delta(fields, minus); + fields.insert(total.to_string(), value.to_string()); + } +} + +/// A snapshot id no snapshot of the table has: 63 random bits, as +/// iceberg-rust's own generator makes them. +fn new_snapshot_id(metadata: &TableMetadata) -> SnapshotId { + loop { + let (a, b) = uuid::Uuid::new_v4().as_u64_pair(); + let id = ((a ^ b) >> 1) as i64; + if metadata.snapshot_by_id(id).is_none() { + return id; + } + } +} + +/// Milliseconds since the Unix epoch, or 0 on a clock set before it. +fn now_ms() -> i64 { + let elapsed = SystemTime::now().duration_since(UNIX_EPOCH); + elapsed.map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX)) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::path::Path; + use std::sync::Arc; + + use futures::TryStreamExt; + use std::str::FromStr; + + use iceberg::spec::{ + DataContentType, DataFileBuilder, DataFileFormat, FormatVersion, MAIN_BRANCH, + ManifestStatus, Operation, SnapshotReference, SnapshotRetention, + }; + use iceberg::{Catalog, MetadataLocation, NamespaceIdent, TableCreation}; + use infostash_model::{Error, TableRef}; + + use super::{PreparedReplace, Replacement}; + use crate::swap::CatalogSwap; + use crate::test_support::{ + all_ids, entries_of, files_of, ids, ids_schema, job, live_files, local_path, manifests_of, + orders, position, referenced_files, two_appends, + }; + use crate::{Warehouse, ident, read_metadata_file, scan_at, write_files}; + + fn swap_in(remove: Vec, values: &[i64]) -> Replacement { + Replacement { + remove, + batches: vec![ids(values)], + ..Replacement::default() + } + } + + /// Prepare a replace of `second`'s file with `values`. + async fn prepare( + wh: &Warehouse, + orders: &TableRef, + second: &crate::Commit, + values: &[i64], + job_id: &str, + ) -> PreparedReplace { + let remove = files_of(wh, orders, second).await; + wh.prepare_replace( + orders, + second.snapshot, + swap_in(remove, values), + &job(job_id), + ) + .await + .unwrap() + } + + #[tokio::test] + async fn a_replace_commits_one_overwrite_snapshot() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (first, second) = two_appends(&wh, &orders).await; + let before = wh.load(&orders).await.unwrap(); + let uuid = before.metadata().uuid(); + let log_len = before.metadata().metadata_log().len(); + let first_files = files_of(&wh, &orders, &first).await; + let first_manifests = manifests_of(&wh, &orders, first.snapshot).await; + assert_eq!(first_manifests.len(), 1); + let remove = files_of(&wh, &orders, &second).await; + assert_eq!(remove.len(), 1); + + let replaced = wh + .replace_if_head( + &orders, + second.snapshot, + swap_in(remove.clone(), &[30, 40]), + &job("job-7"), + ) + .await + .unwrap(); + + assert_eq!(replaced.rows, 2); + let loaded = wh.load(&orders).await.unwrap(); + let metadata = loaded.metadata(); + assert_eq!(metadata.uuid(), uuid); + assert_eq!(metadata.current_snapshot_id(), Some(replaced.snapshot)); + assert_eq!( + loaded.metadata_location(), + Some(replaced.metadata_location.as_str()) + ); + // Three commits before it, so this is the fourth metadata file. + assert!( + replaced.metadata_location.contains("/00003-"), + "{}", + replaced.metadata_location + ); + assert_eq!(metadata.metadata_log().len(), log_len + 1); + assert_eq!( + wh.snapshots(&orders).await.unwrap(), + [first.snapshot, second.snapshot, replaced.snapshot] + ); + let rows = wh.read_all(&orders).await.unwrap(); + assert_eq!(all_ids(&rows), [1, 2, 30, 40]); + + let snapshot = metadata.current_snapshot().unwrap(); + assert_eq!(snapshot.parent_snapshot_id(), Some(second.snapshot)); + assert_eq!(snapshot.sequence_number(), metadata.last_sequence_number()); + let summary = snapshot.summary(); + assert_eq!(summary.operation, Operation::Overwrite); + let field = |key: &str| summary.additional_properties[key].as_str(); + assert_eq!(field("total-records"), "4"); + assert_eq!(field("total-data-files"), "2"); + assert_eq!(field("added-data-files"), "1"); + assert_eq!(field("deleted-data-files"), "1"); + assert_eq!(field("added-records"), "2"); + assert_eq!(field("deleted-records"), "3"); + assert_eq!(field("total-delete-files"), "0"); + assert_eq!(field("infostash.job-id"), "job-7"); + let state = wh.state(&orders).await.unwrap(); + assert_eq!(state.head, Some(replaced.snapshot)); + assert_eq!(state.properties["infostash.job-id"], "job-7"); + assert_eq!(state.properties["infostash.position"], "after job-7"); + + // The first append's manifest is carried over as it is; the second's + // is rewritten with its file marked deleted; the new file has a + // manifest of its own. + let manifests = manifests_of(&wh, &orders, replaced.snapshot).await; + assert_eq!(manifests.len(), 3, "{manifests:?}"); + assert!(manifests.contains(&first_manifests[0]), "{manifests:?}"); + let added = files_of(&wh, &orders, &replaced).await; + assert_eq!(added.len(), 1); + let entries = entries_of(&wh, &orders, replaced.snapshot).await; + assert_eq!(entries.len(), 3, "{entries:?}"); + let now = Some(replaced.snapshot); + // A deleted entry keeps the data sequence number of the file it + // removes; the new file takes the overwrite's own. + for expected in [ + ( + ManifestStatus::Added, + first_files[0].clone(), + Some(first.snapshot), + Some(1), + ), + (ManifestStatus::Deleted, remove[0].clone(), now, Some(2)), + (ManifestStatus::Added, added[0].clone(), now, Some(3)), + ] { + assert!(entries.contains(&expected), "{expected:?} in {entries:?}"); + } + // Removing a file from the table does not remove it from disk. + assert!(local_path(&remove[0]).exists()); + } + + #[tokio::test] + async fn a_partial_replace_keeps_the_rest_of_a_manifest_as_existing_files() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (first, second) = two_appends(&wh, &orders).await; + let first_files = files_of(&wh, &orders, &first).await; + + // Two prebuilt files in one replace share one manifest. + let loaded = wh.load(&orders).await.unwrap(); + let (mut files, _) = write_files(&loaded, &orders, &[ids(&[10])]).await.unwrap(); + files.extend( + write_files(&loaded, &orders, &[ids(&[20, 21])]) + .await + .unwrap() + .0, + ); + let (ten, twenty) = ( + files[0].file_path().to_string(), + files[1].file_path().to_string(), + ); + let built = Replacement { + remove: files_of(&wh, &orders, &second).await, + files, + ..Replacement::default() + }; + let built = wh + .replace_if_head(&orders, second.snapshot, built, &job("a")) + .await + .unwrap(); + assert_eq!(built.rows, 3); + assert_eq!( + all_ids(&wh.read_all(&orders).await.unwrap()), + [1, 2, 10, 20, 21] + ); + let deleted_only = manifests_of(&wh, &orders, built.snapshot).await; + + // Take out one of them. + let partial = wh + .replace_if_head( + &orders, + built.snapshot, + swap_in(vec![ten.clone()], &[11]), + &job("b"), + ) + .await + .unwrap(); + + let rows = wh.read_all(&orders).await.unwrap(); + assert_eq!(all_ids(&rows), [1, 2, 11, 20, 21]); + let entries = entries_of(&wh, &orders, partial.snapshot).await; + let now = Some(partial.snapshot); + // The other file keeps the snapshot that added it, and the earlier + // replace's deletion is not carried into the rewritten manifest. + // Sequence numbers are carried over too: the kept and removed files + // keep the earlier replace's, and the new file takes this one's. + let eleven = files_of(&wh, &orders, &partial).await; + assert_eq!(eleven.len(), 1); + for expected in [ + ( + ManifestStatus::Existing, + twenty, + Some(built.snapshot), + Some(3), + ), + (ManifestStatus::Deleted, ten, now, Some(3)), + (ManifestStatus::Added, eleven[0].clone(), now, Some(4)), + ( + ManifestStatus::Added, + first_files[0].clone(), + Some(first.snapshot), + Some(1), + ), + ] { + assert!(entries.contains(&expected), "{expected:?} in {entries:?}"); + } + // The manifest left holding only the second append's deleted file is + // reused as it is. + let manifests = manifests_of(&wh, &orders, partial.snapshot).await; + let untouched: Vec<_> = deleted_only + .iter() + .filter(|m| manifests.contains(m)) + .collect(); + assert_eq!(untouched.len(), 2, "{deleted_only:?} {manifests:?}"); + let summary = wh + .load(&orders) + .await + .unwrap() + .metadata() + .current_snapshot() + .unwrap() + .summary() + .clone(); + assert_eq!(summary.additional_properties["total-records"], "5"); + assert_eq!(summary.additional_properties["total-data-files"], "3"); + assert_eq!(live_files(&wh, &orders).await.len(), 3); + } + + #[tokio::test] + async fn two_replaces_on_one_head_give_one_commit_and_one_conflict() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + let a = prepare(&wh, &orders, &second, &[30], "a").await; + let b = prepare(&wh, &orders, &second, &[40], "b").await; + assert_ne!(a.new_location, b.new_location); + assert!(local_path(&a.new_location).exists()); + // Neither is visible before its commit. + assert_eq!(wh.state(&orders).await.unwrap().head, Some(second.snapshot)); + + let won = wh.commit_replace(a.clone()).await.unwrap(); + let lost = wh.commit_replace(b.clone()).await; + assert!(matches!(&lost, Err(Error::Conflict(_))), "{lost:?}"); + + let snapshots = wh.snapshots(&orders).await.unwrap(); + assert_eq!(snapshots.len(), 3); + assert_eq!(snapshots[2], won.snapshot); + assert_eq!(all_ids(&wh.read_all(&orders).await.unwrap()), [1, 2, 30]); + let referenced = referenced_files(&wh, &orders).await; + assert!(a.added.iter().all(|f| referenced.contains(f))); + for file in &b.added { + assert!(!referenced.contains(file), "{file} is referenced"); + // The loser's files are orphaned, not deleted. + assert!(local_path(file).exists()); + } + assert!(local_path(&b.new_location).exists()); + // The winner can't land twice either. + let again = wh.commit_replace(a).await; + assert!(matches!(&again, Err(Error::Conflict(_))), "{again:?}"); + assert_eq!(wh.snapshots(&orders).await.unwrap().len(), 3); + } + + #[tokio::test(flavor = "multi_thread")] + async fn concurrent_commits_on_one_head_have_exactly_one_winner() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + let a = prepare(&wh, &orders, &second, &[30], "a").await; + let b = prepare(&wh, &orders, &second, &[40], "b").await; + + let wh = Arc::new(wh); + let spawn = |prepared: PreparedReplace| { + let wh = Arc::clone(&wh); + tokio::spawn(async move { wh.commit_replace(prepared).await }) + }; + let (result_a, result_b) = tokio::join!(spawn(a), spawn(b)); + let results = [result_a.unwrap(), result_b.unwrap()]; + + let won: Vec<_> = results.iter().filter_map(|r| r.as_ref().ok()).collect(); + assert_eq!(won.len(), 1, "{results:?}"); + let conflicts = results + .iter() + .filter(|r| matches!(r, Err(Error::Conflict(_)))) + .count(); + assert_eq!(conflicts, 1, "{results:?}"); + assert_eq!(wh.state(&orders).await.unwrap().head, Some(won[0].snapshot)); + assert_eq!(wh.snapshots(&orders).await.unwrap().len(), 3); + let rows = all_ids(&wh.read_all(&orders).await.unwrap()); + assert!(rows == [1, 2, 30] || rows == [1, 2, 40], "{rows:?}"); + } + + #[tokio::test] + async fn a_replace_racing_an_append_that_lands_after_prepare_is_a_conflict() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + let prepared = prepare(&wh, &orders, &second, &[30], "j").await; + let third = wh + .append_if_head( + &orders, + &[ids(&[6])], + Some(second.snapshot), + &position("p3"), + ) + .await + .unwrap(); + + // The swap alone refuses it, with no requirement check in front. + let (from, to) = (&prepared.base_location, &prepared.new_location); + assert!(!wh.swap.swap(&ident(&orders), from, to).await.unwrap()); + // The commit refuses it at the requirement check. + let lost = wh.commit_replace(prepared.clone()).await; + assert!( + matches!(&lost, Err(Error::Conflict(m)) if m.contains("snapshot has changed")), + "{lost:?}" + ); + let loaded = wh.load(&orders).await.unwrap(); + assert_eq!( + loaded.metadata().current_snapshot_id(), + Some(third.snapshot) + ); + assert!( + loaded + .metadata() + .snapshots() + .all(|s| s.summary().operation == Operation::Append) + ); + let rows = wh.read_all(&orders).await.unwrap(); + assert_eq!(all_ids(&rows), [1, 2, 3, 4, 5, 6]); + let referenced = referenced_files(&wh, &orders).await; + assert!(prepared.added.iter().all(|f| !referenced.contains(f))); + } + + #[tokio::test] + async fn a_table_dropped_and_created_again_is_a_conflict() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + let prepared = prepare(&wh, &orders, &second, &[30], "j").await; + wh.catalog.drop_table(&ident(&orders)).await.unwrap(); + wh.create_table(&orders, &ids_schema()).await.unwrap(); + + let lost = wh.commit_replace(prepared).await; + assert!( + matches!(&lost, Err(Error::Conflict(m)) if m.contains("UUID")), + "{lost:?}" + ); + assert!(wh.snapshots(&orders).await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_swap_that_matches_no_row_on_an_unmoved_table_is_a_storage_error() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + let prepared = prepare(&wh, &orders, &second, &[30], "j").await; + // A swap keyed on the wrong catalog name stands in for a catalog + // whose layout the statement no longer fits. + let uri = format!( + "sqlite:{}", + crate::sqlite_path(&dir.path().canonicalize().unwrap().join(crate::CATALOG_FILE)) + ); + let wh = Warehouse { + catalog: wh.catalog, + swap: CatalogSwap::connect(&uri, "elsewhere").await.unwrap(), + }; + + let err = wh.commit_replace(prepared).await; + assert!( + matches!(&err, Err(Error::Storage(m)) if m.contains("no longer fits")), + "{err:?}" + ); + assert_eq!(wh.state(&orders).await.unwrap().head, Some(second.snapshot)); + } + + #[tokio::test] + async fn a_reader_holding_the_old_metadata_still_reads_the_old_files() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + let reader = wh.load(&orders).await.unwrap(); + let remove = files_of(&wh, &orders, &second).await; + let replaced = wh + .replace_if_head( + &orders, + second.snapshot, + swap_in(remove, &[30, 40]), + &job("j"), + ) + .await + .unwrap(); + + let old: Vec<_> = scan_at(&reader, None, None) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + assert_eq!(all_ids(&old), [1, 2, 3, 4, 5]); + let old_file = read_metadata_file(&orders, &second.metadata_location) + .await + .unwrap(); + assert_eq!(all_ids(&old_file), [1, 2, 3, 4, 5]); + let new_file = read_metadata_file(&orders, &replaced.metadata_location) + .await + .unwrap(); + assert_eq!(all_ids(&new_file), [1, 2, 30, 40]); + } + + /// Count the entries in a directory. + fn count(dir: &Path) -> usize { + std::fs::read_dir(dir).unwrap().count() + } + + #[tokio::test] + async fn a_refused_replace_writes_nothing() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (first, second) = two_appends(&wh, &orders).await; + let first_files = files_of(&wh, &orders, &first).await; + let metadata_dir = local_path(&second.metadata_location) + .parent() + .unwrap() + .to_path_buf(); + let data_dir = metadata_dir.parent().unwrap().join("data"); + let before = (count(&metadata_dir), count(&data_dir)); + + let stale = wh + .prepare_replace( + &orders, + first.snapshot, + swap_in(first_files.clone(), &[30]), + &job("j"), + ) + .await; + assert!(matches!(&stale, Err(Error::Conflict(_))), "{stale:?}"); + + let nowhere = vec!["file:///nowhere/orders.parquet".to_string()]; + let missing = wh + .prepare_replace(&orders, second.snapshot, swap_in(nowhere, &[30]), &job("j")) + .await; + assert!( + matches!(&missing, Err(Error::Storage(m)) if m.contains("not live") && m.contains("/nowhere/")), + "{missing:?}" + ); + + let delete_file = DataFileBuilder::default() + .content(DataContentType::PositionDeletes) + .file_path("file:///nowhere/deletes.parquet".to_string()) + .file_format(DataFileFormat::Parquet) + .record_count(1) + .file_size_in_bytes(1) + .build() + .unwrap(); + let deletes = Replacement { + files: vec![delete_file], + ..Replacement::default() + }; + let deletes = wh + .prepare_replace(&orders, second.snapshot, deletes, &job("j")) + .await; + assert!( + matches!(&deletes, Err(Error::Storage(m)) if m.contains("delete file")), + "{deletes:?}" + ); + + let nothing = Replacement { + batches: vec![ids(&[])], + ..Replacement::default() + }; + let nothing = wh + .prepare_replace(&orders, second.snapshot, nothing, &job("j")) + .await; + assert!( + matches!(¬hing, Err(Error::Storage(m)) if m.contains("nothing to replace")), + "{nothing:?}" + ); + let foreign = HashMap::from([("total-records".to_string(), "9".to_string())]); + let foreign = wh + .prepare_replace( + &orders, + second.snapshot, + swap_in(first_files, &[30]), + &foreign, + ) + .await; + assert!( + matches!(&foreign, Err(Error::Storage(m)) if m.contains("total-records")), + "{foreign:?}" + ); + + assert_eq!((count(&metadata_dir), count(&data_dir)), before); + assert_eq!( + wh.snapshots(&orders).await.unwrap(), + [first.snapshot, second.snapshot] + ); + } + + #[tokio::test] + async fn a_file_removed_earlier_is_not_live() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + let remove = files_of(&wh, &orders, &second).await; + let replaced = wh + .replace_if_head( + &orders, + second.snapshot, + swap_in(remove.clone(), &[30]), + &job("j"), + ) + .await + .unwrap(); + + let twice = wh + .replace_if_head( + &orders, + replaced.snapshot, + swap_in(remove, &[50]), + &job("k"), + ) + .await; + assert!( + matches!(&twice, Err(Error::Storage(m)) if m.contains("not live")), + "{twice:?}" + ); + assert_eq!( + wh.state(&orders).await.unwrap().head, + Some(replaced.snapshot) + ); + } + + #[tokio::test] + async fn a_full_refresh_with_no_rows_still_commits_an_empty_overwrite() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + assert!(live_files(&wh, &orders).await.is_empty()); + let (_, second) = two_appends(&wh, &orders).await; + let live = live_files(&wh, &orders).await; + assert_eq!(live.len(), 2); + + let empty = Replacement { + remove: live, + ..Replacement::default() + }; + let refreshed = wh + .replace_if_head(&orders, second.snapshot, empty, &job("job-7")) + .await + .unwrap(); + + assert_eq!(refreshed.rows, 0); + assert!(all_ids(&wh.read_all(&orders).await.unwrap()).is_empty()); + assert!(live_files(&wh, &orders).await.is_empty()); + let loaded = wh.load(&orders).await.unwrap(); + let summary = loaded.metadata().current_snapshot().unwrap().summary(); + assert_eq!(summary.operation, Operation::Overwrite); + let field = |key: &str| summary.additional_properties[key].as_str(); + assert_eq!(field("total-records"), "0"); + assert_eq!(field("total-data-files"), "0"); + assert_eq!(field("total-files-size"), "0"); + assert_eq!(field("deleted-data-files"), "2"); + assert_eq!(field("deleted-records"), "5"); + let state = wh.state(&orders).await.unwrap(); + assert_eq!(state.properties["infostash.job-id"], "job-7"); + // Both manifests were rewritten, and every entry is a deletion. + let entries = entries_of(&wh, &orders, refreshed.snapshot).await; + assert_eq!(entries.len(), 2, "{entries:?}"); + assert!(entries.iter().all(|(s, ..)| *s == ManifestStatus::Deleted)); + // The next append starts from an empty table. + let after = Some(refreshed.snapshot); + wh.append_if_head(&orders, &[ids(&[7])], after, &position("p3")) + .await + .unwrap(); + assert_eq!(all_ids(&wh.read_all(&orders).await.unwrap()), [7]); + assert_eq!(all_ids(&wh.read_since(&orders, after).await.unwrap()), [7]); + } + + #[tokio::test] + async fn a_file_added_twice_or_already_live_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + let loaded = wh.load(&orders).await.unwrap(); + let (files, _) = write_files(&loaded, &orders, &[ids(&[10])]).await.unwrap(); + let path = files[0].file_path().to_string(); + + let doubled = Replacement { + files: vec![files[0].clone(), files[0].clone()], + ..Replacement::default() + }; + let doubled = wh + .replace_if_head(&orders, second.snapshot, doubled, &job("j")) + .await; + assert!( + matches!(&doubled, Err(Error::Storage(m)) if m.contains(&path) && m.contains("more than once")), + "{doubled:?}" + ); + + let once = Replacement { + files: files.clone(), + ..Replacement::default() + }; + let added = wh + .replace_if_head(&orders, second.snapshot, once.clone(), &job("j")) + .await + .unwrap(); + let again = wh + .replace_if_head(&orders, added.snapshot, once, &job("k")) + .await; + assert!( + matches!(&again, Err(Error::Storage(m)) if m.contains(&path) && m.contains("already live")), + "{again:?}" + ); + + // Removing it and adding it back in one replace is allowed. + let back = Replacement { + remove: vec![path], + files, + ..Replacement::default() + }; + wh.replace_if_head(&orders, added.snapshot, back, &job("k")) + .await + .unwrap(); + assert_eq!( + all_ids(&wh.read_all(&orders).await.unwrap()), + [1, 2, 3, 4, 5, 10] + ); + } + + #[tokio::test] + async fn a_replace_keeps_the_main_branch_retention() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (_, second) = two_appends(&wh, &orders).await; + // Give main a retention policy, committed the way a replace is. + let loaded = wh.load(&orders).await.unwrap(); + let base = loaded.metadata_location().unwrap().to_string(); + let retention = SnapshotRetention::branch(Some(7), Some(86_400_000), None); + let reference = SnapshotReference::new(second.snapshot, retention); + let kept = loaded + .metadata() + .clone() + .into_builder(Some(base.clone())) + .set_ref(MAIN_BRANCH, reference) + .unwrap() + .build() + .unwrap() + .metadata; + let next = MetadataLocation::from_str(&base) + .unwrap() + .with_next_version() + .try_with_new_metadata(&kept) + .unwrap(); + kept.write_to(loaded.file_io(), &next).await.unwrap(); + let next = next.to_string(); + assert!(wh.swap.swap(&ident(&orders), &base, &next).await.unwrap()); + + let prepared = prepare(&wh, &orders, &second, &[30], "j").await; + let replaced = wh.commit_replace(prepared).await.unwrap(); + + let loaded = wh.load(&orders).await.unwrap(); + let main = loaded.metadata().snapshot_for_ref(MAIN_BRANCH).unwrap(); + assert_eq!(main.snapshot_id(), replaced.snapshot); + // iceberg keeps the refs crate-private, so read them from the file. + let text = std::fs::read_to_string(local_path(&replaced.metadata_location)).unwrap(); + for field in [ + "\"min-snapshots-to-keep\":7", + "\"max-snapshot-age-ms\":86400000", + ] { + assert!(text.contains(field), "{field} missing from {text}"); + } + } + + #[tokio::test] + async fn only_format_version_2_tables_are_replaced() { + let dir = tempfile::tempdir().unwrap(); + let (wh, _) = orders(dir.path()).await; + let old = TableRef::new(infostash_model::Layer::Raw, "old"); + let creation = TableCreation::builder() + .name(old.name.clone()) + .schema(iceberg::arrow::arrow_schema_to_schema_auto_assign_ids(&ids_schema()).unwrap()) + .format_version(FormatVersion::V1) + .build(); + let raw = NamespaceIdent::new(old.layer.namespace().to_string()); + wh.catalog.create_table(&raw, creation).await.unwrap(); + let first = wh.append(&old, &[ids(&[1])]).await.unwrap(); + + let refused = wh + .replace_if_head(&old, first.snapshot, swap_in(vec![], &[2]), &job("j")) + .await; + assert!( + matches!(&refused, Err(Error::Storage(m)) if m.contains("format version 2")), + "{refused:?}" + ); + assert_eq!(wh.snapshots(&old).await.unwrap(), [first.snapshot]); + } + + /// The catalog also reads an `iceberg_tables` in its older layout, which + /// has no `iceberg_type` column, and appends to tables in it. The swap + /// must fit that layout too. + #[tokio::test] + async fn a_replace_commits_on_a_catalog_without_the_record_type_column() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (first, second) = two_appends(&wh, &orders).await; + sqlx::query("ALTER TABLE iceberg_tables DROP COLUMN iceberg_type") + .execute(wh.swap.pool()) + .await + .unwrap(); + drop(wh); + + // The catalog detects the old layout on open and still serves the + // table. + let wh = Warehouse::open_local(dir.path()).await.unwrap(); + assert_eq!(wh.state(&orders).await.unwrap().head, Some(second.snapshot)); + let first_files = files_of(&wh, &orders, &first).await; + let replaced = wh + .replace_if_head( + &orders, + second.snapshot, + swap_in(first_files, &[10]), + &job("job-8"), + ) + .await + .unwrap(); + assert_eq!(all_ids(&wh.read_all(&orders).await.unwrap()), [3, 4, 5, 10]); + let state = wh.state(&orders).await.unwrap(); + assert_eq!(state.properties["infostash.job-id"], "job-8"); + + // A stale replace is still a conflict, and the catalog's own commit + // still lands on the old layout. + let stale = wh + .replace_if_head( + &orders, + second.snapshot, + swap_in(vec![], &[11]), + &job("job-9"), + ) + .await; + assert!(matches!(stale, Err(Error::Conflict(_))), "{stale:?}"); + let after = Some(replaced.snapshot); + wh.append_if_head(&orders, &[ids(&[12])], after, &position("p3")) + .await + .unwrap(); + let rows = wh.read_all(&orders).await.unwrap(); + assert_eq!(all_ids(&rows), [3, 4, 5, 10, 12]); + } + + #[test] + fn a_total_the_base_does_not_hold_as_a_number_stays_unset() { + let strings = |pairs: &[(&str, &str)]| -> HashMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + }; + let previous = strings(&[ + ("total-records", "10"), + ("total-data-files", "many"), + ("total-delete-files", "0"), + ]); + let mut fields = strings(&[ + ("added-records", "3"), + ("deleted-records", "4"), + ("added-data-files", "1"), + ("deleted-data-files", "1"), + ("added-files-size", "100"), + ]); + + super::carry_totals(&previous, &mut fields); + + assert_eq!(fields["total-records"], "9"); + assert_eq!(fields["total-delete-files"], "0"); + assert!(!fields.contains_key("total-data-files"), "{fields:?}"); + assert!(!fields.contains_key("total-files-size"), "{fields:?}"); + } +} diff --git a/crates/storage/src/swap.rs b/crates/storage/src/swap.rs new file mode 100644 index 0000000..0bf8c5d --- /dev/null +++ b/crates/storage/src/swap.rs @@ -0,0 +1,106 @@ +//! The compare-and-swap that commits a replace on the SQL catalog. +//! +//! This is the only code that knows the catalog's private `iceberg_tables` +//! layout. A catalog that cannot run the statement, such as the Phase 1 +//! REST catalog, replaces this module and keeps the snapshot building in +//! [`crate::replace`]. See ADR 0007. + +use iceberg::TableIdent; +use infostash_model::Result; +use sqlx::SqlitePool; + +use crate::storage; + +/// Our own connections to the catalog's database, next to the catalog's. +pub(crate) struct CatalogSwap { + pool: SqlitePool, + /// The name the warehouse registers under in the catalog, which keys + /// every row of `iceberg_tables`. + catalog_name: &'static str, +} + +impl CatalogSwap { + /// Connect to the catalog database at `uri`, which the catalog has + /// already created. + pub(crate) async fn connect(uri: &str, catalog_name: &'static str) -> Result { + let pool = SqlitePool::connect(uri).await.map_err(storage)?; + Ok(Self { pool, catalog_name }) + } + + /// The SQL catalog's own commit statement, less its record type filter: + /// point `table` at `to` only if it still points at `from`, and keep + /// `from` as the previous location. Returns whether a row changed. + /// SQLite runs one writer at a time, so of two swaps from one `from` + /// exactly one changes a row. + /// + /// The filter adds nothing: the key is the table's primary key, and a + /// view row never holds a table's metadata location. Without it the + /// statement fits both layouts of `iceberg_tables` the catalog reads, + /// with and without the `iceberg_type` column. Every pin bump must + /// compare this statement with the catalog's `update_table`. + pub(crate) async fn swap(&self, table: &TableIdent, from: &str, to: &str) -> Result { + let updated = sqlx::query( + "UPDATE iceberg_tables \ + SET metadata_location = ?, previous_metadata_location = ? \ + WHERE catalog_name = ? AND table_name = ? AND table_namespace = ? \ + AND metadata_location = ?", + ) + .bind(to) + .bind(from) + .bind(self.catalog_name) + .bind(table.name()) + .bind(table.namespace().join(".")) + .bind(from) + .execute(&self.pool) + .await + .map_err(storage)? + .rows_affected(); + Ok(updated == 1) + } + + /// The pool, for tests that change the catalog's layout. + #[cfg(test)] + pub(crate) fn pool(&self) -> &SqlitePool { + &self.pool + } +} + +#[cfg(test)] +mod tests { + use iceberg::TableIdent; + + use crate::ident; + use crate::test_support::{all_ids, orders, two_appends}; + + #[tokio::test] + async fn the_swap_moves_the_catalog_pointer_once() { + let dir = tempfile::tempdir().unwrap(); + let (wh, orders) = orders(dir.path()).await; + let (first, second) = two_appends(&wh, &orders).await; + let table = ident(&orders); + let (first_file, second_file) = (&first.metadata_location, &second.metadata_location); + + // Pointing the table back at the first metadata file rolls the head + // back, and the same swap again finds no row to update. + assert!(wh.swap.swap(&table, second_file, first_file).await.unwrap()); + assert_eq!(wh.state(&orders).await.unwrap().head, Some(first.snapshot)); + assert!(!wh.swap.swap(&table, second_file, first_file).await.unwrap()); + // Another table's row is not touched. + let other = TableIdent::new(table.namespace().clone(), "other".into()); + assert!(!wh.swap.swap(&other, first_file, second_file).await.unwrap()); + assert_eq!(wh.state(&orders).await.unwrap().head, Some(first.snapshot)); + // Back to where it was, and the previous location is recorded. + assert!(wh.swap.swap(&table, first_file, second_file).await.unwrap()); + assert_eq!(wh.state(&orders).await.unwrap().head, Some(second.snapshot)); + let previous: String = sqlx::query_scalar( + "SELECT previous_metadata_location FROM iceberg_tables WHERE table_name = ?", + ) + .bind(table.name()) + .fetch_one(wh.swap.pool()) + .await + .unwrap(); + assert_eq!(&previous, first_file); + let rows = wh.read_all(&orders).await.unwrap(); + assert_eq!(all_ids(&rows), [1, 2, 3, 4, 5]); + } +} diff --git a/crates/storage/src/test_support.rs b/crates/storage/src/test_support.rs new file mode 100644 index 0000000..1d14212 --- /dev/null +++ b/crates/storage/src/test_support.rs @@ -0,0 +1,183 @@ +//! Fixtures shared by the crate's tests: a one-column `orders` table and +//! ways to look inside its snapshots. + +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::sync::Arc; + +use arrow_array::{Int64Array, RecordBatch}; +use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use futures::TryStreamExt; +use iceberg::spec::ManifestStatus; +use infostash_model::{Layer, SnapshotId, TableRef}; + +use crate::{Commit, Warehouse, files_added_by}; + +pub(crate) fn ids_schema() -> Arc { + Arc::new(ArrowSchema::new(vec![Field::new( + "id", + DataType::Int64, + false, + )])) +} + +pub(crate) fn ids(values: &[i64]) -> RecordBatch { + RecordBatch::try_new( + ids_schema(), + vec![Arc::new(Int64Array::from(values.to_vec()))], + ) + .unwrap() +} + +pub(crate) fn position(value: &str) -> HashMap { + HashMap::from([("infostash.position".to_string(), value.to_string())]) +} + +/// The properties a job commit carries: its id and a checkpoint. +pub(crate) fn job(id: &str) -> HashMap { + HashMap::from([ + ("infostash.job-id".to_string(), id.to_string()), + ("infostash.position".to_string(), format!("after {id}")), + ]) +} + +/// A warehouse in `dir` with an empty `raw.orders` table of ids. +pub(crate) async fn orders(dir: &Path) -> (Warehouse, TableRef) { + let wh = Warehouse::open_local(dir).await.unwrap(); + let orders = TableRef::new(Layer::Raw, "orders"); + wh.create_table(&orders, &ids_schema()).await.unwrap(); + (wh, orders) +} + +/// Every id in `batches`, sorted. +pub(crate) fn all_ids(batches: &[RecordBatch]) -> Vec { + let mut ids: Vec = batches + .iter() + .flat_map(|b| { + b.column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + ids.sort_unstable(); + ids +} + +/// Two appends with a position each: ids 1 and 2, then ids 3, 4 and 5. +pub(crate) async fn two_appends(wh: &Warehouse, orders: &TableRef) -> (Commit, Commit) { + let first = wh + .append_if_head(orders, &[ids(&[1, 2])], None, &position("p1")) + .await + .unwrap(); + let second = wh + .append_if_head( + orders, + &[ids(&[3, 4, 5])], + Some(first.snapshot), + &position("p2"), + ) + .await + .unwrap(); + (first, second) +} + +/// The data files `commit` added, sorted. +pub(crate) async fn files_of(wh: &Warehouse, orders: &TableRef, commit: &Commit) -> Vec { + let loaded = wh.load(orders).await.unwrap(); + let mut files: Vec = + files_added_by(&loaded, commit.snapshot, &HashSet::from([commit.snapshot])) + .await + .unwrap() + .into_iter() + .collect(); + files.sort(); + files +} + +/// Every data file any snapshot of the table references, with any status. +pub(crate) async fn referenced_files(wh: &Warehouse, orders: &TableRef) -> HashSet { + let loaded = wh.load(orders).await.unwrap(); + let mut files = HashSet::new(); + for snapshot in loaded.metadata().snapshots() { + let list = loaded.manifest_list_reader(snapshot).load().await.unwrap(); + for manifest in list.entries() { + let manifest = loaded.manifest_reader().read(manifest).await.unwrap(); + for entry in manifest.entries() { + files.insert(entry.file_path().to_string()); + } + } + } + files +} + +/// Status, path, owning snapshot and data sequence number of every entry +/// in `snapshot`'s manifests. +pub(crate) async fn entries_of( + wh: &Warehouse, + orders: &TableRef, + snapshot: SnapshotId, +) -> Vec<(ManifestStatus, String, Option, Option)> { + let loaded = wh.load(orders).await.unwrap(); + let snapshot = loaded.metadata().snapshot_by_id(snapshot).unwrap(); + let list = loaded.manifest_list_reader(snapshot).load().await.unwrap(); + let mut entries = Vec::new(); + for manifest in list.entries() { + let manifest = loaded.manifest_reader().read(manifest).await.unwrap(); + for entry in manifest.entries() { + let path = entry.file_path().to_string(); + let (by, seq) = (entry.snapshot_id(), entry.sequence_number()); + entries.push((entry.status(), path, by, seq)); + } + } + entries +} + +/// The manifest paths in `snapshot`'s manifest list. +pub(crate) async fn manifests_of( + wh: &Warehouse, + orders: &TableRef, + snapshot: SnapshotId, +) -> Vec { + let loaded = wh.load(orders).await.unwrap(); + let snapshot = loaded.metadata().snapshot_by_id(snapshot).unwrap(); + let list = loaded.manifest_list_reader(snapshot).load().await.unwrap(); + list.entries() + .iter() + .map(|m| m.manifest_path.clone()) + .collect() +} + +/// The local path of a `file://` location. +pub(crate) fn local_path(file: &str) -> &Path { + Path::new(file.strip_prefix("file://").unwrap()) +} + +/// Paths of the data files in the table's current snapshot, sorted: what a +/// replace that empties the table removes. +pub(crate) async fn live_files(wh: &Warehouse, table: &TableRef) -> Vec { + let loaded = wh.load(table).await.unwrap(); + if loaded.metadata().current_snapshot().is_none() { + return Vec::new(); + } + let tasks: Vec<_> = loaded + .scan() + .select_all() + .build() + .unwrap() + .plan_files() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut files: Vec = tasks + .into_iter() + .map(|t| t.data_file_path().to_string()) + .collect(); + files.sort(); + files.dedup(); + files +}