diff --git a/Cargo.lock b/Cargo.lock index f7a41ea..c6d440a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2413,7 +2413,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -2461,7 +2461,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#429f3a0aa236eec59850679c025c068d2ac29d41" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" dependencies = [ "aes-gcm", "anyhow", @@ -2520,7 +2520,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#429f3a0aa236eec59850679c025c068d2ac29d41" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" dependencies = [ "async-trait", "aws-config", @@ -2544,7 +2544,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#429f3a0aa236eec59850679c025c068d2ac29d41" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" dependencies = [ "proc-macro2", "quote", @@ -2554,7 +2554,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#429f3a0aa236eec59850679c025c068d2ac29d41" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#6ce920721007be32c6ec53e6a9ee7d0c83efbead" dependencies = [ "anyhow", "async-trait", @@ -4270,7 +4270,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror", "tokio", "tracing", @@ -4308,7 +4308,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.52.0", ] diff --git a/README.md b/README.md index 328a67c..2b2be4f 100644 --- a/README.md +++ b/README.md @@ -273,6 +273,7 @@ Settings come from environment variables and an optional YAML file: `--config`, | `PG2ICEBERG_LOG_FORMAT` | — | `text` (default) or `json` | | `OTEL_EXPORTER_OTLP_ENDPOINT` | — | Export traces over OTLP/HTTP to this collector. Default: none | | `PG2ICEBERG_MAINTENANCE` | `sink.maintenance` | `pg2iceberg` / `managed` (the catalog compacts, expires and cleans the tables). Default: `pg2iceberg` | +| `PG2ICEBERG_FORMAT_VERSION` | `sink.format_version` | Iceberg format version of the tables pg2iceberg creates: `2` or `3` (row lineage). Default: `2` | | `AWS_REGION`, `AWS_ENDPOINT_URL_S3`, `AWS_ACCESS_KEY_ID`, ... | `sink.s3_*` | Read as AWS tools read them; the file's `sink.s3_*` take precedence | ### Inferred settings diff --git a/config.example.yaml b/config.example.yaml index d43ba11..73c79ee 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -130,6 +130,12 @@ sink: # and `maintain` refuses. # maintenance: managed + # Iceberg format version of the tables pg2iceberg creates: 2 (default), + # or 3, whose rows carry row lineage (`_row_id`, + # `_last_updated_sequence_number`) — check that your engines read v3. + # Tables that already exist keep theirs. + # format_version: 3 + # Free-form REST catalog props passthrough. Useful for # vendor-specific settings the Go config doesn't have a dedicated # field for. diff --git a/crates/pg2iceberg-iceberg/src/compact.rs b/crates/pg2iceberg-iceberg/src/compact.rs index 6e01475..717aacb 100644 --- a/crates/pg2iceberg-iceberg/src/compact.rs +++ b/crates/pg2iceberg-iceberg/src/compact.rs @@ -61,6 +61,10 @@ //! `t` is retired only when no dirty data file with seq `< t` remains //! after this pass. Clean files don't hold it back: by definition no //! live delete applies to any of their rows. +//! - **Moved rows keep their lineage.** On a v3 table, every row has a +//! `_row_id` and a `_last_updated_sequence_number` — stored in its file, +//! or inherited from the file's first row id and data sequence number. +//! A rewritten row keeps both: the outputs store them for every row. //! - **No partial rewrites.** We rewrite a data file fully or not at all. //! Partial rewrites would lose rows. //! - **Atomicity.** Either the entire `Replace` snapshot lands, or none @@ -68,8 +72,11 @@ use crate::reader::{value_at, ArrowBatches}; use crate::writer::{StreamingDataFile, TableWriter, WriterError}; -use crate::{Catalog, DataFile, FileIndex, IcebergError, PkKey, PreparedCompaction, Snapshot}; -use arrow_array::{Array, BooleanArray, RecordBatch}; +use crate::{ + Catalog, DataFile, FileIndex, IcebergError, PkKey, PreparedCompaction, Snapshot, + LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID, ROW_ID_FIELD_ID, +}; +use arrow_array::{Array, BooleanArray, Int64Array, RecordBatch}; use arrow_select::filter::filter_record_batch; use bytes::Bytes; use futures::future::BoxFuture; @@ -79,6 +86,7 @@ use pg2iceberg_stream::{BlobStore, StreamError}; use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque}; use std::future::{poll_fn, Future}; use std::pin::pin; +use std::sync::Arc; use std::task::{Context, Poll}; use thiserror::Error; @@ -243,6 +251,8 @@ struct LiveFile { /// its snapshot's — `snapshots()` lists a file under the snapshot that /// added it — or, for a compaction's output, the one its pass read. seq: i64, + /// A v3 data file's first row id ([`DataFile::first_row_id`]). + first_row_id: Option, } impl LiveFile { @@ -252,10 +262,74 @@ impl LiveFile { record_count: df.record_count, partition_values: df.partition_values.clone(), seq, + first_row_id: df.first_row_id, } } } +/// Row lineage's two columns, as a v3 table's data files store them. +fn lineage_columns() -> [ColumnSchema; 2] { + let column = |name: &str, field_id| ColumnSchema { + name: name.into(), + field_id, + ty: pg2iceberg_core::IcebergType::Long, + nullable: true, + is_primary_key: false, + }; + [ + column("_row_id", ROW_ID_FIELD_ID), + column( + "_last_updated_sequence_number", + LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID, + ), + ] +} + +/// `batch`, read from rows `offset..` of `input`, with its last two +/// columns — row lineage's — filled in for each row the file doesn't +/// store them for: `_row_id` as the file's first row id plus the row's +/// position, `_last_updated_sequence_number` as the file's data sequence +/// number. What a v3 reader reads for the rows, so the rewritten file +/// stores what they had: the spec requires a commit that moves rows to +/// keep both. +fn with_lineage(batch: RecordBatch, input: &LiveFile, offset: u64) -> Result { + let width = batch.num_columns(); + let stored = |i: usize| -> Result { + batch + .column(i) + .as_any() + .downcast_ref::() + .cloned() + .ok_or_else(|| { + WriterError::Encode(format!( + "row lineage column {} isn't a long", + batch.schema().field(i).name() + )) + .into() + }) + }; + let (row_ids, seqs) = (stored(width - 2)?, stored(width - 1)?); + let row_ids: Int64Array = (0..batch.num_rows()) + .map(|i| match row_ids.is_null(i) { + false => Some(row_ids.value(i)), + true => input + .first_row_id + .map(|first| first + offset as i64 + i as i64), + }) + .collect(); + let seqs: Int64Array = (0..batch.num_rows()) + .map(|i| match seqs.is_null(i) { + false => Some(seqs.value(i)), + true => input.first_row_id.map(|_| input.seq), + }) + .collect(); + let mut columns = batch.columns()[..width - 2].to_vec(); + columns.push(Arc::new(row_ids)); + columns.push(Arc::new(seqs)); + RecordBatch::try_new(batch.schema(), columns) + .map_err(|e| WriterError::Encode(format!("row lineage: {e}")).into()) +} + /// What one pass does: which data files it rewrites (in processing /// order) and which delete files it drops. #[derive(Debug, Default)] @@ -399,7 +473,15 @@ where )) .await?; - let writer = TableWriter::new(schema.clone()); + // A v3 table's rows have lineage — its data files take first row ids + // — which the outputs store for each row they move. + let lineage = live_data.values().any(|f| f.first_row_id.is_some()); + let mut written = schema.clone(); + if lineage { + written.columns.extend(lineage_columns()); + } + let writer = TableWriter::new(written); + let read_cols = writer.schema().columns.clone(); let mut out = OutputFiles::default(); let mut rows_rewritten: u64 = 0; let mut rows_removed_by_deletes: u64 = 0; @@ -419,9 +501,15 @@ where while let Some(((path, meta), bytes)) = io.next_input().await? { let source = sources[path.as_str()]; bytes_before += meta.byte_size; - for batch in ArrowBatches::new(bytes, &schema.columns, DECODE_BATCH_ROWS)? { - let batch = batch?; + // The position in the input of the batch's first row. + let mut offset: u64 = 0; + for batch in ArrowBatches::new(bytes, &read_cols, DECODE_BATCH_ROWS)? { + let mut batch = batch?; let n = batch.num_rows(); + if lineage { + batch = with_lineage(batch, meta, offset)?; + } + offset += n as u64; peak_rows = peak_rows.max(n as u64); rows_rewritten += n as u64; // A delete applies iff it is strictly newer than the row. @@ -498,6 +586,7 @@ where removed_paths, // The table as the pass read it: the last snapshot. data_sequence_number: snapshots.last().map(|s| s.id), + stores_row_lineage: lineage, }, outcome: CompactionOutcome { input_data_files: plan.inputs.len(), @@ -552,6 +641,8 @@ where added_data_files: Vec::new(), removed_paths: retired.clone(), data_sequence_number: snapshots.last().map(|s| s.id), + // It moves no rows. + stores_row_lineage: true, }) .await?; Ok(Some(CompactionOutcome { @@ -639,6 +730,7 @@ impl OutputFiles { equality_field_ids: vec![], partition_values, sequence_number: None, + first_row_id: None, }); self.added_pks.push(pks); self.added_sources.push(sources); @@ -1144,6 +1236,7 @@ mod tests { equality_field_ids: vec![], partition_values: vec![], sequence_number: None, + first_row_id: None, } } @@ -1226,6 +1319,7 @@ mod tests { record_count: 2, partition_values: Vec::new(), seq, + first_row_id: None, }; let files = |entries: &[(&str, i64)]| -> BTreeMap { entries diff --git a/crates/pg2iceberg-iceberg/src/lib.rs b/crates/pg2iceberg-iceberg/src/lib.rs index 162c121..d1d5189 100644 --- a/crates/pg2iceberg-iceberg/src/lib.rs +++ b/crates/pg2iceberg-iceberg/src/lib.rs @@ -169,6 +169,38 @@ pub struct PreparedCommit { pub equality_deletes: Vec, } +/// The Iceberg format version of the tables pg2iceberg creates. Tables +/// that already exist keep theirs: pg2iceberg writes each as its own +/// format version says. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FormatVersion { + /// What every engine reads. + #[default] + V2, + /// Adds row lineage: every row has a `_row_id`, and the sequence + /// number of the commit that last changed it. + V3, +} + +impl FormatVersion { + /// The version numbered `n`: 2 or 3. + pub fn from_number(n: u8) -> Result { + match n { + 2 => Ok(Self::V2), + 3 => Ok(Self::V3), + other => Err(IcebergError::Other(format!( + "unsupported Iceberg format version {other}; expected 2 or 3" + ))), + } + } +} + +/// The field id the spec reserves for row lineage's `_row_id`. +pub const ROW_ID_FIELD_ID: i32 = 2147483540; +/// The field id the spec reserves for row lineage's +/// `_last_updated_sequence_number`. +pub const LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID: i32 = 2147483539; + /// Built by [`crate::compact::compact_table`] and consumed by /// [`Catalog::commit_compaction`]. Produces an `Operation::Replace` /// snapshot: drop everything in `removed_paths`, add everything in @@ -190,8 +222,19 @@ pub struct PreparedCompaction { /// committed while the pass ran still apply to them. (With the new /// snapshot's, a row deleted meanwhile would come back.) pub data_sequence_number: Option, + /// Whether the added files store their rows' lineage (`_row_id`, + /// `_last_updated_sequence_number`), as a pass over a v3 table writes + /// them. A pass that read rows without ids writes none — and must not + /// commit if the rows it moves have since been given theirs (the table + /// upgraded to v3 meanwhile): the catalog refuses it, as a conflict. + pub stores_row_lineage: bool, } +/// What a catalog's conflict error says when a compaction that stores no +/// row lineage would move rows that have row ids (see +/// [`PreparedCompaction::stores_row_lineage`]). +pub const ROW_IDS_CONFLICT: &str = "rewrite moves rows that have row ids, without them"; + #[derive(Clone, Debug)] pub struct DataFile { pub path: String, @@ -209,6 +252,12 @@ pub struct DataFile { /// deletes committed while the pass ran still apply to it. `None`: the /// snapshot's (see [`Self::sequence_number_in`]). pub sequence_number: Option, + /// A v3 data file's first row id, as the catalog assigned it when the + /// file was committed: the `_row_id` of each row the file doesn't + /// store one for is this plus the row's position in the file (row + /// lineage). `None` for a file about to be committed, a delete file, + /// and on a table without row lineage (v2). + pub first_row_id: Option, } impl DataFile { diff --git a/crates/pg2iceberg-iceberg/src/orphan.rs b/crates/pg2iceberg-iceberg/src/orphan.rs index 4285bdd..9d48917 100644 --- a/crates/pg2iceberg-iceberg/src/orphan.rs +++ b/crates/pg2iceberg-iceberg/src/orphan.rs @@ -179,6 +179,7 @@ mod tests { equality_field_ids: vec![], partition_values: vec![], sequence_number: None, + first_row_id: None, }; Snapshot { id, diff --git a/crates/pg2iceberg-iceberg/src/prod/catalog.rs b/crates/pg2iceberg-iceberg/src/prod/catalog.rs index c0400c5..9d086a4 100644 --- a/crates/pg2iceberg-iceberg/src/prod/catalog.rs +++ b/crates/pg2iceberg-iceberg/src/prod/catalog.rs @@ -58,6 +58,8 @@ use crate::{ pub struct IcebergRustCatalog { inner: Arc, manifests: Arc, + /// The format version of the tables it creates. + format_version: crate::FormatVersion, } impl IcebergRustCatalog { @@ -65,6 +67,15 @@ impl IcebergRustCatalog { Self { inner, manifests: Arc::default(), + format_version: crate::FormatVersion::default(), + } + } + + /// Create tables in format `version` (v2 by default). + pub fn with_format_version(self, version: crate::FormatVersion) -> Self { + Self { + format_version: version, + ..self } } } @@ -74,6 +85,7 @@ impl Clone for IcebergRustCatalog { Self { inner: Arc::clone(&self.inner), manifests: Arc::clone(&self.manifests), + format_version: self.format_version, } } } @@ -93,7 +105,11 @@ struct ManifestCache { #[derive(Clone, Default)] struct TableManifests { lists: HashMap>, - manifests: HashMap>, + /// By path and the first row id the list naming the manifest gives + /// it, which its data files inherit theirs from on load: a manifest a + /// table wrote before its upgrade to v3 has none until the next + /// commit's list assigns one. + manifests: HashMap<(String, Option), Arc>, } impl ManifestCache { @@ -173,14 +189,19 @@ impl Catalog for IcebergRustCata // build. The `clone()` on `ice_schema` is the cost of avoiding // a more elaborate dynamic-build dance. // - // Format v2, explicitly: what pg2iceberg writes and is tested - // against, and what every engine reads (v3 isn't yet — ClickHouse, - // open-source Trino). + // The format version explicitly, never the catalog's default: v2 + // unless configured — what every engine reads (v3 isn't yet — + // ClickHouse, open-source Trino). (A REST catalog takes it as the + // `format-version` property, which the fork's client sends.) + let format_version = match self.format_version { + crate::FormatVersion::V2 => FormatVersion::V2, + crate::FormatVersion::V3 => FormatVersion::V3, + }; let creation = if schema.partition_spec.is_empty() { TableCreation::builder() .name(schema.ident.name.clone()) .schema(ice_schema) - .format_version(FormatVersion::V2) + .format_version(format_version) .build() } else { let unbound = to_iceberg_unbound_partition_spec(schema)?; @@ -188,7 +209,7 @@ impl Catalog for IcebergRustCata .name(schema.ident.name.clone()) .schema(ice_schema) .partition_spec(unbound) - .format_version(FormatVersion::V2) + .format_version(format_version) .build() }; let table = self @@ -336,6 +357,9 @@ impl Catalog for IcebergRustCata if let Some(seq) = prepared.data_sequence_number { action = action.set_data_sequence_number(seq); } + if !prepared.stores_row_lineage { + action = action.validate_no_row_ids(); + } let action = action // iceberg-rust refuses a snapshot that adds no files and sets // no summary property (apache/iceberg-rust#1548), and a pass @@ -485,6 +509,11 @@ impl Catalog for IcebergRustCata let mut expired_adds: BTreeMap = BTreeMap::new(); let cached = self.manifests.get(ident); let mut read = TableManifests::default(); + // Each data file's first row id, as the latest snapshot listing it + // gives it — what a reader of the table reads. One the table had + // before its upgrade to v3 gets one only in a later snapshot than + // the one that added it. + let mut first_row_ids: HashMap = HashMap::new(); for snap in snaps { // Use the iceberg snapshot_id for manifest filtering (matches the // `added_snapshot_id` field stored in manifest entries), but report @@ -518,8 +547,8 @@ impl Catalog for IcebergRustCata std::collections::BTreeSet::new(); for entry in manifest_list.entries() { // Each manifest list names every manifest before it. - let path = &entry.manifest_path; - let manifest = match read.manifests.get(path).or(cached.manifests.get(path)) { + let key = (entry.manifest_path.clone(), entry.first_row_id); + let manifest = match read.manifests.get(&key).or(cached.manifests.get(&key)) { Some(manifest) => Arc::clone(manifest), None => Arc::new( table @@ -529,13 +558,16 @@ impl Catalog for IcebergRustCata .map_err(map_iceberg_err)?, ), }; - read.manifests.insert(path.clone(), Arc::clone(&manifest)); + read.manifests.insert(key, Arc::clone(&manifest)); // A `DELETED` entry records a file its snapshot removed (other // engines write them; our rewrite omits removed files): no // file the snapshot holds. for me in manifest.entries().iter().filter(|me| me.is_alive()) { let df = me.data_file(); all_paths_this_snap.insert(df.file_path().to_string()); + if let Some(id) = df.first_row_id() { + first_row_ids.insert(df.file_path().to_string(), id); + } let partition_values = iceberg_struct_to_partition_literals(df.partition()); let mut our = DataFile { @@ -545,6 +577,11 @@ impl Catalog for IcebergRustCata equality_field_ids: df.equality_ids().unwrap_or_default(), partition_values, sequence_number: None, + // Inherited on load, as a v3 reader assigns it. + first_row_id: match df.content_type() { + DataContentType::Data => df.first_row_id(), + _ => None, + }, }; // Match the sim's "files added in this commit" // semantics: only surface entries first introduced @@ -631,6 +668,11 @@ impl Catalog for IcebergRustCata } out.extend(stand_ins.into_values()); out.sort_by_key(|s| s.id); + for df in out.iter_mut().flat_map(|s| &mut s.data_files) { + if let Some(&id) = first_row_ids.get(&df.path) { + df.first_row_id = Some(id); + } + } self.manifests.put(ident, read); Ok(out) } @@ -1403,6 +1445,7 @@ mod tests { equality_field_ids: vec![], partition_values: vec![PartitionLiteral::Long(1)], sequence_number: None, + first_row_id: None, }, DataFile { path: "memory:///warehouse/public/orders/data-qty-2.parquet".into(), @@ -1411,6 +1454,7 @@ mod tests { equality_field_ids: vec![], partition_values: vec![PartitionLiteral::Long(2)], sequence_number: None, + first_row_id: None, }, ], equality_deletes: vec![], @@ -1511,6 +1555,7 @@ mod tests { equality_field_ids: vec![], partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }); } c.commit_snapshot(PreparedCommit { @@ -1566,6 +1611,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }) @@ -1632,6 +1678,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }) @@ -1681,6 +1728,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }) @@ -1699,9 +1747,11 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], removed_paths: original_paths.clone(), data_sequence_number: None, + stores_row_lineage: false, }) .await .unwrap(); @@ -1744,6 +1794,7 @@ mod tests { added_data_files: vec![], removed_paths: vec![], data_sequence_number: None, + stores_row_lineage: false, }) .await .unwrap(); @@ -1768,6 +1819,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }) @@ -1779,6 +1831,7 @@ mod tests { added_data_files: vec![], removed_paths: vec![path.into()], data_sequence_number: None, + stores_row_lineage: false, }) .await .unwrap(); @@ -1815,6 +1868,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }) @@ -1876,6 +1930,7 @@ mod tests { equality_field_ids: eq_ids, partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }; let tick = || tokio::time::sleep(std::time::Duration::from_millis(2)); let append = |data: Vec, deletes: Vec| { @@ -1898,6 +1953,7 @@ mod tests { added_data_files: vec![file("c0", vec![])], removed_paths: vec![file("d0", vec![]).path], data_sequence_number: None, + stores_row_lineage: false, }) .await .unwrap(); @@ -1935,6 +1991,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }) @@ -1981,6 +2038,7 @@ mod tests { equality_field_ids: vec![1], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], }) .await @@ -2009,6 +2067,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![DataFile { path: "memory:///warehouse/public/orders/eq-deletes-0.parquet".into(), @@ -2017,6 +2076,7 @@ mod tests { equality_field_ids: vec![1], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], }) .await @@ -2051,6 +2111,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], }) .await @@ -2227,6 +2288,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }], @@ -2284,6 +2346,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }) @@ -2342,6 +2405,7 @@ mod tests { equality_field_ids, partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }; PreparedCommit { ident: ident(), @@ -2539,6 +2603,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }) @@ -2583,6 +2648,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }; diff --git a/crates/pg2iceberg-iceberg/src/verify.rs b/crates/pg2iceberg-iceberg/src/verify.rs index 1d8b336..7c06d09 100644 --- a/crates/pg2iceberg-iceberg/src/verify.rs +++ b/crates/pg2iceberg-iceberg/src/verify.rs @@ -293,6 +293,7 @@ mod tests { equality_field_ids: vec![], partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }); } for (i, chunk) in prepared.equality_deletes.into_iter().enumerate() { @@ -309,6 +310,7 @@ mod tests { equality_field_ids: prepared.pk_field_ids.clone(), partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }); } cat.snaps diff --git a/crates/pg2iceberg-logical/src/catalog_cache.rs b/crates/pg2iceberg-logical/src/catalog_cache.rs index b495587..867b52f 100644 --- a/crates/pg2iceberg-logical/src/catalog_cache.rs +++ b/crates/pg2iceberg-logical/src/catalog_cache.rs @@ -362,6 +362,7 @@ mod tests { equality_field_ids: Vec::new(), partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: Vec::new(), } diff --git a/crates/pg2iceberg-logical/src/materializer.rs b/crates/pg2iceberg-logical/src/materializer.rs index bce54b7..8af31a3 100644 --- a/crates/pg2iceberg-logical/src/materializer.rs +++ b/crates/pg2iceberg-logical/src/materializer.rs @@ -1077,6 +1077,7 @@ impl Materializer { equality_field_ids: vec![], partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }); } self.catalog @@ -2885,6 +2886,7 @@ impl Materializer { equality_field_ids: vec![], partition_values: chunk.partition_values.clone(), sequence_number: None, + first_row_id: None, }); data_pk_groups.push((path, chunk.pk_keys, chunk.partition_values)); } @@ -2904,6 +2906,7 @@ impl Materializer { equality_field_ids: pk_field_ids.clone(), partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }); deleted_pks.extend(chunk.pk_keys); } @@ -3288,6 +3291,7 @@ impl Materializer { equality_field_ids: vec![], partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }); } // Re-emission of an already-emitted marker would conflict on diff --git a/crates/pg2iceberg-sim/src/catalog.rs b/crates/pg2iceberg-sim/src/catalog.rs index b2bb04b..e64b3e2 100644 --- a/crates/pg2iceberg-sim/src/catalog.rs +++ b/crates/pg2iceberg-sim/src/catalog.rs @@ -9,13 +9,21 @@ //! state: files it added stay live until a later snapshot removes them. //! [`ReaderView`] is that state — what a query engine reading the current //! snapshot sees — independent of what [`Catalog::snapshots`] reports. +//! +//! Tables are created format v2 unless the catalog is told otherwise +//! ([`MemoryCatalog::with_format_version`]), and can be upgraded +//! ([`MemoryCatalog::upgrade_format_version`]). A v3 table has row +//! lineage, as the spec requires: every commit gives each data file it +//! adds the next `record_count` row ids, the first one recorded on the +//! file ([`DataFile::first_row_id`]). A row's id is that plus its +//! position in the file, unless the file stores the row's own. use async_trait::async_trait; use pg2iceberg_core::{Namespace, TableIdent, TableSchema}; use pg2iceberg_iceberg::PreparedCommit; use pg2iceberg_iceberg::{ apply_schema_changes, log_ends_property, merge_log_ends, recorded_log_ends, Catalog, DataFile, - IcebergError, LogRange, Result, SchemaChange, Snapshot, TableMetadata, + FormatVersion, IcebergError, LogRange, Result, SchemaChange, Snapshot, TableMetadata, }; use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex}; @@ -36,9 +44,34 @@ struct MemTable { /// Each snapshot's recorded log ends: those carried forward, and its /// own commit's (see [`LogRange`]). log_ends: BTreeMap>, + format_version: FormatVersion, + /// v3: the row id the next data file's first row takes. + next_row_id: i64, } impl MemTable { + /// Give rows their ids, as a v3 table's commit does: each data file + /// the table holds without any yet — one it had before its upgrade to + /// v3 — and then each of `files`, just committed, takes the next + /// `record_count` of them, in order. Every commit does: its manifest + /// list assigns ids to each manifest it names that has none. A v2 + /// table has no row lineage. + fn assign_row_ids(&mut self, files: &mut [DataFile]) { + if self.format_version != FormatVersion::V3 { + return; + } + let live: BTreeSet = self.live_paths().into_iter().map(str::to_string).collect(); + let earlier = self + .snapshots + .iter_mut() + .flat_map(|s| s.data_files.iter_mut()) + .filter(|f| f.first_row_id.is_none() && live.contains(&f.path)); + for f in earlier.chain(files.iter_mut()) { + f.first_row_id = Some(self.next_row_id); + self.next_row_id += f.record_count as i64; + } + } + /// Record snapshot `id`, just committed: it carries the table's log /// ends forward, with `range`'s. fn record_log_ends(&mut self, id: i64, range: Option<&LogRange>) { @@ -89,6 +122,30 @@ impl MemTable { Ok(()) } + /// Like the fork's `RewriteFilesAction::validate_no_row_ids`: a + /// rewrite whose outputs store no row lineage fails if a data file it + /// removes has row ids — its rows would take new ones. + fn check_row_lineage(&self, prepared: &pg2iceberg_iceberg::PreparedCompaction) -> Result<()> { + if prepared.stores_row_lineage { + return Ok(()); + } + let removed: BTreeSet<&str> = prepared.removed_paths.iter().map(String::as_str).collect(); + let with_ids: Vec<&str> = self + .snapshots + .iter() + .flat_map(|s| &s.data_files) + .filter(|f| f.first_row_id.is_some() && removed.contains(f.path.as_str())) + .map(|f| f.path.as_str()) + .collect(); + if !with_ids.is_empty() { + return Err(IcebergError::Conflict(format!( + "{}: {with_ids:?}", + pg2iceberg_iceberg::ROW_IDS_CONFLICT + ))); + } + Ok(()) + } + /// Bring `metadata.log_ends` in line with the table's properties and /// its unexpired snapshots. fn refresh_log_ends(&mut self) { @@ -106,6 +163,8 @@ impl MemTable { #[derive(Default, Clone)] pub struct MemoryCatalog { state: Arc>, + /// The format version of the tables it creates. + format_version: FormatVersion, } impl MemoryCatalog { @@ -113,6 +172,15 @@ impl MemoryCatalog { Self::default() } + /// Create tables in format `version` (v2 by default), as + /// `IcebergRustCatalog::with_format_version` does. + pub fn with_format_version(self, version: FormatVersion) -> Self { + Self { + format_version: version, + ..self + } + } + /// Every snapshot ever committed to `ident`, expired or not: the /// history a reader of the current snapshot effectively sees. pub fn history(&self, ident: &TableIdent) -> Vec { @@ -123,6 +191,44 @@ impl MemoryCatalog { .unwrap_or_default() } + /// Test hook: upgrade `ident` to format `version`, as another engine + /// does (`ALTER TABLE … SET TBLPROPERTIES ('format-version' = '3')`). + /// Upgrading to v3 starts the table's row ids at 0; the rows it holds + /// get theirs at its next commit. + pub fn upgrade_format_version(&self, ident: &TableIdent, version: FormatVersion) -> Result<()> { + let mut s = self.state.lock().unwrap(); + let table = s + .tables + .get_mut(ident) + .ok_or_else(|| IcebergError::NotFound(format!("table: {ident}")))?; + if (table.format_version, version) == (FormatVersion::V3, FormatVersion::V2) { + return Err(IcebergError::Other(format!( + "cannot downgrade {ident} from v3 to v2" + ))); + } + if version == FormatVersion::V3 && table.format_version != FormatVersion::V3 { + table.next_row_id = 0; + } + table.format_version = version; + Ok(()) + } + + /// The table's format version. + pub fn format_version(&self, ident: &TableIdent) -> Option { + let s = self.state.lock().unwrap(); + s.tables.get(ident).map(|t| t.format_version) + } + + /// v3: the row id the table's next data file starts at — above every + /// id its rows have been given. + pub fn next_row_id(&self, ident: &TableIdent) -> Option { + let s = self.state.lock().unwrap(); + s.tables + .get(ident) + .filter(|t| t.format_version == FormatVersion::V3) + .map(|t| t.next_row_id) + } + /// Test hook: stamp a config map onto a table's metadata. The /// vended-credentials router consumes /// [`TableMetadata::config`] to extract S3 creds, but @@ -165,6 +271,7 @@ impl MemoryCatalog { .get_mut(&prepared.ident) .ok_or_else(|| IcebergError::NotFound(format!("table: {}", prepared.ident)))?; table.check_removable(&prepared.removed_paths)?; + table.check_row_lineage(&prepared)?; if own_sequence_number { let read = prepared.data_sequence_number.unwrap_or(0); let since = table @@ -186,7 +293,7 @@ impl MemoryCatalog { } else { prepared.data_sequence_number }; - let added = prepared + let mut added: Vec = prepared .added_data_files .into_iter() .map(|f| DataFile { @@ -194,6 +301,7 @@ impl MemoryCatalog { ..f }) .collect(); + table.assign_row_ids(&mut added); table.snapshots.push(Snapshot { id, data_files: added, @@ -222,6 +330,7 @@ impl MemoryCatalog { .get_mut(ident) .ok_or_else(|| IcebergError::NotFound(format!("table: {ident}")))?; table.check_removable(&paths)?; + table.assign_row_ids(&mut []); let id = table.next_snapshot_id; table.next_snapshot_id += 1; table.snapshots.push(Snapshot { @@ -287,6 +396,8 @@ impl Catalog for MemoryCatalog { expired: BTreeSet::new(), next_snapshot_id: 1, log_ends: BTreeMap::new(), + format_version: self.format_version, + next_row_id: 0, }, ); Ok(metadata) @@ -311,9 +422,11 @@ impl Catalog for MemoryCatalog { // snapshot id (multiplied by 1000 so retention-by-ms tests can // discriminate). Real catalogs surface iceberg's own // wall-clock millis here. + let mut data_files = prepared.data_files; + table.assign_row_ids(&mut data_files); table.snapshots.push(Snapshot { id, - data_files: prepared.data_files, + data_files, delete_files: prepared.equality_deletes, removed_paths: Vec::new(), timestamp_ms: id * 1000, @@ -353,9 +466,11 @@ impl Catalog for MemoryCatalog { } let id = table.next_snapshot_id; table.next_snapshot_id += 1; + let mut data_files = step.data_files; + table.assign_row_ids(&mut data_files); table.snapshots.push(Snapshot { id, - data_files: step.data_files, + data_files, delete_files: step.equality_deletes, removed_paths: Vec::new(), timestamp_ms: id * 1000, @@ -399,10 +514,11 @@ impl Catalog for MemoryCatalog { return Ok(table.metadata.clone()); } table.check_removable(&prepared.removed_paths)?; + table.check_row_lineage(&prepared)?; let id = table.next_snapshot_id; table.next_snapshot_id += 1; - let added = prepared + let mut added: Vec = prepared .added_data_files .into_iter() .map(|f| DataFile { @@ -410,6 +526,7 @@ impl Catalog for MemoryCatalog { ..f }) .collect(); + table.assign_row_ids(&mut added); table.snapshots.push(Snapshot { id, data_files: added, @@ -608,6 +725,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], })) @@ -751,6 +869,7 @@ mod tests { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }], diff --git a/crates/pg2iceberg-sim/src/oracle.rs b/crates/pg2iceberg-sim/src/oracle.rs index 6ce63a6..cdf113d 100644 --- a/crates/pg2iceberg-sim/src/oracle.rs +++ b/crates/pg2iceberg-sim/src/oracle.rs @@ -15,13 +15,23 @@ //! - The live file set comes from the table's actual state — the full //! history ([`live_files_from_history`]) or the current snapshot's //! manifests — never from `Catalog::snapshots`. +//! - On a v3 table, each row's lineage ([`engine_read_lineage`]) is what +//! the spec says it is: the `_row_id` and +//! `_last_updated_sequence_number` the file stores for it, or else +//! inherited — the file's first row id plus the row's position in it, +//! and the file's data sequence number. use pg2iceberg_core::partition::{apply_transform, PartitionLiteral}; -use pg2iceberg_core::{ColumnName, ColumnSchema, Row, TableSchema}; +use pg2iceberg_core::{ColumnName, ColumnSchema, IcebergType, PgValue, Row, TableSchema}; use pg2iceberg_iceberg::{read_data_file_by_field_id, PkKey, Snapshot}; use pg2iceberg_stream::BlobStore; use std::collections::{BTreeMap, BTreeSet}; +/// The field id the spec reserves for `_row_id`. +pub const ROW_ID_FIELD_ID: i32 = 2147483540; +/// The field id the spec reserves for `_last_updated_sequence_number`. +pub const LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID: i32 = 2147483539; + /// A data or equality-delete file in the table's current state. #[derive(Clone, Debug)] pub struct LiveFile { @@ -32,12 +42,30 @@ pub struct LiveFile { pub partition: Vec, /// `Some(equality field ids)` for an equality-delete file. pub equality_ids: Option>, + /// A v3 data file's first row id: the id of each row it doesn't store + /// one for is this plus the row's position in the file. `None` for a + /// delete file, and on a table without row lineage (v2). + pub first_row_id: Option, +} + +/// A row as a v3 engine reads it, with its lineage. +#[derive(Clone, Debug, PartialEq)] +pub struct LineageRow { + pub row: Row, + /// `_row_id`: the one the file stores for the row, else the file's + /// first row id plus the row's position. `None` if neither exists. + pub row_id: Option, + /// `_last_updated_sequence_number`: the one the file stores for the + /// row, else — in a file with row lineage — the file's data sequence + /// number. + pub last_updated: Option, } /// The live files of a table whose full commit history is `snapshots`: /// every file a snapshot added that no snapshot removed, at its data /// sequence number — the snapshot's, unless the file keeps an older one -/// (a compaction's output). +/// (a compaction's output) — and, on a v3 table, at the first row id +/// its commit gave it. pub fn live_files_from_history(snapshots: &[Snapshot]) -> Vec { let removed: BTreeSet<&str> = snapshots .iter() @@ -45,18 +73,23 @@ pub fn live_files_from_history(snapshots: &[Snapshot]) -> Vec { .collect(); let mut out = Vec::new(); for snap in snapshots { - let files = snap.data_files.iter().map(|f| (f, None)).chain( - snap.delete_files - .iter() - .map(|f| (f, Some(f.equality_field_ids.clone()))), - ); - for (f, equality_ids) in files { + let files = snap + .data_files + .iter() + .map(|f| (f, None, f.first_row_id)) + .chain( + snap.delete_files + .iter() + .map(|f| (f, Some(f.equality_field_ids.clone()), None)), + ); + for (f, equality_ids, first_row_id) in files { if !removed.contains(f.path.as_str()) { out.push(LiveFile { path: f.path.clone(), seq: f.sequence_number_in(snap), partition: f.partition_values.clone(), equality_ids, + first_row_id, }); } } @@ -71,6 +104,19 @@ pub async fn engine_read( schema: &TableSchema, files: &[LiveFile], ) -> Result, String> { + Ok(engine_read_lineage(blob, schema, files) + .await? + .into_iter() + .map(|r| r.row) + .collect()) +} + +/// [`engine_read`], with each row's lineage. +pub async fn engine_read_lineage( + blob: &dyn BlobStore, + schema: &TableSchema, + files: &[LiveFile], +) -> Result, String> { let by_id: BTreeMap = schema.columns.iter().map(|c| (c.field_id, c)).collect(); @@ -108,9 +154,20 @@ pub async fn engine_read( }); } + let row_id_col = lineage_column("_row_id", ROW_ID_FIELD_ID); + let seq_col = lineage_column( + "_last_updated_sequence_number", + LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID, + ); + let mut cols = schema.columns.clone(); + cols.extend([row_id_col.clone(), seq_col.clone()]); let mut out = Vec::new(); for f in files.iter().filter(|f| f.equality_ids.is_none()) { - for row in read(blob, &f.path, &schema.columns).await? { + for (pos, mut row) in read(blob, &f.path, &cols).await?.into_iter().enumerate() { + let stored_id = + take_long(&mut row, &row_id_col).map_err(|e| format!("{}: {e}", f.path))?; + let stored_seq = + take_long(&mut row, &seq_col).map_err(|e| format!("{}: {e}", f.path))?; let deleted = deletes.iter().any(|d| { d.seq > f.seq && d.partition == f.partition @@ -126,12 +183,36 @@ pub async fn engine_read( f.path, f.partition )); } - out.push(row); + out.push(LineageRow { + row, + row_id: stored_id.or(f.first_row_id.map(|first| first + pos as i64)), + last_updated: stored_seq.or(f.first_row_id.map(|_| f.seq)), + }); } } Ok(out) } +/// One of the two columns of row lineage the spec reserves. +fn lineage_column(name: &str, field_id: i32) -> ColumnSchema { + ColumnSchema { + name: name.into(), + field_id, + ty: IcebergType::Long, + nullable: true, + is_primary_key: false, + } +} + +/// Remove `col` from `row`: the value the file stores for it, if any. +fn take_long(row: &mut Row, col: &ColumnSchema) -> Result, String> { + match row.remove(&ColumnName(col.name.clone())) { + None | Some(PgValue::Null) => Ok(None), + Some(PgValue::Int8(n)) => Ok(Some(n)), + Some(other) => Err(format!("{} holds {other:?}", col.name)), + } +} + /// The partition `row`'s values compute to under `schema`'s spec. fn partition_of(row: &Row, schema: &TableSchema) -> Result, String> { schema diff --git a/crates/pg2iceberg-tests/tests/cleanup_orphans_e2e.rs b/crates/pg2iceberg-tests/tests/cleanup_orphans_e2e.rs index f2da403..71bc688 100644 --- a/crates/pg2iceberg-tests/tests/cleanup_orphans_e2e.rs +++ b/crates/pg2iceberg-tests/tests/cleanup_orphans_e2e.rs @@ -119,6 +119,7 @@ impl Harness { equality_field_ids: vec![], partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], })) @@ -323,9 +324,11 @@ fn compaction_replaced_files_become_orphans() { equality_field_ids: vec![], partition_values: vec![], sequence_number: None, + first_row_id: None, }], removed_paths: vec![p1.clone(), p2.clone(), p3.clone()], data_sequence_number: None, + stores_row_lineage: false, })) .unwrap(); @@ -445,6 +448,7 @@ fn cleanup_keeps_other_tables_files() { equality_field_ids: vec![], partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], })) diff --git a/crates/pg2iceberg-tests/tests/compact_e2e.rs b/crates/pg2iceberg-tests/tests/compact_e2e.rs index 1e7016a..c7018de 100644 --- a/crates/pg2iceberg-tests/tests/compact_e2e.rs +++ b/crates/pg2iceberg-tests/tests/compact_e2e.rs @@ -147,6 +147,7 @@ impl Harness { equality_field_ids: vec![], partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }); } let mut delete_files = Vec::new(); @@ -161,6 +162,7 @@ impl Harness { equality_field_ids: prepared.pk_field_ids.clone(), partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, }); } block_on(self.cat.commit_snapshot(PreparedCommit { @@ -1114,3 +1116,60 @@ fn deletes_wait_until_they_kill_enough_rows_to_rewrite_for() { assert_eq!(out.input_delete_files, 6); assert_eq!(visible_id_qty(&h, &s).len(), 890); } + +/// Each row of `s`, by id, with its `_row_id` and +/// `_last_updated_sequence_number` as a v3 engine reads them. +fn lineage(h: &Harness, s: &TableSchema) -> BTreeMap { + use pg2iceberg_sim::oracle::{engine_read_lineage, live_files_from_history}; + let files = live_files_from_history(&h.cat.history(&s.ident)); + block_on(engine_read_lineage(h.blob.as_ref(), s, &files)) + .unwrap() + .into_iter() + .map(|r| { + let Some(PgValue::Int4(id)) = r.row.get(&ColumnName("id".into())) else { + panic!("no id in {:?}", r.row); + }; + let lineage = (r.row_id.expect("row id"), r.last_updated.expect("seq")); + (*id, lineage) + }) + .collect() +} + +/// On a v3 table, a pass keeps the lineage of each row it moves, however +/// many record batches it reads them in: a row's position runs on across +/// them. +#[test] +fn compaction_keeps_row_lineage_across_record_batches() { + let h = Harness { + cat: MemoryCatalog::new().with_format_version(pg2iceberg_iceberg::FormatVersion::V3), + ..Harness::new() + }; + let s = schema_id_qty(); + ensure_table(&h, &s); + let n = 3 * pg2iceberg_iceberg::compact::DECODE_BATCH_ROWS as i32 + 7; + let files = h.commit( + &s, + (1..=n) + .map(|i| MaterializedRow { + op: Op::Insert, + row: row_id_qty(i, i), + unchanged_cols: vec![], + unchanged_from: None, + }) + .collect(), + ); + assert_eq!(files.len(), 1, "one input file"); + delete(&h, &s, pk_only(1)); + let before = lineage(&h, &s); + let cfg = CompactionConfig { + data_file_threshold: 1, + delete_file_threshold: 1, + target_size_bytes: 1024 * 1024 * 1024, + ..Default::default() + }; + let out = h.run_compaction(&s, &cfg).expect("compaction should run"); + assert_eq!(out.rows_rewritten, n as u64); + let after = lineage(&h, &s); + assert_eq!(after.len(), n as usize - 1); + assert_eq!(before, after); +} diff --git a/crates/pg2iceberg-tests/tests/dst.rs b/crates/pg2iceberg-tests/tests/dst.rs index 62b2711..8a27002 100644 --- a/crates/pg2iceberg-tests/tests/dst.rs +++ b/crates/pg2iceberg-tests/tests/dst.rs @@ -54,6 +54,16 @@ //! before a commit (`CleanupOrphansMidCommit`) — and ends it once it //! commits or gives them up (19). //! +//! In `FORMAT_V3` cases the tables are Iceberg v3 — created so, or +//! upgraded by another engine — whose row lineage +//! gives every row an id (`_row_id`) and the sequence number of the +//! commit that last changed it (`_last_updated_sequence_number`). Every +//! live row has both, and no two rows share an id (invariant 20); a +//! commit that only moves rows — anyone's compaction, a removal of +//! delete files — keeps each row's (21). An update through an equality +//! delete may give the row a new id: the spec allows it, since the +//! writer never read the old one. +//! //! Each case runs on the sims or — under the `integration` feature — on //! production's catalog and blob store (`IcebergRustCatalog` over //! iceberg-rust's memory catalog, `ObjectStoreBlobStore` over @@ -81,7 +91,7 @@ use pg2iceberg_sim::catalog::MemoryCatalog; use pg2iceberg_sim::clock::TestClock; use pg2iceberg_sim::coord::MemoryCoordinator; use pg2iceberg_sim::id::SeqIdGen; -use pg2iceberg_sim::oracle::{engine_read, live_files_from_history, LiveFile}; +use pg2iceberg_sim::oracle::{engine_read, engine_read_lineage, live_files_from_history, LiveFile}; use pg2iceberg_sim::pgoutput::ReplicaIdentity; use pg2iceberg_sim::postgres::{SimPostgres, SimReplicationStream}; use pg2iceberg_snapshot::Snapshotter; @@ -525,6 +535,9 @@ enum Step { /// The catalog drops the delete files no data file is left for them to /// apply to, in a `delete` snapshot. CatalogRemoveDanglingDeletes, + /// In `V3_UPGRADED` cases, another engine upgrades the tables to v3 + /// now, whatever they hold, if it hasn't yet. + UpgradeToV3, } fn step_strategy() -> impl Strategy { @@ -578,6 +591,7 @@ fn step_strategy() -> impl Strategy { 1 => Just(Step::CatalogCompactMidCommit), 1 => Just(Step::CatalogExpire), 1 => Just(Step::CatalogRemoveDanglingDeletes), + 1 => Just(Step::UpgradeToV3), ] } @@ -647,8 +661,30 @@ thread_local! { /// Whether this case's materializers hold a FileIndex of one key: /// past it, files are evicted, and keys found by reading them. static BOUNDED_INDEX: std::cell::Cell = const { std::cell::Cell::new(false) }; + /// Whether this case's tables are Iceberg v3, with row lineage: + /// pg2iceberg creates them v3 (`sink.format_version`), unless + /// `V3_UPGRADED`. + static FORMAT_V3: std::cell::Cell = const { std::cell::Cell::new(false) }; + /// With `FORMAT_V3`: pg2iceberg creates the tables v2, and another + /// engine upgrades them to v3 while pg2iceberg runs, whatever they + /// hold (`Step::UpgradeToV3`, or at quiescence) — tables pg2iceberg + /// finds v3 whatever its setting. + static V3_UPGRADED: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// The format version pg2iceberg creates this case's tables in. +fn created_format_version() -> pg2iceberg_iceberg::FormatVersion { + if FORMAT_V3.get() && !V3_UPGRADED.get() { + pg2iceberg_iceberg::FormatVersion::V3 + } else { + pg2iceberg_iceberg::FormatVersion::V2 + } } +/// A v3 table's rows by primary key, each with its `_row_id` and +/// `_last_updated_sequence_number`. +type Lineage = BTreeMap; + /// A FileIndex budget of one key (see `FileIndex::with_memory`): the /// workloads' keys are a handful. const TINY_INDEX_BYTES: usize = 64; @@ -738,7 +774,7 @@ impl Storage { } fn sim() -> Self { - let catalog = Arc::new(MemoryCatalog::new()); + let catalog = Arc::new(MemoryCatalog::new().with_format_version(created_format_version())); let blob = Arc::new(MemoryBlobStore::new()); Self { catalog: catalog.clone(), @@ -795,6 +831,80 @@ impl Storage { } } + /// Upgrade `ident`, which has no snapshots yet, to format v3. + async fn upgrade_to_v3(&self, ident: &TableIdent) -> Result<(), String> { + match &self.backend { + Backend::Sim { catalog, .. } => catalog + .upgrade_format_version(ident, pg2iceberg_iceberg::FormatVersion::V3) + .map_err(|e| e.to_string()), + #[cfg(feature = "integration")] + Backend::Prod { iceberg } => prod_backend::upgrade_to_v3(iceberg, ident).await, + } + } + + /// The table's row lineage, if it's a v3 table: its rows, by primary + /// key, each with its row id and last-updated sequence number. + /// + /// 20. Row lineage is whole: every live row of a v3 table has an id + /// and a last-updated sequence number, no two rows share an id, + /// and every id is below the table's next one — which the next + /// commit's rows take theirs from. (A table upgraded from v2 + /// holds rows without, until its next commit gives them theirs.) + /// Rows without are left out. + async fn lineage(&self, ident: &TableIdent) -> Result, String> { + let next_row_id = match &self.backend { + Backend::Sim { catalog, .. } => catalog.next_row_id(ident), + #[cfg(feature = "integration")] + Backend::Prod { iceberg } => prod_backend::next_row_id(iceberg, ident).await?, + }; + let Some(next_row_id) = next_row_id else { + return Ok(None); + }; + let files = self.live_files(ident).await?; + let schema = self + .catalog + .load_table(ident) + .await + .map_err(|e| format!("load_table: {e}"))? + .ok_or_else(|| format!("no table {ident}"))? + .schema; + let rows = engine_read_lineage(self.blob.as_ref(), &schema, &files).await?; + let pk = [ColumnName("id".into())]; + let mut lineage = Lineage::new(); + let mut ids = BTreeMap::new(); + for r in rows { + // A table upgraded from v2 gives its rows ids at its next + // commit; until then, it has given out none. + if (r.row_id, r.last_updated, next_row_id) == (None, None, 0) { + continue; + } + let (Some(row_id), Some(last_updated)) = (r.row_id, r.last_updated) else { + return Err(format!( + "invariant 20 (row lineage): {ident}'s row {:?} has row id {:?} and \ + last-updated sequence number {:?}", + r.row, r.row_id, r.last_updated + )); + }; + if row_id >= next_row_id { + return Err(format!( + "invariant 20 (row lineage): {ident}'s row {:?} has row id {row_id}, but the \ + table's next row id is {next_row_id}", + r.row + )); + } + if let Some(other) = ids.insert(row_id, r.row.clone()) { + return Err(format!( + "invariant 20 (row lineage): {ident}'s rows {other:?} and {:?} share row id \ + {row_id}", + r.row + )); + } + let key = pg2iceberg_iceberg::PkKey::from_row(&r.row, &pk); + lineage.insert(key, (r.row, row_id, last_updated)); + } + Ok(Some(lineage)) + } + /// The current snapshot's summary counts the files it holds. (The sim /// catalog writes no summaries.) #[cfg_attr(not(feature = "integration"), allow(unused_variables))] @@ -842,6 +952,7 @@ impl Storage { if own_sequence_number { None } else { read }, pass.removed_paths.into_iter().collect(), if own_sequence_number { read } else { None }, + !pass.stores_row_lineage, ) .await } @@ -867,6 +978,7 @@ impl Storage { None, paths.into_iter().collect(), None, + false, ) .await } @@ -922,7 +1034,7 @@ impl Storage { mod prod_backend { use super::*; use iceberg::memory::{MemoryCatalogBuilder, MEMORY_CATALOG_WAREHOUSE}; - use iceberg::spec::DataContentType; + use iceberg::spec::{DataContentType, FormatVersion}; use iceberg::{CatalogBuilder, NamespaceIdent}; use pg2iceberg_iceberg::prod::IcebergRustCatalog; use pg2iceberg_stream::ObjectStoreBlobStore; @@ -943,7 +1055,10 @@ mod prod_backend { object_store::memory::InMemory::new(), )))); Storage { - catalog: Arc::new(IcebergRustCatalog::new(iceberg.clone())), + catalog: Arc::new( + IcebergRustCatalog::new(iceberg.clone()) + .with_format_version(created_format_version()), + ), blob, backend: Backend::Prod { iceberg }, } @@ -1037,9 +1152,13 @@ mod prod_backend { .map_err(err)?; for entry in manifest.entries().iter().filter(|e| e.is_alive()) { let df = entry.data_file(); - let equality_ids = match df.content_type() { - DataContentType::Data => None, - DataContentType::EqualityDeletes => Some(df.equality_ids().unwrap_or_default()), + // A v3 data file's first row id: iceberg-rust inherits it on + // load, as Java's `ManifestReader` does. + let (equality_ids, first_row_id) = match df.content_type() { + DataContentType::Data => (None, df.first_row_id()), + DataContentType::EqualityDeletes => { + (Some(df.equality_ids().unwrap_or_default()), None) + } DataContentType::PositionDeletes => { return Err(format!("unexpected position delete {}", df.file_path())) } @@ -1055,12 +1174,54 @@ mod prod_backend { .map(partition_literal) .collect::>()?, equality_ids, + first_row_id, }); } } Ok(out) } + async fn load( + iceberg: &iceberg::memory::MemoryCatalog, + ident: &TableIdent, + ) -> Result { + use iceberg::Catalog as _; + let err = |e: iceberg::Error| e.to_string(); + let ns = NamespaceIdent::from_strs(&ident.namespace.0).map_err(err)?; + iceberg + .load_table(&iceberg::TableIdent::new(ns, ident.name.clone())) + .await + .map_err(err) + } + + /// Upgrade `ident` to format v3, as another engine does. + pub async fn upgrade_to_v3( + iceberg: &Arc, + ident: &TableIdent, + ) -> Result<(), String> { + use iceberg::transaction::{ApplyTransactionAction, Transaction}; + let table = load(iceberg, ident).await?; + let tx = Transaction::new(&table); + tx.upgrade_table_version() + .set_format_version(FormatVersion::V3) + .apply(tx) + .map_err(|e| e.to_string())? + .commit(iceberg.as_ref()) + .await + .map_err(|e| e.to_string())?; + Ok(()) + } + + /// v3: the table's next row id. `None` without row lineage. + pub async fn next_row_id( + iceberg: &iceberg::memory::MemoryCatalog, + ident: &TableIdent, + ) -> Result, String> { + let table = load(iceberg, ident).await?; + let meta = table.metadata(); + Ok((meta.format_version() == FormatVersion::V3).then(|| meta.next_row_id() as i64)) + } + /// The harness partitions by `truncate(qty)`: an int, or null. fn partition_literal( value: Option<&iceberg::spec::Literal>, @@ -1079,10 +1240,18 @@ mod prod_backend { /// `removed` is rewritten with that file as a `DELETED` entry and the /// rest as `EXISTING`; `added` goes in a manifest of its own, at data /// sequence number `added_seq` (`None`: the commit's own); the summary - /// counts the table and holds nothing of pg2iceberg's. With + /// counts the table and holds nothing of pg2iceberg's. On a v3 table, + /// an `EXISTING` data file records the first row id it had (loaded + /// with it), so its rows keep their ids, and the snapshot takes a row + /// range for the rows of the manifests that record none. (`added`'s rows would keep + /// theirs as Java's rewrite keeps them — stored in the files — only if + /// the files store them: they're pg2iceberg's compaction's.) With /// `no_deletes_since`, the commit fails if a live delete file is newer /// than that sequence number, as Java's rewrite validates when its - /// outputs take the commit's own number. + /// outputs take the commit's own number. With `no_row_ids` — `added` + /// stores no row lineage — it fails if a removed data file's rows + /// have ids, as the sim's rewrites and pg2iceberg's do. + #[allow(clippy::too_many_arguments)] pub async fn commit_foreign( iceberg: &Arc, ident: &TableIdent, @@ -1091,6 +1260,7 @@ mod prod_backend { added_seq: Option, removed: BTreeSet, no_deletes_since: Option, + no_row_ids: bool, ) -> pg2iceberg_iceberg::Result<()> { use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::Catalog as _; @@ -1111,6 +1281,7 @@ mod prod_backend { added_seq, removed, no_deletes_since, + no_row_ids, }; action .apply(Transaction::new(&table)) @@ -1127,6 +1298,7 @@ mod prod_backend { added_seq: Option, removed: BTreeSet, no_deletes_since: Option, + no_row_ids: bool, } /// The `total-*` summary properties of a table holding the files @@ -1191,6 +1363,7 @@ mod prod_backend { let file_io = table.file_io(); let schema = meta.current_schema().clone(); let spec = meta.default_partition_spec().as_ref().clone(); + let v3 = meta.format_version() == FormatVersion::V3; let mut written = 0; let mut new_manifest = |content| -> iceberg::Result { written += 1; @@ -1204,9 +1377,11 @@ mod prod_backend { schema.clone(), spec.clone(), ); - Ok(match content { - ManifestContentType::Data => builder.build_v2_data(), - ManifestContentType::Deletes => builder.build_v2_deletes(), + Ok(match (content, v3) { + (ManifestContentType::Data, false) => builder.build_v2_data(), + (ManifestContentType::Deletes, false) => builder.build_v2_deletes(), + (ManifestContentType::Data, true) => builder.build_v3_data(), + (ManifestContentType::Deletes, true) => builder.build_v3_deletes(), }) }; @@ -1238,8 +1413,20 @@ mod prod_backend { } let mut writer = new_manifest(manifest_file.content)?; for e in alive { + // On v3, with the first row id it was loaded with. let df = e.data_file().clone(); let entry_seq = e.sequence_number().expect("inherited on load"); + if self.removed.contains(df.file_path()) + && self.no_row_ids + && df.content_type() == DataContentType::Data + && df.first_row_id().is_some() + { + return Err(conflict(format!( + "{}: {}", + pg2iceberg_iceberg::ROW_IDS_CONFLICT, + df.file_path() + ))); + } if self.removed.contains(df.file_path()) { found.insert(df.file_path().to_string()); writer.add_delete_file(df, entry_seq, e.file_sequence_number)?; @@ -1276,13 +1463,18 @@ mod prod_backend { "{}/metadata/snap-foreign-{snapshot_id}.avro", meta.location() ); - let mut list_writer = ManifestListWriter::v2( - file_io.new_output(&list_path)?.writer().await?, - snapshot_id, - Some(current.snapshot_id()), - seq, - ); + let list_out = file_io.new_output(&list_path)?.writer().await?; + let parent = Some(current.snapshot_id()); + let first_row_id = meta.next_row_id(); + let mut list_writer = if v3 { + ManifestListWriter::v3(list_out, snapshot_id, parent, seq, Some(first_row_id)) + } else { + ManifestListWriter::v2(list_out, snapshot_id, parent, seq) + }; list_writer.add_manifests(manifests.into_iter())?; + let row_range = list_writer + .next_row_id() + .map(|next| (first_row_id, next - first_row_id)); list_writer.close().await?; let now_ms = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -1291,15 +1483,18 @@ mod prod_backend { let snapshot = iceberg::spec::Snapshot::builder() .with_manifest_list(list_path) .with_snapshot_id(snapshot_id) - .with_parent_snapshot_id(Some(current.snapshot_id())) + .with_parent_snapshot_id(parent) .with_sequence_number(seq) .with_summary(Summary { operation: self.operation.clone(), additional_properties: totals.properties(), }) .with_schema_id(meta.current_schema_id()) - .with_timestamp_ms(now_ms) - .build(); + .with_timestamp_ms(now_ms); + let snapshot = match row_range { + Some((first, added)) => snapshot.with_row_range(first, added).build(), + None => snapshot.build(), + }; Ok(iceberg::transaction::ActionCommit::new( vec![ iceberg::TableUpdate::AddSnapshot { snapshot }, @@ -1558,6 +1753,9 @@ struct DstHarness { /// yet committed, and whether its outputs take the commit's sequence /// number (see [`Step::CatalogCompactPlan`]). pending_catalog: Option<(PreparedCompaction, bool)>, + /// Whether another engine has upgraded the tables to v3 + /// (`V3_UPGRADED`). + upgraded: bool, /// The tokio runtime prod-backend cases run iceberg-rust on (see /// [`tokio_runtime`]), entered for the case. Dropped last. #[cfg(feature = "integration")] @@ -1694,6 +1892,7 @@ impl DstHarness { id_gen, pending_external: None, pending_catalog: None, + upgraded: false, #[cfg(feature = "integration")] _tokio: tokio, backfilling: false, @@ -1832,6 +2031,7 @@ impl DstHarness { id_gen, pending_external: None, pending_catalog: None, + upgraded: false, #[cfg(feature = "integration")] _tokio: tokio, backfilling: false, @@ -2081,6 +2281,23 @@ impl DstHarness { } } + /// In `V3_UPGRADED` cases, another engine upgrades the tables to v3, + /// once: pg2iceberg finds them v3 from its next read on. + fn upgrade_to_v3(&mut self) { + if !V3_UPGRADED.get() || self.upgraded { + return; + } + self.upgraded = true; + for table in [ident(), other_ident()] { + if block_on(self.storage.catalog.load_table(&table)) + .unwrap() + .is_some() + { + block_on(self.storage.upgrade_to_v3(&table)).unwrap(); + } + } + } + /// The catalog drops each delete file older than every data file: /// none is left for it to apply to. fn catalog_remove_dangling_deletes(&mut self) { @@ -2101,7 +2318,11 @@ impl DstHarness { if dangling.is_empty() { return; } - match block_on(self.storage.commit_catalog_removal(&ident(), dangling)) { + match block_on(self.audited.keeping_lineage( + &ident(), + "the catalog's removal of delete files", + self.storage.commit_catalog_removal(&ident(), dangling), + )) { Ok(()) | Err(pg2iceberg_iceberg::IcebergError::Conflict(_)) => {} Err(e) => panic!("catalog removal commit: {e}"), } @@ -2194,6 +2415,12 @@ impl DstHarness { if e.to_string() .to_lowercase() .contains("rewrite removes files no longer in the table") => {} + // A pass that read rows before they had ids, which they've had + // since (the table upgraded to v3): it plans again. + Err(e) + if e.to_string() + .to_lowercase() + .contains(pg2iceberg_iceberg::ROW_IDS_CONFLICT) => {} Err(e) => panic!("compaction: {e}"), } // The materializer updates its FileIndex from what the pass @@ -2584,9 +2811,14 @@ impl DstHarness { } => self.catalog_compact_plan(*own_sequence_number), Step::CatalogCompactCommit => { if let Some((pass, own_sequence_number)) = self.pending_catalog.take() { + let table = pass.ident.clone(); match block_on( - self.storage - .commit_catalog_rewrite(pass, own_sequence_number), + self.audited.keeping_lineage( + &table, + "the catalog's compaction", + self.storage + .commit_catalog_rewrite(pass, own_sequence_number), + ), ) { // Lost to another rewrite of its files, or to a // delete: the catalog plans again next time. @@ -2607,6 +2839,7 @@ impl DstHarness { } } Step::CatalogRemoveDanglingDeletes => self.catalog_remove_dangling_deletes(), + Step::UpgradeToV3 => self.upgrade_to_v3(), Step::LoseCommitResponse => self .audited .lose_next_response @@ -3137,7 +3370,14 @@ impl AuditedCatalog { None => return Ok(()), Some(HeldPass::Job(pass)) => { let table = pass.ident.clone(); - let result = self.inner.commit_compaction(pass).await.map(drop); + let result = self + .keeping_lineage( + &table, + "a compaction job", + self.inner.commit_compaction(pass), + ) + .await + .map(drop); self.coord.bump_table_epoch(&table).await.unwrap(); self.coord .end_write(&table, writers::COMPACT_COMMAND) @@ -3147,9 +3387,14 @@ impl AuditedCatalog { } // Another engine's: no pg2iceberg process hears of it. Some(HeldPass::Catalog(pass, own_sequence_number)) => { - self.storage - .commit_catalog_rewrite(pass, own_sequence_number) - .await + let table = pass.ident.clone(); + self.keeping_lineage( + &table, + "the catalog's compaction", + self.storage + .commit_catalog_rewrite(pass, own_sequence_number), + ) + .await } }; match result { @@ -3203,9 +3448,64 @@ impl AuditedCatalog { .unwrap() .push(format!("after a commit: {e}")); } + if let Err(e) = self.storage.lineage(&ident()).await { + self.violations + .lock() + .unwrap() + .push(format!("after a commit: {e}")); + } + } + + /// Run `commit`, `who`'s, which only moves rows: a compaction's, or a + /// removal of delete files. + /// + /// 21. A commit that only moves rows keeps their lineage: on a v3 + /// table, each row it leaves live has the row id and last-updated + /// sequence number it had before. Engines that read the table + /// incrementally tell rows apart by id, and tell what changed by + /// the sequence number. + async fn keeping_lineage( + &self, + ident: &TableIdent, + who: &str, + commit: impl std::future::Future>, + ) -> pg2iceberg_iceberg::Result { + let before = self.storage.lineage(ident).await; + let result = commit.await; + // Whether or not it says it applied: a lost response applied. + let after = self.storage.lineage(ident).await; + let violation = match (before, after) { + (Ok(Some(before)), Ok(Some(after))) => lineage_kept(&before, &after) + .err() + .map(|e| format!("invariant 21 (row lineage): {who} on {ident}: {e}")), + (Ok(_), Ok(_)) => None, + (Err(e), _) => Some(format!("before {who} on {ident}: {e}")), + (_, Err(e)) => Some(format!("after {who} on {ident}: {e}")), + }; + if let Some(v) = violation { + self.violations.lock().unwrap().push(v); + } + result } } +/// Every row live both `before` and `after` keeps its row id and +/// last-updated sequence number. +fn lineage_kept(before: &Lineage, after: &Lineage) -> Result<(), String> { + for (key, (row, id, seq)) in after { + let Some((_, was_id, was_seq)) = before.get(key) else { + continue; + }; + if (id, seq) != (was_id, was_seq) { + return Err(format!( + "row {row:?} had row id {was_id} and last-updated sequence number {was_seq}, \ + now {id} and {seq}" + )); + } + } + Ok(()) +} + #[async_trait::async_trait] impl Catalog for AuditedCatalog { async fn ensure_namespace(&self, ns: &Namespace) -> pg2iceberg_iceberg::Result<()> { @@ -3279,7 +3579,14 @@ impl Catalog for AuditedCatalog { self.land_other_pass().await?; self.check_leaves_maintenance_to_the_catalog(&prepared) .await; - let meta = self.inner.commit_compaction(prepared).await?; + let table = prepared.ident.clone(); + let meta = self + .keeping_lineage( + &table, + "pg2iceberg's compaction", + self.inner.commit_compaction(prepared), + ) + .await?; self.audit().await; let meta = self.respond(meta)?; Ok(meta) @@ -3337,6 +3644,8 @@ fn write_after_schema_change(h: &DstHarness) { } fn check_invariants(h: &mut DstHarness) -> Result<(), String> { + // The upgrade comes, if the workload didn't bring it. + h.upgrade_to_v3(); while h.backfilling { h.backfill_chunk(); } @@ -3645,6 +3954,20 @@ fn check_invariants(h: &mut DstHarness) -> Result<(), String> { block_on(h.storage.summary_counts_live_files(&ident())) .map_err(|e| format!("invariant 15: {e}"))?; + // 20. Row lineage is whole (see `Storage::lineage`), at quiescence as + // after every commit. + let lineage = block_on(h.storage.lineage(&ident()))?; + if FORMAT_V3.get() != lineage.is_some() { + return Err(format!( + "FORMAT_V3 is {}, but the table has row lineage: {}", + FORMAT_V3.get(), + lineage.is_some() + )); + } + if let Some(v) = h.audited.violations.lock().unwrap().first() { + return Err(v.clone()); + } + Ok(()) } @@ -3748,7 +4071,10 @@ proptest! { second_table in 0u8..3, managed in any::(), bounded_index in any::(), + format in 0u8..3, ) { + FORMAT_V3.set(format > 0); + V3_UPGRADED.set(format == 2); MANAGED.set(managed); BOUNDED_INDEX.set(bounded_index); SECOND_TABLE.set(second_table); @@ -6075,3 +6401,311 @@ fn pipeline_crash_then_materializer_catches_up() { h.run_step(&Step::MaterializerCycle); check_invariants(&mut h).unwrap(); } + +// ---------- Iceberg v3: row lineage ---------- + +/// Run `steps` on v3 tables (`FORMAT_V3`), checking the invariants after +/// each step and at quiescence. +fn v3_case(prod: bool, managed: bool, steps: &[Step]) { + FORMAT_V3.set(true); + MANAGED.set(managed); + PROD_BACKEND.set(prod); + let mut h = DstHarness::boot(); + for step in steps { + h.run_step(step); + if let Err(e) = check_step_invariants(&h) { + panic!("after {step:?}: {e}"); + } + } + if let Err(e) = check_invariants(&mut h) { + panic!("at quiescence: {e}"); + } +} + +/// Rounds of random steps, each ending in a commit and a compaction — +/// the catalog's too, where it maintains the table: the random workload +/// compacts too rarely to move many rows (invariant 21). +fn compacting_rounds(managed: bool) -> impl Strategy> { + prop::collection::vec(prop::collection::vec(step_strategy(), 1..=6), 1..=6).prop_map( + move |rounds| { + let mut steps = Vec::new(); + for round in rounds { + steps.extend(round); + steps.extend([Step::DriveFlush, Step::MaterializerCycle, Step::Compact]); + if managed { + steps.extend(catalog_compaction()); + } + } + steps + }, + ) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(64))] + + /// A v3 table keeps its row lineage under random workloads that + /// compact after every round. + #[test] + fn row_lineage_survives_random_workloads_that_compact( + (managed, steps) in any::() + .prop_flat_map(|managed| (Just(managed), compacting_rounds(managed))), + prod_backend in integration_only(), + partitioned in any::(), + upgraded in any::(), + ) { + PARTITIONED.set(partitioned); + FORMAT_V3.set(true); + V3_UPGRADED.set(upgraded); + MANAGED.set(managed); + PROD_BACKEND.set(prod_backend); + let mut h = DstHarness::boot(); + for (i, step) in steps.iter().enumerate() { + h.run_step(step); + if let Err(e) = check_step_invariants(&h) { + panic!("workload {steps:?}\nfailed after step {i}: {e}"); + } + } + if let Err(e) = check_invariants(&mut h) { + panic!("workload {steps:?}\nfailed: {e}"); + } + } +} + +/// Inserts, updates and deletes over several commits, without compaction. +fn writes() -> Vec { + [ + &dirty_file()[..], + &[ + Step::Insert { id: 3, qty: 30 }, + Step::Delete { id: 2 }, + Step::DriveFlush, + Step::MaterializerCycle, + ], + ] + .concat() +} + +/// Every row pg2iceberg writes to a v3 table has its lineage: the +/// commits give its file a row range. +#[test] +fn rows_written_to_a_v3_table_have_row_lineage() { + v3_case(false, false, &writes()); +} + +#[cfg(feature = "integration")] +#[test] +fn rows_written_to_a_v3_table_have_row_lineage_on_iceberg() { + v3_case(true, false, &writes()); +} + +/// pg2iceberg's compaction moves the rows of a dirty file into a new +/// one: they keep their ids and sequence numbers. +#[test] +fn compaction_keeps_row_lineage() { + v3_case( + false, + false, + &[&dirty_file()[..], &[Step::Compact]].concat(), + ); +} + +#[cfg(feature = "integration")] +#[test] +fn compaction_keeps_row_lineage_on_iceberg() { + v3_case(true, false, &[&dirty_file()[..], &[Step::Compact]].concat()); +} + +/// So does the catalog's — as Java's rewrite does, writing the rows' ids +/// into the files it writes, and recording the first row id of each file +/// it carries into a manifest it rewrites. +#[test] +fn the_catalogs_compaction_keeps_row_lineage() { + v3_case( + false, + true, + &[&dirty_file()[..], &catalog_compaction()].concat(), + ); +} + +#[cfg(feature = "integration")] +#[test] +fn the_catalogs_compaction_keeps_row_lineage_on_iceberg() { + v3_case( + true, + true, + &[&dirty_file()[..], &catalog_compaction()].concat(), + ); +} + +/// The catalog's compaction, which counts every file older than a delete +/// as dirty, rewrites its own output: the rows it moves again keep the +/// lineage that output stores. +#[test] +fn the_catalogs_compaction_of_its_own_output_keeps_row_lineage() { + v3_case(false, true, &catalog_compacted_repeatedly()); +} + +#[cfg(feature = "integration")] +#[test] +fn the_catalogs_compaction_of_its_own_output_keeps_row_lineage_on_iceberg() { + v3_case(true, true, &catalog_compacted_repeatedly()); +} + +/// Rounds of changes, each ending in pg2iceberg's compaction and then the +/// catalog's, whose passes merge their earlier outputs with new files +/// (the shrunk case of `row_lineage_survives_random_workloads_that_compact`). +fn catalog_compacted_repeatedly() -> Vec { + let rounds = [ + vec![Step::BigTx { inserts: 3, qty: 0 }], + vec![Step::DrivePartial { n: 1 }], + vec![Step::Insert { id: 2, qty: 0 }], + vec![Step::Update { id: 2, qty: 0 }], + vec![Step::DrivePartial { n: 1 }], + ]; + rounds + .into_iter() + .flat_map(|round| { + round + .into_iter() + .chain([Step::DriveFlush, Step::MaterializerCycle, Step::Compact]) + .chain(catalog_compaction()) + }) + .collect() +} + +/// Compaction rewrites the manifest that lists the file it rewrites, and +/// carries the files it leaves into the new one. Each must record the +/// first row id it had, or the manifest list gives the new manifest's +/// files fresh ones: the rows of a file nothing touched (row 1, alone in +/// its partition) would change ids. Manifests are production's alone. +#[cfg(feature = "integration")] +#[test] +fn compaction_keeps_the_lineage_of_the_files_it_leaves_on_iceberg() { + PARTITIONED.set(true); + v3_case( + true, + false, + &[ + Step::Insert { id: 1, qty: 10 }, + Step::Insert { id: 2, qty: 100 }, + Step::Insert { id: 3, qty: 110 }, + Step::DriveFlush, + Step::MaterializerCycle, + Step::Update { id: 3, qty: 111 }, + Step::DriveFlush, + Step::MaterializerCycle, + Step::Compact, + ], + ); +} + +/// The harness's model of another engine's commit (`commit_foreign`) +/// carries a file it leaves into the manifest it rewrites with the file's +/// first row id, as Java does: the file's rows keep their ids. +#[cfg(feature = "integration")] +#[test] +fn the_catalogs_commit_keeps_the_lineage_of_the_files_it_leaves_on_iceberg() { + PARTITIONED.set(true); + FORMAT_V3.set(true); + MANAGED.set(true); + PROD_BACKEND.set(true); + let mut h = DstHarness::boot(); + // One commit, so one manifest: a file per partition. + for step in [ + Step::Insert { id: 1, qty: 10 }, + Step::Insert { id: 2, qty: 100 }, + Step::DriveFlush, + Step::MaterializerCycle, + ] { + h.run_step(&step); + } + let files = block_on(h.storage.live_files(&ident())).unwrap(); + assert_eq!(files.len(), 2, "a file per partition: {files:?}"); + let second = files + .iter() + .find(|f| f.partition == [pg2iceberg_core::partition::PartitionLiteral::Int(100)]) + .expect("row 2's file") + .path + .clone(); + let before = block_on(h.storage.lineage(&ident())).unwrap().unwrap(); + // The catalog drops row 2's file, keeping row 1's. + block_on(h.storage.commit_catalog_removal(&ident(), vec![second])).unwrap(); + let after = block_on(h.storage.lineage(&ident())).unwrap().unwrap(); + assert_eq!(after.len(), 1, "row 1 is left: {after:?}"); + lineage_kept(&before, &after).unwrap(); +} + +/// A compaction pass planned while the table is still v2 stores no row +/// lineage; another engine upgrades the table, and its next commit gives +/// the rows ids. The pass, committed after, would move them to new ones: +/// the catalog refuses it, and it plans again. +#[test] +fn a_pass_planned_before_an_upgrade_keeps_row_lineage() { + upgraded_while_a_pass_is_in_flight(false); +} + +#[cfg(feature = "integration")] +#[test] +fn a_pass_planned_before_an_upgrade_keeps_row_lineage_on_iceberg() { + upgraded_while_a_pass_is_in_flight(true); +} + +fn upgraded_while_a_pass_is_in_flight(prod: bool) { + PARTITIONED.set(true); + V3_UPGRADED.set(true); + let round = |first: Step| { + [ + first, + Step::DriveFlush, + Step::MaterializerCycle, + Step::Compact, + ] + }; + let steps = [ + &round(Step::Insert { id: 1, qty: 0 })[..], + &catalog_compaction(), + &round(Step::BigTx { inserts: 1, qty: 0 }), + &catalog_compaction(), + &[ + Step::CatalogCompactPlan { + own_sequence_number: false, + }, + Step::UpgradeToV3, + ], + &round(Step::DriveFlush), + &catalog_compaction(), + ] + .concat(); + v3_case(prod, true, &steps); +} + +/// So does a `compact` job's pass, planned before the upgrade and +/// committed after pg2iceberg's next commit gave the rows ids. +#[test] +fn a_jobs_pass_planned_before_an_upgrade_keeps_row_lineage() { + a_jobs_pass_across_an_upgrade(false); +} + +#[cfg(feature = "integration")] +#[test] +fn a_jobs_pass_planned_before_an_upgrade_keeps_row_lineage_on_iceberg() { + a_jobs_pass_across_an_upgrade(true); +} + +fn a_jobs_pass_across_an_upgrade(prod: bool) { + V3_UPGRADED.set(true); + let steps = [ + &dirty_file()[..], + &[ + Step::ExternalCompactPlan, + Step::UpgradeToV3, + Step::Insert { id: 3, qty: 30 }, + Step::DriveFlush, + Step::MaterializerCycle, + Step::ExternalCompactCommit, + ], + ] + .concat(); + v3_case(prod, false, &steps); +} diff --git a/crates/pg2iceberg-tests/tests/expire_e2e.rs b/crates/pg2iceberg-tests/tests/expire_e2e.rs index d914965..1abbb78 100644 --- a/crates/pg2iceberg-tests/tests/expire_e2e.rs +++ b/crates/pg2iceberg-tests/tests/expire_e2e.rs @@ -118,6 +118,7 @@ impl Harness { added_data_files, removed_paths: removed, data_sequence_number: planned_at, + stores_row_lineage: false, })) .unwrap(); paths @@ -141,6 +142,7 @@ impl Harness { equality_field_ids: equality_field_ids.clone(), partition_values: chunk.partition_values, sequence_number: None, + first_row_id: None, } }) .collect() diff --git a/crates/pg2iceberg-tests/tests/file_index_memory.rs b/crates/pg2iceberg-tests/tests/file_index_memory.rs index de5b298..1105b74 100644 --- a/crates/pg2iceberg-tests/tests/file_index_memory.rs +++ b/crates/pg2iceberg-tests/tests/file_index_memory.rs @@ -140,6 +140,7 @@ fn table_with_history( equality_field_ids: vec![], partition_values: c.partition_values.clone(), sequence_number: None, + first_row_id: None, }) .collect(); let equality_deletes = prepared @@ -152,6 +153,7 @@ fn table_with_history( equality_field_ids: prepared.pk_field_ids.clone(), partition_values: c.partition_values.clone(), sequence_number: None, + first_row_id: None, }) .collect(); block_on(cat.commit_snapshot(PreparedCommit { diff --git a/crates/pg2iceberg-tests/tests/integration_lifecycle.rs b/crates/pg2iceberg-tests/tests/integration_lifecycle.rs index d070331..055b651 100644 --- a/crates/pg2iceberg-tests/tests/integration_lifecycle.rs +++ b/crates/pg2iceberg-tests/tests/integration_lifecycle.rs @@ -257,6 +257,7 @@ fn config_for_stack(stack: &Stack, table: &str, slot: &str, publication: &str) - maintenance_retention: String::new(), maintenance_grace: "30m".into(), maintenance: String::new(), + format_version: 2, catalog_props: BTreeMap::new(), meta_namespace: String::new(), }, @@ -329,12 +330,87 @@ async fn iam_credentials_with_a_custom_endpoint_commit_snapshots() { equality_field_ids: vec![], partition_values: Vec::new(), sequence_number: None, + first_row_id: None, }], equality_deletes: vec![], }) .await .expect("commit, writing manifests to MinIO"); } +/// A REST catalog creates each table in the format version pg2iceberg is +/// set to — its create request carries it only as a table property — and, +/// for v3, takes a commit's row range: the next commit's rows get ids. +#[tokio::test] +async fn a_rest_catalog_creates_tables_in_the_configured_format_version() { + use iceberg::Catalog as _; + use pg2iceberg_core::{ColumnSchema, IcebergType, TableSchema}; + use pg2iceberg_iceberg::{Catalog as _, DataFile, FormatVersion, PreparedCommit}; + + let stack = bring_up_stack().await; + let cfg = config_for_stack(&stack, "unused", "unused", "unused"); + for (version, expected) in [ + (FormatVersion::V2, iceberg::spec::FormatVersion::V2), + (FormatVersion::V3, iceberg::spec::FormatVersion::V3), + ] { + let rest = Arc::new(build_rest_catalog(&cfg).await.expect("rest catalog")); + let catalog = IcebergRustCatalog::new(rest.clone()).with_format_version(version); + let ident = TableIdent { + namespace: Namespace(vec!["public".into()]), + name: format!("v{}_{}", expected as u8, uniq()), + }; + catalog + .ensure_namespace(&ident.namespace) + .await + .expect("namespace"); + catalog + .create_table(&TableSchema { + ident: ident.clone(), + columns: vec![ColumnSchema { + name: "id".into(), + field_id: 1, + ty: IcebergType::Int, + nullable: false, + is_primary_key: true, + }], + partition_spec: Vec::new(), + pg_schema: None, + }) + .await + .expect("create table"); + catalog + .commit_snapshot(PreparedCommit { + ident: ident.clone(), + data_files: vec![DataFile { + path: format!("s3://{}/{}/data.parquet", stack.bucket, ident.name), + record_count: 5, + byte_size: 1, + equality_field_ids: vec![], + partition_values: Vec::new(), + sequence_number: None, + first_row_id: None, + }], + equality_deletes: vec![], + }) + .await + .expect("commit"); + let table = rest + .load_table(&iceberg::TableIdent::new( + iceberg::NamespaceIdent::new("public".into()), + ident.name.clone(), + )) + .await + .expect("load table"); + assert_eq!(table.metadata().format_version(), expected, "{ident}"); + if expected == iceberg::spec::FormatVersion::V3 { + assert_eq!( + table.metadata().next_row_id(), + 5, + "{ident}: row ids given out" + ); + } + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn lifecycle_inserts_propagate_pg_to_iceberg() { // Surface lifecycle / pipeline / catalog tracing into test output diff --git a/crates/pg2iceberg-tests/tests/integration_vended_lifecycle.rs b/crates/pg2iceberg-tests/tests/integration_vended_lifecycle.rs index cc4612a..4c6ea63 100644 --- a/crates/pg2iceberg-tests/tests/integration_vended_lifecycle.rs +++ b/crates/pg2iceberg-tests/tests/integration_vended_lifecycle.rs @@ -298,6 +298,7 @@ fn config_for_vended_stack(stack: &VendedStack) -> Config { maintenance_retention: String::new(), maintenance_grace: "30m".into(), maintenance: String::new(), + format_version: 2, // The `header.x-iceberg-access-delegation: vended-credentials` // header is auto-added by `rest_catalog_props()` when // credential_mode=vended; no need to repeat it here. diff --git a/crates/pg2iceberg/src/config.rs b/crates/pg2iceberg/src/config.rs index ebe0420..a595944 100644 --- a/crates/pg2iceberg/src/config.rs +++ b/crates/pg2iceberg/src/config.rs @@ -407,6 +407,14 @@ pub struct SinkConfig { #[serde(default)] pub maintenance: String, + /// The Iceberg format version of the tables pg2iceberg creates: 2 + /// (the default), or 3, whose rows carry row lineage (`_row_id`, + /// `_last_updated_sequence_number`) — but which fewer engines read. + /// Tables that already exist keep theirs. See + /// [`Self::resolved_format_version`]. + #[serde(default = "default_format_version")] + pub format_version: u8, + /// Free-form REST-catalog props passthrough. Useful for /// vendor-specific settings (Polaris OAuth2 /// server URI, etc.) without us having to enumerate every quirk. @@ -459,6 +467,7 @@ impl Default for SinkConfig { maintenance_retention: String::new(), maintenance_grace: default_maintenance_grace(), maintenance: String::new(), + format_version: default_format_version(), catalog_props: BTreeMap::new(), meta_namespace: String::new(), } @@ -520,6 +529,16 @@ impl SinkConfig { } } + /// `format_version`, checked. + pub fn resolved_format_version(&self) -> Result { + pg2iceberg_iceberg::FormatVersion::from_number(self.format_version).map_err(|_| { + anyhow::anyhow!( + "unsupported sink.format_version {}; expected 2 or 3", + self.format_version + ) + }) + } + /// `compaction_delete_ratio` is a share: 0 rewrites a file for any row /// deleted from it; past 1, only `compaction_delete_files` does. pub fn validate_compaction(&self) -> Result<()> { @@ -642,6 +661,10 @@ fn default_target_file_size() -> u64 { 128 * 1024 * 1024 } +fn default_format_version() -> u8 { + 2 +} + fn default_maintenance_grace() -> String { "30m".into() } @@ -733,6 +756,7 @@ impl Config { self.metrics_addr()?; self.liveness_timeout()?; self.sink.resolved_maintenance()?; + self.sink.resolved_format_version()?; Ok(self) } @@ -810,6 +834,7 @@ impl Config { /// | `PG2ICEBERG_METRICS_ADDR` | `metrics_addr` | /// | `PG2ICEBERG_LIVENESS_TIMEOUT` | `liveness_timeout` | /// | `PG2ICEBERG_MAINTENANCE` | `sink.maintenance` | + /// | `PG2ICEBERG_FORMAT_VERSION` | `sink.format_version` | /// /// AWS's own variables fill in only what the file leaves out: /// `AWS_REGION` / `AWS_DEFAULT_REGION` for `sink.s3_region`, and @@ -846,6 +871,14 @@ impl Config { set(&mut self.metrics_addr, "PG2ICEBERG_METRICS_ADDR"); set(&mut self.liveness_timeout, "PG2ICEBERG_LIVENESS_TIMEOUT"); set(&mut self.sink.maintenance, "PG2ICEBERG_MAINTENANCE"); + if let Some(version) = env("PG2ICEBERG_FORMAT_VERSION") { + self.sink.format_version = version.trim().parse().map_err(|_| { + anyhow::anyhow!( + "PG2ICEBERG_FORMAT_VERSION={version:?}: expected an Iceberg format \ + version, 2 or 3" + ) + })?; + } if let Some(tables) = env("PG2ICEBERG_TABLES") { self.tables = tables .split(|c: char| c == ',' || c.is_whitespace()) @@ -1666,6 +1699,30 @@ sink: move |name: &str| map.get(name).cloned() } + /// Tables are created v2 unless told v3, from the file or the + /// environment; another version fails the config load. + #[test] + fn tables_are_created_v2_unless_told_v3() { + use pg2iceberg_iceberg::FormatVersion; + let version = |cfg: Config| cfg.sink.resolved_format_version().unwrap(); + assert_eq!( + version(Config::from_env(&vars(&[])).unwrap()), + FormatVersion::V2 + ); + let env = vars(&[("PG2ICEBERG_FORMAT_VERSION", "3")]); + assert_eq!(version(Config::from_env(&env).unwrap()), FormatVersion::V3); + let yaml = format!("{INTERVALS} format_version: 3\n"); + let env = vars(&[]); + let cfg = Config::parse(&yaml, &env).unwrap().finish(&env).unwrap(); + assert_eq!(version(cfg), FormatVersion::V3); + for bad in ["4", "v3"] { + let err = Config::from_env(&vars(&[("PG2ICEBERG_FORMAT_VERSION", bad)])) + .unwrap_err() + .to_string(); + assert!(err.contains("format"), "{bad}: {err}"); + } + } + /// pg2iceberg maintains the tables unless told the catalog does; a /// value it doesn't know fails the config, not a later `maintain`. #[test] diff --git a/crates/pg2iceberg/src/run.rs b/crates/pg2iceberg/src/run.rs index bf527a1..96edc4c 100644 --- a/crates/pg2iceberg/src/run.rs +++ b/crates/pg2iceberg/src/run.rs @@ -192,7 +192,8 @@ fn data_path(location: &str, kind: &str, partition_segment: &str, id: &str) -> S pub async fn run(cfg: Config, metrics: Arc) -> Result<()> { cfg.require_catalog()?; let catalog = build_rest_catalog(&cfg).await?; - let catalog = IcebergRustCatalog::new(Arc::new(catalog)); + let catalog = IcebergRustCatalog::new(Arc::new(catalog)) + .with_format_version(cfg.sink.resolved_format_version()?); let storage = build_storage(&cfg, &catalog) .await .context("build blob store")?; @@ -378,7 +379,8 @@ fn bucket_from_warehouse(warehouse: &str) -> Result { pub async fn run_stream_only(cfg: Config, metrics: Arc) -> Result<()> { cfg.require_catalog()?; let catalog = build_rest_catalog(&cfg).await?; - let catalog = IcebergRustCatalog::new(Arc::new(catalog)); + let catalog = IcebergRustCatalog::new(Arc::new(catalog)) + .with_format_version(cfg.sink.resolved_format_version()?); let storage = build_storage(&cfg, &catalog) .await .context("build blob store")?; @@ -720,7 +722,8 @@ async fn build_one_shot_materializer( metrics: Arc, ) -> Result> { cfg.require_catalog()?; - let catalog = IcebergRustCatalog::new(Arc::new(build_rest_catalog(&cfg).await?)); + let catalog = IcebergRustCatalog::new(Arc::new(build_rest_catalog(&cfg).await?)) + .with_format_version(cfg.sink.resolved_format_version()?); let storage = build_storage(&cfg, &catalog) .await .context("build blob store")?; @@ -823,7 +826,8 @@ async fn build_one_shot_materializer( pub async fn run_verify(cfg: Config, chunk_size: usize) -> Result<()> { cfg.require_catalog()?; let catalog: Arc> = Arc::new( - IcebergRustCatalog::new(Arc::new(build_rest_catalog(&cfg).await?)), + IcebergRustCatalog::new(Arc::new(build_rest_catalog(&cfg).await?)) + .with_format_version(cfg.sink.resolved_format_version()?), ); let blob = build_storage(&cfg, &catalog) .await @@ -1027,7 +1031,8 @@ async fn snapshot_only(cfg: Config, metrics: Arc) -> Result<()> { use pg2iceberg_validate::LifecycleError; cfg.require_catalog()?; - let catalog = IcebergRustCatalog::new(Arc::new(build_rest_catalog(&cfg).await?)); + let catalog = IcebergRustCatalog::new(Arc::new(build_rest_catalog(&cfg).await?)) + .with_format_version(cfg.sink.resolved_format_version()?); // The snapshot is written straight to the tables: nothing to stage. let Storage { blob, diff --git a/docs/architecture/compaction.md b/docs/architecture/compaction.md index ab200b1..48ac49c 100644 --- a/docs/architecture/compaction.md +++ b/docs/architecture/compaction.md @@ -38,6 +38,12 @@ The ratio bounds the work: rewriting reads at most 1 / `compaction_delete_ratio` Each file is a round trip to the object store, so a pass overlaps them: the inputs are read ahead of the one being rewritten — up to eight files, and `target_file_size` bytes of them — starting while the deletes load, and an output uploads while the next is written, up to `target_file_size` bytes uploading. A pass holds those in memory as well as the file it's reading and the one it's writing: about four `target_file_size`s at most. +## Row lineage + +A v3 table ([`sink.format_version`](../usage/configuration.md#notes-on-individual-fields)) gives every row a `_row_id` and the sequence number of the commit that last changed it, `_last_updated_sequence_number`. A data file doesn't store them unless it has to: the catalog gives each file it commits a range of row ids, and a row's id is the first plus its position in the file; its sequence number is the file's. Moved into a new file, a row would take the new file's — a new id, as if it were a new row, and a later sequence number, as if it had changed. + +So on a v3 table, compaction writes both into the files it writes, for every row: the ones the input stored, else the ones it gave. The row keeps them however many times it's moved, and engines that read the table incrementally see nothing changed. + ## When the catalog compacts With `sink.maintenance: managed`, the catalog compacts the table and pg2iceberg rewrites no data files. Its pass keeps one job: retiring the delete files no data file is left for — the catalog's rewrites leave them behind. See [Table Maintenance](maintenance.md#when-the-catalog-maintains-the-tables). diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index fb31894..5c9fb66 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -49,6 +49,7 @@ sink: | `ICEBERG_NAMESPACE` | `sink.namespace` | | `ICEBERG_CREDENTIAL_MODE` | `sink.credential_mode` | | `PG2ICEBERG_MAINTENANCE` | `sink.maintenance` | +| `PG2ICEBERG_FORMAT_VERSION` | `sink.format_version` | | `PG2ICEBERG_METRICS_ADDR` | `metrics_addr` | | `PG2ICEBERG_LIVENESS_TIMEOUT` | `liveness_timeout` | | `PG2ICEBERG_LOG_FORMAT` | `text` (default) or `json` logs; no config field | @@ -168,6 +169,10 @@ sink: # Data Catalog, Glue's table optimizers. See Table Maintenance. maintenance: pg2iceberg + # Iceberg format version of the tables pg2iceberg creates: 2, or 3 for + # row lineage. Tables that already exist keep theirs. + format_version: 2 + # `pg2iceberg maintain` (snapshot expiry + orphan cleanup) maintenance_retention: 168h # 7 days. Snapshots older than this are dropped. maintenance_grace: 30m # Orphan files younger than this are protected, @@ -207,6 +212,7 @@ snapshot_only: false # legacy field; prefer the `snapshot` subcomman - **`sink.materializer_batch_rows`** — the most change events the materializer folds into one step, which bounds its memory, along with the up to eight staged files it reads ahead (each at most `flush_rows` events). A transaction larger than this is written in several steps committed in one atomic catalog update, so readers of the table never see part of it; catching up, the materializer commits up to four steps at a time anyway. Steps only adding rows share a snapshot ([commit](../architecture/materializer.md)). - **`sink.materializer_concurrency`** — the most tables the materializer applies at once (default 4; 1 applies them one after another). A table's work is mostly waiting on the object store and catalog, so more at once catch up faster on many tables; each holds its own `materializer_batch_rows` events and staged files read ahead, and commits on its own, so memory and the catalog's request rate grow with it. With meta tables on (`meta_namespace`), tables go one at a time. - **`sink.pk_index_memory`** — the most memory, in bytes, the materializer holds of a table's primary keys: it maps each key to the data file holding its row ([file index](../architecture/materializer.md#file-index)). Within it — about 7 million keys at the default 512 MiB, leaving room for the map to grow and a quarter for the filter of evicted keys — nothing changes. Past it, the oldest files' keys are evicted, and changes to their rows cost more: an insert that may be of one writes an equality delete, and an update leaving TOAST columns out, a delete on a partitioned table, a `TRUNCATE` or a column added with a default reads the files that may hold them. Each table has its own budget. +- **`sink.format_version`** — the [Iceberg format version](https://iceberg.apache.org/spec/#format-versioning) of the tables pg2iceberg creates: 2 (the default) or 3. A v3 table has row lineage: every row has a `_row_id`, and the sequence number of the commit that last changed it (`_last_updated_sequence_number`), which engines use to read a table incrementally. Compaction keeps both for the rows it moves ([compaction](../architecture/compaction.md#row-lineage)). An update gives the row a new `_row_id`: pg2iceberg writes it as an equality delete and an insert, which the spec allows to. Not every engine reads v3 tables yet: check the ones reading yours. Tables that already exist keep their version — pg2iceberg writes each as its own version says, including one another engine upgraded to v3 — and pg2iceberg never upgrades one. - **`sink.flush_interval` / `sink.materializer_interval`** — the micro-batch cadence: changes are staged every `flush_interval` (or sooner, once `flush_rows` are buffered) and committed to Iceberg every `materializer_interval` (or back to back, while catching up on a backlog). A change typically reaches Iceberg within about the two added together. Shorter intervals mean fresher tables, and more, smaller Iceberg snapshots and files for compaction to merge. - **`sink.compaction_delete_files` / `sink.compaction_delete_ratio`** — when compaction drops deleted rows. An update or delete leaves the old row in its data file, hidden by an equality delete that every reader applies; dropping it means rewriting the files holding such rows, and every older one, before the delete file can go. Under updates spread across a large table that's most of the table, so pg2iceberg waits until the deleted rows are `compaction_delete_ratio` of the rows those files hold (rewriting then reads at most 10 rows per row deleted, at 0.1), or until there are `compaction_delete_files` delete files for readers to open. Lower values keep reads cheaper; higher ones rewrite less. - **`sink.flush_bytes`** — not yet exposed; flush threshold is `flush_rows` + `flush_interval` only.