Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ Settings come from environment variables and an optional YAML file: `--config`,
| `PG2ICEBERG_LOG_FORMAT` | — | `text` (default) or `json` |
| `OTEL_EXPORTER_OTLP_ENDPOINT` | — | Export traces over OTLP/HTTP to this collector. Default: none |
| `PG2ICEBERG_MAINTENANCE` | `sink.maintenance` | `pg2iceberg` / `managed` (the catalog compacts, expires and cleans the tables). Default: `pg2iceberg` |
| `PG2ICEBERG_FORMAT_VERSION` | `sink.format_version` | Iceberg format version of the tables pg2iceberg creates: `2` or `3` (row lineage). Default: `2` |
| `AWS_REGION`, `AWS_ENDPOINT_URL_S3`, `AWS_ACCESS_KEY_ID`, ... | `sink.s3_*` | Read as AWS tools read them; the file's `sink.s3_*` take precedence |

### Inferred settings
Expand Down
6 changes: 6 additions & 0 deletions config.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,12 @@ sink:
# and `maintain` refuses.
# maintenance: managed

# Iceberg format version of the tables pg2iceberg creates: 2 (default),
# or 3, whose rows carry row lineage (`_row_id`,
# `_last_updated_sequence_number`) — check that your engines read v3.
# Tables that already exist keep theirs.
# format_version: 3

# Free-form REST catalog props passthrough. Useful for
# vendor-specific settings the Go config doesn't have a dedicated
# field for.
Expand Down
104 changes: 99 additions & 5 deletions crates/pg2iceberg-iceberg/src/compact.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,15 +61,22 @@
//! `t` is retired only when no dirty data file with seq `< t` remains
//! after this pass. Clean files don't hold it back: by definition no
//! live delete applies to any of their rows.
//! - **Moved rows keep their lineage.** On a v3 table, every row has a
//! `_row_id` and a `_last_updated_sequence_number` — stored in its file,
//! or inherited from the file's first row id and data sequence number.
//! A rewritten row keeps both: the outputs store them for every row.
//! - **No partial rewrites.** We rewrite a data file fully or not at all.
//! Partial rewrites would lose rows.
//! - **Atomicity.** Either the entire `Replace` snapshot lands, or none
//! of it. The catalog's commit path is the gate.

use crate::reader::{value_at, ArrowBatches};
use crate::writer::{StreamingDataFile, TableWriter, WriterError};
use crate::{Catalog, DataFile, FileIndex, IcebergError, PkKey, PreparedCompaction, Snapshot};
use arrow_array::{Array, BooleanArray, RecordBatch};
use crate::{
Catalog, DataFile, FileIndex, IcebergError, PkKey, PreparedCompaction, Snapshot,
LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID, ROW_ID_FIELD_ID,
};
use arrow_array::{Array, BooleanArray, Int64Array, RecordBatch};
use arrow_select::filter::filter_record_batch;
use bytes::Bytes;
use futures::future::BoxFuture;
Expand All @@ -79,6 +86,7 @@ use pg2iceberg_stream::{BlobStore, StreamError};
use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
use std::future::{poll_fn, Future};
use std::pin::pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use thiserror::Error;

Expand Down Expand Up @@ -243,6 +251,8 @@ struct LiveFile {
/// its snapshot's — `snapshots()` lists a file under the snapshot that
/// added it — or, for a compaction's output, the one its pass read.
seq: i64,
/// A v3 data file's first row id ([`DataFile::first_row_id`]).
first_row_id: Option<i64>,
}

impl LiveFile {
Expand All @@ -252,10 +262,74 @@ impl LiveFile {
record_count: df.record_count,
partition_values: df.partition_values.clone(),
seq,
first_row_id: df.first_row_id,
}
}
}

/// Row lineage's two columns, as a v3 table's data files store them.
fn lineage_columns() -> [ColumnSchema; 2] {
let column = |name: &str, field_id| ColumnSchema {
name: name.into(),
field_id,
ty: pg2iceberg_core::IcebergType::Long,
nullable: true,
is_primary_key: false,
};
[
column("_row_id", ROW_ID_FIELD_ID),
column(
"_last_updated_sequence_number",
LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID,
),
]
}

/// `batch`, read from rows `offset..` of `input`, with its last two
/// columns — row lineage's — filled in for each row the file doesn't
/// store them for: `_row_id` as the file's first row id plus the row's
/// position, `_last_updated_sequence_number` as the file's data sequence
/// number. What a v3 reader reads for the rows, so the rewritten file
/// stores what they had: the spec requires a commit that moves rows to
/// keep both.
fn with_lineage(batch: RecordBatch, input: &LiveFile, offset: u64) -> Result<RecordBatch> {
let width = batch.num_columns();
let stored = |i: usize| -> Result<Int64Array> {
batch
.column(i)
.as_any()
.downcast_ref::<Int64Array>()
.cloned()
.ok_or_else(|| {
WriterError::Encode(format!(
"row lineage column {} isn't a long",
batch.schema().field(i).name()
))
.into()
})
};
let (row_ids, seqs) = (stored(width - 2)?, stored(width - 1)?);
let row_ids: Int64Array = (0..batch.num_rows())
.map(|i| match row_ids.is_null(i) {
false => Some(row_ids.value(i)),
true => input
.first_row_id
.map(|first| first + offset as i64 + i as i64),
})
.collect();
let seqs: Int64Array = (0..batch.num_rows())
.map(|i| match seqs.is_null(i) {
false => Some(seqs.value(i)),
true => input.first_row_id.map(|_| input.seq),
})
.collect();
let mut columns = batch.columns()[..width - 2].to_vec();
columns.push(Arc::new(row_ids));
columns.push(Arc::new(seqs));
RecordBatch::try_new(batch.schema(), columns)
.map_err(|e| WriterError::Encode(format!("row lineage: {e}")).into())
}

/// What one pass does: which data files it rewrites (in processing
/// order) and which delete files it drops.
#[derive(Debug, Default)]
Expand Down Expand Up @@ -399,7 +473,15 @@ where
))
.await?;

let writer = TableWriter::new(schema.clone());
// A v3 table's rows have lineage — its data files take first row ids
// — which the outputs store for each row they move.
let lineage = live_data.values().any(|f| f.first_row_id.is_some());
let mut written = schema.clone();
if lineage {
written.columns.extend(lineage_columns());
}
let writer = TableWriter::new(written);
let read_cols = writer.schema().columns.clone();
let mut out = OutputFiles::default();
let mut rows_rewritten: u64 = 0;
let mut rows_removed_by_deletes: u64 = 0;
Expand All @@ -419,9 +501,15 @@ where
while let Some(((path, meta), bytes)) = io.next_input().await? {
let source = sources[path.as_str()];
bytes_before += meta.byte_size;
for batch in ArrowBatches::new(bytes, &schema.columns, DECODE_BATCH_ROWS)? {
let batch = batch?;
// The position in the input of the batch's first row.
let mut offset: u64 = 0;
for batch in ArrowBatches::new(bytes, &read_cols, DECODE_BATCH_ROWS)? {
let mut batch = batch?;
let n = batch.num_rows();
if lineage {
batch = with_lineage(batch, meta, offset)?;
}
offset += n as u64;
peak_rows = peak_rows.max(n as u64);
rows_rewritten += n as u64;
// A delete applies iff it is strictly newer than the row.
Expand Down Expand Up @@ -498,6 +586,7 @@ where
removed_paths,
// The table as the pass read it: the last snapshot.
data_sequence_number: snapshots.last().map(|s| s.id),
stores_row_lineage: lineage,
},
outcome: CompactionOutcome {
input_data_files: plan.inputs.len(),
Expand Down Expand Up @@ -552,6 +641,8 @@ where
added_data_files: Vec::new(),
removed_paths: retired.clone(),
data_sequence_number: snapshots.last().map(|s| s.id),
// It moves no rows.
stores_row_lineage: true,
})
.await?;
Ok(Some(CompactionOutcome {
Expand Down Expand Up @@ -639,6 +730,7 @@ impl OutputFiles {
equality_field_ids: vec![],
partition_values,
sequence_number: None,
first_row_id: None,
});
self.added_pks.push(pks);
self.added_sources.push(sources);
Expand Down Expand Up @@ -1144,6 +1236,7 @@ mod tests {
equality_field_ids: vec![],
partition_values: vec![],
sequence_number: None,
first_row_id: None,
}
}

Expand Down Expand Up @@ -1226,6 +1319,7 @@ mod tests {
record_count: 2,
partition_values: Vec::new(),
seq,
first_row_id: None,
};
let files = |entries: &[(&str, i64)]| -> BTreeMap<String, LiveFile> {
entries
Expand Down
49 changes: 49 additions & 0 deletions crates/pg2iceberg-iceberg/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,38 @@ pub struct PreparedCommit {
pub equality_deletes: Vec<DataFile>,
}

/// The Iceberg format version of the tables pg2iceberg creates. Tables
/// that already exist keep theirs: pg2iceberg writes each as its own
/// format version says.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum FormatVersion {
/// What every engine reads.
#[default]
V2,
/// Adds row lineage: every row has a `_row_id`, and the sequence
/// number of the commit that last changed it.
V3,
}

impl FormatVersion {
/// The version numbered `n`: 2 or 3.
pub fn from_number(n: u8) -> Result<Self> {
match n {
2 => Ok(Self::V2),
3 => Ok(Self::V3),
other => Err(IcebergError::Other(format!(
"unsupported Iceberg format version {other}; expected 2 or 3"
))),
}
}
}

/// The field id the spec reserves for row lineage's `_row_id`.
pub const ROW_ID_FIELD_ID: i32 = 2147483540;
/// The field id the spec reserves for row lineage's
/// `_last_updated_sequence_number`.
pub const LAST_UPDATED_SEQUENCE_NUMBER_FIELD_ID: i32 = 2147483539;

/// Built by [`crate::compact::compact_table`] and consumed by
/// [`Catalog::commit_compaction`]. Produces an `Operation::Replace`
/// snapshot: drop everything in `removed_paths`, add everything in
Expand All @@ -190,8 +222,19 @@ pub struct PreparedCompaction {
/// committed while the pass ran still apply to them. (With the new
/// snapshot's, a row deleted meanwhile would come back.)
pub data_sequence_number: Option<i64>,
/// Whether the added files store their rows' lineage (`_row_id`,
/// `_last_updated_sequence_number`), as a pass over a v3 table writes
/// them. A pass that read rows without ids writes none — and must not
/// commit if the rows it moves have since been given theirs (the table
/// upgraded to v3 meanwhile): the catalog refuses it, as a conflict.
pub stores_row_lineage: bool,
}

/// What a catalog's conflict error says when a compaction that stores no
/// row lineage would move rows that have row ids (see
/// [`PreparedCompaction::stores_row_lineage`]).
pub const ROW_IDS_CONFLICT: &str = "rewrite moves rows that have row ids, without them";

#[derive(Clone, Debug)]
pub struct DataFile {
pub path: String,
Expand All @@ -209,6 +252,12 @@ pub struct DataFile {
/// deletes committed while the pass ran still apply to it. `None`: the
/// snapshot's (see [`Self::sequence_number_in`]).
pub sequence_number: Option<i64>,
/// A v3 data file's first row id, as the catalog assigned it when the
/// file was committed: the `_row_id` of each row the file doesn't
/// store one for is this plus the row's position in the file (row
/// lineage). `None` for a file about to be committed, a delete file,
/// and on a table without row lineage (v2).
pub first_row_id: Option<i64>,
}

impl DataFile {
Expand Down
1 change: 1 addition & 0 deletions crates/pg2iceberg-iceberg/src/orphan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ mod tests {
equality_field_ids: vec![],
partition_values: vec![],
sequence_number: None,
first_row_id: None,
};
Snapshot {
id,
Expand Down
Loading