Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
703 changes: 579 additions & 124 deletions Cargo.lock

Large diffs are not rendered by default.

18 changes: 10 additions & 8 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,12 @@ all = { level = "warn", priority = -1 }
# `[patch.crates-io]` and matching git overrides for the two ecosystem
# forks pg2iceberg-rust depends on:
#
# 1. `iceberg` → `polynya-dev/iceberg-rust@polynya-patches`. Based on
# upstream v0.9.0 plus three non-invasive patches: `TransactionAction`
# visibility, a new `UpdateSchemaAction`, and a mixed-content
# `FastAppendAction` that accepts both data and equality-delete files.
# 1. `iceberg` → `polynya-dev/iceberg-rust@polynya-patches-v0.10.1`:
# upstream v0.10.1 plus `TransactionAction` visibility, a whole-schema
# `ReplaceSchemaAction`, a `FastAppendAction` that accepts delete files
# as well as data files, a compaction `RewriteFilesAction` (explicit
# data sequence number, removed-file check, summary totals),
# `Table::properties()` for vended credentials, and SigV4 signing.
# See `crates/pg2iceberg-iceberg/src/prod/gap_audit.rs` for context.
#
# 2. `tokio-postgres` + `postgres-replication` → `iambriccardo/rust-postgres`
Expand All @@ -58,15 +60,15 @@ all = { level = "warn", priority = -1 }
# (Supabase's etl maintainer) rather than rolling our own to avoid the
# maintenance burden of two forks.
[patch.crates-io]
iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches" }
iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" }
# `iceberg-catalog-rest` also has to come from the fork: the
# polynya-patches branch threads the REST `loadTable` response's
# `config` map through `Table::builder().properties(response_config)`,
# which the published v0.9.0 omits. Without it, vended-credentials
# which upstream omits. Without it, vended-credentials
# servers (Polaris, Lakekeeper, ...) return per-table S3 STS tokens
# in `config`, but our `extract_creds` sees an empty map because the
# `Table::properties()` accessor surfaces nothing.
iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches" }
iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches" }
iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" }
iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" }
tokio-postgres = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" }
postgres-replication = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" }
2 changes: 1 addition & 1 deletion crates/pg2iceberg-iceberg/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ thiserror.workspace = true

# iceberg-rust prod-impl wrap. Heavy transitive deps (reqwest, opendal,
# arrow) but most of those are already in our graph from parquet.
iceberg = { version = "0.9", optional = true }
iceberg = { version = "0.10", optional = true }
# Used by the vended-credentials S3 router (`prod::vended`) to construct
# per-table object stores from catalog-vended creds.
object_store = { version = "0.11", optional = true, default-features = false, features = ["aws"] }
Expand Down
16 changes: 10 additions & 6 deletions crates/pg2iceberg-iceberg/src/prod/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
//! data and delete manifests at commit time.
//! - **Schema evolution.** Translates our `Vec<SchemaChange>` to a target
//! `iceberg::Schema` (via [`crate::apply_schema_changes`]) and submits
//! via the forked `Transaction::update_schema()`.
//! via the forked `Transaction::replace_schema()`.
//! - **`load_table` not-found.** Maps `ErrorKind::TableNotFound` and
//! `NamespaceNotFound` → `Ok(None)` (the materializer treats not-found
//! distinctly from transient errors).
Expand Down Expand Up @@ -102,7 +102,7 @@ impl<C: IcebergCatalogTrait + Send + Sync + 'static> Catalog for IcebergRustCata
{
Ok(None)
}
// iceberg-rust 0.9 REST catalog returns
// iceberg-rust's REST catalog (as of 0.9) returns
// `ErrorKind::Unexpected` with this exact message on a
// 404 from the REST server (instead of the expected
// `TableNotFound`). Caught by the testcontainers
Expand Down Expand Up @@ -334,7 +334,7 @@ impl<C: IcebergCatalogTrait + Send + Sync + 'static> Catalog for IcebergRustCata
let new_iceberg_schema = to_iceberg_schema(&our_schema)?;

let tx = Transaction::new(&table);
let action = tx.update_schema().set_schema(new_iceberg_schema);
let action = tx.replace_schema().set_schema(new_iceberg_schema);
let tx = action.apply(tx).map_err(map_iceberg_err)?;
let updated = tx
.commit(self.inner.as_ref())
Expand Down Expand Up @@ -386,8 +386,9 @@ impl<C: IcebergCatalogTrait + Send + Sync + 'static> Catalog for IcebergRustCata
let seq_num = snap.sequence_number();
let parent_id = snap.parent_snapshot_id();
let is_replace = matches!(snap.summary().operation, Operation::Replace);
let manifest_list = snap
.load_manifest_list(table.file_io(), table.metadata())
let manifest_list = table
.manifest_list_reader(&snap)
.load()
.await
.map_err(map_iceberg_err)?;
let mut data_files: Vec<DataFile> = Vec::new();
Expand Down Expand Up @@ -603,10 +604,13 @@ impl TransactionAction for ChainedAppendAction {
for update in &step_updates {
builder = update.clone().apply(builder)?;
}
// `local`'s runtime isn't public; pg2iceberg always runs
// inside the tokio runtime iceberg-rust was given.
let mut next = Table::builder()
.identifier(local.identifier().clone())
.file_io(local.file_io().clone())
.metadata(builder.build()?.metadata);
.metadata(builder.build()?.metadata)
.runtime(iceberg::Runtime::try_current()?);
if let Some(location) = local.metadata_location() {
next = next.metadata_location(location);
}
Expand Down
31 changes: 19 additions & 12 deletions crates/pg2iceberg-iceberg/src/prod/gap_audit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
//! create_table → iceberg::Catalog::create_table ✅ wired
//! commit_snapshot (append) → iceberg::transaction::Transaction::fast_append ✅ wired
//! commit_snapshot (deletes) → fork-patched FastAppendAction routes by type ✅ wired (via fork)
//! evolve_schema → iceberg::transaction::Transaction::update_schema ✅ wired (via fork)
//! evolve_schema → iceberg::transaction::Transaction::replace_schema ✅ wired (via fork)
//! commit_compaction → fork's Transaction::rewrite_files ✅ wired (via fork)
//! snapshots → table.metadata().snapshots() + manifest walk ✅ wired
//! ```
//!
Expand All @@ -19,26 +20,32 @@
//! ## We're on a patched fork of iceberg-rust
//!
//! The workspace `Cargo.toml` pins `iceberg` to
//! `polynya-dev/iceberg-rust` (branch `polynya-patches`), based on
//! upstream `v0.9.0`. The fork carries three non-invasive patches:
//! `polynya-dev/iceberg-rust` (branch `polynya-patches-v0.10.1`), based on
//! upstream `v0.10.1`. The fork's patches:
//!
//! 1. **`TransactionAction` visibility.** The trait (and
//! `BoxedTransactionAction`) flipped from `pub(crate)` to `pub`. Lets
//! downstream crates author custom transaction actions.
//! 2. **`UpdateSchemaAction`.** Takes a target `Schema`, emits
//! 2. **`ReplaceSchemaAction`.** Takes a target `Schema`, emits
//! `TableUpdate::AddSchema` + `TableUpdate::SetCurrentSchema(-1)` plus
//! `UuidMatch` / `CurrentSchemaIdMatch` / `LastAssignedFieldIdMatch`
//! requirements. Wrapped by a `Transaction::update_schema()`
//! convenience method matching upstream conventions.
//! requirements, via `Transaction::replace_schema()`. (Upstream's
//! `UpdateSchemaAction` evolves a schema column by column and can't
//! rename, promote or make a column optional.)
//! 3. **`FastAppendAction` accepts mixed content types.** Files passed to
//! `add_data_files` are routed by `content_type()` into
//! `added_data_files` (Data) or `added_delete_files`
//! (EqualityDeletes / PositionDeletes). `SnapshotProducer` writes them
//! to separate manifests (`build_v{2,3}_data` vs
//! `build_v{2,3}_deletes`) per Iceberg-spec rules. The old
//! "Only data content type is allowed for fast append" check is gone.
//!
//! All three are intended to be upstreamed as separate PRs; once they
//! to separate manifests per Iceberg-spec rules.
//! 4. **`RewriteFilesAction`** for compaction: one `Replace` snapshot that
//! removes files by path and adds outputs at an explicit data sequence
//! number, fails if a removed file is already gone, and keeps the
//! summary's running totals.
//! 5. **`Table::properties()`**: the REST `loadTable` response's `config`
//! (vended credentials).
//! 6. **SigV4 request signing** for the REST catalog.
//!
//! All are intended to be upstreamed as separate PRs; once they
//! land we drop the fork and pin to a fresh crates.io release.
//!
//! ## Schema evolution wiring
Expand All @@ -52,7 +59,7 @@
//! catalog) — `AddColumn` appends with `field_id = max + 1`,
//! `DropColumn` is a soft drop that flips `nullable = true`.
//! 3. Translating the evolved `TableSchema` back to an iceberg `Schema`
//! and submitting it via `Transaction::update_schema()` from the fork.
//! and submitting it via `Transaction::replace_schema()` from the fork.
//!
//! Both catalogs share `apply_schema_changes`, so the two backends never
//! drift on field-id allocation or soft-drop semantics. The
Expand Down
12 changes: 3 additions & 9 deletions crates/pg2iceberg-iceberg/src/prod/smoke.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! Smoke tests that exercise `iceberg-rust 0.9` end-to-end against the
//! Smoke tests that exercise `iceberg-rust` end-to-end against the
//! in-process `MemoryCatalog`. These confirm the upstream surface our
//! `IcebergRustCatalog` wrapper relies on:
//!
Expand Down Expand Up @@ -201,10 +201,7 @@ async fn fast_append_round_trip_creates_snapshot_with_data_file() {
.expect("fast append must produce a snapshot");
assert_eq!(snap.summary().operation, iceberg::spec::Operation::Append);

let manifest_list = snap
.load_manifest_list(table.file_io(), table.metadata())
.await
.unwrap();
let manifest_list = table.manifest_list_reader(snap).load().await.unwrap();
assert_eq!(manifest_list.entries().len(), 1);
let manifest = manifest_list.entries()[0]
.load_manifest(table.file_io())
Expand Down Expand Up @@ -272,10 +269,7 @@ async fn fast_append_accepts_mixed_data_and_equality_delete_files() {
let table = tx.commit(&catalog).await.unwrap();

let snap = table.metadata().current_snapshot().unwrap();
let manifest_list = snap
.load_manifest_list(table.file_io(), table.metadata())
.await
.unwrap();
let manifest_list = table.manifest_list_reader(snap).load().await.unwrap();
// Two manifests: one Data, one Deletes.
assert_eq!(manifest_list.entries().len(), 2);
let mut kinds: Vec<ManifestContentType> =
Expand Down
4 changes: 2 additions & 2 deletions crates/pg2iceberg-tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ tokio-postgres = { version = "0.7", optional = true, default-features = false, f
object_store = { version = "0.11", optional = true, default-features = false, features = ["aws"] }
# The DST's production backend reads a table's manifests directly, as
# ground truth independent of pg2iceberg's own `snapshots()`.
iceberg = { version = "0.9", optional = true }
iceberg = { version = "0.10", optional = true }
# Parses pgoutput in the sim-vs-Postgres conformance suite.
postgres-replication = { version = "0.6", optional = true, default-features = false }
async-trait = { workspace = true, optional = true }
Expand Down Expand Up @@ -67,7 +67,7 @@ proptest.workspace = true
# chaos soak's lifecycle-driving DST: virtual time is the only way
# to keep `run_logical_main_loop`'s scheduler-driven ticks
# deterministic under proptest seeds.
tokio = { version = "1", features = ["rt", "macros", "time", "test-util", "sync"] }
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "time", "test-util", "sync"] }

[features]
# Enables the testcontainers-driven integration tests
Expand Down
Loading
Loading