Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 108 additions & 62 deletions Cargo.lock

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,15 @@ all = { level = "warn", priority = -1 }
# (Supabase's etl maintainer) rather than rolling our own to avoid the
# maintenance burden of two forks.
[patch.crates-io]
iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" }
iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" }
# `iceberg-catalog-rest` also has to come from the fork: the
# polynya-patches branch threads the REST `loadTable` response's
# `config` map through `Table::builder().properties(response_config)`,
# which upstream omits. Without it, vended-credentials
# servers (Polaris, Lakekeeper, ...) return per-table S3 STS tokens
# in `config`, but our `extract_creds` sees an empty map because the
# `Table::properties()` accessor surfaces nothing.
iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" }
iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" }
iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" }
iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" }
tokio-postgres = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" }
postgres-replication = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" }
2 changes: 1 addition & 1 deletion crates/pg2iceberg-iceberg/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ futures = "0.3"

# iceberg-rust prod-impl wrap. Heavy transitive deps (reqwest, opendal,
# arrow) but most of those are already in our graph from parquet.
iceberg = { version = "0.10", optional = true }
iceberg = { version = "0.11", optional = true }
# Used by the vended-credentials S3 router (`prod::vended`) to construct
# per-table object stores from catalog-vended creds.
object_store = { version = "0.11", optional = true, default-features = false, features = ["aws"] }
Expand Down
9 changes: 5 additions & 4 deletions crates/pg2iceberg-iceberg/src/prod/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -522,8 +522,9 @@ impl<C: IcebergCatalogTrait + Send + Sync + 'static> Catalog for IcebergRustCata
let manifest = match read.manifests.get(path).or(cached.manifests.get(path)) {
Some(manifest) => Arc::clone(manifest),
None => Arc::new(
entry
.load_manifest(table.file_io())
table
.manifest_reader()
.read(entry)
.await
.map_err(map_iceberg_err)?,
),
Expand Down Expand Up @@ -905,7 +906,7 @@ fn to_iceberg_type(ty: IcebergType) -> Type {
fn from_iceberg_type(ty: &Type) -> Result<IcebergType> {
let p = match ty {
Type::Primitive(p) => p,
Type::Struct(_) | Type::List(_) | Type::Map(_) => {
Type::Struct(_) | Type::List(_) | Type::Map(_) | Type::Variant(_) => {
return Err(IcebergError::Other(format!(
"non-primitive iceberg type encountered: {ty:?}"
)));
Expand Down Expand Up @@ -2090,7 +2091,7 @@ mod tests {
self.inner.write(bs).await
}

async fn close(&mut self) -> iceberg::Result<()> {
async fn close(&mut self) -> iceberg::Result<iceberg::io::FileMetadata> {
tokio::time::sleep(self.latency).await;
self.inner.close().await
}
Expand Down
5 changes: 3 additions & 2 deletions crates/pg2iceberg-iceberg/src/prod/smoke.rs
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,9 @@ async fn fast_append_round_trip_creates_snapshot_with_data_file() {

let manifest_list = table.manifest_list_reader(snap).load().await.unwrap();
assert_eq!(manifest_list.entries().len(), 1);
let manifest = manifest_list.entries()[0]
.load_manifest(table.file_io())
let manifest = table
.manifest_reader()
.read(&manifest_list.entries()[0])
.await
.unwrap();
assert_eq!(manifest.entries().len(), 1);
Expand Down
2 changes: 1 addition & 1 deletion crates/pg2iceberg-tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ tokio-postgres = { version = "0.7", optional = true, default-features = false, f
object_store = { version = "0.11", optional = true, default-features = false, features = ["aws"] }
# The DST's production backend reads a table's manifests directly, as
# ground truth independent of pg2iceberg's own `snapshots()`.
iceberg = { version = "0.10", optional = true }
iceberg = { version = "0.11", optional = true }
# Parses pgoutput in the sim-vs-Postgres conformance suite.
postgres-replication = { version = "0.6", optional = true, default-features = false }
async-trait = { workspace = true, optional = true }
Expand Down
17 changes: 10 additions & 7 deletions crates/pg2iceberg-tests/tests/dst.rs
Original file line number Diff line number Diff line change
Expand Up @@ -973,8 +973,9 @@ mod prod_backend {
.map_err(err)?;
let mut held: HashMap<&str, u64> = HashMap::new();
for manifest_file in list.entries() {
let manifest = manifest_file
.load_manifest(table.file_io())
let manifest = table
.manifest_reader()
.read(manifest_file)
.await
.map_err(err)?;
for entry in manifest.entries().iter().filter(|e| e.is_alive()) {
Expand Down Expand Up @@ -1029,8 +1030,9 @@ mod prod_backend {
.map_err(err)?;
let mut out = Vec::new();
for manifest_file in list.entries() {
let manifest = manifest_file
.load_manifest(table.file_io())
let manifest = table
.manifest_reader()
.read(manifest_file)
.await
.map_err(err)?;
for entry in manifest.entries().iter().filter(|e| e.is_alive()) {
Expand Down Expand Up @@ -1213,7 +1215,7 @@ mod prod_backend {
let mut totals = Totals::default();
let list = table.manifest_list_reader(current).load().await?;
for manifest_file in list.entries() {
let manifest = manifest_file.load_manifest(file_io).await?;
let manifest = table.manifest_reader().read(manifest_file).await?;
// A rewrite drops the entries earlier commits left `DELETED`.
let alive: Vec<_> = manifest.entries().iter().filter(|e| e.is_alive()).collect();
if let Some(read) = self.no_deletes_since {
Expand Down Expand Up @@ -1383,8 +1385,9 @@ mod prod_backend {
.await
.map_err(err)?;
for manifest_file in list.entries() {
let manifest = manifest_file
.load_manifest(table.file_io())
let manifest = table
.manifest_reader()
.read(manifest_file)
.await
.map_err(err)?;
out.extend(
Expand Down
6 changes: 3 additions & 3 deletions crates/pg2iceberg/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,14 @@ pg2iceberg-validate.workspace = true

# Real iceberg + object_store types — same fork pin as the rest of the
# workspace, applied via [patch.crates-io] in the workspace Cargo.toml.
iceberg = { version = "0.10", features = [] }
iceberg-catalog-rest = { version = "0.10", default-features = false }
iceberg = { version = "0.11", features = [] }
iceberg-catalog-rest = { version = "0.11", default-features = false }
# StorageFactory for the REST catalog. iceberg-rust 0.9's REST catalog
# requires a StorageFactory be provided up front (otherwise table ops
# fail at runtime with "StorageFactory must be provided"). We use
# OpenDAL's S3 backend which the upstream integration tests also use.
# Surfaced by the testcontainers integration test.
iceberg-storage-opendal = { version = "0.10", features = ["opendal-s3"] }
iceberg-storage-opendal = { version = "0.11", features = ["opendal-s3"] }
# `aws` enables the S3 backend; `gcp`/`azure` are deferred follow-ons.
object_store = { version = "0.11", default-features = false, features = ["aws"] }

Expand Down
28 changes: 28 additions & 0 deletions crates/pg2iceberg/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -975,11 +975,17 @@ impl Config {
props.insert("warehouse".into(), self.sink.warehouse.clone());
}
let region = self.sink.resolved_region();
// Each with `rest.auth.type` set: iceberg-rust's REST client picks
// its auth manager by it, and only infers one — with a warning —
// when it's unset.
match self.sink.resolved_catalog_auth() {
// A static token is OAuth2's, as in Iceberg Java.
"bearer" if !self.sink.catalog_token.is_empty() => {
props.insert("rest.auth.type".into(), "oauth2".into());
props.insert("token".into(), self.sink.catalog_token.clone());
}
"oauth2" if !self.sink.catalog_client_id.is_empty() => {
props.insert("rest.auth.type".into(), "oauth2".into());
props.insert(
"oauth2-server-uri".into(),
format!(
Expand All @@ -999,6 +1005,7 @@ impl Config {
// credentials, for the service the endpoint names (S3
// Tables' `s3tables`, Glue's `glue`).
"sigv4" => {
props.insert("rest.auth.type".into(), "sigv4".into());
props.insert("rest.sigv4-enabled".into(), "true".into());
props.insert("rest.signing-region".into(), region.clone());
if let Some((service, _)) = aws_endpoint(&self.sink.catalog_uri) {
Expand Down Expand Up @@ -1717,6 +1724,10 @@ sink:
assert_eq!(cfg.sink.resolved_region(), "eu-west-1");
let props = cfg.rest_catalog_props();
assert_eq!(props.get("token").map(String::as_str), Some("t0ken"));
assert_eq!(
props.get("rest.auth.type").map(String::as_str),
Some("oauth2")
);
assert_eq!(
props.get("s3.region").map(String::as_str),
Some("eu-west-1")
Expand Down Expand Up @@ -1835,6 +1846,10 @@ sink:
..Config::default()
};
let props = cfg.rest_catalog_props();
assert_eq!(
props.get("rest.auth.type").map(String::as_str),
Some("sigv4")
);
assert_eq!(
props.get("rest.sigv4-enabled").map(String::as_str),
Some("true")
Expand Down Expand Up @@ -1863,6 +1878,19 @@ sink:
..SinkConfig::default()
};
assert_eq!(sink.resolved_catalog_auth(), "oauth2");
let props = Config {
sink,
..Config::default()
}
.rest_catalog_props();
assert_eq!(
props.get("rest.auth.type").map(String::as_str),
Some("oauth2")
);
assert_eq!(
props.get("credential").map(String::as_str),
Some("id:secret")
);
}

#[test]
Expand Down
Loading