From 23d1fbf3efa91a6b4ee1aaf7ddcf8b0cd6331b2c Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 17 Nov 2026 02:30:55 +0800 Subject: [PATCH 1/2] chore: iceberg-rust 0.11.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Through the fork's patches replayed onto it (polynya-dev/iceberg-rust#7). - Manifests are read through `Table::manifest_reader()`: 0.11 removed `ManifestFile::load_manifest`. - `Variant` columns are refused, as other non-primitive types are. - The REST catalog props set `rest.auth.type` for each catalog auth — `oauth2` for a bearer token and for client credentials, `sigv4` for AWS endpoints: 0.11's REST client picks its auth manager by it, and infers one otherwise, warning at every start. SigV4 signing is the fork's `SigV4AuthManager` now. - The lockfile moves what 0.11 requires: opendal 0.58 among them. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 164 +++++++++++------- Cargo.toml | 6 +- crates/pg2iceberg-iceberg/Cargo.toml | 2 +- crates/pg2iceberg-iceberg/src/prod/catalog.rs | 9 +- crates/pg2iceberg-iceberg/src/prod/smoke.rs | 5 +- crates/pg2iceberg-tests/Cargo.toml | 2 +- crates/pg2iceberg-tests/tests/dst.rs | 17 +- crates/pg2iceberg/Cargo.toml | 6 +- crates/pg2iceberg/src/config.rs | 28 +++ 9 files changed, 159 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd0621f..7ee948f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,7 +247,7 @@ dependencies = [ "arrow-schema 58.4.0", "chrono", "half", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "num-complex", "num-integer", "num-traits", @@ -505,6 +505,16 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + [[package]] name = "asyncband" version = "0.7.3" @@ -1501,12 +1511,13 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin", ] [[package]] @@ -2215,9 +2226,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -2449,8 +2460,8 @@ dependencies = [ [[package]] name = "iceberg" -version = "0.10.1" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#0ded76c9e9f627822b7a7a4d032771c410fc7a48" +version = "0.11.0" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=rebase-v0.11.0#a5b550ddd444e0416bba7e265039808ce56d65dd" dependencies = [ "aes-gcm", "anyhow", @@ -2471,12 +2482,15 @@ dependencies = [ "bimap", "bytes", "chrono", + "crc32fast", "derive_builder", "expect-test", "fastnum", "flate2", "fnv", + "form_urlencoded", "futures", + "iceberg-property-macro", "itertools 0.13.0", "moka", "murmur3", @@ -2505,8 +2519,8 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" -version = "0.10.1" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#0ded76c9e9f627822b7a7a4d032771c410fc7a48" +version = "0.11.0" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=rebase-v0.11.0#a5b550ddd444e0416bba7e265039808ce56d65dd" dependencies = [ "async-trait", "aws-config", @@ -2522,14 +2536,25 @@ dependencies = [ "serde_derive", "serde_json", "tokio", + "tracing", "typed-builder", "uuid", ] +[[package]] +name = "iceberg-property-macro" +version = "0.11.0" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=rebase-v0.11.0#a5b550ddd444e0416bba7e265039808ce56d65dd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "iceberg-storage-opendal" -version = "0.10.1" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#0ded76c9e9f627822b7a7a4d032771c410fc7a48" +version = "0.11.0" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=rebase-v0.11.0#a5b550ddd444e0416bba7e265039808ce56d65dd" dependencies = [ "anyhow", "async-trait", @@ -2678,7 +2703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -2755,10 +2780,11 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.24" +version = "0.2.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +checksum = "ac63a5a611e9d720726d6ab6ee2ee19e6f620503554ea5d303c7cb294ff40c12" dependencies = [ + "jiff-core", "jiff-static", "jiff-tzdb-platform", "js-sys", @@ -2767,15 +2793,25 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", - "windows-sys 0.61.2", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e9635854583b49ce3c6c43a071f70b0a49e79ad3c261c7b3aa76b2280416d2" +dependencies = [ + "log", ] [[package]] name = "jiff-static" -version = "0.2.24" +version = "0.2.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +checksum = "9f9c27796c21cc8b33ce490f148d53fe41d679b887885bf0ef8f662cdb9b35c0" dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.117", @@ -2995,9 +3031,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" [[package]] name = "lru-slab" @@ -3049,15 +3085,6 @@ dependencies = [ "digest 0.11.2", ] -[[package]] -name = "mea" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31fc7d159de0085ab6dd7ff145a9819442cfd3d098f783263120503c3f3e58b0" -dependencies = [ - "slab", -] - [[package]] name = "memchr" version = "2.8.0" @@ -3257,12 +3284,13 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" dependencies = [ "ctor", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -3273,24 +3301,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" dependencies = [ "anyhow", - "base64 0.22.1", + "asyncband 0.6.7", + "base64 0.23.1", "bytes", "futures", "http", - "http-body", "jiff", "log", "md-5 0.11.0", - "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.5", "serde", "serde_json", "tokio", @@ -3299,23 +3325,37 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +dependencies = [ + "bytes", + "futures", + "http", + "http-body", + "opendal-core", + "reqwest 0.13.5", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" dependencies = [ + "asyncband 0.6.7", "futures", "http", - "mea", "opendal-core", ] [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" dependencies = [ "log", "opendal-core", @@ -3323,9 +3363,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" dependencies = [ "backon", "log", @@ -3334,9 +3374,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" dependencies = [ "opendal-core", "tokio", @@ -3344,9 +3384,9 @@ dependencies = [ [[package]] name = "opendal-service-fs" -version = "0.57.0" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e89a665fef0e6bd249cf5ea47fc174b7ba892159bee4b9382528b1ca873a2c" +checksum = "c7ef1e1c45f3f89282a59073897e0d685e51385fed0aea771714789525cff996" dependencies = [ "bytes", "log", @@ -3358,18 +3398,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", - "crc32c", + "crc-fast", "http", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -3565,7 +3605,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "lz4_flex", "num-bigint", "num-integer", @@ -4199,9 +4239,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -4510,7 +4550,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" dependencies = [ "anyhow", - "asyncband", + "asyncband 0.7.3", "base64 0.23.1", "bytes", "hex", @@ -5200,6 +5240,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" diff --git a/Cargo.toml b/Cargo.toml index bdd86ce..4284fbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ all = { level = "warn", priority = -1 } # (Supabase's etl maintainer) rather than rolling our own to avoid the # maintenance burden of two forks. [patch.crates-io] -iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" } +iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "rebase-v0.11.0" } # `iceberg-catalog-rest` also has to come from the fork: the # polynya-patches branch threads the REST `loadTable` response's # `config` map through `Table::builder().properties(response_config)`, @@ -68,7 +68,7 @@ iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polyn # servers (Polaris, Lakekeeper, ...) return per-table S3 STS tokens # in `config`, but our `extract_creds` sees an empty map because the # `Table::properties()` accessor surfaces nothing. -iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" } -iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" } +iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "rebase-v0.11.0" } +iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "rebase-v0.11.0" } tokio-postgres = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" } postgres-replication = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" } diff --git a/crates/pg2iceberg-iceberg/Cargo.toml b/crates/pg2iceberg-iceberg/Cargo.toml index 6d86922..27aa5f6 100644 --- a/crates/pg2iceberg-iceberg/Cargo.toml +++ b/crates/pg2iceberg-iceberg/Cargo.toml @@ -28,7 +28,7 @@ futures = "0.3" # iceberg-rust prod-impl wrap. Heavy transitive deps (reqwest, opendal, # arrow) but most of those are already in our graph from parquet. -iceberg = { version = "0.10", optional = true } +iceberg = { version = "0.11", optional = true } # Used by the vended-credentials S3 router (`prod::vended`) to construct # per-table object stores from catalog-vended creds. object_store = { version = "0.11", optional = true, default-features = false, features = ["aws"] } diff --git a/crates/pg2iceberg-iceberg/src/prod/catalog.rs b/crates/pg2iceberg-iceberg/src/prod/catalog.rs index 6903fc1..c0400c5 100644 --- a/crates/pg2iceberg-iceberg/src/prod/catalog.rs +++ b/crates/pg2iceberg-iceberg/src/prod/catalog.rs @@ -522,8 +522,9 @@ impl Catalog for IcebergRustCata let manifest = match read.manifests.get(path).or(cached.manifests.get(path)) { Some(manifest) => Arc::clone(manifest), None => Arc::new( - entry - .load_manifest(table.file_io()) + table + .manifest_reader() + .read(entry) .await .map_err(map_iceberg_err)?, ), @@ -905,7 +906,7 @@ fn to_iceberg_type(ty: IcebergType) -> Type { fn from_iceberg_type(ty: &Type) -> Result { let p = match ty { Type::Primitive(p) => p, - Type::Struct(_) | Type::List(_) | Type::Map(_) => { + Type::Struct(_) | Type::List(_) | Type::Map(_) | Type::Variant(_) => { return Err(IcebergError::Other(format!( "non-primitive iceberg type encountered: {ty:?}" ))); @@ -2090,7 +2091,7 @@ mod tests { self.inner.write(bs).await } - async fn close(&mut self) -> iceberg::Result<()> { + async fn close(&mut self) -> iceberg::Result { tokio::time::sleep(self.latency).await; self.inner.close().await } diff --git a/crates/pg2iceberg-iceberg/src/prod/smoke.rs b/crates/pg2iceberg-iceberg/src/prod/smoke.rs index a1716e4..0a683c0 100644 --- a/crates/pg2iceberg-iceberg/src/prod/smoke.rs +++ b/crates/pg2iceberg-iceberg/src/prod/smoke.rs @@ -203,8 +203,9 @@ async fn fast_append_round_trip_creates_snapshot_with_data_file() { let manifest_list = table.manifest_list_reader(snap).load().await.unwrap(); assert_eq!(manifest_list.entries().len(), 1); - let manifest = manifest_list.entries()[0] - .load_manifest(table.file_io()) + let manifest = table + .manifest_reader() + .read(&manifest_list.entries()[0]) .await .unwrap(); assert_eq!(manifest.entries().len(), 1); diff --git a/crates/pg2iceberg-tests/Cargo.toml b/crates/pg2iceberg-tests/Cargo.toml index 9383781..9efa7d6 100644 --- a/crates/pg2iceberg-tests/Cargo.toml +++ b/crates/pg2iceberg-tests/Cargo.toml @@ -33,7 +33,7 @@ tokio-postgres = { version = "0.7", optional = true, default-features = false, f object_store = { version = "0.11", optional = true, default-features = false, features = ["aws"] } # The DST's production backend reads a table's manifests directly, as # ground truth independent of pg2iceberg's own `snapshots()`. -iceberg = { version = "0.10", optional = true } +iceberg = { version = "0.11", optional = true } # Parses pgoutput in the sim-vs-Postgres conformance suite. postgres-replication = { version = "0.6", optional = true, default-features = false } async-trait = { workspace = true, optional = true } diff --git a/crates/pg2iceberg-tests/tests/dst.rs b/crates/pg2iceberg-tests/tests/dst.rs index 8814fe6..62b2711 100644 --- a/crates/pg2iceberg-tests/tests/dst.rs +++ b/crates/pg2iceberg-tests/tests/dst.rs @@ -973,8 +973,9 @@ mod prod_backend { .map_err(err)?; let mut held: HashMap<&str, u64> = HashMap::new(); for manifest_file in list.entries() { - let manifest = manifest_file - .load_manifest(table.file_io()) + let manifest = table + .manifest_reader() + .read(manifest_file) .await .map_err(err)?; for entry in manifest.entries().iter().filter(|e| e.is_alive()) { @@ -1029,8 +1030,9 @@ mod prod_backend { .map_err(err)?; let mut out = Vec::new(); for manifest_file in list.entries() { - let manifest = manifest_file - .load_manifest(table.file_io()) + let manifest = table + .manifest_reader() + .read(manifest_file) .await .map_err(err)?; for entry in manifest.entries().iter().filter(|e| e.is_alive()) { @@ -1213,7 +1215,7 @@ mod prod_backend { let mut totals = Totals::default(); let list = table.manifest_list_reader(current).load().await?; for manifest_file in list.entries() { - let manifest = manifest_file.load_manifest(file_io).await?; + let manifest = table.manifest_reader().read(manifest_file).await?; // A rewrite drops the entries earlier commits left `DELETED`. let alive: Vec<_> = manifest.entries().iter().filter(|e| e.is_alive()).collect(); if let Some(read) = self.no_deletes_since { @@ -1383,8 +1385,9 @@ mod prod_backend { .await .map_err(err)?; for manifest_file in list.entries() { - let manifest = manifest_file - .load_manifest(table.file_io()) + let manifest = table + .manifest_reader() + .read(manifest_file) .await .map_err(err)?; out.extend( diff --git a/crates/pg2iceberg/Cargo.toml b/crates/pg2iceberg/Cargo.toml index 8bfeca9..b6d476d 100644 --- a/crates/pg2iceberg/Cargo.toml +++ b/crates/pg2iceberg/Cargo.toml @@ -28,14 +28,14 @@ pg2iceberg-validate.workspace = true # Real iceberg + object_store types — same fork pin as the rest of the # workspace, applied via [patch.crates-io] in the workspace Cargo.toml. -iceberg = { version = "0.10", features = [] } -iceberg-catalog-rest = { version = "0.10", default-features = false } +iceberg = { version = "0.11", features = [] } +iceberg-catalog-rest = { version = "0.11", default-features = false } # StorageFactory for the REST catalog. iceberg-rust 0.9's REST catalog # requires a StorageFactory be provided up front (otherwise table ops # fail at runtime with "StorageFactory must be provided"). We use # OpenDAL's S3 backend which the upstream integration tests also use. # Surfaced by the testcontainers integration test. -iceberg-storage-opendal = { version = "0.10", features = ["opendal-s3"] } +iceberg-storage-opendal = { version = "0.11", features = ["opendal-s3"] } # `aws` enables the S3 backend; `gcp`/`azure` are deferred follow-ons. object_store = { version = "0.11", default-features = false, features = ["aws"] } diff --git a/crates/pg2iceberg/src/config.rs b/crates/pg2iceberg/src/config.rs index 863c3b1..ebe0420 100644 --- a/crates/pg2iceberg/src/config.rs +++ b/crates/pg2iceberg/src/config.rs @@ -975,11 +975,17 @@ impl Config { props.insert("warehouse".into(), self.sink.warehouse.clone()); } let region = self.sink.resolved_region(); + // Each with `rest.auth.type` set: iceberg-rust's REST client picks + // its auth manager by it, and only infers one — with a warning — + // when it's unset. match self.sink.resolved_catalog_auth() { + // A static token is OAuth2's, as in Iceberg Java. "bearer" if !self.sink.catalog_token.is_empty() => { + props.insert("rest.auth.type".into(), "oauth2".into()); props.insert("token".into(), self.sink.catalog_token.clone()); } "oauth2" if !self.sink.catalog_client_id.is_empty() => { + props.insert("rest.auth.type".into(), "oauth2".into()); props.insert( "oauth2-server-uri".into(), format!( @@ -999,6 +1005,7 @@ impl Config { // credentials, for the service the endpoint names (S3 // Tables' `s3tables`, Glue's `glue`). "sigv4" => { + props.insert("rest.auth.type".into(), "sigv4".into()); props.insert("rest.sigv4-enabled".into(), "true".into()); props.insert("rest.signing-region".into(), region.clone()); if let Some((service, _)) = aws_endpoint(&self.sink.catalog_uri) { @@ -1717,6 +1724,10 @@ sink: assert_eq!(cfg.sink.resolved_region(), "eu-west-1"); let props = cfg.rest_catalog_props(); assert_eq!(props.get("token").map(String::as_str), Some("t0ken")); + assert_eq!( + props.get("rest.auth.type").map(String::as_str), + Some("oauth2") + ); assert_eq!( props.get("s3.region").map(String::as_str), Some("eu-west-1") @@ -1835,6 +1846,10 @@ sink: ..Config::default() }; let props = cfg.rest_catalog_props(); + assert_eq!( + props.get("rest.auth.type").map(String::as_str), + Some("sigv4") + ); assert_eq!( props.get("rest.sigv4-enabled").map(String::as_str), Some("true") @@ -1863,6 +1878,19 @@ sink: ..SinkConfig::default() }; assert_eq!(sink.resolved_catalog_auth(), "oauth2"); + let props = Config { + sink, + ..Config::default() + } + .rest_catalog_props(); + assert_eq!( + props.get("rest.auth.type").map(String::as_str), + Some("oauth2") + ); + assert_eq!( + props.get("credential").map(String::as_str), + Some("id:secret") + ); } #[test] From db56cddcf5794a3c8186a973741eda71786ac1a5 Mon Sep 17 00:00:00 2001 From: Hasyimi Bahrudin Date: Tue, 17 Nov 2026 02:37:51 +0800 Subject: [PATCH 2/2] chore: iceberg-rust from polynya-patches-v0.11.0 polynya-dev/iceberg-rust#7 merged. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 14 +++++++------- Cargo.toml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ee948f..f7a41ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2413,7 +2413,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2461,7 +2461,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=rebase-v0.11.0#a5b550ddd444e0416bba7e265039808ce56d65dd" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#429f3a0aa236eec59850679c025c068d2ac29d41" dependencies = [ "aes-gcm", "anyhow", @@ -2520,7 +2520,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=rebase-v0.11.0#a5b550ddd444e0416bba7e265039808ce56d65dd" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#429f3a0aa236eec59850679c025c068d2ac29d41" dependencies = [ "async-trait", "aws-config", @@ -2544,7 +2544,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=rebase-v0.11.0#a5b550ddd444e0416bba7e265039808ce56d65dd" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#429f3a0aa236eec59850679c025c068d2ac29d41" dependencies = [ "proc-macro2", "quote", @@ -2554,7 +2554,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.11.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=rebase-v0.11.0#a5b550ddd444e0416bba7e265039808ce56d65dd" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.11.0#429f3a0aa236eec59850679c025c068d2ac29d41" dependencies = [ "anyhow", "async-trait", @@ -4270,7 +4270,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror", "tokio", "tracing", @@ -4308,7 +4308,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.52.0", ] diff --git a/Cargo.toml b/Cargo.toml index 4284fbc..c8c6359 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ all = { level = "warn", priority = -1 } # (Supabase's etl maintainer) rather than rolling our own to avoid the # maintenance burden of two forks. [patch.crates-io] -iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "rebase-v0.11.0" } +iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" } # `iceberg-catalog-rest` also has to come from the fork: the # polynya-patches branch threads the REST `loadTable` response's # `config` map through `Table::builder().properties(response_config)`, @@ -68,7 +68,7 @@ iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "rebas # servers (Polaris, Lakekeeper, ...) return per-table S3 STS tokens # in `config`, but our `extract_creds` sees an empty map because the # `Table::properties()` accessor surfaces nothing. -iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "rebase-v0.11.0" } -iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "rebase-v0.11.0" } +iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" } +iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.11.0" } tokio-postgres = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" } postgres-replication = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" }