diff --git a/Cargo.lock b/Cargo.lock index 89b43e2..288bd99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.7.8" @@ -130,6 +165,7 @@ checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" dependencies = [ "bigdecimal", "bon", + "crc32fast", "digest 0.10.7", "log", "miniz_oxide", @@ -140,6 +176,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "snap", "strum", "strum_macros", "thiserror", @@ -170,14 +207,14 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow-arith" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", "chrono", "num-traits", ] @@ -200,17 +237,17 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ "ahash 0.8.12", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", "chrono", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "num-complex", "num-integer", "num-traits", @@ -229,9 +266,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", @@ -261,16 +298,16 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", "arrow-ord", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "atoi", "base64 0.22.1", "chrono", @@ -294,12 +331,12 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ - "arrow-buffer 57.3.0", - "arrow-schema 57.3.0", + "arrow-buffer 58.4.0", + "arrow-schema 58.4.0", "half", "num-integer", "num-traits", @@ -320,29 +357,29 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "flatbuffers 25.12.19", ] [[package]] name = "arrow-ord" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", ] [[package]] @@ -353,9 +390,9 @@ checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" [[package]] name = "arrow-schema" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" [[package]] name = "arrow-select" @@ -373,29 +410,29 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" dependencies = [ "ahash 0.8.12", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", "num-traits", ] [[package]] name = "arrow-string" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" dependencies = [ - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-data 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "memchr", "num-traits", "regex", @@ -468,6 +505,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "asyncband" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" + [[package]] name = "atoi" version = "2.0.0" @@ -512,7 +555,7 @@ dependencies = [ "fastrand", "hex", "http", - "sha1", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -943,6 +986,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1295,6 +1344,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "4.6.1" @@ -1356,6 +1415,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1486,6 +1555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1498,6 +1568,25 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -1647,7 +1736,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -2048,6 +2136,16 @@ dependencies = [ "wasip3", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gloo-timers" version = "0.3.0" @@ -2115,12 +2213,6 @@ dependencies = [ "foldhash", ] -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" - [[package]] name = "hashbrown" version = "0.17.0" @@ -2310,7 +2402,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2357,19 +2449,20 @@ dependencies = [ [[package]] name = "iceberg" -version = "0.9.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches#167648a0931c1488cfe03e4949ff4b93db195dba" +version = "0.10.1" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#cc60df5c899beac69a19145c7f18eac86972e316" dependencies = [ + "aes-gcm", "anyhow", "apache-avro", "array-init", "arrow-arith", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-cast 58.4.0", "arrow-ord", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "arrow-string", "as-any", "async-trait", @@ -2389,9 +2482,9 @@ dependencies = [ "murmur3", "once_cell", "ordered-float 4.6.0", - "parquet 57.3.0", - "rand 0.8.6", - "reqwest", + "parquet 58.4.0", + "rand 0.9.4", + "reqwest 0.12.28", "roaring", "serde", "serde_bytes", @@ -2401,17 +2494,19 @@ dependencies = [ "serde_with", "strum", "tokio", + "tracing", "typed-builder", "typetag", "url", "uuid", + "zeroize", "zstd", ] [[package]] name = "iceberg-catalog-rest" -version = "0.9.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches#167648a0931c1488cfe03e4949ff4b93db195dba" +version = "0.10.1" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#cc60df5c899beac69a19145c7f18eac86972e316" dependencies = [ "async-trait", "aws-config", @@ -2422,29 +2517,29 @@ dependencies = [ "http", "iceberg", "itertools 0.13.0", - "reqwest", + "reqwest 0.12.28", "serde", "serde_derive", "serde_json", "tokio", - "tracing", "typed-builder", "uuid", ] [[package]] name = "iceberg-storage-opendal" -version = "0.9.0" -source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches#167648a0931c1488cfe03e4949ff4b93db195dba" +version = "0.10.1" +source = "git+https://github.com/polynya-dev/iceberg-rust?branch=polynya-patches-v0.10.1#cc60df5c899beac69a19145c7f18eac86972e316" dependencies = [ "anyhow", "async-trait", "bytes", "cfg-if", + "futures", "iceberg", "opendal", - "reqsign", - "reqwest", + "reqsign-aws-v4", + "reqsign-core", "serde", "typetag", "url", @@ -2588,6 +2683,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -2657,10 +2761,12 @@ checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", "windows-sys 0.61.2", ] @@ -2690,6 +2796,55 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2805,6 +2960,18 @@ dependencies = [ "redox_syscall 0.7.4", ] +[[package]] +name = "link-section" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" + +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2840,9 +3007,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ "twox-hash 2.1.2", ] @@ -2882,6 +3049,15 @@ dependencies = [ "digest 0.11.2", ] +[[package]] +name = "mea" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31fc7d159de0085ab6dd7ff145a9819442cfd3d098f783263120503c3f3e58b0" +dependencies = [ + "slab", +] + [[package]] name = "memchr" version = "2.8.0" @@ -3050,7 +3226,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.37.5", "rand 0.8.6", - "reqwest", + "reqwest 0.12.28", "ring", "serde", "serde_json", @@ -3073,33 +3249,132 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "opendal" -version = "0.55.0" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +dependencies = [ + "ctor", + "opendal-core", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-fs", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a" +checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" dependencies = [ "anyhow", - "backon", "base64 0.22.1", "bytes", - "crc32c", "futures", - "getrandom 0.2.17", "http", "http-body", "jiff", "log", - "md-5 0.10.6", + "md-5 0.11.0", + "mea", "percent-encoding", - "quick-xml 0.38.4", - "reqsign", - "reqwest", + "quick-xml 0.39.4", + "reqsign-core", + "reqwest 0.13.5", "serde", "serde_json", "tokio", "url", "uuid", + "web-time", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +dependencies = [ + "futures", + "http", + "mea", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-fs" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e89a665fef0e6bd249cf5ea47fc174b7ba892159bee4b9382528b1ca873a2c" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-s3" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "http", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml 0.39.4", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", ] [[package]] @@ -3201,18 +3476,17 @@ dependencies = [ [[package]] name = "parquet" -version = "57.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" dependencies = [ "ahash 0.8.12", - "arrow-array 57.3.0", - "arrow-buffer 57.3.0", - "arrow-cast 57.3.0", - "arrow-data 57.3.0", - "arrow-ipc 57.3.0", - "arrow-schema 57.3.0", - "arrow-select 57.3.0", + "arrow-array 58.4.0", + "arrow-buffer 58.4.0", + "arrow-data 58.4.0", + "arrow-ipc 58.4.0", + "arrow-schema 58.4.0", + "arrow-select 58.4.0", "base64 0.22.1", "brotli", "bytes", @@ -3220,12 +3494,13 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "lz4_flex", "num-bigint", "num-integer", "num-traits", "paste", + "ring", "seq-macro", "simdutf8", "snap", @@ -3480,7 +3755,7 @@ dependencies = [ "pollster", "postgres-replication", "proptest", - "reqwest", + "reqwest 0.12.28", "serde_json", "testcontainers", "testcontainers-modules", @@ -3578,6 +3853,18 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -3819,9 +4106,19 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ "memchr", "serde", @@ -3840,7 +4137,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror", "tokio", "tracing", @@ -3853,6 +4150,7 @@ version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", @@ -3877,7 +4175,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.52.0", ] @@ -4077,31 +4375,70 @@ dependencies = [ ] [[package]] -name = "reqsign" -version = "0.16.5" +name = "reqsign-aws-core" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" dependencies = [ - "anyhow", - "async-trait", - "base64 0.22.1", - "chrono", + "bytes", "form_urlencoded", - "getrandom 0.2.17", "hex", - "hmac 0.12.1", - "home", "http", "log", "percent-encoding", - "quick-xml 0.37.5", - "rand 0.8.6", - "reqwest", + "quick-xml 0.42.0", + "reqsign-core", "rust-ini", "serde", "serde_json", - "sha1", - "sha2 0.10.9", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +dependencies = [ + "bytes", + "http", + "log", + "quick-xml 0.42.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-core" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" +dependencies = [ + "anyhow", + "asyncband", + "base64 0.23.1", + "bytes", + "hex", + "hmac 0.13.0", + "http", + "jiff", + "log", + "percent-encoding", + "sha1 0.11.0", + "sha2 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" +dependencies = [ + "anyhow", + "reqsign-core", "tokio", ] @@ -4143,11 +4480,49 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "webpki-roots 1.0.7", ] +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -4294,6 +4669,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + [[package]] name = "rustls-webpki" version = "0.103.13" @@ -4559,6 +4961,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", +] + [[package]] name = "sha2" version = "0.10.9" @@ -4612,6 +5025,16 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -5414,6 +5837,16 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -5668,6 +6101,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -5700,6 +6146,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.26.11" diff --git a/Cargo.toml b/Cargo.toml index a131059..bdd86ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,10 +45,12 @@ all = { level = "warn", priority = -1 } # `[patch.crates-io]` and matching git overrides for the two ecosystem # forks pg2iceberg-rust depends on: # -# 1. `iceberg` → `polynya-dev/iceberg-rust@polynya-patches`. Based on -# upstream v0.9.0 plus three non-invasive patches: `TransactionAction` -# visibility, a new `UpdateSchemaAction`, and a mixed-content -# `FastAppendAction` that accepts both data and equality-delete files. +# 1. `iceberg` → `polynya-dev/iceberg-rust@polynya-patches-v0.10.1`: +# upstream v0.10.1 plus `TransactionAction` visibility, a whole-schema +# `ReplaceSchemaAction`, a `FastAppendAction` that accepts delete files +# as well as data files, a compaction `RewriteFilesAction` (explicit +# data sequence number, removed-file check, summary totals), +# `Table::properties()` for vended credentials, and SigV4 signing. # See `crates/pg2iceberg-iceberg/src/prod/gap_audit.rs` for context. # # 2. `tokio-postgres` + `postgres-replication` → `iambriccardo/rust-postgres` @@ -58,15 +60,15 @@ all = { level = "warn", priority = -1 } # (Supabase's etl maintainer) rather than rolling our own to avoid the # maintenance burden of two forks. [patch.crates-io] -iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches" } +iceberg = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" } # `iceberg-catalog-rest` also has to come from the fork: the # polynya-patches branch threads the REST `loadTable` response's # `config` map through `Table::builder().properties(response_config)`, -# which the published v0.9.0 omits. Without it, vended-credentials +# which upstream omits. Without it, vended-credentials # servers (Polaris, Lakekeeper, ...) return per-table S3 STS tokens # in `config`, but our `extract_creds` sees an empty map because the # `Table::properties()` accessor surfaces nothing. -iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches" } -iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches" } +iceberg-catalog-rest = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" } +iceberg-storage-opendal = { git = "https://github.com/polynya-dev/iceberg-rust", branch = "polynya-patches-v0.10.1" } tokio-postgres = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" } postgres-replication = { git = "https://github.com/iambriccardo/rust-postgres", rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" } diff --git a/crates/pg2iceberg-iceberg/Cargo.toml b/crates/pg2iceberg-iceberg/Cargo.toml index 9216388..b4b3e8d 100644 --- a/crates/pg2iceberg-iceberg/Cargo.toml +++ b/crates/pg2iceberg-iceberg/Cargo.toml @@ -24,7 +24,7 @@ thiserror.workspace = true # iceberg-rust prod-impl wrap. Heavy transitive deps (reqwest, opendal, # arrow) but most of those are already in our graph from parquet. -iceberg = { version = "0.9", optional = true } +iceberg = { version = "0.10", optional = true } # Used by the vended-credentials S3 router (`prod::vended`) to construct # per-table object stores from catalog-vended creds. object_store = { version = "0.11", optional = true, default-features = false, features = ["aws"] } diff --git a/crates/pg2iceberg-iceberg/src/prod/catalog.rs b/crates/pg2iceberg-iceberg/src/prod/catalog.rs index 49a9c72..ac3231c 100644 --- a/crates/pg2iceberg-iceberg/src/prod/catalog.rs +++ b/crates/pg2iceberg-iceberg/src/prod/catalog.rs @@ -12,7 +12,7 @@ //! data and delete manifests at commit time. //! - **Schema evolution.** Translates our `Vec` to a target //! `iceberg::Schema` (via [`crate::apply_schema_changes`]) and submits -//! via the forked `Transaction::update_schema()`. +//! via the forked `Transaction::replace_schema()`. //! - **`load_table` not-found.** Maps `ErrorKind::TableNotFound` and //! `NamespaceNotFound` → `Ok(None)` (the materializer treats not-found //! distinctly from transient errors). @@ -102,7 +102,7 @@ impl Catalog for IcebergRustCata { Ok(None) } - // iceberg-rust 0.9 REST catalog returns + // iceberg-rust's REST catalog (as of 0.9) returns // `ErrorKind::Unexpected` with this exact message on a // 404 from the REST server (instead of the expected // `TableNotFound`). Caught by the testcontainers @@ -334,7 +334,7 @@ impl Catalog for IcebergRustCata let new_iceberg_schema = to_iceberg_schema(&our_schema)?; let tx = Transaction::new(&table); - let action = tx.update_schema().set_schema(new_iceberg_schema); + let action = tx.replace_schema().set_schema(new_iceberg_schema); let tx = action.apply(tx).map_err(map_iceberg_err)?; let updated = tx .commit(self.inner.as_ref()) @@ -386,8 +386,9 @@ impl Catalog for IcebergRustCata let seq_num = snap.sequence_number(); let parent_id = snap.parent_snapshot_id(); let is_replace = matches!(snap.summary().operation, Operation::Replace); - let manifest_list = snap - .load_manifest_list(table.file_io(), table.metadata()) + let manifest_list = table + .manifest_list_reader(&snap) + .load() .await .map_err(map_iceberg_err)?; let mut data_files: Vec = Vec::new(); @@ -603,10 +604,13 @@ impl TransactionAction for ChainedAppendAction { for update in &step_updates { builder = update.clone().apply(builder)?; } + // `local`'s runtime isn't public; pg2iceberg always runs + // inside the tokio runtime iceberg-rust was given. let mut next = Table::builder() .identifier(local.identifier().clone()) .file_io(local.file_io().clone()) - .metadata(builder.build()?.metadata); + .metadata(builder.build()?.metadata) + .runtime(iceberg::Runtime::try_current()?); if let Some(location) = local.metadata_location() { next = next.metadata_location(location); } diff --git a/crates/pg2iceberg-iceberg/src/prod/gap_audit.rs b/crates/pg2iceberg-iceberg/src/prod/gap_audit.rs index 8753dfc..766bc80 100644 --- a/crates/pg2iceberg-iceberg/src/prod/gap_audit.rs +++ b/crates/pg2iceberg-iceberg/src/prod/gap_audit.rs @@ -9,7 +9,8 @@ //! create_table → iceberg::Catalog::create_table ✅ wired //! commit_snapshot (append) → iceberg::transaction::Transaction::fast_append ✅ wired //! commit_snapshot (deletes) → fork-patched FastAppendAction routes by type ✅ wired (via fork) -//! evolve_schema → iceberg::transaction::Transaction::update_schema ✅ wired (via fork) +//! evolve_schema → iceberg::transaction::Transaction::replace_schema ✅ wired (via fork) +//! commit_compaction → fork's Transaction::rewrite_files ✅ wired (via fork) //! snapshots → table.metadata().snapshots() + manifest walk ✅ wired //! ``` //! @@ -19,26 +20,32 @@ //! ## We're on a patched fork of iceberg-rust //! //! The workspace `Cargo.toml` pins `iceberg` to -//! `polynya-dev/iceberg-rust` (branch `polynya-patches`), based on -//! upstream `v0.9.0`. The fork carries three non-invasive patches: +//! `polynya-dev/iceberg-rust` (branch `polynya-patches-v0.10.1`), based on +//! upstream `v0.10.1`. The fork's patches: //! //! 1. **`TransactionAction` visibility.** The trait (and //! `BoxedTransactionAction`) flipped from `pub(crate)` to `pub`. Lets //! downstream crates author custom transaction actions. -//! 2. **`UpdateSchemaAction`.** Takes a target `Schema`, emits +//! 2. **`ReplaceSchemaAction`.** Takes a target `Schema`, emits //! `TableUpdate::AddSchema` + `TableUpdate::SetCurrentSchema(-1)` plus //! `UuidMatch` / `CurrentSchemaIdMatch` / `LastAssignedFieldIdMatch` -//! requirements. Wrapped by a `Transaction::update_schema()` -//! convenience method matching upstream conventions. +//! requirements, via `Transaction::replace_schema()`. (Upstream's +//! `UpdateSchemaAction` evolves a schema column by column and can't +//! rename, promote or make a column optional.) //! 3. **`FastAppendAction` accepts mixed content types.** Files passed to //! `add_data_files` are routed by `content_type()` into //! `added_data_files` (Data) or `added_delete_files` //! (EqualityDeletes / PositionDeletes). `SnapshotProducer` writes them -//! to separate manifests (`build_v{2,3}_data` vs -//! `build_v{2,3}_deletes`) per Iceberg-spec rules. The old -//! "Only data content type is allowed for fast append" check is gone. -//! -//! All three are intended to be upstreamed as separate PRs; once they +//! to separate manifests per Iceberg-spec rules. +//! 4. **`RewriteFilesAction`** for compaction: one `Replace` snapshot that +//! removes files by path and adds outputs at an explicit data sequence +//! number, fails if a removed file is already gone, and keeps the +//! summary's running totals. +//! 5. **`Table::properties()`**: the REST `loadTable` response's `config` +//! (vended credentials). +//! 6. **SigV4 request signing** for the REST catalog. +//! +//! All are intended to be upstreamed as separate PRs; once they //! land we drop the fork and pin to a fresh crates.io release. //! //! ## Schema evolution wiring @@ -52,7 +59,7 @@ //! catalog) — `AddColumn` appends with `field_id = max + 1`, //! `DropColumn` is a soft drop that flips `nullable = true`. //! 3. Translating the evolved `TableSchema` back to an iceberg `Schema` -//! and submitting it via `Transaction::update_schema()` from the fork. +//! and submitting it via `Transaction::replace_schema()` from the fork. //! //! Both catalogs share `apply_schema_changes`, so the two backends never //! drift on field-id allocation or soft-drop semantics. The diff --git a/crates/pg2iceberg-iceberg/src/prod/smoke.rs b/crates/pg2iceberg-iceberg/src/prod/smoke.rs index c70fa76..a1716e4 100644 --- a/crates/pg2iceberg-iceberg/src/prod/smoke.rs +++ b/crates/pg2iceberg-iceberg/src/prod/smoke.rs @@ -1,4 +1,4 @@ -//! Smoke tests that exercise `iceberg-rust 0.9` end-to-end against the +//! Smoke tests that exercise `iceberg-rust` end-to-end against the //! in-process `MemoryCatalog`. These confirm the upstream surface our //! `IcebergRustCatalog` wrapper relies on: //! @@ -201,10 +201,7 @@ async fn fast_append_round_trip_creates_snapshot_with_data_file() { .expect("fast append must produce a snapshot"); assert_eq!(snap.summary().operation, iceberg::spec::Operation::Append); - let manifest_list = snap - .load_manifest_list(table.file_io(), table.metadata()) - .await - .unwrap(); + let manifest_list = table.manifest_list_reader(snap).load().await.unwrap(); assert_eq!(manifest_list.entries().len(), 1); let manifest = manifest_list.entries()[0] .load_manifest(table.file_io()) @@ -272,10 +269,7 @@ async fn fast_append_accepts_mixed_data_and_equality_delete_files() { let table = tx.commit(&catalog).await.unwrap(); let snap = table.metadata().current_snapshot().unwrap(); - let manifest_list = snap - .load_manifest_list(table.file_io(), table.metadata()) - .await - .unwrap(); + let manifest_list = table.manifest_list_reader(snap).load().await.unwrap(); // Two manifests: one Data, one Deletes. assert_eq!(manifest_list.entries().len(), 2); let mut kinds: Vec = diff --git a/crates/pg2iceberg-tests/Cargo.toml b/crates/pg2iceberg-tests/Cargo.toml index 0abd373..3d06477 100644 --- a/crates/pg2iceberg-tests/Cargo.toml +++ b/crates/pg2iceberg-tests/Cargo.toml @@ -33,7 +33,7 @@ tokio-postgres = { version = "0.7", optional = true, default-features = false, f object_store = { version = "0.11", optional = true, default-features = false, features = ["aws"] } # The DST's production backend reads a table's manifests directly, as # ground truth independent of pg2iceberg's own `snapshots()`. -iceberg = { version = "0.9", optional = true } +iceberg = { version = "0.10", optional = true } # Parses pgoutput in the sim-vs-Postgres conformance suite. postgres-replication = { version = "0.6", optional = true, default-features = false } async-trait = { workspace = true, optional = true } @@ -67,7 +67,7 @@ proptest.workspace = true # chaos soak's lifecycle-driving DST: virtual time is the only way # to keep `run_logical_main_loop`'s scheduler-driven ticks # deterministic under proptest seeds. -tokio = { version = "1", features = ["rt", "macros", "time", "test-util", "sync"] } +tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "time", "test-util", "sync"] } [features] # Enables the testcontainers-driven integration tests diff --git a/crates/pg2iceberg-tests/tests/dst.rs b/crates/pg2iceberg-tests/tests/dst.rs index 33ad8e2..501b4d5 100644 --- a/crates/pg2iceberg-tests/tests/dst.rs +++ b/crates/pg2iceberg-tests/tests/dst.rs @@ -679,6 +679,19 @@ impl Storage { } } + /// The current snapshot's summary counts the files it holds. (The sim + /// catalog writes no summaries.) + #[cfg_attr(not(feature = "integration"), allow(unused_variables))] + async fn summary_counts_live_files(&self, ident: &TableIdent) -> Result<(), String> { + match &self.backend { + Backend::Sim { .. } => Ok(()), + #[cfg(feature = "integration")] + Backend::Prod { iceberg } => { + prod_backend::summary_counts_live_files(iceberg, ident).await + } + } + } + /// Object keys of every stored blob. async fn blob_keys(&self) -> Result, String> { Ok(self @@ -733,6 +746,63 @@ mod prod_backend { } } + /// The current snapshot's summary totals equal what its manifests + /// hold. Engines take them as the table's size: ClickHouse 26.2 + /// answered `count()` with `total-records`. + pub async fn summary_counts_live_files( + iceberg: &iceberg::memory::MemoryCatalog, + ident: &TableIdent, + ) -> Result<(), String> { + use iceberg::Catalog as _; + let err = |e: iceberg::Error| e.to_string(); + let ns = NamespaceIdent::from_strs(&ident.namespace.0).map_err(err)?; + let table = iceberg + .load_table(&iceberg::TableIdent::new(ns, ident.name.clone())) + .await + .map_err(err)?; + let Some(snapshot) = table.metadata().current_snapshot() else { + return Ok(()); + }; + let list = table + .manifest_list_reader(snapshot) + .load() + .await + .map_err(err)?; + let mut held: HashMap<&str, u64> = HashMap::new(); + for manifest_file in list.entries() { + let manifest = manifest_file + .load_manifest(table.file_io()) + .await + .map_err(err)?; + for entry in manifest.entries().iter().filter(|e| e.is_alive()) { + let df = entry.data_file(); + let (files, records) = match df.content_type() { + DataContentType::Data => ("total-data-files", "total-records"), + _ => ("total-delete-files", "total-equality-deletes"), + }; + *held.entry(files).or_default() += 1; + *held.entry(records).or_default() += df.record_count(); + } + } + let summary = &snapshot.summary().additional_properties; + for key in [ + "total-records", + "total-data-files", + "total-delete-files", + "total-equality-deletes", + ] { + let claims = summary.get(key).map(String::as_str).unwrap_or("none"); + let holds = held.get(key).copied().unwrap_or(0); + if claims != holds.to_string() { + return Err(format!( + "snapshot {} summary: {key} = {claims}, its files hold {holds}", + snapshot.sequence_number() + )); + } + } + Ok(()) + } + /// The live files of the table's current snapshot, straight from /// its manifests. pub async fn live_files( @@ -749,8 +819,9 @@ mod prod_backend { let Some(snapshot) = table.metadata().current_snapshot() else { return Ok(Vec::new()); }; - let list = snapshot - .load_manifest_list(table.file_io(), table.metadata()) + let list = table + .manifest_list_reader(snapshot) + .load() .await .map_err(err)?; let mut out = Vec::new(); @@ -916,6 +987,26 @@ struct DstHarness { /// A compaction pass another process (a `pg2iceberg compact` job) /// has planned and written but not yet committed. pending_external: Option, + /// The tokio runtime prod-backend cases run iceberg-rust on (see + /// [`tokio_runtime`]), entered for the case. Dropped last. + #[cfg(feature = "integration")] + _tokio: Option>, +} + +/// iceberg-rust runs its work on a tokio runtime — the catalog defaults +/// to the current one, and spawns onto it — but the harness drives futures +/// with `pollster`. Prod-backend cases enter this runtime, whose worker +/// threads run what iceberg-rust spawns. +#[cfg(feature = "integration")] +fn tokio_runtime() -> &'static tokio::runtime::Runtime { + static RUNTIME: std::sync::OnceLock = std::sync::OnceLock::new(); + RUNTIME.get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .expect("tokio runtime") + }) } impl DstHarness { @@ -950,6 +1041,8 @@ impl DstHarness { CoordSchema::default_name(), arc_clock, )); + #[cfg(feature = "integration")] + let tokio = PROD_BACKEND.get().then(|| tokio_runtime().enter()); let storage = Storage::for_case(); let blob_store = storage.blob.clone(); let catalog = storage.catalog.clone(); @@ -1020,6 +1113,8 @@ impl DstHarness { next_bulk_id: 1000, id_gen, pending_external: None, + #[cfg(feature = "integration")] + _tokio: tokio, backfilling: false, backfill_pipeline: None, other_live: BTreeSet::new(), @@ -1074,6 +1169,8 @@ impl DstHarness { CoordSchema::default_name(), arc_clock, )); + #[cfg(feature = "integration")] + let tokio = PROD_BACKEND.get().then(|| tokio_runtime().enter()); let storage = Storage::for_case(); let blob_store = storage.blob.clone(); let catalog = storage.catalog.clone(); @@ -1144,6 +1241,8 @@ impl DstHarness { next_bulk_id: 1000, id_gen, pending_external: None, + #[cfg(feature = "integration")] + _tokio: tokio, backfilling: false, backfill_pipeline: None, other_live: BTreeSet::new(), @@ -1211,8 +1310,11 @@ impl DstHarness { } /// Worker "b"'s cycle, in distributed mode. + /// `Some(0)` without one; `None` if its commit's response was lost. fn materialize_other(&mut self) -> Option { - let mut other = self.other.take()?; + let Some(mut other) = self.other.take() else { + return Some(0); + }; let before = self.current_snapshot(); let n = cycle(&mut other); if self.current_snapshot() != before { @@ -2100,10 +2202,11 @@ fn check_invariants(h: &mut DstHarness) -> Result<(), String> { h.drive(); h.flush_and_ack(); // Drain materializer; safety bound to catch infinite loops. + // A cycle whose commit response was lost (`None`) did work: retry. for _ in 0..1000 { let a = h.materialize(); - let b = h.materialize_other().unwrap_or(0); - if a == Some(0) && b == 0 { + let b = h.materialize_other(); + if a == Some(0) && b == Some(0) { break; } } @@ -2365,6 +2468,11 @@ fn check_invariants(h: &mut DstHarness) -> Result<(), String> { )); } + // 15. The snapshot summary's totals count the table's files: engines + // plan with them, and some answer `count()` from them. + block_on(h.storage.summary_counts_live_files(&ident())) + .map_err(|e| format!("invariant 15: {e}"))?; + Ok(()) } @@ -2715,6 +2823,26 @@ fn compaction_with_no_surviving_rows_commits_on_iceberg() { check_invariants(&mut h).unwrap(); } +/// The current snapshot's summary counts the table's files, after +/// updates, deletes and a compaction. +#[cfg(feature = "integration")] +#[test] +fn snapshot_summary_counts_the_live_files_on_iceberg() { + PROD_BACKEND.set(true); + let mut h = DstHarness::boot(); + for id in 1..=4 { + h.run_step(&Step::Insert { id, qty: 10 }); + } + h.run_step(&Step::DriveFlush); + h.run_step(&Step::MaterializerCycle); + h.run_step(&Step::Update { id: 1, qty: 11 }); + h.run_step(&Step::Delete { id: 2 }); + h.run_step(&Step::DriveFlush); + h.run_step(&Step::MaterializerCycle); + h.run_step(&Step::Compact); + check_invariants(&mut h).unwrap(); +} + /// An UPDATE that moves a row to another partition must delete it from /// the old one: Iceberg scopes an equality delete to its partition. #[test] @@ -3125,6 +3253,21 @@ fn table_mapped_into_the_sink_namespace_replicates() { check_invariants(&mut h).unwrap(); } +/// Worker "b" loses its commit's response, as the harness drains to +/// quiescence: the drain retries it rather than stop with the rows staged. +#[test] +fn draining_retries_a_second_workers_lost_commit() { + DISTRIBUTED.set(true); + SECOND_TABLE.set(1); + let mut h = DstHarness::boot(); + h.run_step(&Step::LoseCommitResponse); + h.run_step(&Step::OtherInsert { id: 1, qty: 0 }); + h.run_step(&Step::OtherInsert { id: 2, qty: 0 }); + h.run_step(&Step::TruncateBoth); + h.run_step(&Step::OtherWorkerCycle); + check_invariants(&mut h).unwrap(); +} + /// One TRUNCATE naming both tables empties both, in the sim's decoded /// stream and, with `wire`, through production's decoder, which splits /// pgoutput's single message per table. diff --git a/crates/pg2iceberg/Cargo.toml b/crates/pg2iceberg/Cargo.toml index 47f3db7..e4a164c 100644 --- a/crates/pg2iceberg/Cargo.toml +++ b/crates/pg2iceberg/Cargo.toml @@ -28,14 +28,14 @@ pg2iceberg-validate.workspace = true # Real iceberg + object_store types — same fork pin as the rest of the # workspace, applied via [patch.crates-io] in the workspace Cargo.toml. -iceberg = { version = "0.9", features = [] } -iceberg-catalog-rest = { version = "0.9", default-features = false } +iceberg = { version = "0.10", features = [] } +iceberg-catalog-rest = { version = "0.10", default-features = false } # StorageFactory for the REST catalog. iceberg-rust 0.9's REST catalog # requires a StorageFactory be provided up front (otherwise table ops # fail at runtime with "StorageFactory must be provided"). We use # OpenDAL's S3 backend which the upstream integration tests also use. # Surfaced by the testcontainers integration test. -iceberg-storage-opendal = { version = "0.9", features = ["opendal-s3"] } +iceberg-storage-opendal = { version = "0.10", features = ["opendal-s3"] } # `aws` enables the S3 backend; `gcp`/`azure` are deferred follow-ons. object_store = { version = "0.11", default-features = false, features = ["aws"] } diff --git a/crates/pg2iceberg/src/run.rs b/crates/pg2iceberg/src/run.rs index aa2dab9..7d86abf 100644 --- a/crates/pg2iceberg/src/run.rs +++ b/crates/pg2iceberg/src/run.rs @@ -83,7 +83,7 @@ pub async fn build_rest_catalog(cfg: &Config) -> Result = cfg.rest_catalog_props().into_iter().collect(); - // iceberg-rust 0.9's REST catalog requires a StorageFactory or + // iceberg-rust's REST catalog requires a StorageFactory or // every file-IO operation panics with "StorageFactory must be // provided". OpenDAL's S3 backend is what the upstream // integration tests use; it picks up endpoint / credentials @@ -92,7 +92,6 @@ pub async fn build_rest_catalog(cfg: &Config) -> Result